x

Author: alpe

apps/evm/go.mod

@@ -2,10 +2,10 @@ module github.com/evstack/ev-node/apps/evm
 
 go 1.25.7
 
-// replace (
-// 	github.com/evstack/ev-node => ../../
-// 	github.com/evstack/ev-node/execution/evm => ../../execution/evm
-// )
+replace (
+	github.com/evstack/ev-node => ../../
+	github.com/evstack/ev-node/execution/evm => ../../execution/evm
+)
 
 require (
 	github.com/ethereum/go-ethereum v1.17.2

apps/evm/go.sum

@@ -472,12 +472,8 @@ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJ
 github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8=
 github.com/ethereum/go-ethereum v1.17.2 h1:ag6geu0kn8Hv5FLKTpH+Hm2DHD+iuFtuqKxEuwUsDOI=
 github.com/ethereum/go-ethereum v1.17.2/go.mod h1:KHcRXfGOUfUmKg51IhQ0IowiqZ6PqZf08CMtk0g5K1o=
-github.com/evstack/ev-node v1.1.0-rc.1 h1:NtPuuDLqN2h4/edu5zxRlZAxmLkTG3ncXBO2PlCDvVs=
-github.com/evstack/ev-node v1.1.0-rc.1/go.mod h1:6rhWWzuyiqNn/erDmWCk1aLxUuQphyOGIRq56/smSyk=
 github.com/evstack/ev-node/core v1.0.0 h1:s0Tx0uWHme7SJn/ZNEtee4qNM8UO6PIxXnHhPbbKTz8=
 github.com/evstack/ev-node/core v1.0.0/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY=
-github.com/evstack/ev-node/execution/evm v1.0.0 h1:UTAdCrnPsLoGzSgsBx4Kv76jkXpMmHBIpNv3MxyzWPo=
-github.com/evstack/ev-node/execution/evm v1.0.0/go.mod h1:UrqkiepfTMiot6M8jnswgu3VU8SSucZpaMIHIl22/1A=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=

docs/guides/raft_production.md

@@ -33,7 +33,7 @@ Raft is configured via CLI flags or the `config.toml` file under the `[raft]` (o
 | `--evnode.raft.raft_addr` | `raft.raft_addr` | TCP address for Raft transport. | `0.0.0.0:5001` (Bind to private IP) |
 | `--evnode.raft.raft_dir` | `raft.raft_dir` | Directory for Raft data. | `/data/raft` (Must be persistent) |
 | `--evnode.raft.peers` | `raft.peers` | Comma-separated list of peer addresses in format `nodeID@host:port`. | `node-1@10.0.0.1:5001,node-2@10.0.0.2:5001,node-3@10.0.0.3:5001` |
-| `--evnode.raft.bootstrap` | `raft.bootstrap` | Bootstrap the cluster. **Required** for initial setup. | `true` (See Limitations) |
+| `--evnode.raft.bootstrap` | `raft.bootstrap` | Compatibility flag. Startup mode is selected automatically from persisted raft configuration state. | optional |
 
 ### Timeout Tuning
 
@@ -55,11 +55,15 @@ Ideally, a failover should complete within `2 * BlockTime` to minimize user impa
 
 ## Production Deployment Principles
 
-### 1. Static Peering & Bootstrap
-Current implementation requires **Bootstrap Mode** (`--evnode.raft.bootstrap=true`) for all nodes participating in the cluster initialization.
-*   **All nodes** should list the full set of peers in `--evnode.raft.peers`.
+### 1. Static Peering & Automatic Startup Mode
+Use static peering with automatic mode selection from local raft configuration:
+*   If local raft configuration already exists in `--evnode.raft.raft_dir`, the node starts in rejoin mode.
+*   If no local raft configuration exists yet, the node bootstraps from configured peers.
+*   `--evnode.raft.bootstrap` is retained for compatibility but does not control mode selection.
+*   **All configured cluster members** should list the full set of peers in `--evnode.raft.peers`.
 *   The `peers` list format is strict: `NodeID@Host:Port`.
-*   **Limitation**: Dynamic addition of peers (Run-time Membership Changes) via RPC/CLI is not currently exposed. The cluster membership is static based on the initial bootstrap configuration.
+*   **Limitation**: Dynamic addition of peers (run-time membership changes) via RPC/CLI is not currently exposed.
+*   **Not supported**: Joining an existing cluster as a brand-new node that was not part of the initial static membership.
 
 ### 2. Infrastructure Requirements
 *   **Encrypted Network (CRITICAL)**: Raft traffic is **unencrypted** (plain TCP). You **MUST** run the cluster inside a private network, VPN, or encrypted mesh (e.g., WireGuard, Tailscale). **Never expose Raft ports to the public internet**; doing so allows attackers to hijack the cluster consensus.
@@ -86,13 +90,13 @@ Monitor the following metrics (propagated via Prometheus if enabled):
 
 ```bash
 ./ev-node start \
-  --node.aggregator \
-  --raft.enable \
-  --raft.node_id="node-1" \
-  --raft.raft_addr="0.0.0.0:5001" \
-  --raft.raft_dir="/var/lib/ev-node/raft" \
-  --raft.bootstrap=true \
-  --raft.peers="node-1@10.0.1.1:5001,node-2@10.0.1.2:5001,node-3@10.0.1.3:5001" \
-  --p2p.listen_address="/ip4/0.0.0.0/tcp/26656" \
+  --rollkit.node.aggregator=true \
+  --evnode.raft.enable=true \
+  --evnode.raft.node_id="node-1" \
+  --evnode.raft.raft_addr="0.0.0.0:5001" \
+  --evnode.raft.raft_dir="/var/lib/ev-node/raft" \
+  --evnode.raft.bootstrap=true \
+  --evnode.raft.peers="node-1@10.0.1.1:5001,node-2@10.0.1.2:5001,node-3@10.0.1.3:5001" \
+  --rollkit.p2p.listen_address="/ip4/0.0.0.0/tcp/26656" \
   ...other flags
 ```

docs/learn/config.md

@@ -1321,7 +1321,7 @@ _Constant:_ `FlagRaftDir`
 ### Raft Bootstrap
 
 **Description:**
-If true, bootstraps a new Raft cluster. Only set this on the very first node when initializing a new cluster.
+Legacy compatibility flag. Startup mode is now auto-selected from persisted raft configuration state, so this flag is not used to choose bootstrap vs rejoin.
 
 **YAML:**
 
@@ -1352,6 +1352,16 @@ raft:
 _Default:_ `""` (empty)
 _Constant:_ `FlagRaftPeers`
 
+### Raft Startup Mode
+
+Raft startup mode is selected automatically from local raft configuration state:
+
+* If the node already has persisted raft configuration in `raft.raft_dir`, it starts in rejoin mode.
+* If no raft configuration exists yet, it bootstraps a cluster from configured peers.
+* `raft.bootstrap` is retained for compatibility but does not control mode selection.
+
+`--evnode.raft.rejoin` has been removed.
+
 ### Raft Snap Count
 
 **Description:**

node/failover.go

@@ -183,22 +183,29 @@ func setupFailoverState(
 	}, nil
 }
 
+// shouldStartSyncInPublisherMode avoids startup deadlock when a raft leader boots
+// with empty sync stores and no peer can serve height 1 yet.
 func (f *failoverState) shouldStartSyncInPublisherMode(ctx context.Context) bool {
 	if !f.isAggregator || f.raftNode == nil || !f.raftNode.IsLeader() {
 		return false
 	}
 
-	height, err := f.store.Height(ctx)
+	storeHeight, err := f.store.Height(ctx)
 	if err != nil {
-		f.logger.Warn().Err(err).Msg("cannot determine local height; keeping blocking sync startup")
+		f.logger.Warn().Err(err).Msg("cannot determine store height; keeping blocking sync startup")
 		return false
 	}
-	if height > 0 {
+	headerHeight := f.headerSyncService.Store().Height()
+	dataHeight := f.dataSyncService.Store().Height()
+	if headerHeight > 0 || dataHeight > 0 {
 		return false
 	}
 
 	f.logger.Info().
-		Msg("raft leader with empty store: starting sync services in publisher mode")
+		Uint64("store_height", storeHeight).
+		Uint64("header_height", headerHeight).
+		Uint64("data_height", dataHeight).
+		Msg("raft-enabled aggregator with empty sync stores: starting sync services in publisher mode")
 	return true
 }
 

pkg/config/config.go

@@ -400,7 +400,7 @@ type RaftConfig struct {
 	NodeID             string        `mapstructure:"node_id" yaml:"node_id" comment:"Unique identifier for this node in the Raft cluster"`
 	RaftAddr           string        `mapstructure:"raft_addr" yaml:"raft_addr" comment:"Address for Raft communication (host:port)"`
 	RaftDir            string        `mapstructure:"raft_dir" yaml:"raft_dir" comment:"Directory for Raft logs and snapshots"`
-	Bootstrap          bool          `mapstructure:"bootstrap" yaml:"bootstrap" comment:"Bootstrap a new Raft cluster (only for the first node)"`
+	Bootstrap          bool          `mapstructure:"bootstrap" yaml:"bootstrap" comment:"Bootstrap a new static Raft cluster during initial bring-up"`
 	Peers              string        `mapstructure:"peers" yaml:"peers" comment:"Comma-separated list of peer Raft addresses (nodeID@host:port)"`
 	SnapCount          uint64        `mapstructure:"snap_count" yaml:"snap_count" comment:"Number of log entries between snapshots"`
 	SendTimeout        time.Duration `mapstructure:"send_timeout" yaml:"send_timeout" comment:"Max duration to wait for a message to be sent to a peer"`
@@ -646,7 +646,7 @@ func AddFlags(cmd *cobra.Command) {
 	cmd.Flags().String(FlagRaftNodeID, def.Raft.NodeID, "unique identifier for this node in the Raft cluster")
 	cmd.Flags().String(FlagRaftAddr, def.Raft.RaftAddr, "address for Raft communication (host:port)")
 	cmd.Flags().String(FlagRaftDir, def.Raft.RaftDir, "directory for Raft logs and snapshots")
-	cmd.Flags().Bool(FlagRaftBootstrap, def.Raft.Bootstrap, "bootstrap a new Raft cluster (only for the first node)")
+	cmd.Flags().Bool(FlagRaftBootstrap, def.Raft.Bootstrap, "bootstrap a new static Raft cluster during initial bring-up")
 	cmd.Flags().String(FlagRaftPeers, def.Raft.Peers, "comma-separated list of peer Raft addresses (nodeID@host:port)")
 	cmd.Flags().Uint64(FlagRaftSnapCount, def.Raft.SnapCount, "number of log entries between snapshots")
 	cmd.Flags().Duration(FlagRaftSendTimeout, def.Raft.SendTimeout, "max duration to wait for a message to be sent to a peer")

pkg/config/config_test.go

@@ -122,6 +122,18 @@ func TestAddFlags(t *testing.T) {
 	assertFlagValue(t, flags, FlagRPCAddress, DefaultConfig().RPC.Address)
 	assertFlagValue(t, flags, FlagRPCEnableDAVisualization, DefaultConfig().RPC.EnableDAVisualization)
 
+	// Raft flags
+	assertFlagValue(t, flags, FlagRaftEnable, DefaultConfig().Raft.Enable)
+	assertFlagValue(t, flags, FlagRaftNodeID, DefaultConfig().Raft.NodeID)
+	assertFlagValue(t, flags, FlagRaftAddr, DefaultConfig().Raft.RaftAddr)
+	assertFlagValue(t, flags, FlagRaftDir, DefaultConfig().Raft.RaftDir)
+	assertFlagValue(t, flags, FlagRaftBootstrap, DefaultConfig().Raft.Bootstrap)
+	assertFlagValue(t, flags, FlagRaftPeers, DefaultConfig().Raft.Peers)
+	assertFlagValue(t, flags, FlagRaftSnapCount, DefaultConfig().Raft.SnapCount)
+	assertFlagValue(t, flags, FlagRaftSendTimeout, DefaultConfig().Raft.SendTimeout)
+	assertFlagValue(t, flags, FlagRaftHeartbeatTimeout, DefaultConfig().Raft.HeartbeatTimeout)
+	assertFlagValue(t, flags, FlagRaftLeaderLeaseTimeout, DefaultConfig().Raft.LeaderLeaseTimeout)
+
 	// Pruning flags
 	assertFlagValue(t, flags, FlagPruningMode, DefaultConfig().Pruning.Mode)
 	assertFlagValue(t, flags, FlagPruningKeepRecent, DefaultConfig().Pruning.KeepRecent)

pkg/raft/election.go

@@ -64,6 +64,7 @@ func (d *DynamicLeaderElection) Run(ctx context.Context) error {
 		close(errCh)
 	}()
 
+	var runnable Runnable
 	startWorker := func(name string, workerFunc func(ctx context.Context) error) {
 		workerCancel()
 		wg.Wait() // Ensure previous worker is fully stopped
@@ -83,11 +84,26 @@ func (d *DynamicLeaderElection) Run(ctx context.Context) error {
 			}
 		}(workerCtx)
 	}
+	startFollower := func() error {
+		var err error
+		if runnable, err = d.followerFactory(); err != nil {
+			return err
+		}
+		// avoids validating against stale raft state.
+		if err = d.node.waitForMsgsLanded(d.node.Config().SendTimeout); err != nil {
+			// this wait can legitimately time out
+			d.logger.Debug().Err(err).Msg("timed out waiting for raft messages before follower verification; continuing")
+		}
+		if err = d.verifyState(ctx, runnable); err != nil {
+			return err
+		}
+		startWorker("follower", runnable.Run)
+		return nil
+	}
 	ticker := time.NewTicker(300 * time.Millisecond)
 	defer ticker.Stop()
 	d.running.Store(true)
 	defer d.running.Store(false)
-	var runnable Runnable
 	for {
 		select {
 		case becameLeader := <-d.node.leaderCh():
@@ -144,15 +160,9 @@ func (d *DynamicLeaderElection) Run(ctx context.Context) error {
 			} else if !becameLeader && !isCurrentlyLeader && !isStarted { // start as a follower
 				d.logger.Info().Msg("starting follower operations")
 				isStarted = true
-				var err error
-				if runnable, err = d.followerFactory(); err != nil {
+				if err := startFollower(); err != nil {
 					return err
 				}
-
-				if err = d.verifyState(ctx, runnable); err != nil {
-					return err
-				}
-				startWorker("follower", runnable.Run)
 			}
 			// LeaderCh fires only when leader changes not on initial election
 		case <-ticker.C:
@@ -171,11 +181,9 @@ func (d *DynamicLeaderElection) Run(ctx context.Context) error {
 
 			d.logger.Info().Msg("starting follower operations")
 			isStarted = true
-			var err error
-			if runnable, err = d.followerFactory(); err != nil {
+			if err := startFollower(); err != nil {
 				return err
 			}
-			startWorker("follower", runnable.Run)
 		case err := <-errCh:
 			return err
 		case <-ctx.Done():
@@ -189,14 +197,32 @@ func (d *DynamicLeaderElection) verifyState(ctx context.Context, runnable Runnab
 	// Verify sync state before starting follower operations
 	raftState := d.node.GetState()
 	if raftState == nil || raftState.Height == 0 {
-		// Initial/empty raft state - skip recovery and let normal sync handle it.
-		// This can happen during rolling restarts when the Raft FSM hasn't replayed logs yet.
-		d.logger.Info().Msg("raft state at height 0, skipping recovery to allow normal sync")
-		return nil
+		waitTimeout := d.node.Config().SendTimeout
+		deadline := time.NewTimer(waitTimeout)
+		defer deadline.Stop()
+		ticker := time.NewTicker(min(50*time.Millisecond, max(waitTimeout/4, time.Millisecond)))
+		defer ticker.Stop()
+
+		for raftState == nil || raftState.Height == 0 {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-deadline.C:
+				d.logger.Info().Msg("raft state still at height 0 after wait; skipping recovery to allow normal sync")
+				return nil
+			case <-ticker.C:
+				raftState = d.node.GetState()
+			}
+		}
 	}
 	diff, err := runnable.IsSynced(raftState)
 	if err != nil {
-		return err
+		d.logger.Warn().Err(err).Uint64("raft_height", raftState.Height).Msg("sync check failed, attempting recovery from raft canonical state")
+		if recErr := runnable.Recover(ctx, raftState); recErr != nil {
+			return errors.Join(err, fmt.Errorf("recovery after sync-check failure: %w", recErr))
+		}
+		d.logger.Info().Msg("recovery successful after sync-check failure")
+		return nil
 	}
 	if diff == 0 {
 		return nil

pkg/raft/election_test.go

@@ -25,6 +25,9 @@ func TestDynamicLeaderElectionRun(t *testing.T) {
 				m.EXPECT().leaderCh().Return((<-chan bool)(leaderCh))
 				m.EXPECT().leaderID().Return("other")
 				m.EXPECT().NodeID().Return("self")
+				m.EXPECT().Config().Return(testCfg())
+				m.EXPECT().waitForMsgsLanded(2 * time.Millisecond).Return(nil)
+				m.EXPECT().GetState().Return(&RaftBlockState{})
 
 				started := make(chan struct{})
 				follower := &testRunnable{startedCh: started}

pkg/raft/node.go

@@ -109,13 +109,8 @@ func (n *Node) Start(_ context.Context) error {
 	if n == nil {
 		return nil
 	}
-	if !n.config.Bootstrap {
-		// it is intended to fail fast here. at this stage only bootstrap mode is supported.
-		return fmt.Errorf("raft cluster requires bootstrap mode")
-	}
-
 	if future := n.raft.GetConfiguration(); future.Error() == nil && len(future.Configuration().Servers) > 0 {
-		n.logger.Info().Msg("cluster already bootstrapped, skipping")
+		n.logger.Info().Msg("raft node started with existing local state")
 		return nil
 	}