Support for driving multiple EL nodes from a single Nimbus BN (status…

…-im#4465)

* Support for driving multiple EL nodes from a single Nimbus BN

Full list of changes:

* Eth1Monitor has been renamed to ELManager to match its current
  responsibilities better.

* The ELManager is no longer optional in the code (it won't have
  a nil value under any circumstances).

* The support for subscribing for headers was removed as it only
  worked with WebSockets and contributed significant complexity
  while bringing only a very minor advantage.

* The `--web3-url` parameter has been deprecated in favor of a
  new `--el` parameter. The new parameter has a reasonable default
  value and supports specifying a different JWT for each connection.
  Each connection can also be configured with a different set of
  responsibilities (e.g. download deposits, validate blocks and/or
  produce blocks). On the command-line, these properties can be
  configured through URL properties stored in the #anchor part of
  the URL. In TOML files, they come with a very natural syntax
  (althrough the URL scheme is also supported).

* The previously scattered EL-related state and logic is now moved
  to `eth1_monitor.nim` (this module will be renamed to `el_manager.nim`
  in a follow-up commit). State is assigned properly either to the
  `ELManager` or the to individual `ELConnection` objects where
  appropriate.

  The ELManager executes all Engine API requests against all attached
  EL nodes, in parallel. It compares their results and if there is a
  disagreement regarding the validity of a certain payload, this is
  detected and the beacon node is protected from publishing a block
  with a potential execution layer consensus bug in it.

  The BN provides metrics per EL node for the number of successful or
  failed requests for each type Engine API requests. If an EL node
  goes offline and connectivity is resoted later, we report the
  problem and the remedy in edge-triggered fashion.

* More progress towards implementing Deneb block production in the VC
  and comparing the value of blocks produced by the EL and the builder
  API.

* Adds a Makefile target for the zhejiang testnet

.gitmodules

@@ -200,7 +200,6 @@
 [submodule "vendor/gnosis-chain-configs"]
 	path = vendor/gnosis-chain-configs
 	url = https://github.com/gnosischain/configs.git
-[submodule "vendor/capella-testnets"]
-	path = vendor/capella-testnets
+[submodule "vendor/withdrawals-testnets"]
+	path = vendor/withdrawals-testnets
 	url = https://github.com/ethpandaops/withdrawals-testnet.git
-	branch = master

Makefile

@@ -673,13 +673,13 @@ sepolia-dev-deposit: | sepolia-build deposit_contract
 clean-sepolia:
 	$(call CLEAN_NETWORK,sepolia)
 
-### Capella devnets
+### Withdrawals testnets
 
-capella-devnet-3:
-	tmuxinator start -p scripts/tmuxinator-el-cl-pair-in-devnet.yml network="vendor/capella-testnets/withdrawal-devnet-3/custom_config_data"
+zhejiang:
+	tmuxinator start -p scripts/tmuxinator-el-cl-pair-in-devnet.yml network="vendor/withdrawals-testnets/zhejiang-testnet/custom_config_data"
 
-clean-capella-devnet-3:
-	scripts/clean-devnet-dir.sh vendor/capella-testnets/withdrawal-devnet-3/custom_config_data
+clean-zhejiang:
+	scripts/clean-devnet-dir.sh vendor/withdrawals-testnets/zhejiang-testnet/custom_config_data
 
 ###
 ### Gnosis chain binary

beacon_chain/beacon_node.nim

@@ -68,7 +68,7 @@ type
     syncCommitteeMsgPool*: ref SyncCommitteeMsgPool
     lightClientPool*: ref LightClientPool
     validatorChangePool*: ref ValidatorChangePool
-    eth1Monitor*: Eth1Monitor
+    elManager*: ELManager
     payloadBuilderRestClient*: RestClientRef
     restServer*: RestServerRef
     keymanagerHost*: ref KeymanagerHost
@@ -90,7 +90,6 @@ type
     restKeysCache*: Table[ValidatorPubKey, ValidatorIndex]
     validatorMonitor*: ref ValidatorMonitor
     stateTtlCache*: StateTtlCache
-    nextExchangeTransitionConfTime*: Moment
     router*: ref MessageRouter
     dynamicFeeRecipientsStore*: ref DynamicFeeRecipientsStore
     externalBuilderRegistrations*:

beacon_chain/beacon_node_light_client.nim

@@ -8,14 +8,12 @@
 {.push raises: [].}
 
 import
-  chronicles,
+  chronicles, web3/engine_api_types,
   ./beacon_node
 
 logScope: topics = "beacnde"
 
 func shouldSyncOptimistically*(node: BeaconNode, wallSlot: Slot): bool =
-  if node.eth1Monitor == nil:
-    return false
   let optimisticHeader = node.lightClient.optimisticHeader
   withForkyHeader(optimisticHeader):
     when lcDataFork > LightClientDataFork.None:
@@ -41,7 +39,7 @@ proc initLightClient*(
 
   let
     optimisticHandler = proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock):
-        Future[void] {.async.} =
+                             Future[void] {.async.} =
       info "New LC optimistic block",
         opt = signedBlock.toBlockId(),
         dag = node.dag.head.bid,
@@ -51,10 +49,9 @@ proc initLightClient*(
           if blck.message.is_execution_block:
             template payload(): auto = blck.message.body.execution_payload
 
-            let eth1Monitor = node.eth1Monitor
-            if eth1Monitor != nil and not payload.block_hash.isZero:
+            if not payload.block_hash.isZero:
               # engine_newPayloadV1
-              discard await eth1Monitor.newExecutionPayload(payload)
+              discard await node.elManager.newExecutionPayload(payload)
 
               # Retain optimistic head for other `forkchoiceUpdated` callers.
               # May temporarily block `forkchoiceUpdatedV1` calls, e.g., Geth:
@@ -67,7 +64,7 @@ proc initLightClient*(
 
               # engine_forkchoiceUpdatedV1
               let beaconHead = node.attestationPool[].getBeaconHead(nil)
-              discard await eth1Monitor.runForkchoiceUpdated(
+              discard await node.elManager.forkchoiceUpdated(
                 headBlockHash = payload.block_hash,
                 safeBlockHash = beaconHead.safeExecutionPayloadHash,
                 finalizedBlockHash = beaconHead.finalizedExecutionPayloadHash)

beacon_chain/conf.nim

@@ -26,6 +26,7 @@ import
   ./spec/datatypes/base,
   ./networking/network_metadata,
   ./validators/slashing_protection_common,
+  ./eth1/el_conf,
   ./filepath
 
 from consensus_object_pools/block_pools_types_light_client
@@ -35,7 +36,7 @@ export
   uri, nat, enr,
   defaultEth2TcpPort, enabledLogLevel, ValidIpAddress,
   defs, parseCmdArg, completeCmdArg, network_metadata,
-  network, BlockHashOrNumber,
+  el_conf, network, BlockHashOrNumber,
   confTomlDefs, confTomlNet, confTomlUri
 
 declareGauge network_name, "network name", ["name"]
@@ -176,14 +177,17 @@ type
       name: "era-dir" .}: Option[InputDir]
 
     web3Urls* {.
-      desc: "One or more execution layer Web3 provider URLs"
-      name: "web3-url" .}: seq[string]
+      desc: "One or more execution layer Engine API URLs"
+      name: "web3-url" .}: seq[EngineApiUrlConfigValue]
 
-    web3ForcePolling* {.
-      hidden
+    elUrls* {.
+      desc: "One or more execution layer Engine API URLs"
+      name: "el" .}: seq[EngineApiUrlConfigValue]
+
+    noEl* {.
       defaultValue: false
-      desc: "Force the use of polling when determining the head block of Eth1"
-      name: "web3-force-polling" .}: bool
+      desc: "Don't use an EL. The node will remain optimistically synced and won't be able to perform validator duties"
+      name: "no-el" .}: bool
 
     optimistic* {.
       hidden # deprecated > 22.12
@@ -234,7 +238,7 @@ type
     # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.2/src/engine/authentication.md#key-distribution
     jwtSecret* {.
       desc: "A file containing the hex-encoded 256 bit secret key to be used for verifying/generating JWT tokens"
-      name: "jwt-secret" .}: Option[string]
+      name: "jwt-secret" .}: Option[InputFile]
 
     case cmd* {.
       command
@@ -1302,7 +1306,7 @@ func defaultFeeRecipient*(conf: AnyConf): Eth1Address =
 proc loadJwtSecret*(
     rng: var HmacDrbgContext,
     dataDir: string,
-    jwtSecret: Option[string],
+    jwtSecret: Option[InputFile],
     allowCreate: bool): Option[seq[byte]] =
   # Some Web3 endpoints aren't compatible with JWT, but if explicitly chosen,
   # use it regardless.
@@ -1317,8 +1321,18 @@ proc loadJwtSecret*(
   else:
     none(seq[byte])
 
-template loadJwtSecret*(
+proc loadJwtSecret*(
     rng: var HmacDrbgContext,
     config: BeaconNodeConf,
     allowCreate: bool): Option[seq[byte]] =
   rng.loadJwtSecret(string(config.dataDir), config.jwtSecret, allowCreate)
+
+proc engineApiUrls*(config: BeaconNodeConf): seq[EngineApiUrl] =
+  let elUrls = if config.noEl:
+    return newSeq[EngineApiUrl]()
+  elif config.elUrls.len == 0 and config.web3Urls.len == 0:
+    @[defaultEngineApiUrl]
+  else:
+    config.elUrls
+
+  (elUrls & config.web3Urls).toFinalEngineApiUrls(config.jwtSecret)

beacon_chain/conf_light_client.nim

@@ -123,12 +123,21 @@ type LightClientConf* = object
 
   # Execution layer
   web3Urls* {.
-    desc: "One or more execution layer Web3 provider URLs"
-    name: "web3-url" .}: seq[string]
+    desc: "One or more execution layer Engine API URLs"
+    name: "web3-url" .}: seq[EngineApiUrlConfigValue]
+
+  elUrls* {.
+    desc: "One or more execution layer Engine API URLs"
+    name: "el" .}: seq[EngineApiUrlConfigValue]
+
+  noEl* {.
+    defaultValue: false
+    desc: "Don't use an EL. The node will remain optimistically synced and won't be able to perform validator duties"
+    name: "no-el" .}: bool
 
   jwtSecret* {.
     desc: "A file containing the hex-encoded 256 bit secret key to be used for verifying/generating JWT tokens"
-    name: "jwt-secret" .}: Option[string]
+    name: "jwt-secret" .}: Option[InputFile]
 
   # Testing
   stopAtEpoch* {.
@@ -145,3 +154,13 @@ template loadJwtSecret*(
     config: LightClientConf,
     allowCreate: bool): Option[seq[byte]] =
   rng.loadJwtSecret(string(config.dataDir), config.jwtSecret, allowCreate)
+
+proc engineApiUrls*(config: LightClientConf): seq[EngineApiUrl] =
+  let elUrls = if config.noEl:
+    return newSeq[EngineApiUrl]()
+  elif config.elUrls.len == 0 and config.web3Urls.len == 0:
+    @[defaultEngineApiUrl]
+  else:
+    config.elUrls
+
+  (elUrls & config.web3Urls).toFinalEngineApiUrls(config.jwtSecret)