Optimistic parallelization of transactions to improve performance (hy…

…perledger#7296) Optimistic transaction parallelization execution during block processing to improve the performances. This feature can enabled with a flag --Xbonsai-parallel-tx-processing-enabled=true Signed-off-by: Karim Taam <karim.t2am@gmail.com> Co-authored-by: Ameziane H <ameziane.hamlat@consensys.net> Co-authored-by: garyschulte <garyschulte@gmail.com> Signed-off-by: gconnect <agatevureglory@gmail.com>
gconnect · Aug 26, 2024 · 5cdc82c · 5cdc82c
1 parent e85dd37
commit 5cdc82c
Show file tree

Hide file tree

Showing 93 changed files with 2,629 additions and 329 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@
 - Added EIP-7702 [#7237](https://github.com/hyperledger/besu/pull/7237)
 - Implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262)
 - Add trie log pruner metrics [#7352](https://github.com/hyperledger/besu/pull/7352)
+- `--Xbonsai-parallel-tx-processing-enabled` option enables executing transactions in parallel during block processing for Bonsai nodes
 
 ### Bug fixes
 - Fix `eth_call` deserialization to correctly ignore unknown fields in the transaction object. [#7323](https://github.com/hyperledger/besu/pull/7323)

diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
@@ -1901,6 +1901,8 @@ public BesuControllerBuilder getControllerBuilder() {
         .privacyParameters(privacyParameters())
         .clock(Clock.systemUTC())
         .isRevertReasonEnabled(isRevertReasonEnabled)
+        .isParallelTxProcessingEnabled(
+            dataStorageConfiguration.getUnstable().isParallelTxProcessingEnabled())
         .storageProvider(storageProvider)
         .gasLimitCalculator(
             miningParametersSupplier.get().getTargetGasLimit().isPresent()

diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
@@ -122,6 +122,14 @@ public static class Unstable {
             "Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})")
     private boolean bonsaiCodeUsingCodeHashEnabled = DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
 
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-parallel-tx-processing-enabled"},
+        arity = "1",
+        description =
+            "Enables parallelization of transactions to optimize processing speed by concurrently loading and executing necessary data in advance. (default: ${DEFAULT-VALUE})")
+    private Boolean isParallelTxProcessingEnabled = false;
+
     /** Default Constructor. */
     Unstable() {}
   }
@@ -142,40 +150,48 @@ public static DataStorageOptions create() {
    * @param syncMode the sync mode
    */
   public void validate(final CommandLine commandLine, final SyncMode syncMode) {
-    if (DataStorageFormat.BONSAI == dataStorageFormat && bonsaiLimitTrieLogsEnabled) {
-      if (SyncMode.FULL == syncMode) {
-        throw new CommandLine.ParameterException(
-            commandLine,
-            String.format(
-                "Cannot enable %s with sync-mode %s. You must set %s or use a different sync-mode",
-                BONSAI_LIMIT_TRIE_LOGS_ENABLED,
-                SyncMode.FULL,
-                BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false"));
-      }
-      if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) {
-        throw new CommandLine.ParameterException(
-            commandLine,
-            String.format(
-                BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
-                MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT));
-      }
-      if (bonsaiTrieLogPruningWindowSize <= 0) {
-        throw new CommandLine.ParameterException(
-            commandLine,
-            String.format(
-                BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
-                bonsaiTrieLogPruningWindowSize));
+    if (DataStorageFormat.BONSAI == dataStorageFormat) {
+      if (bonsaiLimitTrieLogsEnabled) {
+        if (SyncMode.FULL == syncMode) {
+          throw new CommandLine.ParameterException(
+              commandLine,
+              String.format(
+                  "Cannot enable %s with sync-mode %s. You must set %s or use a different sync-mode",
+                  BONSAI_LIMIT_TRIE_LOGS_ENABLED,
+                  SyncMode.FULL,
+                  BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false"));
+        }
+        if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) {
+          throw new CommandLine.ParameterException(
+              commandLine,
+              String.format(
+                  BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
+                  MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT));
+        }
+        if (bonsaiTrieLogPruningWindowSize <= 0) {
+          throw new CommandLine.ParameterException(
+              commandLine,
+              String.format(
+                  BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
+                  bonsaiTrieLogPruningWindowSize));
+        }
+        if (bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) {
+          throw new CommandLine.ParameterException(
+              commandLine,
+              String.format(
+                  BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
+                      + "=%d must be greater than "
+                      + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
+                      + "=%d",
+                  bonsaiTrieLogPruningWindowSize,
+                  bonsaiMaxLayersToLoad));
+        }
       }
-      if (bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) {
+    } else {
+      if (unstableOptions.isParallelTxProcessingEnabled) {
         throw new CommandLine.ParameterException(
             commandLine,
-            String.format(
-                BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
-                    + "=%d must be greater than "
-                    + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
-                    + "=%d",
-                bonsaiTrieLogPruningWindowSize,
-                bonsaiMaxLayersToLoad));
+            "Transaction parallelization is not supported unless operating in a 'diffbased' mode, such as Bonsai.");
       }
     }
   }
@@ -198,6 +214,8 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai
         domainObject.getUnstable().getBonsaiFullFlatDbEnabled();
     dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
         domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
+    dataStorageOptions.unstableOptions.isParallelTxProcessingEnabled =
+        domainObject.getUnstable().isParallelTxProcessingEnabled();
 
     return dataStorageOptions;
   }
@@ -214,6 +232,7 @@ public DataStorageConfiguration toDomainObject() {
             ImmutableDataStorageConfiguration.Unstable.builder()
                 .bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled)
                 .bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
+                .isParallelTxProcessingEnabled(unstableOptions.isParallelTxProcessingEnabled)
                 .build())
         .build();
   }

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
@@ -203,6 +203,9 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
 
   private int numberOfBlocksToCache = 0;
 
+  /** whether parallel transaction processing is enabled or not */
+  protected boolean isParallelTxProcessingEnabled;
+
   /** Instantiates a new Besu controller builder. */
   protected BesuControllerBuilder() {}
 
@@ -512,6 +515,20 @@ public BesuControllerBuilder randomPeerPriority(final Boolean randomPeerPriority
     return this;
   }
 
+  /**
+   * Sets whether parallel transaction processing is enabled. When parallel transaction processing
+   * is enabled, transactions within a block can be processed in parallel and potentially improving
+   * performance
+   *
+   * @param isParallelTxProcessingEnabled true to enable parallel transaction
+   * @return the besu controller
+   */
+  public BesuControllerBuilder isParallelTxProcessingEnabled(
+      final boolean isParallelTxProcessingEnabled) {
+    this.isParallelTxProcessingEnabled = isParallelTxProcessingEnabled;
+    return this;
+  }
+
   /**
    * Build besu controller.
    *

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilder.java
@@ -136,7 +136,9 @@ protected ProtocolSchedule createProtocolSchedule() {
         isRevertReasonEnabled,
         evmConfiguration,
         miningParameters,
-        badBlockManager);
+        badBlockManager,
+        isParallelTxProcessingEnabled,
+        metricsSystem);
   }
 
   @Override

diff --git a/...src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/...src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java
@@ -358,6 +358,15 @@ public BesuControllerBuilder isRevertReasonEnabled(final boolean isRevertReasonE
     return super.isRevertReasonEnabled(isRevertReasonEnabled);
   }
 
+  @Override
+  public BesuControllerBuilder isParallelTxProcessingEnabled(
+      final boolean isParallelTxProcessingEnabled) {
+    besuControllerBuilderSchedule
+        .values()
+        .forEach(b -> b.isParallelTxProcessingEnabled(isParallelTxProcessingEnabled));
+    return super.isParallelTxProcessingEnabled(isParallelTxProcessingEnabled);
+  }
+
   @Override
   public BesuControllerBuilder gasLimitCalculator(final GasLimitCalculator gasLimitCalculator) {
     besuControllerBuilderSchedule.values().forEach(b -> b.gasLimitCalculator(gasLimitCalculator));

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/IbftBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/IbftBesuControllerBuilder.java
@@ -291,7 +291,9 @@ protected ProtocolSchedule createProtocolSchedule() {
         bftExtraDataCodec().get(),
         evmConfiguration,
         miningParameters,
-        badBlockManager);
+        badBlockManager,
+        isParallelTxProcessingEnabled,
+        metricsSystem);
   }
 
   @Override

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MainnetBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MainnetBesuControllerBuilder.java
@@ -99,7 +99,9 @@ protected ProtocolSchedule createProtocolSchedule() {
         isRevertReasonEnabled,
         evmConfiguration,
         miningParameters,
-        badBlockManager);
+        badBlockManager,
+        isParallelTxProcessingEnabled,
+        metricsSystem);
   }
 
   @Override

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java
@@ -177,7 +177,9 @@ protected ProtocolSchedule createProtocolSchedule() {
         privacyParameters,
         isRevertReasonEnabled,
         miningParameters,
-        badBlockManager);
+        badBlockManager,
+        isParallelTxProcessingEnabled,
+        metricsSystem);
   }
 
   @Override

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java
@@ -329,7 +329,9 @@ protected ProtocolSchedule createProtocolSchedule() {
         bftExtraDataCodec().get(),
         evmConfiguration,
         miningParameters,
-        badBlockManager);
+        badBlockManager,
+        isParallelTxProcessingEnabled,
+        metricsSystem);
   }
 
   @Override

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java
@@ -374,6 +374,13 @@ public BesuControllerBuilder isRevertReasonEnabled(final boolean isRevertReasonE
     return propagateConfig(z -> z.isRevertReasonEnabled(isRevertReasonEnabled));
   }
 
+  @Override
+  public BesuControllerBuilder isParallelTxProcessingEnabled(
+      final boolean isParallelTxProcessingEnabled) {
+    super.isParallelTxProcessingEnabled(isParallelTxProcessingEnabled);
+    return propagateConfig(z -> z.isParallelTxProcessingEnabled(isParallelTxProcessingEnabled));
+  }
+
   @Override
   public BesuControllerBuilder gasLimitCalculator(final GasLimitCalculator gasLimitCalculator) {
     super.gasLimitCalculator(gasLimitCalculator);

diff --git a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java
@@ -35,6 +35,7 @@
 import org.hyperledger.besu.ethereum.forkid.ForkIdManager;
 import org.hyperledger.besu.ethereum.mainnet.DefaultProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule;
+import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 
 import java.util.Collection;
 import java.util.List;
@@ -177,12 +178,21 @@ private static MilestoneStreamingTransitionProtocolSchedule createSchedule(
         new MilestoneStreamingProtocolSchedule(
             (DefaultProtocolSchedule)
                 MainnetProtocolSchedule.fromConfig(
-                    configOptions, MiningParameters.MINING_DISABLED, new BadBlockManager()));
+                    configOptions,
+                    MiningParameters.MINING_DISABLED,
+                    new BadBlockManager(),
+                    false,
+                    new NoOpMetricsSystem()));
     MilestoneStreamingProtocolSchedule postMergeProtocolSchedule =
         new MilestoneStreamingProtocolSchedule(
             (DefaultProtocolSchedule)
                 MergeProtocolSchedule.create(
-                    configOptions, false, MiningParameters.MINING_DISABLED, new BadBlockManager()));
+                    configOptions,
+                    false,
+                    MiningParameters.MINING_DISABLED,
+                    new BadBlockManager(),
+                    false,
+                    new NoOpMetricsSystem()));
     final MilestoneStreamingTransitionProtocolSchedule schedule =
         new MilestoneStreamingTransitionProtocolSchedule(
             preMergeProtocolSchedule, postMergeProtocolSchedule);

diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
@@ -276,6 +276,8 @@ public void initMocks() throws Exception {
     when(mockControllerBuilder.privacyParameters(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.clock(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.isRevertReasonEnabled(false)).thenReturn(mockControllerBuilder);
+    when(mockControllerBuilder.isParallelTxProcessingEnabled(false))
+        .thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.storageProvider(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.gasLimitCalculator(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.requiredBlocks(any())).thenReturn(mockControllerBuilder);

diff --git a/...us/clique/src/main/java/org/hyperledger/besu/consensus/clique/CliqueProtocolSchedule.java b/...us/clique/src/main/java/org/hyperledger/besu/consensus/clique/CliqueProtocolSchedule.java
@@ -36,6 +36,7 @@
 import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
+import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 import java.math.BigInteger;
 import java.util.HashMap;
@@ -64,6 +65,9 @@ public class CliqueProtocolSchedule {
    * @param evmConfiguration the evm configuration
    * @param miningParameters the mining parameters
    * @param badBlockManager the cache to use to keep invalid blocks
+   * @param isParallelTxProcessingEnabled indicates whether parallel transaction is enabled
+   * @param metricsSystem A metricSystem instance to be able to expose metrics in the underlying
+   *     calls
    * @return the protocol schedule
    */
   public static ProtocolSchedule create(
@@ -74,7 +78,9 @@ public static ProtocolSchedule create(
       final boolean isRevertReasonEnabled,
       final EvmConfiguration evmConfiguration,
       final MiningParameters miningParameters,
-      final BadBlockManager badBlockManager) {
+      final BadBlockManager badBlockManager,
+      final boolean isParallelTxProcessingEnabled,
+      final MetricsSystem metricsSystem) {
 
     final CliqueConfigOptions cliqueConfig = config.getCliqueConfigOptions();
 
@@ -110,7 +116,9 @@ public static ProtocolSchedule create(
             isRevertReasonEnabled,
             evmConfiguration,
             miningParameters,
-            badBlockManager)
+            badBlockManager,
+            isParallelTxProcessingEnabled,
+            metricsSystem)
         .createProtocolSchedule();
   }
 
@@ -124,6 +132,9 @@ public static ProtocolSchedule create(
    * @param evmConfiguration the evm configuration
    * @param miningParameters the mining parameters
    * @param badBlockManager the cache to use to keep invalid blocks
+   * @param isParallelTxProcessingEnabled indicates whether parallel transaction is enabled
+   * @param metricsSystem A metricSystem instance to be able to expose metrics in the underlying
+   *     calls
    * @return the protocol schedule
    */
   @VisibleForTesting
@@ -134,7 +145,9 @@ public static ProtocolSchedule create(
       final boolean isRevertReasonEnabled,
       final EvmConfiguration evmConfiguration,
       final MiningParameters miningParameters,
-      final BadBlockManager badBlockManager) {
+      final BadBlockManager badBlockManager,
+      final boolean isParallelTxProcessingEnabled,
+      final MetricsSystem metricsSystem) {
     return create(
         config,
         forksSchedule,
@@ -143,7 +156,9 @@ public static ProtocolSchedule create(
         isRevertReasonEnabled,
         evmConfiguration,
         miningParameters,
-        badBlockManager);
+        badBlockManager,
+        isParallelTxProcessingEnabled,
+        metricsSystem);
   }
 
   private static ProtocolSpecBuilder applyCliqueSpecificModifications(