From e85dd37ac76f22062cd6856098db73180d71cb1f Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Fri, 19 Jul 2024 03:21:08 +1000 Subject: [PATCH] Add metrics for trie log pruner (#7352) --------- Signed-off-by: Simon Dudley Signed-off-by: gconnect --- CHANGELOG.md | 9 ++++- .../controller/BesuControllerBuilder.java | 3 +- .../common/trielog/TrieLogPruner.java | 23 ++++++++++++- .../bonsai/trielog/TrieLogPrunerTest.java | 34 +++++++++++++++---- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fb35c2e0b0..7710f8e7b95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Next release +### Upcoming Breaking Changes +- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. +- --Xbonsai-limit-trie-logs-enabled is deprecated, use --bonsai-limit-trie-logs-enabled instead +- --Xbonsai-trie-logs-pruning-window-size is deprecated, use --bonsai-trie-logs-pruning-window-size instead +- `besu storage x-trie-log` subcommand is deprecated, use `besu storage trie-log` instead + ### Breaking Changes - Remove deprecated sync modes (X_SNAP and X_CHECKPOINT). Use SNAP and CHECKPOINT instead [#7309](https://github.com/hyperledger/besu/pull/7309) - Remove PKI-backed QBFT (deprecated in 24.5.1) Other forms of QBFT remain unchanged. [#7293](https://github.com/hyperledger/besu/pull/7293) @@ -15,7 +21,8 @@ - Implement gnark-crypto for eip-2537 [#7316](https://github.com/hyperledger/besu/pull/7316) - Improve blob size transaction selector [#7312](https://github.com/hyperledger/besu/pull/7312) - Added EIP-7702 [#7237](https://github.com/hyperledger/besu/pull/7237) -- implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262) +- Implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262) +- Add trie log pruner metrics [#7352](https://github.com/hyperledger/besu/pull/7352) ### Bug fixes - Fix `eth_call` deserialization to correctly ignore unknown fields in the transaction object. [#7323](https://github.com/hyperledger/besu/pull/7323) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 8f26909a512..20f3ca8ef61 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -793,7 +793,8 @@ private TrieLogPruner createTrieLogPruner( scheduler::executeServiceTask, dataStorageConfiguration.getBonsaiMaxLayersToLoad(), dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize(), - isProofOfStake); + isProofOfStake, + metricsSystem); trieLogPruner.initialize(); return trieLogPruner; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index 6f347be0334..cea5c1a327d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -19,6 +19,9 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Comparator; @@ -45,6 +48,9 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { private final Consumer executeAsync; private final long numBlocksToRetain; private final boolean requireFinalizedBlock; + private final Counter addedToPruneQueueCounter; + private final Counter prunedFromQueueCounter; + private final Counter prunedOrphanCounter; private final Multimap trieLogBlocksAndForksByDescendingBlockNumber = TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder()); @@ -55,7 +61,8 @@ public TrieLogPruner( final Consumer executeAsync, final long numBlocksToRetain, final int pruningLimit, - final boolean requireFinalizedBlock) { + final boolean requireFinalizedBlock, + final MetricsSystem metricsSystem) { this.rootWorldStateStorage = rootWorldStateStorage; this.blockchain = blockchain; this.executeAsync = executeAsync; @@ -63,6 +70,17 @@ public TrieLogPruner( this.pruningLimit = pruningLimit; this.loadingLimit = pruningLimit; // same as pruningLimit for now this.requireFinalizedBlock = requireFinalizedBlock; + this.addedToPruneQueueCounter = + metricsSystem.createCounter( + BesuMetricCategory.PRUNER, + "trie_log_added_to_prune_queue", + "trie log added to prune queue"); + this.prunedFromQueueCounter = + metricsSystem.createCounter( + BesuMetricCategory.PRUNER, "trie_log_pruned_from_queue", "trie log pruned from queue"); + this.prunedOrphanCounter = + metricsSystem.createCounter( + BesuMetricCategory.PRUNER, "trie_log_pruned_orphan", "trie log pruned orphan"); } public int initialize() { @@ -88,6 +106,7 @@ private int preloadQueue() { // prune orphaned blocks (sometimes created during block production) rootWorldStateStorage.pruneTrieLog(blockHash); orphansPruned.getAndIncrement(); + prunedOrphanCounter.inc(); } }); LOG.atDebug().log("Pruned {} orphaned trie logs from database...", orphansPruned.intValue()); @@ -106,6 +125,7 @@ public synchronized void addToPruneQueue(final long blockNumber, final Hash bloc .addArgument(blockHash) .log(); trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash); + addedToPruneQueueCounter.inc(); } public synchronized int pruneFromQueue() { @@ -155,6 +175,7 @@ public synchronized int pruneFromQueue() { }); wasPruned.keySet().forEach(trieLogBlocksAndForksByDescendingBlockNumber::removeAll); + prunedFromQueueCounter.inc(wasPruned.size()); LOG.atTrace() .setMessage("pruned {} trie logs for blocks {}") diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java index bca954a4935..621e73711ab 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.Optional; import java.util.function.Consumer; @@ -72,7 +73,8 @@ public void initialize_preloads_queue_and_prunes_orphaned_blocks() { // When TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, executeAsync, 3, loadingLimit, false); + new TrieLogPruner( + worldState, blockchain, executeAsync, 3, loadingLimit, false, new NoOpMetricsSystem()); trieLogPruner.initialize(); // Then @@ -92,7 +94,13 @@ public void trieLogs_pruned_in_reverse_order_within_pruning_window() { // requireFinalizedBlock = false means this is not a PoS chain TrieLogPruner trieLogPruner = new TrieLogPruner( - worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, false); + worldState, + blockchain, + executeAsync, + blocksToRetain, + pruningWindowSize, + false, + new NoOpMetricsSystem()); trieLogPruner.addToPruneQueue(0, key(0)); // older block outside prune window trieLogPruner.addToPruneQueue(1, key(1)); // block inside the prune window @@ -201,7 +209,13 @@ public void skip_pruning_when_finalized_block_required_but_not_present() { when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight); TrieLogPruner trieLogPruner = new TrieLogPruner( - worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true); + worldState, + blockchain, + executeAsync, + blocksToRetain, + pruningWindowSize, + true, + new NoOpMetricsSystem()); trieLogPruner.addToPruneQueue(1, key(1)); trieLogPruner.addToPruneQueue(2, key(2)); @@ -240,7 +254,8 @@ public void onTrieLogAdded_should_prune() { // Given final TriggerableConsumer triggerableConsumer = new TriggerableConsumer(); TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, triggerableConsumer, 0, 1, false); + new TrieLogPruner( + worldState, blockchain, triggerableConsumer, 0, 1, false, new NoOpMetricsSystem()); assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0); final TrieLogLayer layer = new TrieLogLayer(); @@ -261,7 +276,8 @@ public void onTrieLogAdded_should_prune() { public void onTrieLogAdded_should_not_prune_when_no_blockNumber() { // Given TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, executeAsync, 0, 1, false); + new TrieLogPruner( + worldState, blockchain, executeAsync, 0, 1, false, new NoOpMetricsSystem()); assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0); final TrieLogLayer layer = new TrieLogLayer(); @@ -289,7 +305,13 @@ private TrieLogPruner setupPrunerAndFinalizedBlock( when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight); TrieLogPruner trieLogPruner = new TrieLogPruner( - worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true); + worldState, + blockchain, + executeAsync, + blocksToRetain, + pruningWindowSize, + true, + new NoOpMetricsSystem()); trieLogPruner.addToPruneQueue(1, key(1)); trieLogPruner.addToPruneQueue(2, key(2));