From c659e28aa0e3ea5e83341b8a9eead0f0c89c296a Mon Sep 17 00:00:00 2001 From: Kyrylo Riabov Date: Tue, 23 Apr 2024 22:10:45 +0300 Subject: [PATCH] feat(storage): replace Tree generic with Arc (#7810) Co-authored-by: Matthias Seitz --- .../src/commands/debug_cmd/build_block.rs | 2 +- .../src/commands/debug_cmd/replay_engine.rs | 5 +- .../consensus/beacon/src/engine/test_utils.rs | 12 +-- crates/e2e-test-utils/src/lib.rs | 8 +- crates/node-builder/src/builder.rs | 67 ++++++------- crates/storage/provider/src/providers/mod.rs | 94 ++++++++----------- crates/storage/provider/src/traits/mod.rs | 3 + .../provider/src/traits/tree_viewer.rs | 22 +++++ examples/rpc-db/src/main.rs | 2 +- 9 files changed, 102 insertions(+), 113 deletions(-) create mode 100644 crates/storage/provider/src/traits/tree_viewer.rs diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index 7aaef00fa274..aee51ee793da 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -174,7 +174,7 @@ impl Command { EvmProcessorFactory::new(self.chain.clone(), evm_config), ); let tree = BlockchainTree::new(tree_externals, BlockchainTreeConfig::default(), None)?; - let blockchain_tree = ShareableBlockchainTree::new(tree); + let blockchain_tree = Arc::new(ShareableBlockchainTree::new(tree)); // fetch the best block from the database let best_block = diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index 841b9e3c6767..d9b6e98659cb 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -136,11 +136,10 @@ impl Command { EvmProcessorFactory::new(self.chain.clone(), evm_config), ); let tree = BlockchainTree::new(tree_externals, BlockchainTreeConfig::default(), None)?; - let blockchain_tree = ShareableBlockchainTree::new(tree); + let blockchain_tree = Arc::new(ShareableBlockchainTree::new(tree)); // Set up the blockchain provider - let blockchain_db = - BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; + let blockchain_db = BlockchainProvider::new(provider_factory.clone(), blockchain_tree)?; // Set up network let network_secret_path = diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index ff36e871b028..67225b7c71ce 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -43,13 +43,7 @@ type DatabaseEnv = TempDatabase; type TestBeaconConsensusEngine = BeaconConsensusEngine< Arc, - BlockchainProvider< - Arc, - ShareableBlockchainTree< - Arc, - EitherExecutorFactory>, - >, - >, + BlockchainProvider>, Arc>, EthEngineTypes, >; @@ -423,9 +417,9 @@ where // Setup blockchain tree let externals = TreeExternals::new(provider_factory.clone(), consensus, executor_factory); let config = BlockchainTreeConfig::new(1, 2, 3, 2); - let tree = ShareableBlockchainTree::new( + let tree = Arc::new(ShareableBlockchainTree::new( BlockchainTree::new(externals, config, None).expect("failed to create tree"), - ); + )); let latest = self.base_config.chain_spec.genesis_header().seal_slow(); let blockchain_provider = BlockchainProvider::with_latest(provider_factory.clone(), tree, latest); diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 043d1e0c6645..3c34f76e57aa 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -1,15 +1,13 @@ use node::NodeHelper; use reth::{ args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}, - blockchain_tree::ShareableBlockchainTree, builder::{NodeBuilder, NodeConfig, NodeHandle}, - revm::EvmProcessorFactory, tasks::TaskManager, }; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; use reth_node_builder::{ components::{NetworkBuilder, PayloadServiceBuilder, PoolBuilder}, - FullNodeComponentsAdapter, FullNodeTypesAdapter, NodeTypes, + FullNodeComponentsAdapter, FullNodeTypesAdapter, }; use reth_primitives::ChainSpec; use reth_provider::providers::BlockchainProvider; @@ -100,12 +98,10 @@ where // Type aliases type TmpDB = Arc>; -type EvmType = EvmProcessorFactory<::Evm>; -type RethProvider = BlockchainProvider>>; type TmpPool = <>>::PoolBuilder as PoolBuilder< TmpNodeAdapter, >>::Pool; -type TmpNodeAdapter = FullNodeTypesAdapter>; +type TmpNodeAdapter = FullNodeTypesAdapter>; /// Type alias for a type of NodeHelper pub type NodeHelperType = NodeHelper, TmpPool>>; diff --git a/crates/node-builder/src/builder.rs b/crates/node-builder/src/builder.rs index 3d7d3a04462b..327d906b3484 100644 --- a/crates/node-builder/src/builder.rs +++ b/crates/node-builder/src/builder.rs @@ -63,11 +63,9 @@ use tokio::sync::{mpsc::unbounded_channel, oneshot}; /// The builtin provider type of the reth node. // Note: we need to hardcode this because custom components might depend on it in associated types. -type RethFullProviderType = - BlockchainProvider>>; +type RethFullProviderType = BlockchainProvider; -type RethFullAdapter = - FullNodeTypesAdapter::Evm>>; +type RethFullAdapter = FullNodeTypesAdapter>; #[cfg_attr(doc, aquamarine::aquamarine)] /// Declaratively construct a node. @@ -278,7 +276,7 @@ where >, > where - N: Node::Evm>>>, + N: Node>>, N::PoolBuilder: PoolBuilder>, N::NetworkBuilder: crate::components::NetworkBuilder< RethFullAdapter, @@ -308,15 +306,14 @@ where Types, Components, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, > where - Components: NodeComponentsBuilder< - FullNodeTypesAdapter>, - >, + Components: + NodeComponentsBuilder>>, { NodeBuilder { config: self.config, @@ -339,7 +336,7 @@ impl Types, Components, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -347,9 +344,7 @@ impl where DB: Database + DatabaseMetrics + DatabaseMetadata + Clone + Unpin + 'static, Types: NodeTypes, - Components: NodeComponentsBuilder< - FullNodeTypesAdapter>, - >, + Components: NodeComponentsBuilder>>, { /// Apply a function to the components builder. pub fn map_components(self, f: impl FnOnce(Components) -> Components) -> Self { @@ -371,7 +366,7 @@ where where F: Fn( FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, ) -> eyre::Result<()> @@ -388,7 +383,7 @@ where F: Fn( FullNode< FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -407,7 +402,7 @@ where RpcContext< '_, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -427,7 +422,7 @@ where RpcContext< '_, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -449,7 +444,7 @@ where F: Fn( ExExContext< FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -476,7 +471,7 @@ where ) -> eyre::Result< NodeHandle< FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -556,7 +551,7 @@ where .with_sync_metrics_tx(sync_metrics_tx.clone()); let canon_state_notification_sender = tree.canon_state_notification_sender(); - let blockchain_tree = ShareableBlockchainTree::new(tree); + let blockchain_tree = Arc::new(ShareableBlockchainTree::new(tree)); debug!(target: "reth::cli", "configured blockchain tree"); // fetch the head block from the database @@ -995,7 +990,7 @@ where >, > where - N: Node::Evm>>>, + N: Node>>, N::PoolBuilder: PoolBuilder>, N::NetworkBuilder: crate::components::NetworkBuilder< RethFullAdapter, @@ -1032,7 +1027,7 @@ where >, > where - N: Node::Evm>>>, + N: Node>>, N::PoolBuilder: PoolBuilder>, N::NetworkBuilder: crate::components::NetworkBuilder< RethFullAdapter, @@ -1065,15 +1060,14 @@ where Types, Components, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, > where - Components: NodeComponentsBuilder< - FullNodeTypesAdapter>, - >, + Components: + NodeComponentsBuilder>>, { WithLaunchContext { builder: self.builder.with_components(components_builder), @@ -1090,7 +1084,7 @@ impl Types, Components, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -1098,9 +1092,7 @@ impl where DB: Database + DatabaseMetrics + DatabaseMetadata + Clone + Unpin + 'static, Types: NodeTypes, - Components: NodeComponentsBuilder< - FullNodeTypesAdapter>, - >, + Components: NodeComponentsBuilder>>, { /// Apply a function to the components builder. pub fn map_components(self, f: impl FnOnce(Components) -> Components) -> Self { @@ -1116,7 +1108,7 @@ where where F: Fn( FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, ) -> eyre::Result<()> @@ -1133,7 +1125,7 @@ where F: Fn( FullNode< FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -1152,7 +1144,7 @@ where RpcContext< '_, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -1172,7 +1164,7 @@ where RpcContext< '_, FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -1190,7 +1182,7 @@ where F: Fn( ExExContext< FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -1210,7 +1202,7 @@ where ) -> eyre::Result< NodeHandle< FullNodeComponentsAdapter< - FullNodeTypesAdapter>, + FullNodeTypesAdapter>, Components::Pool, >, >, @@ -1391,13 +1383,12 @@ impl std::fmt::Debug for BuilderContext { pub struct InitState; /// The state after all types of the node have been configured. -#[derive(Debug)] pub struct TypesState where DB: Database + Clone + 'static, Types: NodeTypes, { - adapter: FullNodeTypesAdapter>, + adapter: FullNodeTypesAdapter>, } /// The state of the node builder process after the node's components have been configured. diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index b3011a9b02cf..c9ebd042cdd2 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -4,7 +4,8 @@ use crate::{ CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, EvmEnvProvider, HeaderProvider, ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProviderBox, - StateProviderFactory, TransactionVariant, TransactionsProvider, WithdrawalsProvider, + StateProviderFactory, TransactionVariant, TransactionsProvider, TreeViewer, + WithdrawalsProvider, }; use reth_db::{ database::Database, @@ -67,31 +68,36 @@ use reth_rpc_types::engine::ForkchoiceState; /// This type serves as the main entry point for interacting with the blockchain and provides data /// from database storage and from the blockchain tree (pending state etc.) It is a simple wrapper /// type that holds an instance of the database and the blockchain tree. -#[derive(Clone, Debug)] -pub struct BlockchainProvider { +#[derive(Clone)] +#[allow(missing_debug_implementations)] +pub struct BlockchainProvider { /// Provider type used to access the database. database: ProviderFactory, /// The blockchain tree instance. - tree: Tree, + tree: Arc, /// Tracks the chain info wrt forkchoice updates chain_info: ChainInfoTracker, } -impl BlockchainProvider { +impl BlockchainProvider { /// Create new provider instance that wraps the database and the blockchain tree, using the /// provided latest header to initialize the chain info tracker. - pub fn with_latest(database: ProviderFactory, tree: Tree, latest: SealedHeader) -> Self { + pub fn with_latest( + database: ProviderFactory, + tree: Arc, + latest: SealedHeader, + ) -> Self { Self { database, tree, chain_info: ChainInfoTracker::new(latest) } } } -impl BlockchainProvider +impl BlockchainProvider where DB: Database, { /// Create a new provider using only the database and the tree, fetching the latest header from /// the database to initialize the provider. - pub fn new(database: ProviderFactory, tree: Tree) -> ProviderResult { + pub fn new(database: ProviderFactory, tree: Arc) -> ProviderResult { let provider = database.provider()?; let best: ChainInfo = provider.chain_info()?; match provider.header_by_number(best.best_number)? { @@ -104,10 +110,9 @@ where } } -impl BlockchainProvider +impl BlockchainProvider where DB: Database, - Tree: BlockchainTreeViewer, { /// Ensures that the given block number is canonical (synced) /// @@ -128,7 +133,7 @@ where } } -impl DatabaseProviderFactory for BlockchainProvider +impl DatabaseProviderFactory for BlockchainProvider where DB: Database, { @@ -137,10 +142,9 @@ where } } -impl HeaderProvider for BlockchainProvider +impl HeaderProvider for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { self.database.header(block_hash) @@ -182,10 +186,9 @@ where } } -impl BlockHashReader for BlockchainProvider +impl BlockHashReader for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn block_hash(&self, number: u64) -> ProviderResult> { self.database.block_hash(number) @@ -200,10 +203,9 @@ where } } -impl BlockNumReader for BlockchainProvider +impl BlockNumReader for BlockchainProvider where DB: Database, - Tree: BlockchainTreeViewer + Send + Sync, { fn chain_info(&self) -> ProviderResult { Ok(self.chain_info.chain_info()) @@ -222,10 +224,9 @@ where } } -impl BlockIdReader for BlockchainProvider +impl BlockIdReader for BlockchainProvider where DB: Database, - Tree: BlockchainTreeViewer + Send + Sync, { fn pending_block_num_hash(&self) -> ProviderResult> { Ok(self.tree.pending_block_num_hash()) @@ -240,10 +241,9 @@ where } } -impl BlockReader for BlockchainProvider +impl BlockReader for BlockchainProvider where DB: Database, - Tree: BlockchainTreeViewer + Send + Sync, { fn find_block_by_hash(&self, hash: B256, source: BlockSource) -> ProviderResult> { let block = match source { @@ -320,10 +320,9 @@ where } } -impl TransactionsProvider for BlockchainProvider +impl TransactionsProvider for BlockchainProvider where DB: Database, - Tree: BlockchainTreeViewer + Send + Sync, { fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { self.database.transaction_id(tx_hash) @@ -388,10 +387,9 @@ where } } -impl ReceiptProvider for BlockchainProvider +impl ReceiptProvider for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn receipt(&self, id: TxNumber) -> ProviderResult> { self.database.receipt(id) @@ -412,10 +410,10 @@ where self.database.receipts_by_tx_range(range) } } -impl ReceiptProviderIdExt for BlockchainProvider + +impl ReceiptProviderIdExt for BlockchainProvider where DB: Database, - Tree: BlockchainTreeViewer + Send + Sync, { fn receipts_by_block_id(&self, block: BlockId) -> ProviderResult>> { match block { @@ -440,10 +438,9 @@ where } } -impl WithdrawalsProvider for BlockchainProvider +impl WithdrawalsProvider for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn withdrawals_by_block( &self, @@ -458,10 +455,9 @@ where } } -impl StageCheckpointReader for BlockchainProvider +impl StageCheckpointReader for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { self.database.provider()?.get_stage_checkpoint(id) @@ -472,10 +468,9 @@ where } } -impl EvmEnvProvider for BlockchainProvider +impl EvmEnvProvider for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn fill_env_at( &self, @@ -544,10 +539,9 @@ where } } -impl PruneCheckpointReader for BlockchainProvider +impl PruneCheckpointReader for BlockchainProvider where DB: Database, - Tree: Send + Sync, { fn get_prune_checkpoint( &self, @@ -557,20 +551,18 @@ where } } -impl ChainSpecProvider for BlockchainProvider +impl ChainSpecProvider for BlockchainProvider where DB: Send + Sync, - Tree: Send + Sync, { fn chain_spec(&self) -> Arc { self.database.chain_spec() } } -impl StateProviderFactory for BlockchainProvider +impl StateProviderFactory for BlockchainProvider where DB: Database, - Tree: BlockchainTreePendingStateProvider + BlockchainTreeViewer, { /// Storage provider for latest block fn latest(&self) -> ProviderResult { @@ -644,10 +636,9 @@ where } } -impl BlockchainTreeEngine for BlockchainProvider +impl BlockchainTreeEngine for BlockchainProvider where DB: Send + Sync, - Tree: BlockchainTreeEngine, { fn buffer_block(&self, block: SealedBlockWithSenders) -> Result<(), InsertBlockError> { self.tree.buffer_block(block) @@ -681,10 +672,9 @@ where } } -impl BlockchainTreeViewer for BlockchainProvider +impl BlockchainTreeViewer for BlockchainProvider where DB: Send + Sync, - Tree: BlockchainTreeViewer, { fn blocks(&self) -> BTreeMap> { self.tree.blocks() @@ -743,10 +733,9 @@ where } } -impl CanonChainTracker for BlockchainProvider +impl CanonChainTracker for BlockchainProvider where DB: Send + Sync, - Tree: Send + Sync, Self: BlockReader, { fn on_forkchoice_update_received(&self, _update: &ForkchoiceState) { @@ -779,10 +768,9 @@ where } } -impl BlockReaderIdExt for BlockchainProvider +impl BlockReaderIdExt for BlockchainProvider where Self: BlockReader + BlockIdReader + ReceiptProviderIdExt, - Tree: BlockchainTreeEngine, { fn block_by_id(&self, id: BlockId) -> ProviderResult> { match id { @@ -859,10 +847,9 @@ where } } -impl BlockchainTreePendingStateProvider for BlockchainProvider +impl BlockchainTreePendingStateProvider for BlockchainProvider where DB: Send + Sync, - Tree: BlockchainTreePendingStateProvider, { fn find_pending_state_provider( &self, @@ -872,20 +859,18 @@ where } } -impl CanonStateSubscriptions for BlockchainProvider +impl CanonStateSubscriptions for BlockchainProvider where DB: Send + Sync, - Tree: CanonStateSubscriptions, { fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { self.tree.subscribe_to_canonical_state() } } -impl ChangeSetReader for BlockchainProvider +impl ChangeSetReader for BlockchainProvider where DB: Database, - Tree: Sync + Send, { fn account_block_changeset( &self, @@ -895,10 +880,9 @@ where } } -impl AccountReader for BlockchainProvider +impl AccountReader for BlockchainProvider where DB: Database + Sync + Send, - Tree: Sync + Send, { /// Get basic account information. fn basic_account(&self, address: Address) -> ProviderResult> { diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index c9623cb0c21a..44884acb019a 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -80,3 +80,6 @@ pub use stats::StatsReader; mod full; pub use full::FullProvider; + +mod tree_viewer; +pub use tree_viewer::TreeViewer; diff --git a/crates/storage/provider/src/traits/tree_viewer.rs b/crates/storage/provider/src/traits/tree_viewer.rs new file mode 100644 index 000000000000..db3b19c4d611 --- /dev/null +++ b/crates/storage/provider/src/traits/tree_viewer.rs @@ -0,0 +1,22 @@ +use crate::{BlockchainTreePendingStateProvider, CanonStateSubscriptions}; + +use reth_interfaces::blockchain_tree::{BlockchainTreeEngine, BlockchainTreeViewer}; + +/// Helper trait to combine all the traits we need for the BlockchainProvider +/// +/// This is a temporary solution +pub trait TreeViewer: + BlockchainTreeViewer + + BlockchainTreePendingStateProvider + + CanonStateSubscriptions + + BlockchainTreeEngine +{ +} + +impl TreeViewer for T where + T: BlockchainTreeViewer + + BlockchainTreePendingStateProvider + + CanonStateSubscriptions + + BlockchainTreeEngine +{ +} diff --git a/examples/rpc-db/src/main.rs b/examples/rpc-db/src/main.rs index b8286e51be79..627da093c591 100644 --- a/examples/rpc-db/src/main.rs +++ b/examples/rpc-db/src/main.rs @@ -49,7 +49,7 @@ async fn main() -> eyre::Result<()> { // 2. Setup the blockchain provider using only the database provider and a noop for the tree to // satisfy trait bounds. Tree is not used in this example since we are only operating on the // disk and don't handle new blocks/live sync etc, which is done by the blockchain tree. - let provider = BlockchainProvider::new(factory, NoopBlockchainTree::default())?; + let provider = BlockchainProvider::new(factory, Arc::new(NoopBlockchainTree::default()))?; let rpc_builder = RpcModuleBuilder::default() .with_provider(provider.clone())