From 41fede870690d859862fddc3532e0d30a6c6c945 Mon Sep 17 00:00:00 2001 From: sslivkoff Date: Sat, 30 Dec 2023 23:15:18 -0800 Subject: [PATCH 1/3] support multiple provider types --- crates/cli/src/parse/args.rs | 2 +- crates/cli/src/parse/blocks.rs | 182 +++++++++++------- crates/cli/src/parse/partitions.rs | 14 +- crates/cli/src/parse/query.rs | 10 +- crates/cli/src/parse/source.rs | 9 +- crates/cli/src/parse/timestamps.rs | 179 +++++++++-------- .../src/datasets/address_appearances.rs | 12 +- crates/freeze/src/datasets/balance_diffs.rs | 4 +- crates/freeze/src/datasets/balance_reads.rs | 7 +- crates/freeze/src/datasets/balances.rs | 3 +- crates/freeze/src/datasets/blocks.rs | 3 - crates/freeze/src/datasets/code_diffs.rs | 4 +- crates/freeze/src/datasets/code_reads.rs | 7 +- crates/freeze/src/datasets/codes.rs | 3 +- crates/freeze/src/datasets/contracts.rs | 4 +- crates/freeze/src/datasets/erc20_balances.rs | 2 +- crates/freeze/src/datasets/erc20_metadata.rs | 6 +- crates/freeze/src/datasets/erc20_supplies.rs | 2 +- crates/freeze/src/datasets/erc20_transfers.rs | 4 +- crates/freeze/src/datasets/erc721_metadata.rs | 4 +- .../freeze/src/datasets/erc721_transfers.rs | 4 +- crates/freeze/src/datasets/eth_calls.rs | 2 +- .../freeze/src/datasets/four_byte_counts.rs | 3 +- crates/freeze/src/datasets/geth_calls.rs | 3 +- crates/freeze/src/datasets/geth_opcodes.rs | 6 +- .../freeze/src/datasets/javascript_traces.rs | 2 - crates/freeze/src/datasets/logs.rs | 4 +- .../freeze/src/datasets/native_transfers.rs | 4 +- crates/freeze/src/datasets/nonce_diffs.rs | 4 +- crates/freeze/src/datasets/nonce_reads.rs | 7 +- crates/freeze/src/datasets/nonces.rs | 6 +- crates/freeze/src/datasets/slots.rs | 1 - crates/freeze/src/datasets/storage_diffs.rs | 4 +- crates/freeze/src/datasets/storage_reads.rs | 7 +- crates/freeze/src/datasets/trace_calls.rs | 1 - crates/freeze/src/datasets/traces.rs | 4 +- crates/freeze/src/datasets/transactions.rs | 5 +- crates/freeze/src/datasets/vm_traces.rs | 4 +- crates/freeze/src/freeze.rs | 2 +- .../multi_datasets/blocks_and_transactions.rs | 1 - .../multi_datasets/call_trace_derivatives.rs | 4 +- .../src/multi_datasets/geth_state_diffs.rs | 3 +- .../freeze/src/multi_datasets/state_diffs.rs | 4 +- .../freeze/src/multi_datasets/state_reads.rs | 7 +- crates/freeze/src/types/sources.rs | 174 ++++++++++++----- 45 files changed, 415 insertions(+), 312 deletions(-) diff --git a/crates/cli/src/parse/args.rs b/crates/cli/src/parse/args.rs index f2b03e2..2e6ad87 100644 --- a/crates/cli/src/parse/args.rs +++ b/crates/cli/src/parse/args.rs @@ -12,7 +12,7 @@ pub async fn parse_args( args: &Args, ) -> Result<(Query, Source, FileOutput, ExecutionEnv), ParseError> { let source = source::parse_source(args).await?; - let query = query::parse_query(args, Arc::clone(&source.fetcher)).await?; + let query = query::parse_query(args, Arc::new(source.clone())).await?; let sink = file_output::parse_file_output(args, &source)?; let env = execution::parse_execution_env(args, query.n_tasks() as u64)?; Ok((query, source, sink, env)) diff --git a/crates/cli/src/parse/blocks.rs b/crates/cli/src/parse/blocks.rs index 8cd355c..38e69ee 100644 --- a/crates/cli/src/parse/blocks.rs +++ b/crates/cli/src/parse/blocks.rs @@ -1,14 +1,13 @@ -use ethers::prelude::*; use polars::prelude::*; use std::collections::HashMap; -use cryo_freeze::{BlockChunk, ChunkData, Datatype, Fetcher, ParseError, Subchunk, Table}; +use cryo_freeze::{BlockChunk, ChunkData, Datatype, ParseError, Source, Subchunk, Table}; use crate::args::Args; -pub(crate) async fn parse_blocks( +pub(crate) async fn parse_blocks( args: &Args, - fetcher: Arc>, + source: Arc, ) -> Result<(Option>>, Option>), ParseError> { let (files, explicit_numbers): (Vec<&String>, Vec<&String>) = match &args.blocks { Some(blocks) => blocks.iter().partition(|tx| std::path::Path::new(tx).exists()), @@ -45,10 +44,10 @@ pub(crate) async fn parse_blocks( // parse inputs into BlockChunks let mut block_chunks = Vec::new(); for explicit_number in explicit_numbers { - let outputs = parse_block_inputs(explicit_number, &fetcher).await?; + let outputs = parse_block_inputs(explicit_number, source.clone()).await?; block_chunks.extend(outputs); } - postprocess_block_chunks(block_chunks, args, fetcher).await? + postprocess_block_chunks(block_chunks, args, source).await? } else { Vec::new() }; @@ -105,10 +104,10 @@ fn read_integer_column(path: &str, column: &str) -> Result, ParseError> } } -pub(crate) async fn postprocess_block_chunks( +pub(crate) async fn postprocess_block_chunks( block_chunks: Vec, args: &Args, - fetcher: Arc>, + source: Arc, ) -> Result, ParseError> { // align let block_chunks = if args.align { @@ -124,14 +123,14 @@ pub(crate) async fn postprocess_block_chunks( }; // apply reorg buffer - let block_chunks = apply_reorg_buffer(block_chunks, args.reorg_buffer, &fetcher).await?; + let block_chunks = apply_reorg_buffer(block_chunks, args.reorg_buffer, source).await?; Ok(block_chunks) } -pub(crate) async fn get_default_block_chunks( +pub(crate) async fn get_default_block_chunks( args: &Args, - fetcher: Arc>, + source: Arc, schemas: &HashMap, ) -> Result, ParseError> { let default_blocks = match schemas @@ -142,14 +141,14 @@ pub(crate) async fn get_default_block_chunks( Some(Some(blocks)) => blocks, _ => "0:latest".to_string(), }; - let block_chunks = parse_block_inputs(&default_blocks, &fetcher).await?; - postprocess_block_chunks(block_chunks, args, fetcher).await + let block_chunks = parse_block_inputs(&default_blocks, source.clone()).await?; + postprocess_block_chunks(block_chunks, args, source).await } /// parse block numbers to freeze -async fn parse_block_inputs( +async fn parse_block_inputs( inputs: &str, - fetcher: &Fetcher

, + source: Arc, ) -> Result, ParseError> { let parts: Vec<&str> = inputs.split(' ').collect(); match parts.len() { @@ -157,12 +156,12 @@ async fn parse_block_inputs( let first_input = parts.first().ok_or_else(|| { ParseError::ParseError("Failed to get the first input".to_string()) })?; - parse_block_token(first_input, true, fetcher).await.map(|x| vec![x]) + parse_block_token(first_input, true, source).await.map(|x| vec![x]) } _ => { let mut chunks = Vec::new(); for part in parts { - chunks.push(parse_block_token(part, false, fetcher).await?); + chunks.push(parse_block_token(part, false, source.clone()).await?); } Ok(chunks) } @@ -175,17 +174,17 @@ enum RangePosition { None, } -async fn parse_block_token( +async fn parse_block_token( s: &str, as_range: bool, - fetcher: &Fetcher

, + source: Arc, ) -> Result { let s = s.replace('_', ""); let parts: Vec<&str> = s.split(':').collect(); match parts.as_slice() { [block_ref] => { - let block = parse_block_number(block_ref, RangePosition::None, fetcher).await?; + let block = parse_block_number(block_ref, RangePosition::None, source).await?; Ok(BlockChunk::Numbers(vec![block])) } [first_ref, second_ref] => { @@ -199,13 +198,11 @@ async fn parse_block_token( (*second_ref, None) }; - let (start_block, end_block) = - parse_block_range(first_ref, second_ref, fetcher).await?; + let (start_block, end_block) = parse_block_range(first_ref, second_ref, source).await?; block_range_to_block_chunk(start_block, end_block, as_range, None, n_keep) } [first_ref, second_ref, third_ref] => { - let (start_block, end_block) = - parse_block_range(first_ref, second_ref, fetcher).await?; + let (start_block, end_block) = parse_block_range(first_ref, second_ref, source).await?; let range_size = third_ref .parse::() .map_err(|_e| ParseError::ParseError("start_block parse error".to_string()))?; @@ -251,11 +248,11 @@ pub(crate) fn block_range_to_block_chunk( fn evenly_spaced_subset(items: Vec, subset_length: usize) -> Vec { if subset_length == 0 || items.is_empty() { - return Vec::new() + return Vec::new(); } if subset_length >= items.len() { - return items.to_vec() + return items.to_vec(); } let original_length = items.len(); @@ -273,17 +270,14 @@ fn evenly_spaced_subset(items: Vec, subset_length: usize) -> Vec subset } -async fn parse_block_range

( +async fn parse_block_range( first_ref: &str, second_ref: &str, - fetcher: &Fetcher

, -) -> Result<(u64, u64), ParseError> -where - P: JsonRpcClient, -{ + source: Arc, +) -> Result<(u64, u64), ParseError> { let (start_block, end_block) = match (first_ref, second_ref) { _ if first_ref.starts_with('-') => { - let end_block = parse_block_number(second_ref, RangePosition::Last, fetcher).await?; + let end_block = parse_block_number(second_ref, RangePosition::Last, source).await?; let start_block = end_block .checked_sub(first_ref[1..].parse::().map_err(|_e| { @@ -293,7 +287,7 @@ where (start_block, end_block) } _ if second_ref.starts_with('+') => { - let start_block = parse_block_number(first_ref, RangePosition::First, fetcher).await?; + let start_block = parse_block_number(first_ref, RangePosition::First, source).await?; let end_block = start_block .checked_add(second_ref[1..].parse::().map_err(|_e| { @@ -303,8 +297,9 @@ where (start_block, end_block) } _ => { - let start_block = parse_block_number(first_ref, RangePosition::First, fetcher).await?; - let end_block = parse_block_number(second_ref, RangePosition::Last, fetcher).await?; + let start_block = + parse_block_number(first_ref, RangePosition::First, source.clone()).await?; + let end_block = parse_block_number(second_ref, RangePosition::Last, source).await?; (start_block, end_block) } }; @@ -321,18 +316,18 @@ where Ok((start_block, end_block)) } -async fn parse_block_number( +async fn parse_block_number( block_ref: &str, range_position: RangePosition, - fetcher: &Fetcher

, + source: Arc, ) -> Result { match (block_ref, range_position) { - ("latest", _) => fetcher.get_block_number().await.map(|n| n.as_u64()).map_err(|_e| { + ("latest", _) => source.get_block_number().await.map(|n| n.as_u64()).map_err(|_e| { ParseError::ParseError("Error retrieving latest block number".to_string()) }), ("", RangePosition::First) => Ok(0), ("", RangePosition::Last) => { - fetcher.get_block_number().await.map(|n| n.as_u64()).map_err(|_e| { + source.get_block_number().await.map(|n| n.as_u64()).map_err(|_e| { ParseError::ParseError("Error retrieving last block number".to_string()) }) } @@ -362,15 +357,15 @@ async fn parse_block_number( } } -async fn apply_reorg_buffer( +async fn apply_reorg_buffer( block_chunks: Vec, reorg_filter: u64, - fetcher: &Fetcher

, + source: Arc, ) -> Result, ParseError> { match reorg_filter { 0 => Ok(block_chunks), reorg_filter => { - let latest_block = match fetcher.get_block_number().await { + let latest_block = match source.get_block_number().await { Ok(result) => result.as_u64(), Err(_e) => { return Err(ParseError::ParseError("reorg buffer parse error".to_string())) @@ -388,10 +383,8 @@ async fn apply_reorg_buffer( } } -pub(crate) async fn get_latest_block_number( - fetcher: &Fetcher

, -) -> Result { - fetcher +pub(crate) async fn get_latest_block_number(source: Arc) -> Result { + source .get_block_number() .await .map(|n| n.as_u64()) @@ -401,6 +394,7 @@ pub(crate) async fn get_latest_block_number( #[cfg(test)] mod tests { use super::*; + use ethers::prelude::*; enum BlockTokenTest<'a> { WithoutMock((&'a str, BlockChunk)), // Token | Expected @@ -409,28 +403,44 @@ mod tests { async fn block_token_test_helper(tests: Vec<(BlockTokenTest<'_>, bool)>) { let (provider, mock) = Provider::mocked(); - let fetcher = Fetcher { provider, semaphore: None, rate_limiter: None }; + let source = Source { + provider: provider.into(), + semaphore: Arc::new(None), + rate_limiter: Arc::new(None), + chain_id: 1, + inner_request_size: 1, + max_concurrent_chunks: None, + rpc_url: "".to_string(), + labels: cryo_freeze::SourceLabels::default(), + }; + let source = Arc::new(source); for (test, res) in tests { match test { BlockTokenTest::WithMock((token, expected, latest)) => { mock.push(U64::from(latest)).unwrap(); - assert_eq!(block_token_test_executor(token, expected, &fetcher).await, res); + assert_eq!( + block_token_test_executor(token, expected, source.clone()).await, + res + ); } BlockTokenTest::WithoutMock((token, expected)) => { - assert_eq!(block_token_test_executor(token, expected, &fetcher).await, res); + assert_eq!( + block_token_test_executor(token, expected, source.clone()).await, + res + ); } } } } - async fn block_token_test_executor( + async fn block_token_test_executor( token: &str, expected: BlockChunk, - fetcher: &Fetcher

, + source: Arc, ) -> bool { match expected { BlockChunk::Numbers(expected_block_numbers) => { - let block_chunks = parse_block_token(token, false, fetcher).await.unwrap(); + let block_chunks = parse_block_token(token, false, source).await.unwrap(); assert!(matches!(block_chunks, BlockChunk::Numbers { .. })); let BlockChunk::Numbers(block_numbers) = block_chunks else { panic!("Unexpected shape") @@ -438,7 +448,7 @@ mod tests { block_numbers == expected_block_numbers } BlockChunk::Range(expected_range_start, expected_range_end) => { - let block_chunks = parse_block_token(token, true, fetcher).await.unwrap(); + let block_chunks = parse_block_token(token, true, source).await.unwrap(); assert!(matches!(block_chunks, BlockChunk::Range { .. })); let BlockChunk::Range(range_start, range_end) = block_chunks else { panic!("Unexpected shape") @@ -455,18 +465,30 @@ mod tests { async fn block_input_test_helper(tests: Vec<(BlockInputTest<'_>, bool)>) { let (provider, mock) = Provider::mocked(); - let fetcher = Fetcher { provider, semaphore: None, rate_limiter: None }; + let source = Arc::new(Source { + provider: provider.into(), + chain_id: 1, + rpc_url: "".to_string(), + inner_request_size: 1, + semaphore: Arc::new(None), + max_concurrent_chunks: Some(1), + rate_limiter: Arc::new(None), + labels: cryo_freeze::SourceLabels::default(), + }); for (test, res) in tests { match test { BlockInputTest::WithMock((inputs, expected, latest)) => { mock.push(U64::from(latest)).unwrap(); - assert_eq!(block_input_test_executor(inputs, expected, &fetcher).await, res); + assert_eq!( + block_input_test_executor(inputs, expected, source.clone()).await, + res + ); } BlockInputTest::WithoutMock((inputs, expected)) => { println!("RES {:?}", res); println!("inputs {:?}", inputs); println!("EXPECTED {:?}", expected); - let actual = block_input_test_executor(inputs, expected, &fetcher).await; + let actual = block_input_test_executor(inputs, expected, source.clone()).await; println!("ACTUAL {:?}", actual); assert_eq!(actual, res); } @@ -474,12 +496,12 @@ mod tests { } } - async fn block_input_test_executor( + async fn block_input_test_executor( inputs: &str, expected: Vec, - fetcher: &Fetcher

, + source: Arc, ) -> bool { - let block_chunks = parse_block_inputs(inputs, fetcher).await.unwrap(); + let block_chunks = parse_block_inputs(inputs, source).await.unwrap(); assert_eq!(block_chunks.len(), expected.len()); for (i, block_chunk) in block_chunks.iter().enumerate() { let expected_chunk = &expected[i]; @@ -492,7 +514,7 @@ mod tests { panic!("Unexpected shape") }; if expected_block_numbers != block_numbers { - return false + return false; } } BlockChunk::Range(expected_range_start, expected_range_end) => { @@ -501,7 +523,7 @@ mod tests { panic!("Unexpected shape") }; if expected_range_start != range_start || expected_range_end != range_end { - return false + return false; } } } @@ -516,21 +538,41 @@ mod tests { async fn block_number_test_helper(tests: Vec<(BlockNumberTest<'_>, bool)>) { let (provider, mock) = Provider::mocked(); - let fetcher = Fetcher { provider, semaphore: None, rate_limiter: None }; + let source = Source { + provider: provider.into(), + semaphore: Arc::new(None), + rate_limiter: Arc::new(None), + chain_id: 1, + inner_request_size: 1, + max_concurrent_chunks: Some(1), + rpc_url: "".to_string(), + labels: cryo_freeze::SourceLabels::default(), + }; + let source = Arc::new(source); for (test, res) in tests { match test { BlockNumberTest::WithMock((block_ref, range_position, expected, latest)) => { mock.push(U64::from(latest)).unwrap(); assert_eq!( - block_number_test_executor(block_ref, range_position, expected, &fetcher) - .await, + block_number_test_executor( + block_ref, + range_position, + expected, + source.clone() + ) + .await, res ); } BlockNumberTest::WithoutMock((block_ref, range_position, expected)) => { assert_eq!( - block_number_test_executor(block_ref, range_position, expected, &fetcher) - .await, + block_number_test_executor( + block_ref, + range_position, + expected, + source.clone() + ) + .await, res ); } @@ -538,13 +580,13 @@ mod tests { } } - async fn block_number_test_executor( + async fn block_number_test_executor( block_ref: &str, range_position: RangePosition, expected: u64, - fetcher: &Fetcher

, + source: Arc, ) -> bool { - let block_number = parse_block_number(block_ref, range_position, fetcher).await.unwrap(); + let block_number = parse_block_number(block_ref, range_position, source).await.unwrap(); block_number == expected } diff --git a/crates/cli/src/parse/partitions.rs b/crates/cli/src/parse/partitions.rs index 4acaa92..bada931 100644 --- a/crates/cli/src/parse/partitions.rs +++ b/crates/cli/src/parse/partitions.rs @@ -5,8 +5,8 @@ use super::{ }; use crate::args::Args; use cryo_freeze::{ - AddressChunk, CallDataChunk, Datatype, Dim, Fetcher, ParseError, Partition, PartitionLabels, - SlotChunk, Table, TimeDimension, TopicChunk, TransactionChunk, + AddressChunk, CallDataChunk, Datatype, Dim, ParseError, Partition, PartitionLabels, SlotChunk, + Source, Table, TimeDimension, TopicChunk, TransactionChunk, }; use ethers::prelude::*; use rand::{seq::SliceRandom, thread_rng}; @@ -14,18 +14,18 @@ use std::{collections::HashMap, str::FromStr, sync::Arc}; type ChunkLabels = Vec>; -pub(crate) async fn parse_partitions( +pub(crate) async fn parse_partitions( args: &Args, - fetcher: Arc>, + source: Arc, schemas: &HashMap, ) -> Result<(Vec, Vec, TimeDimension), ParseError> { // TODO: if wanting to chunk these non-block dimensions, do it in parse_binary_arg() // TODO: map from args to dim is not exhaustive // parse chunk data - let (block_number_labels, block_numbers) = blocks::parse_blocks(args, fetcher.clone()).await?; + let (block_number_labels, block_numbers) = blocks::parse_blocks(args, source.clone()).await?; let (block_number_labels, block_numbers) = if block_numbers.is_none() { - timestamps::parse_timestamps(args, fetcher.clone()).await? + timestamps::parse_timestamps(args, source.clone()).await? } else { (block_number_labels, block_numbers) }; @@ -46,7 +46,7 @@ pub(crate) async fn parse_partitions( // set default blocks let block_numbers = if block_numbers.is_none() && transactions.is_none() { - Some(blocks::get_default_block_chunks(args, fetcher, schemas).await?) + Some(blocks::get_default_block_chunks(args, source, schemas).await?) } else { block_numbers }; diff --git a/crates/cli/src/parse/query.rs b/crates/cli/src/parse/query.rs index f5f14a1..329319e 100644 --- a/crates/cli/src/parse/query.rs +++ b/crates/cli/src/parse/query.rs @@ -1,14 +1,10 @@ use super::{parse_schemas, partitions}; use crate::args::Args; -use cryo_freeze::{Dim, Fetcher, ParseError, Query, QueryLabels, Schemas}; -use ethers::prelude::*; +use cryo_freeze::{Dim, ParseError, Query, QueryLabels, Schemas, Source}; use std::sync::Arc; /// parse Query struct from cli Args -pub async fn parse_query( - args: &Args, - fetcher: Arc>, -) -> Result { +pub async fn parse_query(args: &Args, source: Arc) -> Result { let (datatypes, schemas) = parse_schemas(args)?; let arg_aliases = find_arg_aliases(args, &schemas); @@ -17,7 +13,7 @@ pub async fn parse_query( let args = new_args.as_ref().unwrap_or(args); let (partitions, partitioned_by, time_dimension) = - partitions::parse_partitions(args, fetcher, &schemas).await?; + partitions::parse_partitions(args, source, &schemas).await?; let datatypes = cryo_freeze::cluster_datatypes(datatypes); let labels = QueryLabels { align: args.align, reorg_buffer: args.reorg_buffer }; Ok(Query { diff --git a/crates/cli/src/parse/source.rs b/crates/cli/src/parse/source.rs index 2b861f8..284cede 100644 --- a/crates/cli/src/parse/source.rs +++ b/crates/cli/src/parse/source.rs @@ -1,7 +1,7 @@ use std::env; use crate::args::Args; -use cryo_freeze::{Fetcher, ParseError, Source, SourceLabels}; +use cryo_freeze::{ParseError, Source, SourceLabels}; use ethers::prelude::*; use governor::{Quota, RateLimiter}; use polars::prelude::*; @@ -35,15 +35,16 @@ pub(crate) async fn parse_source(args: &Args) -> Result { }; let semaphore = tokio::sync::Semaphore::new(max_concurrent_requests as usize); - let semaphore = Some(semaphore); + let semaphore = Arc::new(Some(semaphore)); - let fetcher = Fetcher { provider, semaphore, rate_limiter }; let output = Source { - fetcher: Arc::new(fetcher), chain_id, inner_request_size: args.inner_request_size, max_concurrent_chunks, + semaphore, + rate_limiter: rate_limiter.into(), rpc_url, + provider: provider.into(), labels: SourceLabels { max_concurrent_requests: args.requests_per_second.map(|x| x as u64), max_requests_per_second: args.requests_per_second.map(|x| x as u64), diff --git a/crates/cli/src/parse/timestamps.rs b/crates/cli/src/parse/timestamps.rs index 78bc629..926216e 100644 --- a/crates/cli/src/parse/timestamps.rs +++ b/crates/cli/src/parse/timestamps.rs @@ -1,5 +1,4 @@ -use cryo_freeze::{BlockChunk, Fetcher, ParseError}; -use ethers::prelude::*; +use cryo_freeze::{BlockChunk, ParseError, Source}; use polars::prelude::*; use crate::{ @@ -9,9 +8,9 @@ use crate::{ use super::blocks::get_latest_block_number; -pub(crate) async fn parse_timestamps( +pub(crate) async fn parse_timestamps( args: &Args, - fetcher: Arc>, + source: Arc, ) -> Result<(Option>>, Option>), ParseError> { let (files, explicit_numbers): (Vec<&String>, Vec<&String>) = match &args.timestamps { Some(timestamp) => timestamp.iter().partition(|tx| std::path::Path::new(tx).exists()), @@ -48,10 +47,10 @@ pub(crate) async fn parse_timestamps( // parse inputs into BlockChunks let mut block_chunks = Vec::new(); for explicit_number in explicit_numbers { - let outputs = parse_timestamp_inputs(explicit_number, &fetcher).await?; + let outputs = parse_timestamp_inputs(explicit_number, source.clone()).await?; block_chunks.extend(outputs); } - postprocess_block_chunks(block_chunks, args, fetcher).await? + postprocess_block_chunks(block_chunks, args, source).await? } else { Vec::new() }; @@ -107,9 +106,9 @@ fn read_integer_column(path: &str, column: &str) -> Result, ParseError> } /// parse timestamp numbers to freeze -async fn parse_timestamp_inputs( +async fn parse_timestamp_inputs( inputs: &str, - fetcher: &Fetcher

, + source: Arc, ) -> Result, ParseError> { let parts: Vec<&str> = inputs.split(' ').collect(); match parts.len() { @@ -117,12 +116,12 @@ async fn parse_timestamp_inputs( let first_input = parts.first().ok_or_else(|| { ParseError::ParseError("Failed to get the first input".to_string()) })?; - parse_timestamp_token(first_input, true, fetcher).await.map(|x| vec![x]) + parse_timestamp_token(first_input, true, source).await.map(|x| vec![x]) } _ => { let mut chunks = Vec::new(); for part in parts { - chunks.push(parse_timestamp_token(part, false, fetcher).await?); + chunks.push(parse_timestamp_token(part, false, source.clone()).await?); } Ok(chunks) } @@ -134,10 +133,10 @@ enum RangePosition { None, } -async fn parse_timestamp_token( +async fn parse_timestamp_token( s: &str, as_range: bool, - fetcher: &Fetcher

, + source: Arc, ) -> Result { let s = s.replace('_', ""); @@ -145,8 +144,8 @@ async fn parse_timestamp_token( match parts.as_slice() { [timestamp_ref] => { let timestamp = - parse_timestamp_number(timestamp_ref, RangePosition::None, fetcher).await?; - let block = timestamp_to_block_number(timestamp, fetcher).await?; + parse_timestamp_number(timestamp_ref, RangePosition::None, source.clone()).await?; + let block = timestamp_to_block_number(timestamp, source).await?; Ok(BlockChunk::Numbers(vec![block])) } @@ -162,10 +161,10 @@ async fn parse_timestamp_token( }; let (start_timestamp, end_timestamp) = - parse_timestamp_range(first_ref, second_ref, fetcher).await?; + parse_timestamp_range(first_ref, second_ref, source.clone()).await?; let (start_block, end_block) = ( - timestamp_to_block_number(start_timestamp, fetcher).await?, - timestamp_to_block_number(end_timestamp, fetcher).await?, + timestamp_to_block_number(start_timestamp, source.clone()).await?, + timestamp_to_block_number(end_timestamp, source).await?, ); block_range_to_block_chunk(start_block, end_block, as_range, None, n_keep) } @@ -175,22 +174,19 @@ async fn parse_timestamp_token( } } -async fn parse_timestamp_range

( +async fn parse_timestamp_range( first_ref: &str, second_ref: &str, - fetcher: &Fetcher

, -) -> Result<(u64, u64), ParseError> -where - P: JsonRpcClient, -{ + source: Arc, +) -> Result<(u64, u64), ParseError> { let (start_timestamp, end_timestamp) = match (first_ref, second_ref) { _ if first_ref.starts_with('-') => { let end_timestamp = - parse_timestamp_number(second_ref, RangePosition::Last, fetcher).await?; + parse_timestamp_number(second_ref, RangePosition::Last, source.clone()).await?; let start_timestamp = end_timestamp .checked_sub( - parse_timestamp_number(&first_ref[1..], RangePosition::None, fetcher).await?, + parse_timestamp_number(&first_ref[1..], RangePosition::None, source).await?, ) .ok_or_else(|| ParseError::ParseError("start_timestamp underflow".to_string()))?; @@ -198,11 +194,11 @@ where } _ if second_ref.starts_with('+') => { let start_timestamp = - parse_timestamp_number(first_ref, RangePosition::First, fetcher).await?; + parse_timestamp_number(first_ref, RangePosition::First, source.clone()).await?; let end_timestamp = start_timestamp .checked_add( - parse_timestamp_number(&second_ref[1..], RangePosition::None, fetcher).await?, + parse_timestamp_number(&second_ref[1..], RangePosition::None, source).await?, ) .ok_or_else(|| ParseError::ParseError("end_timestamp overflow".to_string()))?; @@ -210,10 +206,10 @@ where } _ => { let start_timestamp = - parse_timestamp_number(first_ref, RangePosition::First, fetcher).await?; + parse_timestamp_number(first_ref, RangePosition::First, source.clone()).await?; let end_timestamp = - parse_timestamp_number(second_ref, RangePosition::Last, fetcher).await?; + parse_timestamp_number(second_ref, RangePosition::Last, source).await?; (start_timestamp, end_timestamp) } @@ -229,15 +225,15 @@ where Ok((start_timestamp, end_timestamp)) } -async fn parse_timestamp_number( +async fn parse_timestamp_number( timestamp_ref: &str, range_position: RangePosition, - fetcher: &Fetcher

, + source: Arc, ) -> Result { match (timestamp_ref, range_position) { - ("latest", _) => get_latest_timestamp(fetcher).await, + ("latest", _) => get_latest_timestamp(source).await, ("", RangePosition::First) => Ok(0), - ("", RangePosition::Last) => get_latest_timestamp(fetcher).await, + ("", RangePosition::Last) => get_latest_timestamp(source).await, ("", RangePosition::None) => Err(ParseError::ParseError("invalid input".to_string())), _ if timestamp_ref.ends_with('m') => scale_timestamp_str_by_metric_unit(timestamp_ref, 60), _ if timestamp_ref.ends_with('h') => { @@ -274,16 +270,13 @@ fn scale_timestamp_str_by_metric_unit( // perform binary search to determine the closest block number smaller than or equal to a given // timestamp -async fn timestamp_to_block_number( - timestamp: u64, - fetcher: &Fetcher

, -) -> Result { - let latest_block_number = get_latest_block_number(fetcher).await?; +async fn timestamp_to_block_number(timestamp: u64, source: Arc) -> Result { + let latest_block_number = get_latest_block_number(source.clone()).await?; let mut l = 0; let mut r = latest_block_number; let mut mid = (l + r) / 2; - let mut block = fetcher + let mut block = source .get_block(mid) .await .map_err(|_e| ParseError::ParseError("Error fetching block for timestamp".to_string()))? @@ -291,7 +284,7 @@ async fn timestamp_to_block_number( while l <= r { mid = (l + r) / 2; - block = fetcher + block = source .get_block(mid) .await .map_err(|_e| ParseError::ParseError("Error fetching block for timestamp".to_string()))? @@ -315,9 +308,9 @@ async fn timestamp_to_block_number( } } -async fn get_latest_timestamp(fetcher: &Fetcher

) -> Result { - let latest_block_number = get_latest_block_number(fetcher).await?; - let latest_block = fetcher +async fn get_latest_timestamp(source: Arc) -> Result { + let latest_block_number = get_latest_block_number(source.clone()).await?; + let latest_block = source .get_block(latest_block_number) .await .map_err(|_e| ParseError::ParseError("Error fetching latest block".to_string()))? @@ -333,8 +326,10 @@ mod tests { use governor::{Quota, RateLimiter}; use super::*; + use cryo_freeze::SourceLabels; + use ethers::prelude::*; - async fn setup_fetcher() -> Fetcher> { + async fn setup_source() -> Source { let rpc_url = crate::parse::source::parse_rpc_url(&Args::default()).unwrap(); let max_retry = 5; let initial_backoff = 500; @@ -349,128 +344,150 @@ mod tests { let rate_limiter = Some(RateLimiter::direct(quota)); let semaphore = tokio::sync::Semaphore::new(max_concurrent_requests as usize); - Fetcher { provider, semaphore: Some(semaphore), rate_limiter } + Source { + provider: provider.into(), + semaphore: Arc::new(Some(semaphore)), + rate_limiter: Arc::new(rate_limiter), + chain_id: 1, + inner_request_size: 1, + max_concurrent_chunks: None, + rpc_url: "".to_string(), + labels: SourceLabels::default(), + } } #[tokio::test] async fn test_extrema_timestamp_to_block_number() { - let fetcher = setup_fetcher().await; + let source = setup_source().await; + let source = Arc::new(source); // Before genesis block - assert!(timestamp_to_block_number(1438260000, &fetcher).await.unwrap() == 0); + assert!(timestamp_to_block_number(1438260000, source).await.unwrap() == 0); } #[tokio::test] async fn test_latest_timestamp_to_block_number() { - let fetcher = setup_fetcher().await; - let latest_block_number = get_latest_block_number(&fetcher).await.unwrap(); - let latest_block = fetcher.get_block(latest_block_number).await.unwrap().unwrap(); + let source = setup_source().await; + let source = Arc::new(source); + let latest_block_number = get_latest_block_number(source.clone()).await.unwrap(); + let latest_block = source.get_block(latest_block_number).await.unwrap().unwrap(); let latest_timestamp = latest_block.timestamp.as_u64(); assert_eq!( - timestamp_to_block_number(latest_timestamp, &fetcher).await.unwrap(), + timestamp_to_block_number(latest_timestamp, source).await.unwrap(), latest_block_number ); } #[tokio::test] async fn test_timestamp_between_blocks() { - let fetcher = setup_fetcher().await; + let source = setup_source().await; + let source = Arc::new(source); // Block 1000, and the timestamp surrounding block 1020 - assert!(timestamp_to_block_number(1438272177, &fetcher).await.unwrap() == 1020); - assert!(timestamp_to_block_number(1438272178, &fetcher).await.unwrap() == 1020); + assert!(timestamp_to_block_number(1438272177, source.clone()).await.unwrap() == 1020); + assert!(timestamp_to_block_number(1438272178, source.clone()).await.unwrap() == 1020); // Timestamp 1438272176 is 1 seconds after block 1019 and 1 second before block 1020. Lower // block is returned - assert!(timestamp_to_block_number(1438272176, &fetcher).await.unwrap() == 1019); + assert!(timestamp_to_block_number(1438272176, source.clone()).await.unwrap() == 1019); // Timestamp 1438272187 is 1 seconds after block 1024 and 1 second before block 1025. Lower // block is returned - assert!(timestamp_to_block_number(1438272187, &fetcher).await.unwrap() == 1024); + assert!(timestamp_to_block_number(1438272187, source.clone()).await.unwrap() == 1024); // Timestamp 1438272169 is 4 seconds after block 1016 and 4 seconds before block 1017. Lower // block is returned - assert!(timestamp_to_block_number(1438272169, &fetcher).await.unwrap() == 1016); + assert!(timestamp_to_block_number(1438272169, source.clone()).await.unwrap() == 1016); } #[tokio::test] async fn test_parse_timestamp_number() { - let fetcher = setup_fetcher().await; + let source = setup_source().await; + let source = Arc::new(source); let latest_timestamp = - parse_timestamp_number("latest", RangePosition::None, &fetcher).await.unwrap(); - assert_eq!(latest_timestamp, get_latest_timestamp(&fetcher).await.unwrap()); - - assert_eq!(parse_timestamp_number("", RangePosition::First, &fetcher).await.unwrap(), 0); + parse_timestamp_number("latest", RangePosition::None, source.clone()).await.unwrap(); + assert_eq!(latest_timestamp, get_latest_timestamp(source.clone()).await.unwrap()); assert_eq!( - parse_timestamp_number("", RangePosition::Last, &fetcher).await.unwrap(), - get_latest_timestamp(&fetcher).await.unwrap() + parse_timestamp_number("", RangePosition::First, source.clone()).await.unwrap(), + 0 ); assert_eq!( - parse_timestamp_number("1700000000", RangePosition::None, &fetcher).await.unwrap(), + parse_timestamp_number("", RangePosition::Last, source.clone()).await.unwrap(), + get_latest_timestamp(source.clone()).await.unwrap() + ); + + assert_eq!( + parse_timestamp_number("1700000000", RangePosition::None, source.clone()) + .await + .unwrap(), 1700000000 ); - assert_eq!(parse_timestamp_number("1m", RangePosition::None, &fetcher).await.unwrap(), 60); + assert_eq!( + parse_timestamp_number("1m", RangePosition::None, source.clone()).await.unwrap(), + 60 + ); assert_eq!( - parse_timestamp_number("8760h", RangePosition::None, &fetcher).await.unwrap(), + parse_timestamp_number("8760h", RangePosition::None, source.clone()).await.unwrap(), 8760 * 3600 ); assert_eq!( - parse_timestamp_number("365d", RangePosition::None, &fetcher).await.unwrap(), + parse_timestamp_number("365d", RangePosition::None, source.clone()).await.unwrap(), 365 * 86400 ); assert_eq!( - parse_timestamp_number("52w", RangePosition::None, &fetcher).await.unwrap(), + parse_timestamp_number("52w", RangePosition::None, source.clone()).await.unwrap(), 52 * 86400 * 7 ); assert_eq!( - parse_timestamp_number("12M", RangePosition::None, &fetcher).await.unwrap(), + parse_timestamp_number("12M", RangePosition::None, source.clone()).await.unwrap(), 12 * 86400 * 30 ); assert_eq!( - parse_timestamp_number("1y", RangePosition::None, &fetcher).await.unwrap(), + parse_timestamp_number("1y", RangePosition::None, source.clone()).await.unwrap(), 86400 * 365 ); } #[tokio::test] async fn test_parse_timestamp_range_to_block_number_range() { - let fetcher = setup_fetcher().await; + let source = setup_source().await; + let source = Arc::new(source); let (start_timestamp, end_timestamp) = - parse_timestamp_range("1700000000", "1700000015", &fetcher).await.unwrap(); + parse_timestamp_range("1700000000", "1700000015", source.clone()).await.unwrap(); assert_eq!( ( - timestamp_to_block_number(start_timestamp, &fetcher).await.unwrap(), - timestamp_to_block_number(end_timestamp, &fetcher).await.unwrap() + timestamp_to_block_number(start_timestamp, source.clone()).await.unwrap(), + timestamp_to_block_number(end_timestamp, source.clone()).await.unwrap() ), (18573050, 18573051) ); let (start_timestamp, end_timestamp) = - parse_timestamp_range("-15", "1700000015", &fetcher).await.unwrap(); + parse_timestamp_range("-15", "1700000015", source.clone()).await.unwrap(); assert_eq!( ( - timestamp_to_block_number(start_timestamp, &fetcher).await.unwrap(), - timestamp_to_block_number(end_timestamp, &fetcher).await.unwrap() + timestamp_to_block_number(start_timestamp, source.clone()).await.unwrap(), + timestamp_to_block_number(end_timestamp, source.clone()).await.unwrap() ), (18573050, 18573052) ); let (start_timestamp, end_timestamp) = - parse_timestamp_range("1700000000", "+15", &fetcher).await.unwrap(); + parse_timestamp_range("1700000000", "+15", source.clone()).await.unwrap(); assert_eq!( ( - timestamp_to_block_number(start_timestamp, &fetcher).await.unwrap(), - timestamp_to_block_number(end_timestamp, &fetcher).await.unwrap() + timestamp_to_block_number(start_timestamp, source.clone()).await.unwrap(), + timestamp_to_block_number(end_timestamp, source.clone()).await.unwrap() ), (18573050, 18573051) ); diff --git a/crates/freeze/src/datasets/address_appearances.rs b/crates/freeze/src/datasets/address_appearances.rs index 683379a..4e76b61 100644 --- a/crates/freeze/src/datasets/address_appearances.rs +++ b/crates/freeze/src/datasets/address_appearances.rs @@ -42,7 +42,7 @@ impl CollectByBlock for AddressAppearances { async fn extract(request: Params, source: Arc, _: Arc) -> R { let block_number = request.ethers_block_number()?; - let block = source.fetcher.get_block(request.block_number()?).await?; + let block = source.get_block(request.block_number()?).await?; let block = block.ok_or(CollectError::CollectError("block not found".to_string()))?; let filter = Filter { block_option: FilterBlockOption::Range { @@ -51,8 +51,8 @@ impl CollectByBlock for AddressAppearances { }, ..Default::default() }; - let logs = source.fetcher.get_logs(&filter).await?; - let traces = source.fetcher.trace_block(request.block_number()?.into()).await?; + let logs = source.get_logs(&filter).await?; + let traces = source.trace_block(request.block_number()?.into()).await?; Ok((block, logs, traces)) } @@ -69,7 +69,7 @@ impl CollectByTransaction for AddressAppearances { async fn extract(request: Params, source: Arc, _: Arc) -> R { let tx_hash = request.ethers_transaction_hash()?; - let tx_data = source.fetcher.get_transaction(tx_hash).await?.ok_or_else(|| { + let tx_data = source.get_transaction(tx_hash).await?.ok_or_else(|| { CollectError::CollectError("could not find transaction data".to_string()) })?; @@ -78,21 +78,19 @@ impl CollectByTransaction for AddressAppearances { .ok_or_else(|| CollectError::CollectError("block not found".to_string()))? .as_u64(); let block = source - .fetcher .get_block(block_number) .await? .ok_or(CollectError::CollectError("could not get block".to_string()))?; // logs let logs = source - .fetcher .get_transaction_receipt(tx_hash) .await? .ok_or(CollectError::CollectError("could not get tx receipt".to_string()))? .logs; // traces - let traces = source.fetcher.trace_transaction(request.ethers_transaction_hash()?).await?; + let traces = source.trace_transaction(request.ethers_transaction_hash()?).await?; Ok((block, logs, traces)) } diff --git a/crates/freeze/src/datasets/balance_diffs.rs b/crates/freeze/src/datasets/balance_diffs.rs index 5d48471..231a707 100644 --- a/crates/freeze/src/datasets/balance_diffs.rs +++ b/crates/freeze/src/datasets/balance_diffs.rs @@ -29,7 +29,7 @@ impl CollectByBlock for BalanceDiffs { let schema = query.schemas.get(&Datatype::BalanceDiffs).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source.fetcher.trace_block_state_diffs(request.block_number()? as u32, include_txs).await + source.trace_block_state_diffs(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -42,7 +42,7 @@ impl CollectByTransaction for BalanceDiffs { type Response = BlockTxsTraces; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction_state_diffs(request.transaction_hash()?).await + source.trace_transaction_state_diffs(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/balance_reads.rs b/crates/freeze/src/datasets/balance_reads.rs index 8109b9f..a812862 100644 --- a/crates/freeze/src/datasets/balance_reads.rs +++ b/crates/freeze/src/datasets/balance_reads.rs @@ -29,10 +29,7 @@ impl CollectByBlock for BalanceReads { let schema = query.schemas.get(&Datatype::BalanceReads).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source - .fetcher - .geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs) - .await + source.geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -49,7 +46,7 @@ impl CollectByTransaction for BalanceReads { query.schemas.get(&Datatype::BalanceReads).ok_or(err("schema not provided"))?; let include_block_number = schema.has_column("block_number"); let tx = request.transaction_hash()?; - source.fetcher.geth_debug_trace_transaction_prestate(tx, include_block_number).await + source.geth_debug_trace_transaction_prestate(tx, include_block_number).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/balances.rs b/crates/freeze/src/datasets/balances.rs index fdb4f96..4a2d36a 100644 --- a/crates/freeze/src/datasets/balances.rs +++ b/crates/freeze/src/datasets/balances.rs @@ -37,8 +37,7 @@ impl CollectByBlock for Balances { async fn extract(request: Params, source: Arc, _: Arc) -> R { let address = request.address()?; let block_number = request.block_number()? as u32; - let balance = - source.fetcher.get_balance(H160::from_slice(&address), block_number.into()).await?; + let balance = source.get_balance(H160::from_slice(&address), block_number.into()).await?; Ok((block_number, None, address, balance)) } diff --git a/crates/freeze/src/datasets/blocks.rs b/crates/freeze/src/datasets/blocks.rs index 6a93444..b23d823 100644 --- a/crates/freeze/src/datasets/blocks.rs +++ b/crates/freeze/src/datasets/blocks.rs @@ -46,7 +46,6 @@ impl CollectByBlock for Blocks { async fn extract(request: Params, source: Arc, _: Arc) -> R { let block = source - .fetcher .get_block(request.block_number()?) .await? .ok_or(CollectError::CollectError("block not found".to_string()))?; @@ -65,12 +64,10 @@ impl CollectByTransaction for Blocks { async fn extract(request: Params, source: Arc, _: Arc) -> R { let transaction = source - .fetcher .get_transaction(request.ethers_transaction_hash()?) .await? .ok_or(CollectError::CollectError("transaction not found".to_string()))?; let block = source - .fetcher .get_block_by_hash(transaction.block_hash.ok_or(err("no block block_hash found"))?) .await? .ok_or(CollectError::CollectError("block not found".to_string()))?; diff --git a/crates/freeze/src/datasets/code_diffs.rs b/crates/freeze/src/datasets/code_diffs.rs index a0dcc27..2d2a262 100644 --- a/crates/freeze/src/datasets/code_diffs.rs +++ b/crates/freeze/src/datasets/code_diffs.rs @@ -28,7 +28,7 @@ impl CollectByBlock for CodeDiffs { async fn extract(request: Params, source: Arc, query: Arc) -> R { let schema = query.schemas.get(&Datatype::CodeDiffs).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source.fetcher.trace_block_state_diffs(request.block_number()? as u32, include_txs).await + source.trace_block_state_diffs(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -41,7 +41,7 @@ impl CollectByTransaction for CodeDiffs { type Response = BlockTxTraces; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction_state_diffs(request.transaction_hash()?).await + source.trace_transaction_state_diffs(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/code_reads.rs b/crates/freeze/src/datasets/code_reads.rs index 12be1be..0b0aca3 100644 --- a/crates/freeze/src/datasets/code_reads.rs +++ b/crates/freeze/src/datasets/code_reads.rs @@ -28,10 +28,7 @@ impl CollectByBlock for CodeReads { async fn extract(request: Params, source: Arc, query: Arc) -> R { let schema = query.schemas.get(&Datatype::CodeReads).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source - .fetcher - .geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs) - .await + source.geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -47,7 +44,7 @@ impl CollectByTransaction for CodeReads { let schema = query.schemas.get(&Datatype::CodeReads).ok_or(err("schema not provided"))?; let include_block_number = schema.has_column("block_number"); let tx = request.transaction_hash()?; - source.fetcher.geth_debug_trace_transaction_prestate(tx, include_block_number).await + source.geth_debug_trace_transaction_prestate(tx, include_block_number).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/codes.rs b/crates/freeze/src/datasets/codes.rs index 5c3ecb6..1b1b6bf 100644 --- a/crates/freeze/src/datasets/codes.rs +++ b/crates/freeze/src/datasets/codes.rs @@ -41,8 +41,7 @@ impl CollectByBlock for Codes { async fn extract(request: Params, source: Arc, _: Arc) -> R { let address = request.address()?; let block_number = request.block_number()? as u32; - let output = - source.fetcher.get_code(H160::from_slice(&address), block_number.into()).await?; + let output = source.get_code(H160::from_slice(&address), block_number.into()).await?; Ok((block_number, None, address, output.to_vec())) } diff --git a/crates/freeze/src/datasets/contracts.rs b/crates/freeze/src/datasets/contracts.rs index 0dd3a03..4ea8a26 100644 --- a/crates/freeze/src/datasets/contracts.rs +++ b/crates/freeze/src/datasets/contracts.rs @@ -35,7 +35,7 @@ impl CollectByBlock for Contracts { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_block(request.ethers_block_number()?).await + source.trace_block(request.ethers_block_number()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -50,7 +50,7 @@ impl CollectByTransaction for Contracts { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction(request.ethers_transaction_hash()?).await + source.trace_transaction(request.ethers_transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/erc20_balances.rs b/crates/freeze/src/datasets/erc20_balances.rs index 60794a5..a4cf593 100644 --- a/crates/freeze/src/datasets/erc20_balances.rs +++ b/crates/freeze/src/datasets/erc20_balances.rs @@ -36,7 +36,7 @@ impl CollectByBlock for Erc20Balances { call_data.extend(request.address()?); let block_number = request.ethers_block_number()?; let contract = request.ethers_contract()?; - let balance = source.fetcher.call2(contract, call_data, block_number).await.ok(); + let balance = source.call2(contract, call_data, block_number).await.ok(); let balance = balance.map(|x| x.to_vec().as_slice().into()); Ok((request.block_number()? as u32, request.contract()?, request.address()?, balance)) } diff --git a/crates/freeze/src/datasets/erc20_metadata.rs b/crates/freeze/src/datasets/erc20_metadata.rs index 4fdccc3..cc6f154 100644 --- a/crates/freeze/src/datasets/erc20_metadata.rs +++ b/crates/freeze/src/datasets/erc20_metadata.rs @@ -48,7 +48,7 @@ impl CollectByBlock for Erc20Metadata { // name let call_data = FUNCTION_ERC20_NAME.clone(); - let name = match source.fetcher.call2(address, call_data, block_number).await { + let name = match source.call2(address, call_data, block_number).await { Ok(output) => { String::from_utf8(output.to_vec()).ok().map(|s| remove_control_characters(&s)) } @@ -57,7 +57,7 @@ impl CollectByBlock for Erc20Metadata { // symbol let call_data = FUNCTION_ERC20_SYMBOL.clone(); - let symbol = match source.fetcher.call2(address, call_data, block_number).await { + let symbol = match source.call2(address, call_data, block_number).await { Ok(output) => { String::from_utf8(output.to_vec()).ok().map(|s| remove_control_characters(&s)) } @@ -66,7 +66,7 @@ impl CollectByBlock for Erc20Metadata { // decimals let call_data = FUNCTION_ERC20_DECIMALS.clone(); - let decimals = match source.fetcher.call2(address, call_data, block_number).await { + let decimals = match source.call2(address, call_data, block_number).await { Ok(output) => bytes_to_u32(output).ok(), Err(_) => None, }; diff --git a/crates/freeze/src/datasets/erc20_supplies.rs b/crates/freeze/src/datasets/erc20_supplies.rs index be37b62..4d46366 100644 --- a/crates/freeze/src/datasets/erc20_supplies.rs +++ b/crates/freeze/src/datasets/erc20_supplies.rs @@ -42,7 +42,7 @@ impl CollectByBlock for Erc20Supplies { call_data.extend(request.address()?); let block_number = request.ethers_block_number()?; let contract = request.ethers_address()?; - let output = source.fetcher.call2(contract, call_data, block_number).await.ok(); + let output = source.call2(contract, call_data, block_number).await.ok(); let output = output.map(|x| x.to_vec().as_slice().into()); Ok((request.block_number()? as u32, request.address()?, output)) } diff --git a/crates/freeze/src/datasets/erc20_transfers.rs b/crates/freeze/src/datasets/erc20_transfers.rs index 7870bf4..be23086 100644 --- a/crates/freeze/src/datasets/erc20_transfers.rs +++ b/crates/freeze/src/datasets/erc20_transfers.rs @@ -66,7 +66,7 @@ impl CollectByBlock for Erc20Transfers { topics[2] = Some(ValueOrArray::Value(Some(H256::from_slice(&v[..])))); } let filter = Filter { topics, ..request.ethers_log_filter()? }; - let logs = source.fetcher.get_logs(&filter).await?; + let logs = source.get_logs(&filter).await?; Ok(logs.into_iter().filter(|x| x.topics.len() == 3 && x.data.len() == 32).collect()) } @@ -82,7 +82,7 @@ impl CollectByTransaction for Erc20Transfers { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - let logs = source.fetcher.get_transaction_logs(request.transaction_hash()?).await?; + let logs = source.get_transaction_logs(request.transaction_hash()?).await?; Ok(logs.into_iter().filter(is_erc20_transfer).collect()) } diff --git a/crates/freeze/src/datasets/erc721_metadata.rs b/crates/freeze/src/datasets/erc721_metadata.rs index ef6b7b3..16dbf55 100644 --- a/crates/freeze/src/datasets/erc721_metadata.rs +++ b/crates/freeze/src/datasets/erc721_metadata.rs @@ -38,7 +38,7 @@ impl CollectByBlock for Erc721Metadata { // name let call_data = FUNCTION_ERC20_NAME.clone(); - let name = match source.fetcher.call2(address, call_data, block_number).await { + let name = match source.call2(address, call_data, block_number).await { Ok(output) => { String::from_utf8(output.to_vec()).ok().map(|s| remove_control_characters(&s)) } @@ -47,7 +47,7 @@ impl CollectByBlock for Erc721Metadata { // symbol let call_data = FUNCTION_ERC20_SYMBOL.clone(); - let symbol = match source.fetcher.call2(address, call_data, block_number).await { + let symbol = match source.call2(address, call_data, block_number).await { Ok(output) => { String::from_utf8(output.to_vec()).ok().map(|s| remove_control_characters(&s)) } diff --git a/crates/freeze/src/datasets/erc721_transfers.rs b/crates/freeze/src/datasets/erc721_transfers.rs index 7d00f30..e6c3fc2 100644 --- a/crates/freeze/src/datasets/erc721_transfers.rs +++ b/crates/freeze/src/datasets/erc721_transfers.rs @@ -67,7 +67,7 @@ impl CollectByBlock for Erc721Transfers { topics[2] = Some(ValueOrArray::Value(Some(H256::from_slice(&v[..])))); } let filter = Filter { topics, ..request.ethers_log_filter()? }; - let logs = source.fetcher.get_logs(&filter).await?; + let logs = source.get_logs(&filter).await?; Ok(logs.into_iter().filter(|x| x.topics.len() == 4 && x.data.len() == 0).collect()) } @@ -83,7 +83,7 @@ impl CollectByTransaction for Erc721Transfers { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - let logs = source.fetcher.get_transaction_logs(request.transaction_hash()?).await?; + let logs = source.get_transaction_logs(request.transaction_hash()?).await?; Ok(logs.into_iter().filter(is_erc721_transfer).collect()) } diff --git a/crates/freeze/src/datasets/eth_calls.rs b/crates/freeze/src/datasets/eth_calls.rs index 5f19392..9555975 100644 --- a/crates/freeze/src/datasets/eth_calls.rs +++ b/crates/freeze/src/datasets/eth_calls.rs @@ -52,7 +52,7 @@ impl CollectByBlock for EthCalls { ..Default::default() }; let number = request.block_number()?; - let output = source.fetcher.call(transaction, number.into()).await?; + let output = source.call(transaction, number.into()).await?; Ok((number as u32, request.contract()?, request.call_data()?, output.to_vec())) } diff --git a/crates/freeze/src/datasets/four_byte_counts.rs b/crates/freeze/src/datasets/four_byte_counts.rs index 2903675..01d7145 100644 --- a/crates/freeze/src/datasets/four_byte_counts.rs +++ b/crates/freeze/src/datasets/four_byte_counts.rs @@ -34,7 +34,6 @@ impl CollectByBlock for FourByteCounts { query.schemas.get(&Datatype::FourByteCounts).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); source - .fetcher .geth_debug_trace_block_4byte_traces(request.block_number()? as u32, include_txs) .await } @@ -53,7 +52,7 @@ impl CollectByTransaction for FourByteCounts { query.schemas.get(&Datatype::FourByteCounts).ok_or(err("schema not provided"))?; let include_block_number = schema.has_column("block_number"); let tx = request.transaction_hash()?; - source.fetcher.geth_debug_trace_transaction_4byte_traces(tx, include_block_number).await + source.geth_debug_trace_transaction_4byte_traces(tx, include_block_number).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/geth_calls.rs b/crates/freeze/src/datasets/geth_calls.rs index af14eef..a0dc80c 100644 --- a/crates/freeze/src/datasets/geth_calls.rs +++ b/crates/freeze/src/datasets/geth_calls.rs @@ -34,7 +34,7 @@ impl CollectByBlock for GethCalls { let schema = query.schemas.get_schema(&Datatype::GethCalls)?; let include_transaction = schema.has_column("block_number"); let block_number = request.block_number()? as u32; - source.fetcher.geth_debug_trace_block_calls(block_number, include_transaction).await + source.geth_debug_trace_block_calls(block_number, include_transaction).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -50,7 +50,6 @@ impl CollectByTransaction for GethCalls { let schema = query.schemas.get_schema(&Datatype::GethCalls)?; let include_block_number = schema.has_column("block_number"); source - .fetcher .geth_debug_trace_transaction_calls(request.transaction_hash()?, include_block_number) .await } diff --git a/crates/freeze/src/datasets/geth_opcodes.rs b/crates/freeze/src/datasets/geth_opcodes.rs index b320728..1f71831 100644 --- a/crates/freeze/src/datasets/geth_opcodes.rs +++ b/crates/freeze/src/datasets/geth_opcodes.rs @@ -49,10 +49,7 @@ impl CollectByBlock for GethOpcodes { }; let include_transaction = schema.has_column("block_number"); let block_number = request.block_number()? as u32; - source - .fetcher - .geth_debug_trace_block_opcodes(block_number, include_transaction, options) - .await + source.geth_debug_trace_block_opcodes(block_number, include_transaction, options).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -75,7 +72,6 @@ impl CollectByTransaction for GethOpcodes { }; let include_block_number = schema.has_column("block_number"); source - .fetcher .geth_debug_trace_transaction_opcodes( request.transaction_hash()?, include_block_number, diff --git a/crates/freeze/src/datasets/javascript_traces.rs b/crates/freeze/src/datasets/javascript_traces.rs index 04468e9..056d354 100644 --- a/crates/freeze/src/datasets/javascript_traces.rs +++ b/crates/freeze/src/datasets/javascript_traces.rs @@ -33,7 +33,6 @@ impl CollectByBlock for JavascriptTraces { let block = request.block_number()? as u32; if let Some(js_tracer) = &query.js_tracer { source - .fetcher .geth_debug_trace_block_javascript_traces(js_tracer.clone(), block, include_txs) .await } else { @@ -57,7 +56,6 @@ impl CollectByTransaction for JavascriptTraces { let tx = request.transaction_hash()?; if let Some(js_tracer) = &query.js_tracer { source - .fetcher .geth_debug_trace_transaction_javascript_traces( js_tracer.clone(), tx, diff --git a/crates/freeze/src/datasets/logs.rs b/crates/freeze/src/datasets/logs.rs index c4be632..666d7c9 100644 --- a/crates/freeze/src/datasets/logs.rs +++ b/crates/freeze/src/datasets/logs.rs @@ -64,7 +64,7 @@ impl CollectByBlock for Logs { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.get_logs(&request.ethers_log_filter()?).await + source.get_logs(&request.ethers_log_filter()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -78,7 +78,7 @@ impl CollectByTransaction for Logs { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.get_transaction_logs(request.transaction_hash()?).await + source.get_transaction_logs(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/native_transfers.rs b/crates/freeze/src/datasets/native_transfers.rs index 950bcde..c8fde7e 100644 --- a/crates/freeze/src/datasets/native_transfers.rs +++ b/crates/freeze/src/datasets/native_transfers.rs @@ -30,7 +30,7 @@ impl CollectByBlock for NativeTransfers { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - let traces = source.fetcher.trace_block(request.block_number()?.into()).await?; + let traces = source.trace_block(request.block_number()?.into()).await?; Ok(filter_traces_by_from_to_addresses(traces, &request.from_address, &request.to_address)) } @@ -46,7 +46,7 @@ impl CollectByTransaction for NativeTransfers { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - let traces = source.fetcher.trace_transaction(request.ethers_transaction_hash()?).await?; + let traces = source.trace_transaction(request.ethers_transaction_hash()?).await?; Ok(filter_traces_by_from_to_addresses(traces, &request.from_address, &request.to_address)) } diff --git a/crates/freeze/src/datasets/nonce_diffs.rs b/crates/freeze/src/datasets/nonce_diffs.rs index 90c77fc..ac863f2 100644 --- a/crates/freeze/src/datasets/nonce_diffs.rs +++ b/crates/freeze/src/datasets/nonce_diffs.rs @@ -28,7 +28,7 @@ impl CollectByBlock for NonceDiffs { async fn extract(request: Params, source: Arc, query: Arc) -> R { let schema = query.schemas.get_schema(&Datatype::NonceDiffs)?; let include_txs = schema.has_column("transaction_hash"); - source.fetcher.trace_block_state_diffs(request.block_number()? as u32, include_txs).await + source.trace_block_state_diffs(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -41,7 +41,7 @@ impl CollectByTransaction for NonceDiffs { type Response = BlockTxsTraces; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction_state_diffs(request.transaction_hash()?).await + source.trace_transaction_state_diffs(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/nonce_reads.rs b/crates/freeze/src/datasets/nonce_reads.rs index 2d478b8..fdc8d89 100644 --- a/crates/freeze/src/datasets/nonce_reads.rs +++ b/crates/freeze/src/datasets/nonce_reads.rs @@ -28,10 +28,7 @@ impl CollectByBlock for NonceReads { async fn extract(request: Params, source: Arc, query: Arc) -> R { let schema = query.schemas.get(&Datatype::NonceReads).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source - .fetcher - .geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs) - .await + source.geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -47,7 +44,7 @@ impl CollectByTransaction for NonceReads { let schema = query.schemas.get(&Datatype::NonceReads).ok_or(err("schema not provided"))?; let include_block_number = schema.has_column("block_number"); let tx = request.transaction_hash()?; - source.fetcher.geth_debug_trace_transaction_prestate(tx, include_block_number).await + source.geth_debug_trace_transaction_prestate(tx, include_block_number).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/nonces.rs b/crates/freeze/src/datasets/nonces.rs index d1a163e..2948d1b 100644 --- a/crates/freeze/src/datasets/nonces.rs +++ b/crates/freeze/src/datasets/nonces.rs @@ -37,10 +37,8 @@ impl CollectByBlock for Nonces { async fn extract(request: Params, source: Arc, _: Arc) -> R { let address = request.address()?; let block_number = request.block_number()? as u32; - let output = source - .fetcher - .get_transaction_count(H160::from_slice(&address), block_number.into()) - .await?; + let output = + source.get_transaction_count(H160::from_slice(&address), block_number.into()).await?; Ok((block_number, None, address, output.as_u64())) } diff --git a/crates/freeze/src/datasets/slots.rs b/crates/freeze/src/datasets/slots.rs index 7bc6ba6..1641e44 100644 --- a/crates/freeze/src/datasets/slots.rs +++ b/crates/freeze/src/datasets/slots.rs @@ -48,7 +48,6 @@ impl CollectByBlock for Slots { let block_number = request.block_number()? as u32; let slot = request.slot()?; let output = source - .fetcher .get_storage_at( H160::from_slice(&address), H256::from_slice(&slot), diff --git a/crates/freeze/src/datasets/storage_diffs.rs b/crates/freeze/src/datasets/storage_diffs.rs index 217faad..3f1cb18 100644 --- a/crates/freeze/src/datasets/storage_diffs.rs +++ b/crates/freeze/src/datasets/storage_diffs.rs @@ -33,7 +33,7 @@ impl CollectByBlock for StorageDiffs { async fn extract(request: Params, source: Arc, query: Arc) -> R { let schema = query.schemas.get_schema(&Datatype::StorageDiffs)?; let include_txs = schema.has_column("transaction_hash"); - source.fetcher.trace_block_state_diffs(request.block_number()? as u32, include_txs).await + source.trace_block_state_diffs(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -46,7 +46,7 @@ impl CollectByTransaction for StorageDiffs { type Response = BlockTxsTraces; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction_state_diffs(request.transaction_hash()?).await + source.trace_transaction_state_diffs(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/storage_reads.rs b/crates/freeze/src/datasets/storage_reads.rs index f4d13cb..874919a 100644 --- a/crates/freeze/src/datasets/storage_reads.rs +++ b/crates/freeze/src/datasets/storage_reads.rs @@ -34,10 +34,7 @@ impl CollectByBlock for StorageReads { let schema = query.schemas.get(&Datatype::StorageReads).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source - .fetcher - .geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs) - .await + source.geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -54,7 +51,7 @@ impl CollectByTransaction for StorageReads { query.schemas.get(&Datatype::StorageReads).ok_or(err("schema not provided"))?; let include_block_number = schema.has_column("block_number"); let tx = request.transaction_hash()?; - source.fetcher.geth_debug_trace_transaction_prestate(tx, include_block_number).await + source.geth_debug_trace_transaction_prestate(tx, include_block_number).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/datasets/trace_calls.rs b/crates/freeze/src/datasets/trace_calls.rs index e97392f..5af951e 100644 --- a/crates/freeze/src/datasets/trace_calls.rs +++ b/crates/freeze/src/datasets/trace_calls.rs @@ -51,7 +51,6 @@ impl CollectByBlock for TraceCalls { async fn extract(request: Params, source: Arc, _: Arc) -> R { let traces: Vec = source - .fetcher .trace_call2( request.ethers_contract()?, request.call_data()?, diff --git a/crates/freeze/src/datasets/traces.rs b/crates/freeze/src/datasets/traces.rs index d619cdb..2ce7dd4 100644 --- a/crates/freeze/src/datasets/traces.rs +++ b/crates/freeze/src/datasets/traces.rs @@ -42,7 +42,7 @@ impl CollectByBlock for Traces { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - let traces = source.fetcher.trace_block(request.block_number()?.into()).await?; + let traces = source.trace_block(request.block_number()?.into()).await?; Ok(filter_traces_by_from_to_addresses(traces, &request.from_address, &request.to_address)) } @@ -58,7 +58,7 @@ impl CollectByTransaction for Traces { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - let traces = source.fetcher.trace_transaction(request.ethers_transaction_hash()?).await?; + let traces = source.trace_transaction(request.ethers_transaction_hash()?).await?; Ok(filter_traces_by_from_to_addresses(traces, &request.from_address, &request.to_address)) } diff --git a/crates/freeze/src/datasets/transactions.rs b/crates/freeze/src/datasets/transactions.rs index 896b12a..c72c61c 100644 --- a/crates/freeze/src/datasets/transactions.rs +++ b/crates/freeze/src/datasets/transactions.rs @@ -68,7 +68,6 @@ impl CollectByBlock for Transactions { async fn extract(request: Params, source: Arc, query: Arc) -> R { let block = source - .fetcher .get_block_with_txs(request.block_number()?) .await? .ok_or(CollectError::CollectError("block not found".to_string()))?; @@ -137,12 +136,11 @@ impl CollectByTransaction for Transactions { let tx_hash = request.ethers_transaction_hash()?; let schema = query.schemas.get_schema(&Datatype::Transactions)?; let transaction = source - .fetcher .get_transaction(tx_hash) .await? .ok_or(CollectError::CollectError("transaction not found".to_string()))?; let receipt = if schema.has_column("gas_used") { - source.fetcher.get_transaction_receipt(tx_hash).await? + source.get_transaction_receipt(tx_hash).await? } else { None }; @@ -152,7 +150,6 @@ impl CollectByTransaction for Transactions { .ok_or(CollectError::CollectError("no block number for tx".to_string()))?; let block = source - .fetcher .get_block(block_number.as_u64()) .await? .ok_or(CollectError::CollectError("block not found".to_string()))?; diff --git a/crates/freeze/src/datasets/vm_traces.rs b/crates/freeze/src/datasets/vm_traces.rs index 18466d4..b06b465 100644 --- a/crates/freeze/src/datasets/vm_traces.rs +++ b/crates/freeze/src/datasets/vm_traces.rs @@ -42,7 +42,7 @@ impl CollectByBlock for VmTraces { type Response = (Option, Option>, Vec); async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_block_vm_traces(request.block_number()? as u32).await + source.trace_block_vm_traces(request.block_number()? as u32).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -55,7 +55,7 @@ impl CollectByTransaction for VmTraces { type Response = (Option, Option>, Vec); async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction_vm_traces(request.transaction_hash()?).await + source.trace_transaction_vm_traces(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/freeze.rs b/crates/freeze/src/freeze.rs index a9b3f82..8fae5ad 100644 --- a/crates/freeze/src/freeze.rs +++ b/crates/freeze/src/freeze.rs @@ -85,7 +85,7 @@ fn get_payloads( let semaphore = source .max_concurrent_chunks .map(|x| std::sync::Arc::new(tokio::sync::Semaphore::new(x as usize))); - let source = Arc::new(source.clone()); + let source: Arc = Arc::new(source.clone()); let arc_query = Arc::new(query.clone()); let mut payloads = Vec::new(); let mut skipping = Vec::new(); diff --git a/crates/freeze/src/multi_datasets/blocks_and_transactions.rs b/crates/freeze/src/multi_datasets/blocks_and_transactions.rs index 1fd604a..1201259 100644 --- a/crates/freeze/src/multi_datasets/blocks_and_transactions.rs +++ b/crates/freeze/src/multi_datasets/blocks_and_transactions.rs @@ -50,7 +50,6 @@ impl CollectByTransaction for BlocksAndTransactions { ::extract(request, source.clone(), query).await?; let block_number = tx.block_number.ok_or(err("no block number for tx"))?.as_u64(); let block = source - .fetcher .get_block(block_number) .await? .ok_or(CollectError::CollectError("block not found".to_string()))?; diff --git a/crates/freeze/src/multi_datasets/call_trace_derivatives.rs b/crates/freeze/src/multi_datasets/call_trace_derivatives.rs index df53eb0..90148f2 100644 --- a/crates/freeze/src/multi_datasets/call_trace_derivatives.rs +++ b/crates/freeze/src/multi_datasets/call_trace_derivatives.rs @@ -37,7 +37,7 @@ impl CollectByBlock for CallTraceDerivatives { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_block(request.block_number()?.into()).await + source.trace_block(request.block_number()?.into()).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -52,7 +52,7 @@ impl CollectByTransaction for CallTraceDerivatives { type Response = Vec; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction(request.ethers_transaction_hash()?).await + source.trace_transaction(request.ethers_transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/multi_datasets/geth_state_diffs.rs b/crates/freeze/src/multi_datasets/geth_state_diffs.rs index dad661c..fc1c04a 100644 --- a/crates/freeze/src/multi_datasets/geth_state_diffs.rs +++ b/crates/freeze/src/multi_datasets/geth_state_diffs.rs @@ -55,7 +55,7 @@ impl CollectByBlock for GethStateDiffs { async fn extract(request: Params, source: Arc, query: Arc) -> R { let block_number = request.block_number()? as u32; let include_txs = query.schemas.values().any(|x| x.has_column("transaction_hash")); - source.fetcher.geth_debug_trace_block_diffs(block_number, include_txs).await + source.geth_debug_trace_block_diffs(block_number, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -79,7 +79,6 @@ impl CollectByTransaction for GethStateDiffs { async fn extract(request: Params, source: Arc, query: Arc) -> R { let include_block_number = query.schemas.values().any(|x| x.has_column("transaction_hash")); source - .fetcher .geth_debug_trace_transaction_diffs(request.transaction_hash()?, include_block_number) .await } diff --git a/crates/freeze/src/multi_datasets/state_diffs.rs b/crates/freeze/src/multi_datasets/state_diffs.rs index 858fee4..31cff4f 100644 --- a/crates/freeze/src/multi_datasets/state_diffs.rs +++ b/crates/freeze/src/multi_datasets/state_diffs.rs @@ -35,7 +35,7 @@ impl CollectByBlock for StateDiffs { async fn extract(request: Params, source: Arc, query: Arc) -> R { let include_txs = query.schemas.values().any(|x| x.has_column("transaction_hash")); - source.fetcher.trace_block_state_diffs(request.block_number()? as u32, include_txs).await + source.trace_block_state_diffs(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -48,7 +48,7 @@ impl CollectByTransaction for StateDiffs { type Response = BlockTxsTraces; async fn extract(request: Params, source: Arc, _: Arc) -> R { - source.fetcher.trace_transaction_state_diffs(request.transaction_hash()?).await + source.trace_transaction_state_diffs(request.transaction_hash()?).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/multi_datasets/state_reads.rs b/crates/freeze/src/multi_datasets/state_reads.rs index 339bb49..9f92264 100644 --- a/crates/freeze/src/multi_datasets/state_reads.rs +++ b/crates/freeze/src/multi_datasets/state_reads.rs @@ -38,10 +38,7 @@ impl CollectByBlock for StateReads { let schema = query.schemas.get(&Datatype::StorageReads).ok_or(err("schema not provided"))?; let include_txs = schema.has_column("transaction_hash"); - source - .fetcher - .geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs) - .await + source.geth_debug_trace_block_prestate(request.block_number()? as u32, include_txs).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { @@ -58,7 +55,7 @@ impl CollectByTransaction for StateReads { query.schemas.get(&Datatype::StorageReads).ok_or(err("schema not provided"))?; let include_block_number = schema.has_column("block_number"); let tx = request.transaction_hash()?; - source.fetcher.geth_debug_trace_transaction_prestate(tx, include_block_number).await + source.geth_debug_trace_transaction_prestate(tx, include_block_number).await } fn transform(response: Self::Response, columns: &mut Self, query: &Arc) -> R<()> { diff --git a/crates/freeze/src/types/sources.rs b/crates/freeze/src/types/sources.rs index 7e58f15..f45d774 100644 --- a/crates/freeze/src/types/sources.rs +++ b/crates/freeze/src/types/sources.rs @@ -19,8 +19,8 @@ pub type RateLimiter = governor::RateLimiter>>, + /// provider + pub provider: ProviderWrapper, /// chain_id of network pub chain_id: u64, /// number of blocks per log request @@ -29,10 +29,92 @@ pub struct Source { pub max_concurrent_chunks: Option, /// Rpc Url pub rpc_url: String, + /// semaphore for controlling concurrency + pub semaphore: Arc>, + /// rate limiter for controlling request rate + pub rate_limiter: Arc>, /// Labels (these are non-functional) pub labels: SourceLabels, } +// impl { +// fn default() -> Self { +// pub struct Source { +// pub inner_request_size: DEFAULT_INNER_REQUEST_SIZE, +// pub max_concurrent_chunks: DEFAULT_MAX_CONCURRENT_CHUNKS, +// pub rpc_url: String, +// pub semaphore: Option>, +// pub rate_limiter: Option>, +// pub labels: SourceLabels, +// } +// } +// } + +/// A non-generic wrapper over different provider types for use as a trait object +#[derive(Clone, Debug)] +pub enum ProviderWrapper { + /// mock provider + MockProvider(Arc>), + /// http client + RetryClientHttp(Arc>>), + /// websocket client + WsClient(Arc>), +} + +impl From>> for ProviderWrapper { + fn from(value: Provider>) -> ProviderWrapper { + ProviderWrapper::RetryClientHttp(Arc::new(value)) + } +} + +impl From> for ProviderWrapper { + fn from(value: Provider) -> ProviderWrapper { + ProviderWrapper::WsClient(Arc::new(value)) + } +} + +impl From> for ProviderWrapper { + fn from(value: Provider) -> ProviderWrapper { + ProviderWrapper::MockProvider(Arc::new(value)) + } +} + +/// extract the provider from a source and run specified method +#[macro_export] +macro_rules! source_provider { + ($source:expr, $method:ident($($arg:expr),*)) => { + match &$source.provider { + ProviderWrapper::MockProvider(provider) => provider.$method($($arg),*), + ProviderWrapper::RetryClientHttp(provider) => provider.$method($($arg),*), + ProviderWrapper::WsClient(provider) => provider.$method($($arg),*), + } + }; +} + +// impl Source { +// // pub async fn get_block(&self, block_num: u64) -> Result>> { +// // let _permit = self.permit_request().await; +// // Self::map_err(source_provider!(self, get_block(block_num)).await) +// // } + +// async fn permit_request( +// &self, +// ) -> Option<::core::result::Result, AcquireError>> { +// let permit = match &self.semaphore { +// Some(semaphore) => Some(semaphore.acquire().await), +// _ => None, +// }; +// if let Some(limiter) = &self.rate_limiter { +// limiter.until_ready().await; +// } +// permit +// } + +// fn map_err(res: ::core::result::Result) -> Result { +// res.map_err(CollectError::ProviderError) +// } +// } + impl Source { /// Returns all receipts for a block. /// Tries to use `eth_getBlockReceipts` first, and falls back to `eth_getTransactionReceipt` @@ -42,7 +124,7 @@ impl Source { ) -> Result> { let block_number = block.number.ok_or(CollectError::CollectError("no block number".to_string()))?.as_u64(); - if let Ok(receipts) = self.fetcher.get_block_receipts(block_number).await { + if let Ok(receipts) = self.get_block_receipts(block_number).await { return Ok(receipts); } @@ -57,9 +139,9 @@ impl Source { let mut tasks = Vec::new(); for tx in transactions { let tx_hash = tx.hash; - let fetcher = self.fetcher.clone(); + let source = self.clone(); let task = task::spawn(async move { - match fetcher.get_transaction_receipt(tx_hash).await? { + match source.get_transaction_receipt(tx_hash).await? { Some(receipt) => Ok(receipt), None => { Err(CollectError::CollectError("could not find tx receipt".to_string())) @@ -105,10 +187,16 @@ impl Source { let rate_limiter = None; let semaphore = None; - let fetcher = Fetcher { provider, semaphore, rate_limiter }; + + let provider = Provider::>::new_client( + &rpc_url, + DEFAULT_MAX_RETRIES, + DEFAULT_INTIAL_BACKOFF, + ) + .map_err(|_| CollectError::RPCError("could not connect to provider".to_string()))?; let source = Source { - fetcher: Arc::new(fetcher), + provider: ProviderWrapper::RetryClientHttp(Arc::new(provider)), chain_id, inner_request_size: DEFAULT_INNER_REQUEST_SIZE, max_concurrent_chunks: Some(DEFAULT_MAX_CONCURRENT_CHUNKS), @@ -119,6 +207,8 @@ impl Source { max_retries: Some(DEFAULT_MAX_RETRIES), initial_backoff: Some(DEFAULT_INTIAL_BACKOFF), }, + rate_limiter: rate_limiter.into(), + semaphore: semaphore.into(), }; Ok(source) @@ -173,7 +263,7 @@ fn parse_rpc_url(rpc_url: Option) -> String { // } /// source labels (non-functional) -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct SourceLabels { /// Maximum requests collected concurrently pub max_concurrent_requests: Option, @@ -198,11 +288,12 @@ pub struct Fetcher

{ type Result = ::core::result::Result; -impl Fetcher

{ +// impl Fetcher

{ +impl Source { /// Returns an array (possibly empty) of logs that match the filter pub async fn get_logs(&self, filter: &Filter) -> Result> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_logs(filter).await) + Self::map_err(source_provider!(self, get_logs(filter)).await) } /// Replays all transactions in a block returning the requested traces for each transaction @@ -212,7 +303,9 @@ impl Fetcher

{ trace_types: Vec, ) -> Result> { let _permit = self.permit_request().await; - Self::map_err(self.provider.trace_replay_block_transactions(block, trace_types).await) + Self::map_err( + source_provider!(self, trace_replay_block_transactions(block, trace_types)).await, + ) } /// Get state diff traces of block @@ -263,7 +356,7 @@ impl Fetcher

{ trace_types: Vec, ) -> Result { let _permit = self.permit_request().await; - Self::map_err(self.provider.trace_replay_transaction(tx_hash, trace_types).await) + Self::map_err(source_provider!(self, trace_replay_transaction(tx_hash, trace_types)).await) } /// Get state diff traces of transaction @@ -297,7 +390,7 @@ impl Fetcher

{ /// Gets the transaction with transaction_hash pub async fn get_transaction(&self, tx_hash: TxHash) -> Result> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_transaction(tx_hash).await) + Self::map_err(source_provider!(self, get_transaction(tx_hash)).await) } /// Gets the transaction receipt with transaction_hash @@ -306,25 +399,25 @@ impl Fetcher

{ tx_hash: TxHash, ) -> Result> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_transaction_receipt(tx_hash).await) + Self::map_err(source_provider!(self, get_transaction_receipt(tx_hash)).await) } /// Gets the block at `block_num` (transaction hashes only) pub async fn get_block(&self, block_num: u64) -> Result>> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_block(block_num).await) + Self::map_err(source_provider!(self, get_block(block_num)).await) } /// Gets the block at `block_num` (transaction hashes only) pub async fn get_block_by_hash(&self, block_hash: H256) -> Result>> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_block(BlockId::Hash(block_hash)).await) + Self::map_err(source_provider!(self, get_block(BlockId::Hash(block_hash))).await) } /// Gets the block at `block_num` (full transactions included) pub async fn get_block_with_txs(&self, block_num: u64) -> Result>> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_block_with_txs(block_num).await) + Self::map_err(source_provider!(self, get_block_with_txs(block_num)).await) } /// Returns all receipts for a block. @@ -333,19 +426,21 @@ impl Fetcher

{ /// `eth_getTransactionReceipt` if `eth_getBlockReceipts` is not supported. pub async fn get_block_receipts(&self, block_num: u64) -> Result> { let _permit = self.permit_request().await; - Self::map_err(self.provider.get_block_receipts(block_num).await) + Self::map_err(source_provider!(self, get_block_receipts(block_num)).await) } /// Returns traces created at given block pub async fn trace_block(&self, block_num: BlockNumber) -> Result> { let _permit = self.permit_request().await; - Self::map_err(self.provider.trace_block(block_num).await) + Self::map_err(source_provider!(self, trace_block(block_num)).await) } /// Returns all traces of a given transaction pub async fn trace_transaction(&self, tx_hash: TxHash) -> Result> { let _permit = self.permit_request().await; - self.provider.trace_transaction(tx_hash).await.map_err(CollectError::ProviderError) + source_provider!(self, trace_transaction(tx_hash)) + .await + .map_err(CollectError::ProviderError) } /// Deprecated @@ -355,8 +450,8 @@ impl Fetcher

{ block_number: BlockNumber, ) -> Result { let _permit = self.permit_request().await; - self.provider - .call(&transaction.into(), Some(block_number.into())) + let tx: ethers::core::types::transaction::eip2718::TypedTransaction = transaction.into(); + source_provider!(self, call(&tx, Some(block_number.into()))) .await .map_err(CollectError::ProviderError) } @@ -369,8 +464,7 @@ impl Fetcher

{ block_number: Option, ) -> Result { let _permit = self.permit_request().await; - self.provider - .trace_call(transaction, trace_type, block_number) + source_provider!(self, trace_call(transaction, trace_type, block_number)) .await .map_err(CollectError::ProviderError) } @@ -382,8 +476,7 @@ impl Fetcher

{ block_number: BlockNumber, ) -> Result { let _permit = self.permit_request().await; - self.provider - .get_transaction_count(address, Some(block_number.into())) + source_provider!(self, get_transaction_count(address, Some(block_number.into()))) .await .map_err(CollectError::ProviderError) } @@ -391,8 +484,7 @@ impl Fetcher

{ /// Get code at address pub async fn get_balance(&self, address: H160, block_number: BlockNumber) -> Result { let _permit = self.permit_request().await; - self.provider - .get_balance(address, Some(block_number.into())) + source_provider!(self, get_balance(address, Some(block_number.into()))) .await .map_err(CollectError::ProviderError) } @@ -400,8 +492,7 @@ impl Fetcher

{ /// Get code at address pub async fn get_code(&self, address: H160, block_number: BlockNumber) -> Result { let _permit = self.permit_request().await; - self.provider - .get_code(address, Some(block_number.into())) + source_provider!(self, get_code(address, Some(block_number.into()))) .await .map_err(CollectError::ProviderError) } @@ -414,15 +505,14 @@ impl Fetcher

{ block_number: BlockNumber, ) -> Result { let _permit = self.permit_request().await; - self.provider - .get_storage_at(address, slot, Some(block_number.into())) + source_provider!(self, get_storage_at(address, slot, Some(block_number.into()))) .await .map_err(CollectError::ProviderError) } /// Get the block number pub async fn get_block_number(&self) -> Result { - Self::map_err(self.provider.get_block_number().await) + Self::map_err(source_provider!(self, get_block_number()).await) } // extra helpers below @@ -459,8 +549,8 @@ impl Fetcher

{ ..Default::default() }; let _permit = self.permit_request().await; - self.provider - .call(&transaction.into(), Some(block_number.into())) + let tx: ethers::core::types::transaction::eip2718::TypedTransaction = transaction.into(); + source_provider!(self, call(&tx, Some(block_number.into()))) .await .map_err(CollectError::ProviderError) } @@ -479,8 +569,7 @@ impl Fetcher

{ ..Default::default() }; let _permit = self.permit_request().await; - self.provider - .trace_call(transaction, trace_type, block_number) + source_provider!(self, trace_call(transaction, trace_type, block_number)) .await .map_err(CollectError::ProviderError) } @@ -494,8 +583,7 @@ impl Fetcher

{ ) -> Result<(Option, Vec>>, Vec)> { let traces = { let _permit = self.permit_request().await; - self.provider - .debug_trace_block_by_number(Some(block_number.into()), options) + source_provider!(self, debug_trace_block_by_number(Some(block_number.into()), options)) .await .map_err(CollectError::ProviderError)? }; @@ -683,8 +771,7 @@ impl Fetcher

{ let trace = { let _permit = self.permit_request().await; - self.provider - .debug_trace_transaction(ethers_tx, options) + source_provider!(self, debug_trace_transaction(ethers_tx, options)) .await .map_err(CollectError::ProviderError)? }; @@ -861,11 +948,11 @@ impl Fetcher

{ async fn permit_request( &self, ) -> Option<::core::result::Result, AcquireError>> { - let permit = match &self.semaphore { + let permit = match &*self.semaphore { Some(semaphore) => Some(semaphore.acquire().await), _ => None, }; - if let Some(limiter) = &self.rate_limiter { + if let Some(limiter) = &*self.rate_limiter { limiter.until_ready().await; } permit @@ -882,7 +969,6 @@ use std::collections::BTreeMap; fn parse_geth_diff_object( map: ethers::utils::__serde_json::Map, ) -> Result { - println!("HERE {:?}", map); let pre: BTreeMap = serde_json::from_value(map["pre"].clone()) .map_err(|_| err("cannot deserialize pre diff"))?; let post: BTreeMap = serde_json::from_value(map["post"].clone()) From f405eb8ee046172e9f6545565a949f22fe7def2f Mon Sep 17 00:00:00 2001 From: sslivkoff Date: Sun, 31 Dec 2023 00:09:54 -0800 Subject: [PATCH 2/3] add websocket support --- crates/cli/src/parse/source.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/cli/src/parse/source.rs b/crates/cli/src/parse/source.rs index 284cede..e79bdb6 100644 --- a/crates/cli/src/parse/source.rs +++ b/crates/cli/src/parse/source.rs @@ -1,7 +1,7 @@ use std::env; use crate::args::Args; -use cryo_freeze::{ParseError, Source, SourceLabels}; +use cryo_freeze::{sources::ProviderWrapper, ParseError, Source, SourceLabels}; use ethers::prelude::*; use governor::{Quota, RateLimiter}; use polars::prelude::*; @@ -10,10 +10,24 @@ use std::num::NonZeroU32; pub(crate) async fn parse_source(args: &Args) -> Result { // parse network info let rpc_url = parse_rpc_url(args)?; - let provider = - Provider::>::new_client(&rpc_url, args.max_retries, args.initial_backoff) - .map_err(|_e| ParseError::ParseError("could not connect to provider".to_string()))?; - let chain_id = provider.get_chainid().await.map_err(ParseError::ProviderError)?.as_u64(); + let (provider, chain_id): (ProviderWrapper, u64) = if rpc_url.starts_with("http") { + let provider = Provider::>::new_client( + &rpc_url, + args.max_retries, + args.initial_backoff, + ) + .map_err(|_e| ParseError::ParseError("could not connect to provider".to_string()))?; + let chain_id = provider.get_chainid().await.map_err(ParseError::ProviderError)?.as_u64(); + (provider.into(), chain_id) + } else if rpc_url.starts_with("ws") { + let provider = Provider::::connect(&rpc_url).await.map_err(|_| { + ParseError::ParseError("could not instantiate HTTP Provider".to_string()) + })?; + let chain_id = provider.get_chainid().await.map_err(ParseError::ProviderError)?.as_u64(); + (provider.into(), chain_id) + } else { + return Err(ParseError::ParseError(format!("invalid rpc url: {}", rpc_url))); + }; let rate_limiter = match args.requests_per_second { Some(rate_limit) => match (NonZeroU32::new(1), NonZeroU32::new(rate_limit)) { @@ -44,7 +58,7 @@ pub(crate) async fn parse_source(args: &Args) -> Result { semaphore, rate_limiter: rate_limiter.into(), rpc_url, - provider: provider.into(), + provider, labels: SourceLabels { max_concurrent_requests: args.requests_per_second.map(|x| x as u64), max_requests_per_second: args.requests_per_second.map(|x| x as u64), @@ -67,7 +81,7 @@ pub(crate) fn parse_rpc_url(args: &Args) -> Result { } }, }; - if !url.starts_with("http") { + if !url.starts_with("http") & !url.starts_with("ws") { url = "http://".to_string() + url.as_str(); }; Ok(url) From 331e59a8dbac8ac1a02b1db69797b548810949db Mon Sep 17 00:00:00 2001 From: sslivkoff Date: Sun, 31 Dec 2023 12:03:25 -0800 Subject: [PATCH 3/3] add ipc support --- crates/cli/src/parse/source.rs | 8 ++++- crates/freeze/src/types/sources.rs | 52 +++++++----------------------- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/crates/cli/src/parse/source.rs b/crates/cli/src/parse/source.rs index e79bdb6..a778037 100644 --- a/crates/cli/src/parse/source.rs +++ b/crates/cli/src/parse/source.rs @@ -25,6 +25,12 @@ pub(crate) async fn parse_source(args: &Args) -> Result { })?; let chain_id = provider.get_chainid().await.map_err(ParseError::ProviderError)?.as_u64(); (provider.into(), chain_id) + } else if rpc_url.ends_with(".ipc") { + let provider: Provider = Provider::connect_ipc(&rpc_url).await.map_err(|_| { + ParseError::ParseError("could not instantiate HTTP Provider".to_string()) + })?; + let chain_id = provider.get_chainid().await.map_err(ParseError::ProviderError)?.as_u64(); + (provider.into(), chain_id) } else { return Err(ParseError::ParseError(format!("invalid rpc url: {}", rpc_url))); }; @@ -81,7 +87,7 @@ pub(crate) fn parse_rpc_url(args: &Args) -> Result { } }, }; - if !url.starts_with("http") & !url.starts_with("ws") { + if !url.starts_with("http") & !url.starts_with("ws") & !url.ends_with(".ipc") { url = "http://".to_string() + url.as_str(); }; Ok(url) diff --git a/crates/freeze/src/types/sources.rs b/crates/freeze/src/types/sources.rs index f45d774..4c5107b 100644 --- a/crates/freeze/src/types/sources.rs +++ b/crates/freeze/src/types/sources.rs @@ -37,19 +37,6 @@ pub struct Source { pub labels: SourceLabels, } -// impl { -// fn default() -> Self { -// pub struct Source { -// pub inner_request_size: DEFAULT_INNER_REQUEST_SIZE, -// pub max_concurrent_chunks: DEFAULT_MAX_CONCURRENT_CHUNKS, -// pub rpc_url: String, -// pub semaphore: Option>, -// pub rate_limiter: Option>, -// pub labels: SourceLabels, -// } -// } -// } - /// A non-generic wrapper over different provider types for use as a trait object #[derive(Clone, Debug)] pub enum ProviderWrapper { @@ -59,6 +46,14 @@ pub enum ProviderWrapper { RetryClientHttp(Arc>>), /// websocket client WsClient(Arc>), + /// ipc client + IpcClient(Arc>), +} + +impl From> for ProviderWrapper { + fn from(value: Provider) -> ProviderWrapper { + ProviderWrapper::MockProvider(Arc::new(value)) + } } impl From>> for ProviderWrapper { @@ -73,9 +68,9 @@ impl From> for ProviderWrapper { } } -impl From> for ProviderWrapper { - fn from(value: Provider) -> ProviderWrapper { - ProviderWrapper::MockProvider(Arc::new(value)) +impl From> for ProviderWrapper { + fn from(value: Provider) -> ProviderWrapper { + ProviderWrapper::IpcClient(Arc::new(value)) } } @@ -87,34 +82,11 @@ macro_rules! source_provider { ProviderWrapper::MockProvider(provider) => provider.$method($($arg),*), ProviderWrapper::RetryClientHttp(provider) => provider.$method($($arg),*), ProviderWrapper::WsClient(provider) => provider.$method($($arg),*), + ProviderWrapper::IpcClient(provider) => provider.$method($($arg),*), } }; } -// impl Source { -// // pub async fn get_block(&self, block_num: u64) -> Result>> { -// // let _permit = self.permit_request().await; -// // Self::map_err(source_provider!(self, get_block(block_num)).await) -// // } - -// async fn permit_request( -// &self, -// ) -> Option<::core::result::Result, AcquireError>> { -// let permit = match &self.semaphore { -// Some(semaphore) => Some(semaphore.acquire().await), -// _ => None, -// }; -// if let Some(limiter) = &self.rate_limiter { -// limiter.until_ready().await; -// } -// permit -// } - -// fn map_err(res: ::core::result::Result) -> Result { -// res.map_err(CollectError::ProviderError) -// } -// } - impl Source { /// Returns all receipts for a block. /// Tries to use `eth_getBlockReceipts` first, and falls back to `eth_getTransactionReceipt`