diff --git a/common/proto/erc20.proto b/common/proto/erc20.proto index 2c0b99cf..da0e6d9f 100644 --- a/common/proto/erc20.proto +++ b/common/proto/erc20.proto @@ -11,6 +11,8 @@ message ERC20Token { string name = 2; string symbol = 3; uint64 decimals = 4; + string tx_created = 5; + uint64 block_created = 6; } message TransferEvents { @@ -19,20 +21,26 @@ message TransferEvents { message TransferEvent { string tx_hash = 1; - uint32 log_index = 2; - uint64 log_ordinal = 3; - string token_address = 4; - string from = 5; - string to = 6; - string amount = 7; // BigInt, in token's native amount + uint64 block_number = 2; + uint64 timestamp = 3; + uint32 log_index = 4; + optional uint64 log_ordinal = 5; + string token_address = 6; + string from = 7; + string to = 8; + string amount = 9; // BigInt, in token's native amount + repeated TokenBalance balance_changes = 10; } message TokenBalance { - string token_address = 1; - string balance = 2; // BigInt, in token's native amount + optional uint64 log_ordinal = 1; + ERC20Token token = 2; + string address = 3; // account address of the balance change + optional string old_balance = 4; // BigInt, in token's native amount + string new_balance = 5; // BigInt, in token's native amount + optional int32 reason = 6; } -message Account { - string address = 1; - repeated TokenBalance balances = 2; +message TokenBalances { + TokenBalance items = 1; } diff --git a/erc20-holdings/Makefile b/erc20-holdings/Makefile index a5bf0781..52b9b93e 100644 --- a/erc20-holdings/Makefile +++ b/erc20-holdings/Makefile @@ -6,4 +6,4 @@ build: .PHONY: run run: - substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_block_to_erc20_contracts -s 1 + substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_block_to_erc20_contracts -s 10606500 -t +100 diff --git a/erc20-holdings/src/lib.rs b/erc20-holdings/src/lib.rs index 6419a124..322e7706 100644 --- a/erc20-holdings/src/lib.rs +++ b/erc20-holdings/src/lib.rs @@ -4,6 +4,7 @@ pub mod abi; pub mod pb; mod keyer; +mod rpc; use pb::common::v1 as common; use pb::erc20::v1 as erc20; @@ -22,10 +23,13 @@ use substreams::store::StoreSetBigDecimal; use substreams::store::StoreSetRaw; use substreams::{hex, log, proto, store, Hex}; use substreams_ethereum::{pb::eth as pbeth, Event, NULL_ADDRESS}; +use substreams_helper::erc20::Erc20Token; use substreams_helper::keyer::chainlink_asset_key; use substreams_helper::types::Address; -fn contract_bytecode_len(call: &pbeth::v2::Call) -> usize { +const INITIALIZE_METHOD_HASH: [u8; 4] = hex!("1459457a"); + +fn code_len(call: &pbeth::v2::Call) -> usize { let mut len = 0; for code_change in &call.code_changes { len += code_change.new_code.len() @@ -38,30 +42,108 @@ fn contract_bytecode_len(call: &pbeth::v2::Call) -> usize { #[substreams::handlers::map] fn map_block_to_erc20_contracts( block: pbeth::v2::Block, -) -> Result { - let mut erc20_contracts = common::Addresses { items: vec![] }; - - for call_view in block.calls() { - let call = call_view.call; - if call.call_type == pbeth::v2::CallType::Create as i32 { - // skipping contracts that are too short to be an erc20 token - if contract_bytecode_len(call) < 150 { +) -> Result { + let mut erc20_tokens = erc20::Erc20Tokens { items: vec![] }; + + for tx in block.transaction_traces { + for call in tx.calls { + if call.state_reverted { continue; } - let address = Hex(call.address.clone()).to_string(); + if call.call_type == pbeth::v2::CallType::Create as i32 + || call.call_type == pbeth::v2::CallType::Call as i32 + // proxy contract creation + { + let call_input_len = call.input.len(); + if call.call_type == pbeth::v2::CallType::Call as i32 + && (call_input_len < 4 || call.input[0..4] != INITIALIZE_METHOD_HASH) + { + // this will check if a proxy contract has been called to create a ERC20 contract. + // if that is the case the Proxy contract will call the initialize function on the ERC20 contract + // this is part of the OpenZeppelin Proxy contract standard + continue; + } - // check if contract is an erc20 token - if substreams_helper::erc20::get_erc20_token(address.clone()).is_none() { - continue; - } + // Contract creation not from proxy contract + if call.call_type == pbeth::v2::CallType::Create as i32 { + let mut code_change_len = 0; + for code_change in &call.code_changes { + code_change_len += code_change.new_code.len() + } + + if code_change_len <= 150 { + // skipping contracts with less than 150 bytes of code + log::info!( + "Skipping contract {}. Contract code is less than 150 bytes.", + Hex::encode(&call.address) + ); + continue; + } + } - log::info!("Create {}, len {}", address, contract_bytecode_len(call)); - erc20_contracts.items.push(common::Address { address }); + let mut decimals = 18_u64; + let decimal_result = rpc::get_erc20_decimals(&call.address); + match decimal_result { + Ok(_decimals) => decimals = _decimals, + Err(e) => continue, + }; + + let mut symbol = "".to_string(); + let symbaol_result = rpc::get_erc20_symbol(&call.address); + match symbaol_result { + Ok(_symbol) => symbol = _symbol, + Err(e) => continue, + }; + + let mut name = "".to_string(); + let name_result = rpc::get_erc20_name(&call.address); + match name_result { + Ok(_name) => name = _name, + Err(e) => continue, + }; + + erc20_tokens.items.push(erc20::Erc20Token { + address: Hex::encode(call.address.clone()), + name: name, + symbol: symbol, + decimals: decimals, + tx_created: Hex::encode(&tx.hash), + block_created: block.number, + }); + } } } - Ok(erc20_contracts) + // for call_view in block.calls() { + // let call = call_view.call; + // if call.call_type == pbeth::v2::CallType::Create as i32 { + // // skipping contracts that are too short to be an erc20 token + // if code_len(call) < 150 { + // continue; + // } + // + // let address = Hex::encode(call.address.clone()); + // + // // check if contract is an erc20 token + // let erc20_struct = substreams_helper::erc20::get_erc20_token(address.clone()); + // if erc20_struct.is_none() { + // continue; + // } + // + // log::info!("Create {}, len {}", address, code_len(call)); + // erc20_tokens.items.push(erc20::Erc20Token { + // address: address, + // name: erc20_struct.as_ref().unwrap().name.clone(), + // symbol: erc20_struct.as_ref().unwrap().symbol.clone(), + // decimals: erc20_struct.as_ref().unwrap().decimals, + // tx_created: "TODO".to_string(), + // block_created: block.number, + // }); + // } + // } + + Ok(erc20_tokens) } /// Extracts transfer events from the blocks @@ -81,13 +163,23 @@ fn map_block_to_transfers( } transfer_events.items.push(erc20::TransferEvent { - tx_hash: Hex(log.receipt.transaction.clone().hash).to_string(), + tx_hash: Hex::encode(log.receipt.transaction.clone().hash), + block_number: block.number, + timestamp: block + .header + .as_ref() + .unwrap() + .timestamp + .as_ref() + .unwrap() + .seconds as u64, log_index: log.index(), - log_ordinal: log.ordinal(), - token_address: Hex(log.address()).to_string(), - from: Hex(event.from).to_string(), - to: Hex(event.to).to_string(), + log_ordinal: Some(log.ordinal()), + token_address: Hex::encode(log.address()), + from: Hex::encode(event.from), + to: Hex::encode(event.to), amount: event.value.to_string(), + balance_changes: vec![], }) } } @@ -100,7 +192,7 @@ fn store_transfers(transfers: erc20::TransferEvents, output: store::StoreSetRaw) log::info!("Stored events {}", transfers.items.len()); for transfer in transfers.items { output.set( - transfer.log_ordinal, + transfer.log_ordinal.unwrap(), Hex::encode(&transfer.token_address), &proto::encode(&transfer).unwrap(), ); @@ -112,14 +204,14 @@ fn store_balance(transfers: erc20::TransferEvents, output: store::StoreAddBigInt log::info!("Stored events {}", transfers.items.len()); for transfer in transfers.items { output.add( - transfer.log_ordinal, + transfer.log_ordinal.unwrap(), keyer::account_balance_key(&transfer.to), &BigInt::from_str(transfer.amount.as_str()).unwrap(), ); if Hex::decode(transfer.from.clone()).unwrap() != NULL_ADDRESS { output.add( - transfer.log_ordinal, + transfer.log_ordinal.unwrap(), keyer::account_balance_key(&transfer.from), &BigInt::from_str((transfer.amount).as_str()).unwrap().neg(), ); @@ -148,7 +240,7 @@ fn store_balance_usd( match balances.get_last(keyer::account_balance_key(&transfer.to)) { Some(balance) => output.set( - transfer.log_ordinal, + transfer.log_ordinal.unwrap(), keyer::account_balance_usd_key(&transfer.to), &(token_price.clone() * balance.to_decimal(token_decimals.into())), ), @@ -158,7 +250,7 @@ fn store_balance_usd( if Hex::decode(transfer.from.clone()).unwrap() != NULL_ADDRESS { match balances.get_last(keyer::account_balance_key(&transfer.from)) { Some(balance) => output.set( - transfer.log_ordinal, + transfer.log_ordinal.unwrap(), keyer::account_balance_usd_key(&transfer.from), &(token_price.clone() * balance.to_decimal(token_decimals.into())), ), diff --git a/erc20-holdings/src/rpc.rs b/erc20-holdings/src/rpc.rs new file mode 100644 index 00000000..6b4c5516 --- /dev/null +++ b/erc20-holdings/src/rpc.rs @@ -0,0 +1,75 @@ +use std::fmt::Error; +use substreams::{log, Hex}; +use substreams_ethereum::pb::eth; +use substreams_ethereum::rpc::eth_call; +use substreams_helper::utils::{read_string, read_uint32}; + +// Functions to attempt to get erc20 contract calls + +pub const DECIMALS: &str = "313ce567"; +pub const NAME: &str = "06fdde03"; +pub const SYMBOL: &str = "95d89b41"; + +pub fn get_erc20_decimals(call_addr: &Vec) -> Result { + let rpc_call_decimal = create_rpc_calls(call_addr, vec![DECIMALS]); + let rpc_responses_unmarshalled_decimal = eth_call(&rpc_call_decimal); + let response_decimal = rpc_responses_unmarshalled_decimal.responses; + if response_decimal.len() < 1 || response_decimal[0].failed { + return Err(Error); + } + + let decoded_decimals = read_uint32(response_decimal[0].raw.as_ref()); + if decoded_decimals.is_err() { + log::info!("Failed to decode decimals"); + return Err(Error); + } + + return Ok(decoded_decimals.unwrap() as u64); +} + +pub fn get_erc20_symbol(call_addr: &Vec) -> Result { + let rpc_call_symbol = create_rpc_calls(call_addr, vec![SYMBOL]); + let rpc_responses_unmarshalled = eth_call(&rpc_call_symbol); + let responses = rpc_responses_unmarshalled.responses; + if responses.len() < 2 || responses[1].failed { + log::info!("Failed to get symbol"); + return Err(Error); + }; + + let decoded_symbol = read_string(responses[2].raw.as_ref()); + if decoded_symbol.is_err() { + log::info!("Failed to decode symbol"); + return Err(Error); + } + + return Ok(decoded_symbol.unwrap()); +} + +pub fn get_erc20_name(call_addr: &Vec) -> Result { + let rpc_call_name = create_rpc_calls(call_addr, vec![NAME]); + let rpc_responses_unmarshalled = eth_call(&rpc_call_name); + let responses = rpc_responses_unmarshalled.responses; + if responses.len() < 1 || responses[0].failed { + return Err(Error); + }; + + let decoded_name = read_string(responses[1].raw.as_ref()); + if decoded_name.is_err() { + return Err(Error); + } + + return Ok(decoded_name.unwrap()); +} + +fn create_rpc_calls(addr: &Vec, method_signatures: Vec<&str>) -> eth::rpc::RpcCalls { + let mut rpc_calls = eth::rpc::RpcCalls { calls: vec![] }; + + for method_signature in method_signatures { + rpc_calls.calls.push(eth::rpc::RpcCall { + to_addr: Vec::from(addr.clone()), + data: Hex::decode(method_signature).unwrap(), + }) + } + + return rpc_calls; +} diff --git a/erc20-holdings/substreams.yaml b/erc20-holdings/substreams.yaml index 7353d0de..c72d7109 100644 --- a/erc20-holdings/substreams.yaml +++ b/erc20-holdings/substreams.yaml @@ -20,13 +20,14 @@ binaries: file: ../target/wasm32-unknown-unknown/release/substreams_erc20_holdings.wasm modules: + # This should be a store module since it is needed by multiple modules - name: map_block_to_erc20_contracts kind: map initialBlock: 1 inputs: - source: sf.ethereum.type.v2.Block output: - type: proto:messari.common.v1.Addresses + type: proto:messari.erc20.v1.ERC20Tokens - name: map_block_to_transfers kind: map diff --git a/erc20-market-cap/src/pb.rs b/erc20-market-cap/src/pb.rs index 9a8fb3eb..eb61bfbe 100644 --- a/erc20-market-cap/src/pb.rs +++ b/erc20-market-cap/src/pb.rs @@ -37,3 +37,13 @@ pub mod erc20_price { pub use super::super::erc20_price_v1::*; } } + +#[rustfmt::skip] +#[path = "../target/pb/messari.uniswap.v1.rs"] +pub(in crate::pb) mod uniswap_v1; + +pub mod uniswap { + pub mod v1 { + pub use super::super::uniswap_v1::*; + } +} diff --git a/erc20-price/src/modules/1_store_chainlink_aggregator.rs b/erc20-price/src/modules/1_store_chainlink_aggregator.rs index f8d8f0ab..8515aa3e 100644 --- a/erc20-price/src/modules/1_store_chainlink_aggregator.rs +++ b/erc20-price/src/modules/1_store_chainlink_aggregator.rs @@ -78,12 +78,16 @@ fn store_chainlink_aggregator(block: eth::Block, output: StoreSetProto Result Result<(), anyhow::Error> { codegen::generate(None)?; - codegen::generate_abi(None)?; Ok(()) } diff --git a/substreams-helper/src/pb.rs b/substreams-helper/src/pb.rs index 21c620eb..cd2c33a1 100644 --- a/substreams-helper/src/pb.rs +++ b/substreams-helper/src/pb.rs @@ -1,9 +1,9 @@ #[rustfmt::skip] -#[path = "../target/pb/messari.eth_balance.v1.rs"] -pub(in crate::pb) mod eth_balance_v1; +#[path = "../target/pb/messari.erc20.v1.rs"] +pub(in crate::pb) mod erc20_v1; -pub mod eth_balance { +pub mod erc20 { pub mod v1 { - pub use super::super::eth_balance_v1::*; + pub use super::super::erc20_v1::*; } } diff --git a/substreams-helper/src/token.rs b/substreams-helper/src/token.rs index 9517ff58..b79a5cad 100644 --- a/substreams-helper/src/token.rs +++ b/substreams-helper/src/token.rs @@ -1,21 +1,21 @@ -use crate::pb::eth_balance::v1::Token; +use crate::pb::erc20::v1::Erc20Token; use num_bigint; use substreams::scalar::BigInt; use substreams_ethereum::pb::eth as pbeth; -pub fn get_eth_token() -> Option { - let eth_token = Token { +pub fn get_eth_token() -> Option { + let eth_token = Erc20Token { address: "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE".to_string(), name: "Ethereum".to_string(), symbol: "ETH".to_string(), decimals: 18_u64, + tx_created: "".to_string(), + block_created: 0_u64, }; Some(eth_token) } -// TODO: replace this with substreams::scalar::BigInt once the wrapper is integrated -// TODO: make this an impl of fmt::Display pub fn bigint_to_string(number: Option) -> String { number .as_ref() diff --git a/substreams-helper/substreams.yaml b/substreams-helper/substreams.yaml index a79c6089..fd59c8ab 100644 --- a/substreams-helper/substreams.yaml +++ b/substreams-helper/substreams.yaml @@ -8,10 +8,9 @@ imports: protobuf: files: - - eth_balance.proto - - + - erc20.proto importPaths: - - ../eth-balance/proto + - ../common/proto binaries: default: