Skip to content

Commit

Permalink
Amend erc20 proto to support all fields we want
Browse files Browse the repository at this point in the history
  • Loading branch information
melotik committed Feb 1, 2023
1 parent 5684f3c commit 5babb04
Show file tree
Hide file tree
Showing 16 changed files with 253 additions and 57 deletions.
30 changes: 19 additions & 11 deletions common/proto/erc20.proto
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ message ERC20Token {
string name = 2;
string symbol = 3;
uint64 decimals = 4;
string tx_created = 5;
uint64 block_created = 6;
}

message TransferEvents {
Expand All @@ -19,20 +21,26 @@ message TransferEvents {

message TransferEvent {
string tx_hash = 1;
uint32 log_index = 2;
uint64 log_ordinal = 3;
string token_address = 4;
string from = 5;
string to = 6;
string amount = 7; // BigInt, in token's native amount
uint64 block_number = 2;
uint64 timestamp = 3;
uint32 log_index = 4;
optional uint64 log_ordinal = 5;
string token_address = 6;
string from = 7;
string to = 8;
string amount = 9; // BigInt, in token's native amount
repeated TokenBalance balance_changes = 10;
}

message TokenBalance {
string token_address = 1;
string balance = 2; // BigInt, in token's native amount
optional uint64 log_ordinal = 1;
ERC20Token token = 2;
string address = 3; // account address of the balance change
optional string old_balance = 4; // BigInt, in token's native amount
string new_balance = 5; // BigInt, in token's native amount
optional int32 reason = 6;
}

message Account {
string address = 1;
repeated TokenBalance balances = 2;
message TokenBalances {
TokenBalance items = 1;
}
2 changes: 1 addition & 1 deletion erc20-holdings/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ build:

.PHONY: run
run:
substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_block_to_erc20_contracts -s 1
substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_block_to_erc20_contracts -s 10606500 -t +100
146 changes: 119 additions & 27 deletions erc20-holdings/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub mod abi;
pub mod pb;

mod keyer;
mod rpc;

use pb::common::v1 as common;
use pb::erc20::v1 as erc20;
Expand All @@ -22,10 +23,13 @@ use substreams::store::StoreSetBigDecimal;
use substreams::store::StoreSetRaw;
use substreams::{hex, log, proto, store, Hex};
use substreams_ethereum::{pb::eth as pbeth, Event, NULL_ADDRESS};
use substreams_helper::erc20::Erc20Token;
use substreams_helper::keyer::chainlink_asset_key;
use substreams_helper::types::Address;

fn contract_bytecode_len(call: &pbeth::v2::Call) -> usize {
const INITIALIZE_METHOD_HASH: [u8; 4] = hex!("1459457a");

fn code_len(call: &pbeth::v2::Call) -> usize {
let mut len = 0;
for code_change in &call.code_changes {
len += code_change.new_code.len()
Expand All @@ -38,30 +42,108 @@ fn contract_bytecode_len(call: &pbeth::v2::Call) -> usize {
#[substreams::handlers::map]
fn map_block_to_erc20_contracts(
block: pbeth::v2::Block,
) -> Result<common::Addresses, substreams::errors::Error> {
let mut erc20_contracts = common::Addresses { items: vec![] };

for call_view in block.calls() {
let call = call_view.call;
if call.call_type == pbeth::v2::CallType::Create as i32 {
// skipping contracts that are too short to be an erc20 token
if contract_bytecode_len(call) < 150 {
) -> Result<erc20::Erc20Tokens, substreams::errors::Error> {
let mut erc20_tokens = erc20::Erc20Tokens { items: vec![] };

for tx in block.transaction_traces {
for call in tx.calls {
if call.state_reverted {
continue;
}

let address = Hex(call.address.clone()).to_string();
if call.call_type == pbeth::v2::CallType::Create as i32
|| call.call_type == pbeth::v2::CallType::Call as i32
// proxy contract creation
{
let call_input_len = call.input.len();
if call.call_type == pbeth::v2::CallType::Call as i32
&& (call_input_len < 4 || call.input[0..4] != INITIALIZE_METHOD_HASH)
{
// this will check if a proxy contract has been called to create a ERC20 contract.
// if that is the case the Proxy contract will call the initialize function on the ERC20 contract
// this is part of the OpenZeppelin Proxy contract standard
continue;
}

// check if contract is an erc20 token
if substreams_helper::erc20::get_erc20_token(address.clone()).is_none() {
continue;
}
// Contract creation not from proxy contract
if call.call_type == pbeth::v2::CallType::Create as i32 {
let mut code_change_len = 0;
for code_change in &call.code_changes {
code_change_len += code_change.new_code.len()
}

if code_change_len <= 150 {
// skipping contracts with less than 150 bytes of code
log::info!(
"Skipping contract {}. Contract code is less than 150 bytes.",
Hex::encode(&call.address)
);
continue;
}
}

log::info!("Create {}, len {}", address, contract_bytecode_len(call));
erc20_contracts.items.push(common::Address { address });
let mut decimals = 18_u64;
let decimal_result = rpc::get_erc20_decimals(&call.address);
match decimal_result {
Ok(_decimals) => decimals = _decimals,
Err(e) => continue,
};

let mut symbol = "".to_string();
let symbaol_result = rpc::get_erc20_symbol(&call.address);
match symbaol_result {
Ok(_symbol) => symbol = _symbol,
Err(e) => continue,
};

let mut name = "".to_string();
let name_result = rpc::get_erc20_name(&call.address);
match name_result {
Ok(_name) => name = _name,
Err(e) => continue,
};

erc20_tokens.items.push(erc20::Erc20Token {
address: Hex::encode(call.address.clone()),
name: name,
symbol: symbol,
decimals: decimals,
tx_created: Hex::encode(&tx.hash),
block_created: block.number,
});
}
}
}

Ok(erc20_contracts)
// for call_view in block.calls() {
// let call = call_view.call;
// if call.call_type == pbeth::v2::CallType::Create as i32 {
// // skipping contracts that are too short to be an erc20 token
// if code_len(call) < 150 {
// continue;
// }
//
// let address = Hex::encode(call.address.clone());
//
// // check if contract is an erc20 token
// let erc20_struct = substreams_helper::erc20::get_erc20_token(address.clone());
// if erc20_struct.is_none() {
// continue;
// }
//
// log::info!("Create {}, len {}", address, code_len(call));
// erc20_tokens.items.push(erc20::Erc20Token {
// address: address,
// name: erc20_struct.as_ref().unwrap().name.clone(),
// symbol: erc20_struct.as_ref().unwrap().symbol.clone(),
// decimals: erc20_struct.as_ref().unwrap().decimals,
// tx_created: "TODO".to_string(),
// block_created: block.number,
// });
// }
// }

Ok(erc20_tokens)
}

/// Extracts transfer events from the blocks
Expand All @@ -81,13 +163,23 @@ fn map_block_to_transfers(
}

transfer_events.items.push(erc20::TransferEvent {
tx_hash: Hex(log.receipt.transaction.clone().hash).to_string(),
tx_hash: Hex::encode(log.receipt.transaction.clone().hash),
block_number: block.number,
timestamp: block
.header
.as_ref()
.unwrap()
.timestamp
.as_ref()
.unwrap()
.seconds as u64,
log_index: log.index(),
log_ordinal: log.ordinal(),
token_address: Hex(log.address()).to_string(),
from: Hex(event.from).to_string(),
to: Hex(event.to).to_string(),
log_ordinal: Some(log.ordinal()),
token_address: Hex::encode(log.address()),
from: Hex::encode(event.from),
to: Hex::encode(event.to),
amount: event.value.to_string(),
balance_changes: vec![],
})
}
}
Expand All @@ -100,7 +192,7 @@ fn store_transfers(transfers: erc20::TransferEvents, output: store::StoreSetRaw)
log::info!("Stored events {}", transfers.items.len());
for transfer in transfers.items {
output.set(
transfer.log_ordinal,
transfer.log_ordinal.unwrap(),
Hex::encode(&transfer.token_address),
&proto::encode(&transfer).unwrap(),
);
Expand All @@ -112,14 +204,14 @@ fn store_balance(transfers: erc20::TransferEvents, output: store::StoreAddBigInt
log::info!("Stored events {}", transfers.items.len());
for transfer in transfers.items {
output.add(
transfer.log_ordinal,
transfer.log_ordinal.unwrap(),
keyer::account_balance_key(&transfer.to),
&BigInt::from_str(transfer.amount.as_str()).unwrap(),
);

if Hex::decode(transfer.from.clone()).unwrap() != NULL_ADDRESS {
output.add(
transfer.log_ordinal,
transfer.log_ordinal.unwrap(),
keyer::account_balance_key(&transfer.from),
&BigInt::from_str((transfer.amount).as_str()).unwrap().neg(),
);
Expand Down Expand Up @@ -148,7 +240,7 @@ fn store_balance_usd(

match balances.get_last(keyer::account_balance_key(&transfer.to)) {
Some(balance) => output.set(
transfer.log_ordinal,
transfer.log_ordinal.unwrap(),
keyer::account_balance_usd_key(&transfer.to),
&(token_price.clone() * balance.to_decimal(token_decimals.into())),
),
Expand All @@ -158,7 +250,7 @@ fn store_balance_usd(
if Hex::decode(transfer.from.clone()).unwrap() != NULL_ADDRESS {
match balances.get_last(keyer::account_balance_key(&transfer.from)) {
Some(balance) => output.set(
transfer.log_ordinal,
transfer.log_ordinal.unwrap(),
keyer::account_balance_usd_key(&transfer.from),
&(token_price.clone() * balance.to_decimal(token_decimals.into())),
),
Expand Down
75 changes: 75 additions & 0 deletions erc20-holdings/src/rpc.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
use std::fmt::Error;
use substreams::{log, Hex};
use substreams_ethereum::pb::eth;
use substreams_ethereum::rpc::eth_call;
use substreams_helper::utils::{read_string, read_uint32};

// Functions to attempt to get erc20 contract calls

pub const DECIMALS: &str = "313ce567";
pub const NAME: &str = "06fdde03";
pub const SYMBOL: &str = "95d89b41";

pub fn get_erc20_decimals(call_addr: &Vec<u8>) -> Result<u64, Error> {
let rpc_call_decimal = create_rpc_calls(call_addr, vec![DECIMALS]);
let rpc_responses_unmarshalled_decimal = eth_call(&rpc_call_decimal);
let response_decimal = rpc_responses_unmarshalled_decimal.responses;
if response_decimal.len() < 1 || response_decimal[0].failed {
return Err(Error);
}

let decoded_decimals = read_uint32(response_decimal[0].raw.as_ref());
if decoded_decimals.is_err() {
log::info!("Failed to decode decimals");
return Err(Error);
}

return Ok(decoded_decimals.unwrap() as u64);
}

pub fn get_erc20_symbol(call_addr: &Vec<u8>) -> Result<String, Error> {
let rpc_call_symbol = create_rpc_calls(call_addr, vec![SYMBOL]);
let rpc_responses_unmarshalled = eth_call(&rpc_call_symbol);
let responses = rpc_responses_unmarshalled.responses;
if responses.len() < 2 || responses[1].failed {
log::info!("Failed to get symbol");
return Err(Error);
};

let decoded_symbol = read_string(responses[2].raw.as_ref());
if decoded_symbol.is_err() {
log::info!("Failed to decode symbol");
return Err(Error);
}

return Ok(decoded_symbol.unwrap());
}

pub fn get_erc20_name(call_addr: &Vec<u8>) -> Result<String, Error> {
let rpc_call_name = create_rpc_calls(call_addr, vec![NAME]);
let rpc_responses_unmarshalled = eth_call(&rpc_call_name);
let responses = rpc_responses_unmarshalled.responses;
if responses.len() < 1 || responses[0].failed {
return Err(Error);
};

let decoded_name = read_string(responses[1].raw.as_ref());
if decoded_name.is_err() {
return Err(Error);
}

return Ok(decoded_name.unwrap());
}

fn create_rpc_calls(addr: &Vec<u8>, method_signatures: Vec<&str>) -> eth::rpc::RpcCalls {
let mut rpc_calls = eth::rpc::RpcCalls { calls: vec![] };

for method_signature in method_signatures {
rpc_calls.calls.push(eth::rpc::RpcCall {
to_addr: Vec::from(addr.clone()),
data: Hex::decode(method_signature).unwrap(),
})
}

return rpc_calls;
}
3 changes: 2 additions & 1 deletion erc20-holdings/substreams.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@ binaries:
file: ../target/wasm32-unknown-unknown/release/substreams_erc20_holdings.wasm

modules:
# This should be a store module since it is needed by multiple modules
- name: map_block_to_erc20_contracts
kind: map
initialBlock: 1
inputs:
- source: sf.ethereum.type.v2.Block
output:
type: proto:messari.common.v1.Addresses
type: proto:messari.erc20.v1.ERC20Tokens

- name: map_block_to_transfers
kind: map
Expand Down
10 changes: 10 additions & 0 deletions erc20-market-cap/src/pb.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,13 @@ pub mod erc20_price {
pub use super::super::erc20_price_v1::*;
}
}

#[rustfmt::skip]
#[path = "../target/pb/messari.uniswap.v1.rs"]
pub(in crate::pb) mod uniswap_v1;

pub mod uniswap {
pub mod v1 {
pub use super::super::uniswap_v1::*;
}
}
4 changes: 4 additions & 0 deletions erc20-price/src/modules/1_store_chainlink_aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,16 @@ fn store_chainlink_aggregator(block: eth::Block, output: StoreSetProto<Aggregato
name: base_asset.name,
symbol: base_asset.symbol,
decimals: base_asset.decimals,
tx_created: "TODO".to_string(),
block_created: 0, // TODO this should input store_tokens
}),
quote_asset: Some(Erc20Token {
address: quote_asset.address,
name: quote_asset.name,
symbol: quote_asset.symbol,
decimals: quote_asset.decimals,
tx_created: "TODO".to_string(),
block_created: 0, // TODO this should input store_tokens
}),
decimals: decimals.to_u64(),
};
Expand Down
Loading

0 comments on commit 5babb04

Please sign in to comment.