From 9759d0490411c4afd150abd18a0602ce69986b6f Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Wed, 26 Jun 2024 19:43:14 +0800 Subject: [PATCH] Refactor tokenlon_ethereum_trades --- .../_projects/tokenlon/ethereum/_schema.yml | 58 ++++----- .../ethereum/tokenlon_ethereum_trades.sql | 110 ++++++++++++------ .../tokenlon_v5_ethereum_amm_v1_trades.sql | 34 ------ .../tokenlon_v5_ethereum_amm_v2_trades.sql | 36 +----- .../tokenlon_v5_ethereum_pmm_v5_trades.sql | 36 +----- .../tokenlon_v5_ethereum_rfq_v1_trades.sql | 36 +----- .../tokenlon_v5_ethereum_rfq_v2_trades.sql | 31 +---- 7 files changed, 100 insertions(+), 241 deletions(-) diff --git a/dex/models/_projects/tokenlon/ethereum/_schema.yml b/dex/models/_projects/tokenlon/ethereum/_schema.yml index 9da92a0c311..df031aae3e5 100644 --- a/dex/models/_projects/tokenlon/ethereum/_schema.yml +++ b/dex/models/_projects/tokenlon/ethereum/_schema.yml @@ -1,16 +1,16 @@ version: 2 models: - - name: tokenlon_v5_ethereum_amm_v1_trades + - name: tokenlon_ethereum_trades meta: blockchain: ethereum sector: dex project: tokenlon contributors: izayl config: - tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl', 'metadata'] + tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl'] description: > - Tokenlon AMM v1 trades + Tokenlon v5 all trades tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -21,6 +21,10 @@ models: - tx_hash - evt_index - trace_address + - check_dex_aggregator_seed: + blockchain: ethereum + project: tokenlon + version: 5 columns: - name: blockchain description: 'Blockchain which the DEX is deployed' @@ -30,8 +34,6 @@ models: description: 'Version of the contract built and deployed by the DEX project' - name: block_date description: 'UTC event block date of each DEX trade' - - name: block_month - description: "UTC event block month of each DEX trade" - name: block_time description: 'UTC event block time of each DEX trade' - name: token_bought_symbol @@ -71,16 +73,16 @@ models: - name: evt_index description: 'Index of the corresponding trade event' - - name: tokenlon_v5_ethereum_amm_v2_trades + - name: tokenlon_v5_ethereum_amm_v1_trades meta: blockchain: ethereum sector: dex project: tokenlon contributors: izayl config: - tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl'] + tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl', 'metadata'] description: > - Tokenlon AMM v2 trades + Tokenlon AMM v1 trades tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -100,6 +102,8 @@ models: description: 'Version of the contract built and deployed by the DEX project' - name: block_date description: 'UTC event block date of each DEX trade' + - name: block_month + description: "UTC event block month of each DEX trade" - name: block_time description: 'UTC event block time of each DEX trade' - name: token_bought_symbol @@ -116,8 +120,6 @@ models: description: 'Raw value of the token bought at time of execution in the original currency' - name: token_sold_amount_raw description: 'Raw value of the token sold at time of execution in the original currency' - - name: amount_usd - description: 'USD value of the trade at time of execution' - name: token_bought_address description: 'Contract address of the token bought' - name: token_sold_address @@ -139,7 +141,7 @@ models: - name: evt_index description: 'Index of the corresponding trade event' - - name: tokenlon_v5_ethereum_pmm_v5_trades + - name: tokenlon_v5_ethereum_amm_v2_trades meta: blockchain: ethereum sector: dex @@ -148,7 +150,7 @@ models: config: tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl'] description: > - Tokenlon PMM v5 trades + Tokenlon AMM v2 trades tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -184,8 +186,6 @@ models: description: 'Raw value of the token bought at time of execution in the original currency' - name: token_sold_amount_raw description: 'Raw value of the token sold at time of execution in the original currency' - - name: amount_usd - description: 'USD value of the trade at time of execution' - name: token_bought_address description: 'Contract address of the token bought' - name: token_sold_address @@ -207,7 +207,7 @@ models: - name: evt_index description: 'Index of the corresponding trade event' - - name: tokenlon_v5_ethereum_rfq_v1_trades + - name: tokenlon_v5_ethereum_pmm_v5_trades meta: blockchain: ethereum sector: dex @@ -216,7 +216,7 @@ models: config: tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl'] description: > - Tokenlon RFQ v1 trades + Tokenlon PMM v5 trades tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -252,8 +252,6 @@ models: description: 'Raw value of the token bought at time of execution in the original currency' - name: token_sold_amount_raw description: 'Raw value of the token sold at time of execution in the original currency' - - name: amount_usd - description: 'USD value of the trade at time of execution' - name: token_bought_address description: 'Contract address of the token bought' - name: token_sold_address @@ -275,16 +273,16 @@ models: - name: evt_index description: 'Index of the corresponding trade event' - - name: tokenlon_v5_ethereum_rfq_v2_trades + - name: tokenlon_v5_ethereum_rfq_v1_trades meta: blockchain: ethereum sector: dex project: tokenlon - contributors: keen + contributors: izayl config: - tags: [ 'ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'keen' ] + tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl'] description: > - Tokenlon RFQ v2 trades + Tokenlon RFQ v1 trades tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -320,8 +318,6 @@ models: description: 'Raw value of the token bought at time of execution in the original currency' - name: token_sold_amount_raw description: 'Raw value of the token sold at time of execution in the original currency' - - name: amount_usd - description: 'USD value of the trade at time of execution' - name: token_bought_address description: 'Contract address of the token bought' - name: token_sold_address @@ -343,16 +339,16 @@ models: - name: evt_index description: 'Index of the corresponding trade event' - - name: tokenlon_ethereum_trades + - name: tokenlon_v5_ethereum_rfq_v2_trades meta: blockchain: ethereum sector: dex project: tokenlon - contributors: izayl + contributors: keen config: - tags: ['ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'izayl'] + tags: [ 'ethereum', 'dex', 'trades', 'tokenlon', 'aggregator', 'keen' ] description: > - Tokenlon v5 all trades + Tokenlon RFQ v2 trades tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -363,10 +359,6 @@ models: - tx_hash - evt_index - trace_address - - check_dex_aggregator_seed: - blockchain: ethereum - project: tokenlon - version: 5 columns: - name: blockchain description: 'Blockchain which the DEX is deployed' @@ -392,8 +384,6 @@ models: description: 'Raw value of the token bought at time of execution in the original currency' - name: token_sold_amount_raw description: 'Raw value of the token sold at time of execution in the original currency' - - name: amount_usd - description: 'USD value of the trade at time of execution' - name: token_bought_address description: 'Contract address of the token bought' - name: token_sold_address diff --git a/dex/models/_projects/tokenlon/ethereum/tokenlon_ethereum_trades.sql b/dex/models/_projects/tokenlon/ethereum/tokenlon_ethereum_trades.sql index b4e84971d72..42ac140f75d 100644 --- a/dex/models/_projects/tokenlon/ethereum/tokenlon_ethereum_trades.sql +++ b/dex/models/_projects/tokenlon/ethereum/tokenlon_ethereum_trades.sql @@ -1,10 +1,10 @@ {{ config( schema = 'tokenlon_ethereum', alias = 'trades', - post_hook='{{ expose_spells(\'["ethereum"]\', - "project", - "tokenlon", - \'["izayl"]\') }}' + post_hook='{{ expose_spells(blockchains = \'["ethereum"]\', + spell_type = "project", + spell_name = "tokenlon", + contributors = \'["izayl"]\') }}' ) }} @@ -17,38 +17,72 @@ ref('tokenlon_v5_ethereum_rfq_v2_trades'), ] %} +WITH trades AS ( + SELECT * + FROM ( + {% for dex_model in tokenlon_models %} + SELECT + blockchain, + project, + version, + block_date, + block_month, + block_time, + token_bought_symbol, + token_sold_symbol, + token_pair, + token_bought_amount, + token_sold_amount, + token_bought_amount_raw, + token_sold_amount_raw, + token_bought_address, + token_sold_address, + taker, + maker, + project_contract_address, + tx_hash, + tx_from, + tx_to, + trace_address, + evt_index + FROM {{ dex_model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} + ) +), + +, enrichments_with_prices AS ( + {{ + add_amount_usd( + trades_cte = 'trades' + ) + }} +) -SELECT * -FROM ( - {% for dex_model in tokenlon_models %} - SELECT - blockchain, - project, - version, - block_date, - block_month, - block_time, - token_bought_symbol, - token_sold_symbol, - token_pair, - token_bought_amount, - token_sold_amount, - token_bought_amount_raw, - token_sold_amount_raw, - amount_usd, - token_bought_address, - token_sold_address, - taker, - maker, - project_contract_address, - tx_hash, - tx_from, - tx_to, - trace_address, - evt_index - FROM {{ dex_model }} - {% if not loop.last %} - UNION ALL - {% endif %} - {% endfor %} -) \ No newline at end of file +SELECT blockchain, + project, + version, + block_month, + block_date, + block_time, + token_bought_symbol, + token_sold_symbol, + token_pair, + token_bought_amount, + token_sold_amount, + token_bought_amount_raw, + token_sold_amount_raw, + amount_usd, + token_bought_address, + token_sold_address, + taker, + maker, + project_contract_address, + tx_hash, + tx_from, + tx_to, + trace_address, + evt_index +FROM enrichments_with_prices \ No newline at end of file diff --git a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v1_trades.sql b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v1_trades.sql index 7943fba682f..e4e401e8ac6 100644 --- a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v1_trades.sql +++ b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v1_trades.sql @@ -46,15 +46,6 @@ WITH dexs AS ( {% if is_incremental() %} WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} -), prices AS ( - SELECT DISTINCT - DATE_TRUNC('hour', minute) AS hour, - contract_address, - blockchain, - decimals, - AVG(price) AS price - FROM {{ source('prices', 'usd') }} - GROUP BY DATE_TRUNC('hour', minute), contract_address,blockchain,decimals ) SELECT @@ -74,11 +65,6 @@ SELECT dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, CAST(dexs.token_bought_amount_raw AS UINT256) AS token_bought_amount_raw, CAST(dexs.token_sold_amount_raw AS UINT256) AS token_sold_amount_raw, - COALESCE(dexs. - amount_usd, - (dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price, - (dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price - ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, COALESCE(dexs.taker, tx."from") AS taker, @@ -104,23 +90,3 @@ LEFT JOIN {{ source('tokens', 'erc20') }} erc20a LEFT JOIN {{ source('tokens', 'erc20') }} erc20b ON erc20b.contract_address = dexs.token_sold_address AND erc20b.blockchain = 'ethereum' -LEFT JOIN prices p_bought - ON p_bought.hour = date_trunc('hour', dexs.block_time) - AND p_bought.contract_address = dexs.token_bought_address - AND p_bought.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_bought.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_bought.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} -LEFT JOIN prices p_sold - ON p_sold.hour = date_trunc('hour', dexs.block_time) - AND p_sold.contract_address = dexs.token_sold_address - AND p_sold.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_sold.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_sold.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} \ No newline at end of file diff --git a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v2_trades.sql b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v2_trades.sql index 80f4137e6e0..8a27365673d 100644 --- a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v2_trades.sql +++ b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_amm_v2_trades.sql @@ -46,15 +46,6 @@ WITH dexs AS ( {% if is_incremental() %} WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} -), prices AS ( - SELECT DISTINCT - DATE_TRUNC('hour', minute) AS hour, - contract_address, - blockchain, - decimals, - AVG(price) AS price - FROM {{ source('prices', 'usd') }} - GROUP BY DATE_TRUNC('hour', minute), contract_address,blockchain,decimals ) SELECT @@ -74,11 +65,6 @@ SELECT dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, CAST(dexs.token_bought_amount_raw AS UINT256) AS token_bought_amount_raw, CAST(dexs.token_sold_amount_raw AS UINT256) AS token_sold_amount_raw, - coalesce(dexs. - amount_usd, - (dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price, - (dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price - ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, coalesce(dexs.taker, tx."from") AS taker, @@ -103,24 +89,4 @@ LEFT JOIN {{ source('tokens', 'erc20') }} erc20a AND erc20a.blockchain = 'ethereum' LEFT JOIN {{ source('tokens', 'erc20') }} erc20b ON erc20b.contract_address = dexs.token_sold_address - AND erc20b.blockchain = 'ethereum' -LEFT JOIN prices p_bought - ON p_bought.hour = date_trunc('hour', dexs.block_time) - AND p_bought.contract_address = dexs.token_bought_address - AND p_bought.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_bought.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_bought.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} -LEFT JOIN prices p_sold - ON p_sold.hour = date_trunc('hour', dexs.block_time) - AND p_sold.contract_address = dexs.token_sold_address - AND p_sold.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_sold.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_sold.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} \ No newline at end of file + AND erc20b.blockchain = 'ethereum' \ No newline at end of file diff --git a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_pmm_v5_trades.sql b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_pmm_v5_trades.sql index 8910fbea764..8bbae3275af 100644 --- a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_pmm_v5_trades.sql +++ b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_pmm_v5_trades.sql @@ -38,15 +38,6 @@ WITH dexs AS ( {% if is_incremental() %} WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} -), prices AS ( - SELECT DISTINCT - DATE_TRUNC('hour', minute) AS hour, - contract_address, - blockchain, - decimals, - AVG(price) AS price - FROM {{ source('prices', 'usd') }} - GROUP BY DATE_TRUNC('hour', minute), contract_address,blockchain,decimals ) SELECT @@ -66,11 +57,6 @@ SELECT dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, CAST(dexs.token_bought_amount_raw AS UINT256) AS token_bought_amount_raw, CAST(dexs.token_sold_amount_raw AS UINT256) AS token_sold_amount_raw, - COALESCE(dexs. - amount_usd, - (dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price, - (dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price - ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, COALESCE(dexs.taker, tx."from") AS taker, @@ -95,24 +81,4 @@ LEFT JOIN {{ source('tokens', 'erc20') }} erc20a AND erc20a.blockchain = 'ethereum' LEFT JOIN {{ source('tokens', 'erc20') }} erc20b ON erc20b.contract_address = dexs.token_sold_address - AND erc20b.blockchain = 'ethereum' -LEFT JOIN prices p_bought - ON p_bought.hour = date_trunc('hour', dexs.block_time) - AND p_bought.contract_address = dexs.token_bought_address - AND p_bought.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_bought.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_bought.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} -LEFT JOIN prices p_sold - ON p_sold.hour = date_trunc('hour', dexs.block_time) - AND p_sold.contract_address = dexs.token_sold_address - AND p_sold.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_sold.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_sold.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} \ No newline at end of file + AND erc20b.blockchain = 'ethereum' \ No newline at end of file diff --git a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v1_trades.sql b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v1_trades.sql index 0e89ee3f4c2..97c4391dda2 100644 --- a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v1_trades.sql +++ b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v1_trades.sql @@ -38,15 +38,6 @@ WITH dexs AS ( {% if is_incremental() %} WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} -), prices AS ( - SELECT DISTINCT - DATE_TRUNC('hour', minute) AS hour, - contract_address, - blockchain, - decimals, - AVG(price) AS price - FROM {{ source('prices', 'usd') }} - GROUP BY DATE_TRUNC('hour', minute), contract_address,blockchain,decimals ) SELECT @@ -66,11 +57,6 @@ SELECT dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, CAST(dexs.token_bought_amount_raw AS UINT256) AS token_bought_amount_raw, CAST(dexs.token_sold_amount_raw AS UINT256) AS token_sold_amount_raw, - coalesce(dexs. - amount_usd, - (dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price, - (dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price - ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, coalesce(dexs.taker, tx."from") AS taker, @@ -95,24 +81,4 @@ LEFT JOIN {{ source('tokens', 'erc20') }} erc20a AND erc20a.blockchain = 'ethereum' LEFT JOIN {{ source('tokens', 'erc20') }} erc20b ON erc20b.contract_address = dexs.token_sold_address - AND erc20b.blockchain = 'ethereum' -LEFT JOIN prices p_bought - ON p_bought.hour = date_trunc('hour', dexs.block_time) - AND p_bought.contract_address = dexs.token_bought_address - AND p_bought.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_bought.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_bought.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} -LEFT JOIN prices p_sold - ON p_sold.hour = date_trunc('hour', dexs.block_time) - AND p_sold.contract_address = dexs.token_sold_address - AND p_sold.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_sold.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_sold.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} \ No newline at end of file + AND erc20b.blockchain = 'ethereum' \ No newline at end of file diff --git a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v2_trades.sql b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v2_trades.sql index b9fdafad155..80dc70832c1 100644 --- a/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v2_trades.sql +++ b/dex/models/_projects/tokenlon/ethereum/tokenlon_v5_ethereum_rfq_v2_trades.sql @@ -38,15 +38,6 @@ WITH dexs AS ( {% if is_incremental() %} WHERE {{ incremental_predicate('evt_block_time') }} {% endif %} -), prices AS ( - SELECT DISTINCT - DATE_TRUNC('hour', minute) AS hour, - contract_address, - blockchain, - decimals, - AVG(price) AS price - FROM {{ source('prices', 'usd') }} - GROUP BY DATE_TRUNC('hour', minute), contract_address,blockchain,decimals ) SELECT @@ -101,24 +92,4 @@ LEFT JOIN {{ source('tokens', 'erc20') }} erc20a AND erc20a.blockchain = 'ethereum' LEFT JOIN {{ source('tokens', 'erc20') }} erc20b ON erc20b.contract_address = dexs.token_sold_address - AND erc20b.blockchain = 'ethereum' -LEFT JOIN prices p_bought - ON p_bought.hour = date_trunc('hour', dexs.block_time) - AND p_bought.contract_address = dexs.token_bought_address - AND p_bought.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_bought.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_bought.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} -LEFT JOIN prices p_sold - ON p_sold.hour = date_trunc('hour', dexs.block_time) - AND p_sold.contract_address = dexs.token_sold_address - AND p_sold.blockchain = 'ethereum' - {% if not is_incremental() %} - AND p_sold.hour >= {{project_start_date}} - {% endif %} - {% if is_incremental() %} - AND p_sold.hour >= date_trunc('day', now() - interval '7' day) - {% endif %} \ No newline at end of file + AND erc20b.blockchain = 'ethereum' \ No newline at end of file