From 9bf7433d0f1a4db8a6173f0d515ef7dcee136595 Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Mon, 16 Sep 2024 22:23:43 +0800 Subject: [PATCH] Introduce dex_aggregator_base_trades (#6653) * Introduce dex_aggregator_base_trades * Fix typo * Refactor lifi_trades * Add lifi to base_trades * Fix config blocks * Fix typo * Fix typo * Apply suggestions from code review Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> * Draft enrich_dex_aggregator_trades macro * Remove 5 columns * Remove 5 columns * Recover add_tx_columns --------- Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../models/enrich_dex_aggregator_trades.sql | 99 ++++++++++++++ .../dex/models/_projects/lifi/_schema.yml | 32 ++++- .../models/_projects/lifi/fantom/_schema.yml | 26 +--- ...trades.sql => lifi_fantom_base_trades.sql} | 8 +- .../lifi/fantom/lifi_v2_fantom_trades.sql | 39 ------ .../_projects/lifi/lifi_base_trades.sql | 40 ++++++ .../dex/models/_projects/lifi/lifi_trades.sql | 49 +------ .../_projects/lifi/optimism/_schema.yml | 26 +--- ...ades.sql => lifi_optimism_base_trades.sql} | 8 +- .../lifi/optimism/lifi_v2_optimism_trades.sql | 39 ------ .../dex/models/aggregator_trades/_schema.yml | 45 ++++++- .../dex_aggregator_base_trades.sql | 58 +++++++++ .../dex_aggregator_trades.sql | 123 ++++++++++++------ 13 files changed, 368 insertions(+), 224 deletions(-) create mode 100644 dbt_subprojects/dex/macros/models/enrich_dex_aggregator_trades.sql rename dbt_subprojects/dex/models/_projects/lifi/fantom/{lifi_fantom_trades.sql => lifi_fantom_base_trades.sql} (80%) create mode 100644 dbt_subprojects/dex/models/_projects/lifi/lifi_base_trades.sql rename dbt_subprojects/dex/models/_projects/lifi/optimism/{lifi_optimism_trades.sql => lifi_optimism_base_trades.sql} (80%) create mode 100644 dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_base_trades.sql diff --git a/dbt_subprojects/dex/macros/models/enrich_dex_aggregator_trades.sql b/dbt_subprojects/dex/macros/models/enrich_dex_aggregator_trades.sql new file mode 100644 index 00000000000..fe88afc97dc --- /dev/null +++ b/dbt_subprojects/dex/macros/models/enrich_dex_aggregator_trades.sql @@ -0,0 +1,99 @@ +{% macro enrich_dex_aggregator_trades( + base_trades = null + , filter = null + , tokens_erc20_model = null + ) +%} + +WITH base_trades as ( + SELECT + * + FROM + {{ base_trades }} + WHERE + {{ filter }} + {% if is_incremental() %} + AND + {{ incremental_predicate('block_time') }} + {% endif %} +) + +, enrichments AS ( + SELECT + base_trades.blockchain + , base_trades.project + , base_trades.version + , base_trades.block_month + , base_trades.block_date + , base_trades.block_time +-- , base_trades.block_number -- temporary missing + , erc20_bought.symbol AS token_bought_symbol + , erc20_sold.symbol AS token_sold_symbol + , case + when lower(erc20_bought.symbol) > lower(erc20_sold.symbol) then concat(erc20_sold.symbol, '-', erc20_bought.symbol) + else concat(erc20_bought.symbol, '-', erc20_sold.symbol) + end AS token_pair + , base_trades.token_bought_amount_raw / power(10, erc20_bought.decimals) AS token_bought_amount + , base_trades.token_sold_amount_raw / power(10, erc20_sold.decimals) AS token_sold_amount + , base_trades.token_bought_amount_raw + , base_trades.token_sold_amount_raw + , base_trades.token_bought_address + , base_trades.token_sold_address + , coalesce(base_trades.taker, base_trades.tx_from) AS taker + , base_trades.maker + , base_trades.project_contract_address + , base_trades.tx_hash + , base_trades.tx_from + , base_trades.tx_to + , base_trades.trace_address + , base_trades.evt_index + FROM + base_trades + LEFT JOIN + {{ tokens_erc20_model }} as erc20_bought + ON erc20_bought.contract_address = base_trades.token_bought_address + AND erc20_bought.blockchain = base_trades.blockchain + LEFT JOIN + {{ tokens_erc20_model }} as erc20_sold + ON erc20_sold.contract_address = base_trades.token_sold_address + AND erc20_sold.blockchain = base_trades.blockchain +) + +, enrichments_with_prices AS ( + {{ + add_amount_usd( + trades_cte = 'enrichments' + ) + }} +) + +SELECT + blockchain + , project + , version + , block_month + , block_date + , block_time +-- , block_number + , token_bought_symbol + , token_sold_symbol + , token_pair + , token_bought_amount + , token_sold_amount + , token_bought_amount_raw + , token_sold_amount_raw + , amount_usd + , token_bought_address + , token_sold_address + , taker + , maker + , project_contract_address + , tx_hash + , tx_from + , tx_to + , trace_address + , evt_index +FROM + enrichments_with_prices + +{% endmacro %} \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/lifi/_schema.yml b/dbt_subprojects/dex/models/_projects/lifi/_schema.yml index c4c22570073..60d070724c5 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/lifi/_schema.yml @@ -83,4 +83,34 @@ models: description: "the position of this event log within this transaction" - &block_month name: block_month - description: "UTC event block month of each DEX trade" \ No newline at end of file + description: "UTC event block month of each DEX trade" + + - name: lifi_base_trades + meta: + blockchain: fantom, optimism + sector: dex + project: lifi + contributors: Henrystats, hosuke + config: + tags: [ 'fantom', 'optimism', 'lifi','trades', 'dex', 'aggregator', 'Henrystats','cross-chain' ] + description: > + lifi aggregator base trades on all chains across all contracts and versions. This table will load dex_aggregator trades downstream. + columns: + - *blockchain + - *project + - *version + - *block_date + - *block_time + - *token_bought_amount_raw + - *token_sold_amount_raw + - *token_bought_address + - *token_sold_address + - *taker + - *maker + - *project_contract_address + - *tx_hash + - *tx_from + - *tx_to + - *trace_address + - *evt_index + - *block_month diff --git a/dbt_subprojects/dex/models/_projects/lifi/fantom/_schema.yml b/dbt_subprojects/dex/models/_projects/lifi/fantom/_schema.yml index 474dbc07b1f..dca0d119c25 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/fantom/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/lifi/fantom/_schema.yml @@ -41,30 +41,12 @@ models: - &block_time name: block_time description: "UTC event block time of each DEX trade" - - &token_bought_symbol - name: token_bought_symbol - description: "Token symbol for token bought in the transaction" - - &token_sold_symbol - name: token_sold_symbol - description: "Token symbol for token sold in the transaction" - - &token_pair - name: token_pair - description: "Token symbol pair for each token involved in the transaction" - - &token_bought_amount - name: token_bought_amount - description: "Value of the token bought at time of execution in the original currency" - - &token_sold_amount - name: token_sold_amount - description: "Value of the token sold at time of execution in the original currency" - &token_bought_amount_raw name: token_bought_amount_raw description: "Raw value of the token bought at time of execution in the original currency" - &token_sold_amount_raw name: token_sold_amount_raw description: "Raw value of the token sold at time of execution in the original currency" - - &amount_usd - name: amount_usd - description: "USD value of the trade at time of execution" - &token_bought_address name: token_bought_address description: "Contract address of the token bought" @@ -99,7 +81,7 @@ models: name: block_month description: "UTC event block month of each DEX trade" - - name: lifi_fantom_trades + - name: lifi_fantom_base_trades meta: blockchain: fantom sector: dex @@ -115,14 +97,8 @@ models: - *version - *block_date - *block_time - - *token_bought_symbol - - *token_sold_symbol - - *token_pair - - *token_bought_amount - - *token_sold_amount - *token_bought_amount_raw - *token_sold_amount_raw - - *amount_usd - *token_bought_address - *token_sold_address - *taker diff --git a/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_fantom_trades.sql b/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_fantom_base_trades.sql similarity index 80% rename from dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_fantom_trades.sql rename to dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_fantom_base_trades.sql index 83ab544ad74..b860a1a1403 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_fantom_trades.sql +++ b/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_fantom_base_trades.sql @@ -1,7 +1,7 @@ {{ config( schema = 'lifi_fantom', - alias = 'trades' + alias = 'base_trades' ) }} @@ -20,14 +20,8 @@ FROM ( block_date, block_month, block_time, - token_bought_symbol, - token_sold_symbol, - token_pair, - token_bought_amount, - token_sold_amount, token_bought_amount_raw, token_sold_amount_raw, - amount_usd, token_bought_address, token_sold_address, taker, diff --git a/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_v2_fantom_trades.sql b/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_v2_fantom_trades.sql index 2b555de05d5..33940dd5079 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_v2_fantom_trades.sql +++ b/dbt_subprojects/dex/models/_projects/lifi/fantom/lifi_v2_fantom_trades.sql @@ -61,21 +61,8 @@ SELECT cast(date_trunc('DAY', dexs.block_time) as date) as block_date, cast(date_trunc('MONTH', dexs.block_time) as date) as block_month, dexs.block_time, - erc20a.symbol as token_bought_symbol, - erc20b.symbol as token_sold_symbol, - case - when lower(erc20a.symbol) > lower(erc20b.symbol) then concat(erc20b.symbol, '-', erc20a.symbol) - else concat(erc20a.symbol, '-', erc20b.symbol) - end as token_pair, - dexs.token_bought_amount_raw / power(10, erc20a.decimals) AS token_bought_amount, - dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, dexs.token_bought_amount_raw, dexs.token_sold_amount_raw, - coalesce( - dexs.amount_usd - ,(dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price - ,(dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price - ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, tx."from" AS taker, -- no taker in swap event @@ -95,29 +82,3 @@ inner join {{ source('fantom', 'transactions') }} tx {% if is_incremental() %} and tx.block_time >= date_trunc('day', now() - interval '7' Day) {% endif %} -left join {{ source('tokens', 'erc20') }} erc20a - on erc20a.contract_address = dexs.token_bought_address - and erc20a.blockchain = 'fantom' -left join {{ source('tokens', 'erc20') }} erc20b - on erc20b.contract_address = dexs.token_sold_address - and erc20b.blockchain = 'fantom' -left join {{ source('prices', 'usd') }} p_bought - on p_bought.minute = date_trunc('minute', dexs.block_time) - and p_bought.contract_address = dexs.token_bought_address - and p_bought.blockchain = 'fantom' - {% if not is_incremental() %} - and p_bought.minute >= TIMESTAMP '{{project_start_date}}' - {% endif %} - {% if is_incremental() %} - and p_bought.minute >= date_trunc('day', now() - interval '7' Day) - {% endif %} -left join {{ source('prices', 'usd') }} p_sold - on p_sold.minute = date_trunc('minute', dexs.block_time) - and p_sold.contract_address = dexs.token_sold_address - and p_sold.blockchain = 'fantom' - {% if not is_incremental() %} - and p_sold.minute >= TIMESTAMP '{{project_start_date}}' - {% endif %} - {% if is_incremental() %} - and p_sold.minute >= date_trunc('day', now() - interval '7' Day) - {% endif %} \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/lifi/lifi_base_trades.sql b/dbt_subprojects/dex/models/_projects/lifi/lifi_base_trades.sql new file mode 100644 index 00000000000..493843fab6d --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/lifi/lifi_base_trades.sql @@ -0,0 +1,40 @@ +{{ config( + schema = 'lifi', + alias = 'base_trades' + ) +}} + +{% set lifi_models = [ +ref('lifi_fantom_base_trades') +, ref('lifi_optimism_base_trades') +] %} + + +SELECT * +FROM ( + {% for dex_model in lifi_models %} + SELECT + blockchain, + project, + version, + block_month, + block_date, + block_time, + token_bought_amount_raw, + token_sold_amount_raw, + token_bought_address, + token_sold_address, + taker, + maker, + project_contract_address, + tx_hash, + tx_from, + tx_to, + trace_address, --ensure field is explicitly cast as array in base models + evt_index + FROM {{ dex_model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/lifi/lifi_trades.sql b/dbt_subprojects/dex/models/_projects/lifi/lifi_trades.sql index d91055d86c9..55538339cea 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/lifi_trades.sql +++ b/dbt_subprojects/dex/models/_projects/lifi/lifi_trades.sql @@ -1,50 +1,13 @@ {{ config( schema = 'lifi', alias = 'trades', - post_hook='{{ expose_spells(\'["fantom", "optimism"]\', - "project", - "lifi", - \'["Henrystats"]\') }}' + post_hook='{{ expose_spells(blockchains = \'["fantom", "optimism"]\', + spell_type = "project", + spell_name = "lifi", + contributors = \'["Henrystats", "hosuke"]\') }}' ) }} -{% set lifi_models = [ -ref('lifi_fantom_trades') -,ref('lifi_optimism_trades') -] %} - - SELECT * -FROM ( - {% for dex_model in lifi_models %} - SELECT - blockchain, - project, - version, - block_month, - block_date, - block_time, - token_bought_symbol, - token_sold_symbol, - token_pair, - token_bought_amount, - token_sold_amount, - token_bought_amount_raw, - token_sold_amount_raw, - amount_usd, - token_bought_address, - token_sold_address, - taker, - maker, - project_contract_address, - tx_hash, - tx_from, - tx_to, - trace_address, --ensure field is explicitly cast as array in base models - evt_index - FROM {{ dex_model }} - {% if not loop.last %} - UNION ALL - {% endif %} - {% endfor %} -) \ No newline at end of file +FROM {{ ref('dex_aggregator_trades') }} +WHERE project = 'lifi' \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/lifi/optimism/_schema.yml b/dbt_subprojects/dex/models/_projects/lifi/optimism/_schema.yml index 0fb8ef06ae3..7fd22619823 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/optimism/_schema.yml +++ b/dbt_subprojects/dex/models/_projects/lifi/optimism/_schema.yml @@ -41,30 +41,12 @@ models: - &block_time name: block_time description: "UTC event block time of each DEX trade" - - &token_bought_symbol - name: token_bought_symbol - description: "Token symbol for token bought in the transaction" - - &token_sold_symbol - name: token_sold_symbol - description: "Token symbol for token sold in the transaction" - - &token_pair - name: token_pair - description: "Token symbol pair for each token involved in the transaction" - - &token_bought_amount - name: token_bought_amount - description: "Value of the token bought at time of execution in the original currency" - - &token_sold_amount - name: token_sold_amount - description: "Value of the token sold at time of execution in the original currency" - &token_bought_amount_raw name: token_bought_amount_raw description: "Raw value of the token bought at time of execution in the original currency" - &token_sold_amount_raw name: token_sold_amount_raw description: "Raw value of the token sold at time of execution in the original currency" - - &amount_usd - name: amount_usd - description: "USD value of the trade at time of execution" - &token_bought_address name: token_bought_address description: "Contract address of the token bought" @@ -99,7 +81,7 @@ models: name: block_month description: "UTC event block month of each DEX trade" - - name: lifi_optimism_trades + - name: lifi_optimism_base_trades meta: blockchain: optimism sector: dex @@ -115,14 +97,8 @@ models: - *version - *block_date - *block_time - - *token_bought_symbol - - *token_sold_symbol - - *token_pair - - *token_bought_amount - - *token_sold_amount - *token_bought_amount_raw - *token_sold_amount_raw - - *amount_usd - *token_bought_address - *token_sold_address - *taker diff --git a/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_optimism_trades.sql b/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_optimism_base_trades.sql similarity index 80% rename from dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_optimism_trades.sql rename to dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_optimism_base_trades.sql index d373f3f66a2..4423e749258 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_optimism_trades.sql +++ b/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_optimism_base_trades.sql @@ -1,7 +1,7 @@ {{ config( schema = 'lifi_optimism', - alias = 'trades' + alias = 'base_trades' ) }} @@ -20,14 +20,8 @@ FROM ( block_date, block_month, block_time, - token_bought_symbol, - token_sold_symbol, - token_pair, - token_bought_amount, - token_sold_amount, token_bought_amount_raw, token_sold_amount_raw, - amount_usd, token_bought_address, token_sold_address, taker, diff --git a/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_v2_optimism_trades.sql b/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_v2_optimism_trades.sql index e05c6eb96fc..b6d26eda580 100644 --- a/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_v2_optimism_trades.sql +++ b/dbt_subprojects/dex/models/_projects/lifi/optimism/lifi_v2_optimism_trades.sql @@ -61,21 +61,8 @@ SELECT cast(date_trunc('DAY', dexs.block_time) as date) as block_date, cast(date_trunc('MONTH', dexs.block_time) as date) as block_month, dexs.block_time, - erc20a.symbol as token_bought_symbol, - erc20b.symbol as token_sold_symbol, - case - when lower(erc20a.symbol) > lower(erc20b.symbol) then concat(erc20b.symbol, '-', erc20a.symbol) - else concat(erc20a.symbol, '-', erc20b.symbol) - end as token_pair, - dexs.token_bought_amount_raw / power(10, erc20a.decimals) AS token_bought_amount, - dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, dexs.token_bought_amount_raw, dexs.token_sold_amount_raw, - coalesce( - dexs.amount_usd - ,(dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price - ,(dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price - ) AS amount_usd, dexs.token_bought_address, dexs.token_sold_address, tx."from" AS taker, -- no taker in swap event @@ -95,29 +82,3 @@ inner join {{ source('optimism', 'transactions') }} tx {% if is_incremental() %} and tx.block_time >= date_trunc('day', now() - interval '7' Day) {% endif %} -left join {{ source('tokens', 'erc20') }} erc20a - on erc20a.contract_address = dexs.token_bought_address - and erc20a.blockchain = 'optimism' -left join {{ source('tokens', 'erc20') }} erc20b - on erc20b.contract_address = dexs.token_sold_address - and erc20b.blockchain = 'optimism' -left join {{ source('prices', 'usd') }} p_bought - on p_bought.minute = date_trunc('minute', dexs.block_time) - and p_bought.contract_address = dexs.token_bought_address - and p_bought.blockchain = 'optimism' - {% if not is_incremental() %} - and p_bought.minute >= TIMESTAMP '{{project_start_date}}' - {% endif %} - {% if is_incremental() %} - and p_bought.minute >= date_trunc('day', now() - interval '7' Day) - {% endif %} -left join {{ source('prices', 'usd') }} p_sold - on p_sold.minute = date_trunc('minute', dexs.block_time) - and p_sold.contract_address = dexs.token_sold_address - and p_sold.blockchain = 'optimism' - {% if not is_incremental() %} - and p_sold.minute >= TIMESTAMP '{{project_start_date}}' - {% endif %} - {% if is_incremental() %} - and p_sold.minute >= date_trunc('day', now() - interval '7' Day) - {% endif %} \ No newline at end of file diff --git a/dbt_subprojects/dex/models/aggregator_trades/_schema.yml b/dbt_subprojects/dex/models/aggregator_trades/_schema.yml index 9615ea7bfaa..c2f424433db 100644 --- a/dbt_subprojects/dex/models/aggregator_trades/_schema.yml +++ b/dbt_subprojects/dex/models/aggregator_trades/_schema.yml @@ -92,4 +92,47 @@ models: name: trace_address - &evt_index name: evt_index - description: "Index of the corresponding trade event" \ No newline at end of file + description: "Index of the corresponding trade event" + + - name: dex_aggregator_base_trades + meta: + blockchain: ethereum, gnosis, avalanche_c, fantom, optimism, arbitrum, bnb + sector: dex_aggregator + contributors: hosuke + config: + tags: ['ethereum', 'gnosis', 'avalanche_c', 'fantom', 'aggregator', 'dex', 'trades', 'cross-chain'] + description: > + Base trades data without amount_usd column + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - blockchain + - project + - version + - tx_hash + - evt_index + - trace_address + columns: + - *blockchain + - *project + - *version + - *block_date + - *block_time + - *token_bought_symbol + - *token_sold_symbol + - *token_pair + - *token_bought_amount + - *token_sold_amount + - *token_bought_amount_raw + - *token_sold_amount_raw + - *token_bought_address + - *token_sold_address + - *taker + - *maker + - *project_contract_address + - *tx_hash + - *tx_from + - *tx_to + - *trace_address + - *evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_base_trades.sql b/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_base_trades.sql new file mode 100644 index 00000000000..9b7ee783f7e --- /dev/null +++ b/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_base_trades.sql @@ -0,0 +1,58 @@ +{{ config( + schema ='dex_aggregator' + , alias = 'base_trades' + , partition_by = ['block_month', 'blockchain', 'project'] + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'project', 'version', 'tx_hash', 'evt_index', 'trace_address'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set base_trade_models = [ + ref('lifi_base_trades') +] %} + +with base_union as ( + SELECT * + FROM + ( + {% for model in base_trade_models %} + SELECT + blockchain + , project + , version + , block_date + , block_month + , block_time + -- , block_number -- missing yet + , cast(token_bought_amount_raw as uint256) as token_bought_amount_raw + , cast(token_sold_amount_raw as uint256) as token_sold_amount_raw + , token_bought_address + , token_sold_address + , taker + , maker + , project_contract_address + , tx_hash + , tx_from + , tx_to + , trace_address + , evt_index + FROM + {{ model }} + WHERE + token_sold_amount_raw >= 0 and token_bought_amount_raw >= 0 + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} + ) +) +select + * +from + base_union diff --git a/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_trades.sql b/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_trades.sql index e67cb143484..39a47f57cad 100644 --- a/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_trades.sql +++ b/dbt_subprojects/dex/models/aggregator_trades/dex_aggregator_trades.sql @@ -1,4 +1,3 @@ - {{ config( schema ='dex_aggregator', alias = 'trades', @@ -10,14 +9,13 @@ post_hook='{{ expose_spells(\'["ethereum", "gnosis", "avalanche_c", "fantom", "bnb", "optimism", "arbitrum"]\', "sector", "dex_aggregator", - \'["bh2smith", "Henrystats", "jeff-dude", "rantum" ]\') }}' + \'["bh2smith", "Henrystats", "jeff-dude", "rantum", "hosuke"]\') }}' ) }} -{% set dex_aggregator_models = [ +{% set as_is_models = [ ref('cow_protocol_trades') ,ref('paraswap_trades') - ,ref('lifi_trades') ,ref('yield_yak_trades') ,ref('bebop_trades') ,ref('dodo_aggregator_trades') @@ -30,37 +28,88 @@ ,ref('odos_trades') ] %} -{% for aggregator_model in dex_aggregator_models %} -SELECT - blockchain - , project - , version - , block_date - , block_month - , block_time - , token_bought_symbol - , token_sold_symbol - , token_pair - , token_bought_amount - , token_sold_amount - , try_cast(token_bought_amount_raw as uint256) as token_bought_amount_raw - , try_cast(token_sold_amount_raw as uint256) as token_sold_amount_raw - , amount_usd - , token_bought_address - , token_sold_address - , taker - , maker - , project_contract_address - , tx_hash - , tx_from - , tx_to - , trace_address - , evt_index -FROM {{ aggregator_model }} -{% if is_incremental() %} -WHERE block_date >= date_trunc('day', now() - interval '7' day) -{% endif %} -{% if not loop.last %} -UNION ALL -{% endif %} +WITH enriched_aggregator_base_trades AS ( + {{ + enrich_dex_aggregator_trades( + base_trades = ref('dex_aggregator_base_trades') + , filter = "1 = 1" + , tokens_erc20_model = source('tokens', 'erc20') + ) + }} +) + +, as_is_dexs AS ( + {% for model in as_is_models %} + SELECT + blockchain + , project + , version + , block_date + , block_month + , block_time + , token_bought_symbol + , token_sold_symbol + , token_pair + , token_bought_amount + , token_sold_amount + , cast(token_bought_amount_raw as uint256) as token_bought_amount_raw + , cast(token_sold_amount_raw as uint256) as token_sold_amount_raw + , amount_usd + , token_bought_address + , token_sold_address + , taker + , maker + , project_contract_address + , tx_hash + , tx_from + , tx_to + , trace_address + , evt_index + FROM + {{ model }} + {% if is_incremental() %} + WHERE {{ incremental_predicate('block_time') }} + {% endif %} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) +{% set cte_to_union = [ + 'enriched_aggregator_base_trades' + , 'as_is_dexs' + ] +%} + +{% for cte in cte_to_union %} + SELECT + blockchain + , project + , version + , block_date + , block_month + , block_time + , token_bought_symbol + , token_sold_symbol + , token_pair + , token_bought_amount + , token_sold_amount + , token_bought_amount_raw + , token_sold_amount_raw + , amount_usd + , token_bought_address + , token_sold_address + , taker + , maker + , project_contract_address + , tx_hash + , tx_from + , tx_to + , trace_address + , evt_index + FROM + {{ cte }} + {% if not loop.last %} + UNION ALL + {% endif %} {% endfor %}