Update nft trades to match dex trades materialization design & ensure…

… incremental / partition design in place (#1396) * add block_date, update config block to partition by block_date and add to unique key, update incremental logic * change materialization to view on top of base incremental tables * clean up incremental logic filters * comment out sudoswap until tokenid array is exploded into rows * uncomment sudoswap in nft schema
duneanalytics · Aug 18, 2022 · c7983cb · c7983cb
1 parent 8aad705
commit c7983cb
Show file tree

Hide file tree

Showing 9 changed files with 311 additions and 289 deletions.
diff --git a/spellbook/models/looksrare/ethereum/looksrare_ethereum_events.sql b/spellbook/models/looksrare/ethereum/looksrare_ethereum_events.sql
@@ -1,10 +1,11 @@
 {{ config(
-        alias ='events',
-        materialized ='incremental',
-        file_format ='delta',
-        incremental_strategy='merge',
-        unique_key='unique_trade_id'
-        )
+    alias = 'events',
+    partition_by = ['block_date'],
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['block_date', 'unique_trade_id']
+    )
 }}
 
 WITH looks_rare AS (
@@ -104,6 +105,7 @@ SELECT DISTINCT
     'ethereum' as blockchain,
     'looksrare' as project,
     'v1' as version,
+    TRY_CAST(date_trunc('DAY', looks_rare.block_time) AS date) AS block_date,
     looks_rare.block_time,
     token_id,
     tokens.name AS collection,
@@ -157,7 +159,12 @@ SELECT DISTINCT
     'looksrare' || '-' || tx_hash || '-' ||  token_id::string || '-' ||  seller::string || '-' || COALESCE(erc.contract_address, nft_contract_address) || '-' || looks_rare.evt_index::string || '-' || COALESCE(evt_type::string, 'Other')  || '-' || COALESCE(case when erc.value_unique::string is null then '0' ELSE '1' end, '1') as unique_trade_id
 FROM looks_rare
 INNER JOIN {{ source('ethereum','transactions') }} tx ON tx_hash = tx.hash
-AND tx.block_time > '2022-01-01'
+    {% if not is_incremental() %}
+    AND tx.block_time > '2022-01-01'
+    {% endif %}
+    {% if is_incremental() %}
+    AND TRY_CAST(date_trunc('DAY', tx.block_time) AS date) = TRY_CAST(date_trunc('DAY', looks_rare.block_time) AS date)
+    {% endif %}
 LEFT JOIN erc_transfers erc ON erc.evt_tx_hash = tx_hash AND erc.token_id_erc = token_id
 LEFT JOIN {{ ref('tokens_ethereum_nft') }} tokens ON tokens.contract_address =  nft_contract_address
 LEFT JOIN  {{ ref('nft_ethereum_aggregators') }} agg ON agg.contract_address = tx.to

diff --git a/spellbook/models/magiceden/magiceden_events.sql b/spellbook/models/magiceden/magiceden_events.sql
@@ -1,52 +1,45 @@
 {{ config(
-        alias ='events',
-        materialized ='incremental',
-        file_format ='delta',
-        incremental_strategy='merge',
-        unique_key='unique_trade_id'
+        alias ='events'
         )
 }}
 
-SELECT blockchain,
-project,
-version,
-block_time,
-token_id,
-NULL::string as collection,
-amount_usd,
-token_standard,
-trade_type,
-number_of_items,
-NULL::string as trade_category,
-evt_type,
-seller,
-buyer,
-amount_original,
-amount_raw,
-currency_symbol,
-currency_contract,
-NULL::string as nft_contract_address,
-project_contract_address,
-NULL::string as aggregator_name,
-NULL::string as aggregator_address,
-tx_hash,
-block_number,
-NULL::string as tx_from,
-NULL::string as tx_to,
-platform_fee_amount_raw,
-platform_fee_amount,
-platform_fee_amount_usd,
-platform_fee_percentage,
-royalty_fee_amount_raw,
-royalty_fee_amount,
-royalty_fee_amount_usd,
-royalty_fee_percentage,
-NULL::string as royalty_fee_receive_address,
-royalty_fee_currency_symbol,
-unique_trade_id
-FROM {{ ref('magiceden_solana_events') }}
-
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-WHERE block_time >= (select max(block_time) from {{ this }})
-{% endif %} 
+SELECT
+        blockchain,
+        project,
+        version,
+        block_date,
+        block_time,
+        token_id,
+        NULL::string as collection,
+        amount_usd,
+        token_standard,
+        trade_type,
+        number_of_items,
+        NULL::string as trade_category,
+        evt_type,
+        seller,
+        buyer,
+        amount_original,
+        amount_raw,
+        currency_symbol,
+        currency_contract,
+        NULL::string as nft_contract_address,
+        project_contract_address,
+        NULL::string as aggregator_name,
+        NULL::string as aggregator_address,
+        tx_hash,
+        block_number,
+        NULL::string as tx_from,
+        NULL::string as tx_to,
+        platform_fee_amount_raw,
+        platform_fee_amount,
+        platform_fee_amount_usd,
+        platform_fee_percentage,
+        royalty_fee_amount_raw,
+        royalty_fee_amount,
+        royalty_fee_amount_usd,
+        royalty_fee_percentage,
+        NULL::string as royalty_fee_receive_address,
+        royalty_fee_currency_symbol,
+        unique_trade_id
+FROM {{ ref('magiceden_solana_events') }}
diff --git a/spellbook/models/magiceden/solana/magiceden_solana_events.sql b/spellbook/models/magiceden/solana/magiceden_solana_events.sql
@@ -1,10 +1,11 @@
 {{ config(
-        alias ='events',
-        materialized ='incremental',
-        file_format ='delta',
-        incremental_strategy='merge',
-        unique_key='unique_trade_id'
-        )
+    alias = 'events',
+    partition_by = ['block_date'],
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['block_date', 'unique_trade_id']
+    )
 }}
 
 SELECT 
@@ -14,6 +15,7 @@ SELECT
   WHEN (array_contains(account_keys, 'CMZYPASGWeTz7RNGHaRJfCq2XQ5pYK6nDvVQxzkH51zb')) THEN 'launchpad_v3'
   END as version,
   signatures[0] as tx_hash, 
+  block_date,
   block_time,
   block_slot::string as block_number,
   abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) * p.price AS amount_usd,
@@ -94,13 +96,16 @@ FROM {{ source('solana','transactions') }}
 LEFT JOIN prices.usd p 
   ON p.minute = date_trunc('minute', block_time)
   AND p.symbol = 'SOL'        
-WHERE (array_contains(account_keys, 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K') -- magic eden v2
-OR array_contains(account_keys, 'CMZYPASGWeTz7RNGHaRJfCq2XQ5pYK6nDvVQxzkH51zb'))
-AND success = 'True'
-AND block_date > '2022-01-05'
-AND block_slot > 114980355
-
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-AND block_date >= (select max(block_time) from {{ this }})
-{% endif %} 
+WHERE (
+     array_contains(account_keys, 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K') -- magic eden v2
+     OR array_contains(account_keys, 'CMZYPASGWeTz7RNGHaRJfCq2XQ5pYK6nDvVQxzkH51zb')
+     )
+     AND success = 'True'
+     {% if not is_incremental() %}
+     AND block_date > '2022-01-05'
+     AND block_slot > 114980355
+     {% endif %} 
+     {% if is_incremental() %}
+     -- this filter will only be applied on an incremental run
+     AND block_date >= (select max(block_date) from {{ this }})
+     {% endif %}
diff --git a/spellbook/models/opensea/ethereum/opensea_ethereum_events.sql b/spellbook/models/opensea/ethereum/opensea_ethereum_events.sql
@@ -1,94 +1,88 @@
 {{ config(
-        alias ='events',
-        materialized ='incremental',
-        file_format ='delta',
-        incremental_strategy='merge',
-        unique_key='unique_trade_id'
-        )
+        alias ='events'
+)
 }}
 
-SELECT blockchain,
-project,
-version,
-block_time,
-token_id,
-collection,
-amount_usd,
-token_standard,
-trade_type,
-number_of_items,
-trade_category,
-evt_type,
-seller,
-buyer,
-amount_original,
-amount_raw,
-currency_symbol,
-currency_contract,
-nft_contract_address,
-project_contract_address,
-aggregator_name,
-aggregator_address,
-block_number,
-tx_hash,
-tx_from,
-tx_to,
-platform_fee_amount_raw,
-platform_fee_amount,
-platform_fee_amount_usd,
-platform_fee_percentage,
-royalty_fee_amount_raw,
-royalty_fee_amount,
-royalty_fee_amount_usd,
-royalty_fee_percentage,
-royalty_fee_receive_address,
-royalty_fee_currency_symbol,
-unique_trade_id 
-FROM ({{ ref('opensea_v1_ethereum_events') }})
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-WHERE block_time > now() - interval 2 days
-{% endif %} 
-                UNION
-SELECT blockchain,
-'opensea' as project,
-'v3' as version,
-block_time,
-token_id,
-collection,
-amount_usd,
-token_standard,
-trade_type,
-number_of_items,
-trade_category,
-evt_type,
-seller,
-buyer,
-amount_original,
-amount_raw,
-currency_symbol,
-currency_contract,
-nft_contract_address,
-project_contract_address,
-aggregator_name,
-aggregator_address,
-block_number,
-tx_hash,
-tx_from,
-tx_to,
-platform_fee_amount_raw,
-platform_fee_amount,
-platform_fee_amount_usd,
-platform_fee_percentage,
-royalty_fee_amount_raw,
-royalty_fee_amount,
-royalty_fee_amount_usd,
-royalty_fee_percentage,
-royalty_fee_receive_address,
-royalty_fee_currency_symbol,
-unique_trade_id 
-FROM ({{ ref('opensea_v3_ethereum_events') }})
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-WHERE block_time > now() - interval 2 days
-{% endif %} 
+SELECT *
+FROM
+(
+        SELECT 
+                blockchain,
+                project,
+                version,
+                block_time,
+                token_id,
+                collection,
+                amount_usd,
+                token_standard,
+                trade_type,
+                number_of_items,
+                trade_category,
+                evt_type,
+                seller,
+                buyer,
+                amount_original,
+                amount_raw,
+                currency_symbol,
+                currency_contract,
+                nft_contract_address,
+                project_contract_address,
+                aggregator_name,
+                aggregator_address,
+                tx_hash,
+                block_number,
+                tx_from,
+                tx_to,
+                platform_fee_amount_raw,
+                platform_fee_amount,
+                platform_fee_amount_usd,
+                platform_fee_percentage,
+                royalty_fee_amount_raw,
+                royalty_fee_amount,
+                royalty_fee_amount_usd,
+                royalty_fee_percentage,
+                royalty_fee_receive_address,
+                royalty_fee_currency_symbol,
+                unique_trade_id 
+        FROM ({{ ref('opensea_v1_ethereum_events') }})
+        UNION
+        SELECT
+                blockchain,
+                'opensea' as project,
+                'v3' as version,
+                block_time,
+                token_id,
+                collection,
+                amount_usd,
+                token_standard,
+                trade_type,
+                number_of_items,
+                trade_category,
+                evt_type,
+                seller,
+                buyer,
+                amount_original,
+                amount_raw,
+                currency_symbol,
+                currency_contract,
+                nft_contract_address,
+                project_contract_address,
+                aggregator_name,
+                aggregator_address,
+                tx_hash,
+                block_number,
+                tx_from,
+                tx_to,
+                platform_fee_amount_raw,
+                platform_fee_amount,
+                platform_fee_amount_usd,
+                platform_fee_percentage,
+                royalty_fee_amount_raw,
+                royalty_fee_amount,
+                royalty_fee_amount_usd,
+                royalty_fee_percentage,
+                royalty_fee_receive_address,
+                royalty_fee_currency_symbol,
+                unique_trade_id 
+        FROM ({{ ref('opensea_v3_ethereum_events') }})
+)