T 12704 add magiceden trades abstraction (#1084)

In this PR, I'm adding magic eden trades based on the solana.transactions table
- Creating magiceden_solana.trades, magiceden.trades, and nft.trades composed on both opensea.trades and magiceden.trades
This is part of a bigger attempt to create nft abstractions so wizards can build on top of it.

I've checked that:
* [* ] I tested the query on dune.com after compiling the model with dbt compile (compiled queries are written to the target directory)
* [*] I used "refs" to reference other models in this repo and "sources" to reference raw or decoded tables 
* [*] the directory tree matches the pattern /sector/blockchain/ e.g. /tokens/ethereum
* [*] if adding a new model, I added a test
* [*] the filename is unique and ends with .sql
* [*] each file has only one view, table or function defined  
* [*] column names are `lowercase_snake_cased`

spellbook/dbt_project.yml

@@ -26,6 +26,9 @@ clean-targets:         # directories to be removed by `dbt clean`
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
 models:
   spellbook:
+    nft:
+      +schema: nft
+      +materialized: view
     opensea:
       +schema: opensea
       +materialized: view
@@ -35,6 +38,12 @@ models:
       solana:
         +schema: opensea_solana
         +materialized: view
+    magiceden:
+      +schema: magiceden
+      +materialized: view
+      solana:
+        +schema: magiceden_solana
+        +materialized: view
 
     balances:
       +schema: balances
@@ -81,15 +90,24 @@ seeds:
       ethereum:
         +enabled: true
         +schema: test_data
-        opensea_ethereum_trades_caladan:
+        opensea_ethereum_trades_postgres:
           +column_types:
             evt_block_times: timestamp
             evt_tx_hash: string
             price: string
       solana:
         +enabled: true
         +schema: test_data
-        opensea_solana_trades_caladan:
+        opensea_solana_trades_postgres:
+          +column_types:
+            block_time: timestamp
+            tx_hash: string
+            amount: string
+    magiceden:
+      solana:
+        +enabled: true
+        +schema: test_data
+        magiceden_solana_trades_postgres:
           +column_types:
             block_time: timestamp
             tx_hash: string
@@ -104,6 +122,8 @@ on-run-end:
   - "{{ alter_tblproperties_balances_ethereum_erc20_day() }}"
   - "{{ alter_tblproperties_balances_ethereum_erc20_latest() }}"
 
+  - "{{ alter_tblproperties_nft_trades() }}"
+  - "{{ alter_tblproperties_magiceden_trades() }}"
   - "{{ alter_tblproperties_opensea_trades() }}"
   - "{{ alter_tblproperties_opensea_volume_day() }}"
   - "{{ alter_tblproperties_opensea_txns_day() }}"

spellbook/macros/alter_tblproperties/alter_tblproperties_magiceden_trades.sql

@@ -0,0 +1,11 @@
+{% macro alter_tblproperties_magiceden_trades() -%}
+{%- if target.name == 'prod'-%}
+ALTER VIEW magiceden.trades SET TBLPROPERTIES('dune.public'='true',
+                                                    'dune.data_explorer.blockchains'='["solana"]',
+                                                    'dune.data_explorer.category'='abstraction',
+                                                    'dune.data_explorer.abstraction.type'='project',
+                                                    'dune.data_explorer.abstraction.name'='magiceden',
+                                                    'dune.data_explorer.contributors'='["soispoke"]');
+{%- else -%}
+{%- endif -%}
+{%- endmacro %}

spellbook/macros/alter_tblproperties/alter_tblproperties_nft_trades.sql

@@ -0,0 +1,11 @@
+{% macro alter_tblproperties_nft_trades() -%}
+{%- if target.name == 'prod'-%}
+ALTER VIEW nft.trades SET TBLPROPERTIES('dune.public'='true',
+                                                    'dune.data_explorer.blockchains'='["solana"]',
+                                                    'dune.data_explorer.category'='abstraction',
+                                                    'dune.data_explorer.abstraction.type'='sector',
+                                                    'dune.data_explorer.abstraction.name'='nft',
+                                                    'dune.data_explorer.contributors'='["soispoke"]');
+{%- else -%}
+{%- endif -%}
+{%- endmacro %}

spellbook/models/magiceden/magiceden_schema.yml

@@ -0,0 +1,32 @@
+version: 2
+
+models:
+  - name: magiceden_trades
+    meta:
+      blockchain: solana
+      project: magiceden
+      contributors: soispoke
+    config:
+      tags: ['magiceden','solana','trades']
+    description: >
+        Magic Eden trades
+    columns:
+      - &blockchain
+        name: blockchain
+        description: "Blockchain"
+      - name: project
+        description: "Project"
+      - name: version
+        description: "Project version"
+      - name: tx_hash
+        description: "Transaction hash"
+      - name: block_time
+        description: "UTC event block time"
+      - name: token_symbol
+        description: "Token Symbol"
+      - name: amount
+        description: "Value of the trade at time of execution in the original currency"
+      - name: amount_usd
+        description:  "USD value of the trade at time of execution"
+      - name: trade_id
+        description:  "Trade ID (derived from id in solana transactions)"

spellbook/models/magiceden/magiceden_trades.sql

@@ -0,0 +1,14 @@
+{{ config(
+        alias='trades'
+        )
+}}
+
+SELECT blockchain, 'magiceden' as project, '' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM 
+(
+        SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id 
+        FROM {{ ref('magiceden_solana_trades') }}
+) 
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+where block_time > (select max(block_time) from {{ this }})
+{% endif %} 

spellbook/models/magiceden/solana/magiceden_solana_schema.yml

@@ -0,0 +1,34 @@
+version: 2
+
+models:
+  - name: magiceden_solana_trades
+    meta:
+      blockchain: solana
+      project: magiceden
+      contributors: soispoke
+    config:
+      tags: ['solana','magiceden','trades']
+    description: >
+        MagicEden trades on Solana
+    columns:
+      - name: unique_id
+        description:  "Unique id, combination of signatures[0] and id"
+        tests:
+          - unique
+      - &blockchain
+        name: blockchain
+        description: "Blockchain"
+      - name: tx_hash
+        description: "Transaction hash"
+      - name: block_time
+        description: "UTC event block time"
+      - name: token_symbol
+        description: "Token Symbol"
+      - name: amount
+        description: "Value of the trade at time of execution in the original currency"
+      - name: amount_usd
+        description:  "USD value of the trade at time of execution"
+      - name: trader
+        description:  "Trader"
+      - name: trade_id
+        description:  "Trade ID (derived from id in solana transactions)"

spellbook/models/magiceden/solana/magiceden_solana_trades.sql

@@ -0,0 +1,33 @@
+ {{
+  config(
+        alias='trades',
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_id'
+  )
+}}
+
+SELECT 
+  signatures[0] || id as unique_id,
+  'solana' as blockchain,
+  signatures[0] as tx_hash, 
+  block_time,
+  abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) * p.price AS amount_usd,
+  abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) AS amount,
+  p.symbol as token_symbol,
+  p.contract_address as token_address,
+  account_keys[0] as traders,
+  id as trade_id
+FROM {{ source('solana','transactions') }}
+LEFT JOIN {{ source('prices', 'usd') }} p 
+  ON p.minute = date_trunc('minute', block_time)
+  AND p.symbol = 'SOL'        
+WHERE (array_contains(account_keys, 'MEisE1HzehtrDpAAT8PnLHjpSSkRYakotTuJRPjTpo8') -- magic eden v1
+       OR array_contains(account_keys, 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K'))  -- magic eden v2
+AND ARRAY_CONTAINS(log_messages, 'Program log: Instruction: ExecuteSale')
+AND block_time > '2021-09-01'
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+AND block_time > (select max(block_time) from {{ this }})
+{% endif %} 

spellbook/models/nft/nft_schema.yml

@@ -0,0 +1,26 @@
+version: 2
+
+models:
+  - name: nft_trades
+    meta:
+      blockchain: ethereum, solana
+      sector: nft
+      contributors: soispoke
+    config:
+      tags: ['nft', 'opensea', 'magiceden', 'ethereum', 'solana', 'trades']
+    description: >
+        NFT trades
+    columns:
+      - &blockchain
+        name: blockchain
+        description: "Blockchain"
+      - name: tx_hash
+        description: "Transaction hash"
+      - name: block_time
+        description: "UTC event block time"
+      - name: token_symbol
+        description: "Token Symbol"
+      - name: amount
+        description: "Value of the trade at time of execution in the original currency"
+      - name: amount_usd
+        description:  "USD value of the trade at time of execution"

spellbook/models/nft/nft_trades.sql

@@ -0,0 +1,13 @@
+{{ config(
+        alias='trades'
+        )
+}}
+
+SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address,trade_id FROM 
+(SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_trades') }} 
+UNION ALL
+SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('magiceden_trades') }}) 
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+where block_time > (select max(block_time) from {{ this }})
+{% endif %} 

spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml

@@ -11,12 +11,16 @@ models:
     description: >
         OpenSea trades on Ethereum
     columns:
+      - name: unique_id
+        description:  "Unique id, combination of tx_hash and trade_id"
+        tests:
+          - unique
       - &blockchain
         name: blockchain
         description: "Blockchain"
-      - name: evt_tx_hash
+      - name: tx_hash
         description: "Transaction hash"
-      - name: evt_block_time
+      - name: block_time
         description: "UTC event block time"
       - name: token_symbol
         description: "Token Symbol"
@@ -28,7 +32,9 @@ models:
         description:  "Maker"
       - name: taker
         description:  "Taker"
-
+      - name: trade_id
+        description:  "Trade id (derived from evt_index for ethereum nft trades)"
+
   - name: opensea_ethereum_volume_day
     meta:
       blockchain: ethereum

spellbook/models/opensea/ethereum/opensea_ethereum_trades.sql

@@ -3,11 +3,13 @@
         alias='trades',
         materialized ='incremental',
         file_format ='delta',
-        incremental_strategy='merge'
+        incremental_strategy='merge',
+        unique_key='unique_id'
   )
 }}
 
 SELECT
+evt_tx_hash || evt_index::string as unique_id,
 'ethereum' as blockchain,
 evt_tx_hash as tx_hash,
 evt_block_time as block_time,
@@ -17,15 +19,16 @@ om.price AS amount_raw,
 terc20.symbol as token_symbol,
 wam.token_address as token_address,
 maker,
-taker
+taker,
+evt_index as trade_id
 FROM
   {{ source('opensea_ethereum','wyvernexchange_evt_ordersmatched') }} om
-  INNER JOIN {{ ref('opensea_ethereum_wyvern_atomic_match') }} wam ON wam.tx_hash = om.evt_tx_hash
-  INNER JOIN tokens_ethereum.erc20 terc20 ON terc20.contract_address = wam.token_address
-  INNER JOIN {{ source('prices', 'usd') }} p ON p.minute = date_trunc('minute', evt_block_time)
+  LEFT JOIN {{ ref('opensea_ethereum_wyvern_atomic_match') }} wam ON wam.tx_hash = om.evt_tx_hash
+  LEFT JOIN {{ ref('tokens_ethereum_erc20') }} terc20 ON terc20.contract_address = wam.token_address
+  LEFT JOIN {{ source('prices', 'usd') }} p ON p.minute = date_trunc('minute', evt_block_time)
       AND p.blockchain = 'ethereum'
       AND p.contract_address = wam.token_address
-  AND maker != taker
+  WHERE maker != taker
   AND date_trunc('day', p.minute) >= '2018-06-01'
   AND evt_tx_hash not in (
     SELECT
@@ -35,5 +38,5 @@ FROM
   )
   {% if is_incremental() %}
   -- this filter will only be applied on an incremental run
-  WHERE evt_block_time > (select max(block_time) from {{ this }})
+  AND evt_block_time > (select max(block_time) from {{ this }})
   {% endif %} 

spellbook/models/opensea/opensea_schema.yml

@@ -14,16 +14,18 @@ models:
       - &blockchain
         name: blockchain
         description: "Blockchain"
-      - name: evt_tx_hash
+      - name: tx_hash
         description: "Transaction hash"
-      - name: evt_block_time
+      - name: block_time
         description: "UTC event block time"
       - name: token_symbol
         description: "Token Symbol"
       - name: amount
         description: "Value of the trade at time of execution in the original currency"
       - name: amount_usd
         description:  "USD value of the trade at time of execution"
+      - name: trade_id
+        description:  "Trade ID"
 
   - name: opensea_volume_day
     meta:

spellbook/models/opensea/opensea_trades.sql

@@ -3,10 +3,10 @@
         )
 }}
 
-SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address FROM 
-(SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address FROM {{ ref('opensea_ethereum_trades') }} 
+SELECT blockchain, 'opensea' as project, 'v1' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM 
+(SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_ethereum_trades') }} 
 UNION ALL
-SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address FROM {{ ref('opensea_solana_trades') }}) 
+SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_solana_trades') }}) 
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
 where block_time > (select max(block_time) from {{ this }})

spellbook/models/opensea/solana/opensea_solana_schema.yml

@@ -11,12 +11,16 @@ models:
     description: >
         OpenSea trades on Solana
     columns:
+      - name: unique_id
+        description:  "Unique id, combination of signatures[0] and id"
+        tests:
+          - unique
       - &blockchain
         name: blockchain
         description: "Blockchain"
-      - name: evt_tx_hash
+      - name: tx_hash
         description: "Transaction hash"
-      - name: evt_block_time
+      - name: block_time
         description: "UTC event block time"
       - name: token_symbol
         description: "Token Symbol"
@@ -26,7 +30,9 @@ models:
         description:  "USD value of the trade at time of execution"
       - name: trader
         description:  "Trader"
-
+      - name: trade_id
+        description:  "Trade ID (derived from id in solana transactions)"
+
   - name: opensea_solana_volume_day
     meta:
       blockchain: solana