Merge branch 'main' into op-cur

.github/labeler.yml

@@ -0,0 +1,23 @@
+"dbt: daily":
+- changed-files:
+    - any-glob-to-any-file: 'dbt_subprojects/daily_spellbook/**'
+
+"dbt: dex":
+- changed-files:
+    - any-glob-to-any-file: 'dbt_subprojects/dex/**'
+
+"dbt: hourly":
+- changed-files:
+    - any-glob-to-any-file: 'dbt_subprojects/hourly_spellbook/**'
+
+"dbt: nft":
+- changed-files:
+    - any-glob-to-any-file: 'dbt_subprojects/nft/**'
+
+"dbt: solana":
+- changed-files:
+    - any-glob-to-any-file: 'dbt_subprojects/solana/**'
+
+"dbt: tokens":
+- changed-files:
+    - any-glob-to-any-file: 'dbt_subprojects/tokens/**'

.github/workflows/pr_automation.yml

@@ -0,0 +1,46 @@
+name: PR automation
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - ready_for_review
+      - converted_to_draft
+      - synchronize
+      - labeled
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  pr-automation:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_REPO: ${{ github.repository }}
+      NUMBER: ${{ github.event.number }}
+    steps:
+      - name: Mark new PR as Draft
+        if: (github.event.action == 'opened' && github.event.pull_request.draft == false)
+        run: gh pr ready "$NUMBER" --undo
+
+      - name: Set labels on Open
+        if: github.event.action == 'opened'
+        run: gh pr edit "$NUMBER" --add-label "WIP"
+
+      - name: Set labels on Draft
+        if: github.event.action == 'converted_to_draft'
+        run: gh pr edit "$NUMBER" --add-label "WIP" --remove-label "ready-for-review"
+
+      - name: Set labels on ready-for-review
+        if: github.event.action == 'ready_for_review'
+        run: gh pr edit "$NUMBER" --add-label "ready-for-review" --remove-label "WIP"
+
+      - name: Add subproject labels
+        if: contains('synchronize,opened,reopened', github.event.action)
+        uses: actions/labeler@v5
+        with:
+          sync-labels: true
+
+

CONTRIBUTING.md

@@ -19,7 +19,7 @@ Contributions in the form of issues and pull requests are very much welcome here
 
 ## [BETA] Pre-push hooks
 
-UPDATE: These pre-push hooks require running `dbt compile` which is a fairly slow step due to the size of our project. We intend to rewrite these hooks to be more efficient but for the time being they remain cumbersome. Feel free to use them if you find them useful but the same checks will run in a Github Action when you commit your code. Feel free to uninstall if they do not bring joy, we'll let wizards know when we think we've improved them enought to warrant making them part of the general development flow.
+UPDATE: These pre-push hooks require running `dbt compile` which is a fairly slow step due to the size of our project. We intend to rewrite these hooks to be more efficient but for the time being they remain cumbersome. Feel free to use them if you find them useful but the same checks will run in a Github Action when you commit your code. Feel free to uninstall if they do not bring joy, we'll let wizards know when we think we've improved them enough to warrant making them part of the general development flow.
 
 We are testing out adding pre-push hooks to our workflow. The goal is to catch common errors before code is pushed and
 streamline the pull request review process.
@@ -126,13 +126,13 @@ example custom test:
 ```sql
 with unit_test1 as
 (select
-    case when col1 == 2 and col2 == 'moon' then True else False end as test
+    case when col1 = 2 and col2 = 'moon' then True else False end as test
     from {{ ref('mock_table' )}}
     where tx_id = '102'),
 
 unit_test2 as
 (select
-    case when col1 == 2 and col2 == 'moon' then True else False end as test
+    case when col1 = 2 and col2 = 'moon' then True else False end as test
     from {{ ref('mock_table' )}}
     where tx_id = '103'),
 

Pipfile

@@ -7,7 +7,7 @@ name = "pypi"
 numpy = "2.0.12"
 pre-commit = "2.20.0"
 pytest = "7.1.3"
-dbt-trino = "1.8.2"
+dbt-trino = "1.9.0"
 
 [requires]
-python_version = "3.9"
+python_version = "3.9"

README.md

@@ -11,10 +11,10 @@ Welcome to [Spellbook](https://youtu.be/o7p0BNt7NHs). Cast a magical incantation
 - Are you building something new? **Please make sure to open a Draft PR**, so we minimize duplicated work, and other wizards can help you if you need
 - Don't know where to start? The docs below will guide you, but as a summary:
   - Want to make an incremental improvement to one of our spells? (add a new project, fix a bug you found), simply open a PR with your changes.
-    - Follow the guide for [Submitting a PR](), [Setting up your dev environment]() and [Using dbt to write spells]() if you find yourself lost.
-    - Not sure how to start? Follow the walkthrough [here](https://dune.com/docs/spellbook/).
+    - Follow the guide for [Submitting a PR](#submitting-a-pr), [Setting up your dev environment](#setting-up-your-local-dev-environment) and [Using dbt to write spells](#how-to-use-dbt-to-create-spells) if you find yourself lost.
+    - Not sure how to start? Follow the walkthrough [here](#introduction).
     - Make sure to open a draft PR if you will work on your spell for longer than a few days, to avoid duplicated work
-  - Do you want to get started building spells and you don't know what to build? Check [Issues]() to see what the community needs.
+  - Do you want to get started building spells and you don't know what to build? Check [Issues](https://github.com/duneanalytics/spellbook/issues) to see what the community needs.
   - Check the Discussions section to see what problems the community is trying to solve (i.e. non-incremental changes) or propose your own!
 - Have questions? Head over to #spellbook on our [discord](https://discord.com/channels/757637422384283659/999683200563564655) and the community will be happy to help out!
 - Like with most OSS projects, your contributions to Spellbook are your own IP, you can find more details in the [Contributor License Agreement](CLA.md)

dbt_macros/shared/balancer/balancer_bpt_prices_macro.sql

@@ -349,3 +349,278 @@ WITH pool_labels AS (
     WHERE supply > 0
 
     {% endmacro %}
+
+    {# ######################################################################### #}
+
+ {% macro
+    balancer_v3_compatible_bpt_prices_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
+    )
+%}
+
+WITH pool_labels AS (
+        SELECT
+            address AS pool_id,
+            name AS pool_symbol,
+            pool_type
+        FROM {{ pool_labels_spell }}
+        WHERE blockchain = '{{blockchain}}'
+    ),
+
+    token_data AS (
+        SELECT
+            pool,
+            ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens 
+        FROM (
+            SELECT
+                pool,
+                tokenConfig,
+                SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array
+            FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }}
+        ) AS pool_data
+        CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index)
+        GROUP BY 1
+    ),
+
+-- liquidity formulation, with a few simplifications, compared to liquidity spell
+
+    prices AS (
+        SELECT
+            date_trunc('day', minute) AS day,
+            contract_address AS token,
+            decimals,
+            AVG(price) AS price
+        FROM {{ source('prices', 'usd') }}
+        WHERE blockchain = '{{blockchain}}'
+        GROUP BY 1, 2, 3
+    ),
+
+    erc4626_prices AS(
+        SELECT
+            date_trunc('day', minute) AS day,
+            wrapped_token AS token,
+            decimals,
+            APPROX_PERCENTILE(median_price, 0.5) AS price,
+            DATE_TRUNC ('day', next_change) AS next_change
+        FROM {{ ref('balancer_v3_erc4626_token_prices') }}
+        WHERE blockchain = '{{blockchain}}'
+        GROUP BY 1, 2, 3, 5
+    ),
+
+    swaps_changes AS (
+        SELECT
+            day,
+            pool_id,
+            token,
+            SUM(COALESCE(delta, INT256 '0')) AS delta
+        FROM
+            (
+                SELECT
+                    date_trunc('day', evt_block_time) AS day,
+                    pool AS pool_id,
+                    tokenIn AS token,
+                    CAST(amountIn AS INT256) AS delta
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
+
+                UNION ALL
+
+                SELECT
+                    date_trunc('day', evt_block_time) AS day,
+                    pool AS pool_id,
+                    tokenOut AS token,
+                    -CAST(amountOut AS INT256) AS delta
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
+            ) swaps
+        GROUP BY 1, 2, 3
+    ),
+
+    balance_changes AS(
+        SELECT
+            evt_block_time,
+            pool_id,
+            category,
+            deltas,
+            swapFeeAmountsRaw
+        FROM
+            (
+                SELECT
+                    evt_block_time,
+                    pool AS pool_id,
+                    'add' AS category,
+                    amountsAddedRaw AS deltas,
+                    swapFeeAmountsRaw
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityAdded') }}
+
+                UNION ALL
+
+                SELECT
+                    evt_block_time,
+                    pool AS pool_id,
+                    'remove' AS category,
+                    amountsRemovedRaw AS deltas,
+                    swapFeeAmountsRaw
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityRemoved') }}
+            ) adds_and_removes
+    ),
+
+    zipped_balance_changes AS (
+        SELECT
+            date_trunc('day', evt_block_time) AS day,
+            pool_id,
+            t.tokens,
+            CASE WHEN b.category = 'add'
+            THEN d.deltas
+            WHEN b.category = 'remove'
+            THEN -d.deltas
+            END AS deltas,
+            p.swapFeeAmountsRaw
+        FROM balance_changes b
+        JOIN token_data td ON b.pool_id = td.pool
+        CROSS JOIN UNNEST (td.tokens) WITH ORDINALITY as t(tokens,i)
+        CROSS JOIN UNNEST (b.deltas) WITH ORDINALITY as d(deltas,i)
+        CROSS JOIN UNNEST (b.swapFeeAmountsRaw) WITH ORDINALITY as p(swapFeeAmountsRaw,i)
+        WHERE t.i = d.i
+        AND d.i = p.i
+        ORDER BY 1,2,3
+    ),
+
+    balances_changes AS (
+        SELECT
+            day,
+            pool_id,
+            tokens AS token,
+            deltas - CAST(swapFeeAmountsRaw as int256) AS delta
+        FROM zipped_balance_changes
+        ORDER BY 1, 2, 3
+    ),
+
+    daily_delta_balance AS (
+        SELECT
+            day,
+            pool_id,
+            token,
+            SUM(COALESCE(amount, INT256 '0')) AS amount
+        FROM
+            (
+                SELECT
+                    day,
+                    pool_id,
+                    token,
+                    SUM(COALESCE(delta, INT256 '0')) AS amount
+                FROM balances_changes
+                GROUP BY 1, 2, 3
+
+                UNION ALL
+
+                SELECT
+                    day,
+                    pool_id,
+                    token,
+                    delta AS amount
+                FROM
+                    swaps_changes
+            ) balance
+        GROUP BY 1, 2, 3
+    ),
+
+
+    cumulative_balance AS (
+        SELECT
+            DAY,
+            pool_id,
+            token,
+            LEAD(DAY, 1, NOW()) OVER (PARTITION BY token, pool_id ORDER BY DAY) AS day_of_next_change,
+            SUM(amount) OVER (PARTITION BY pool_id, token ORDER BY DAY ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_amount
+        FROM daily_delta_balance
+    ),
+
+    calendar AS (
+        SELECT date_sequence AS day
+        FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence)
+    ),
+
+   cumulative_usd_balance AS (
+        SELECT
+            c.day,
+            '{{blockchain}}' as blockchain,
+            b.pool_id,
+            b.token,
+            cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p4.decimals)) * COALESCE(p1.price, p4.price, 0) AS protocol_liquidity_usd
+        FROM calendar c
+        LEFT JOIN cumulative_balance b ON b.day <= c.day
+        AND c.day < b.day_of_next_change
+        LEFT JOIN {{ source('tokens', 'erc20') }} t ON t.contract_address = b.token
+        AND blockchain = '{{blockchain}}'
+        LEFT JOIN prices p1 ON p1.day = b.day
+        AND p1.token = b.token
+        LEFT JOIN erc4626_prices p4 ON p4.day <= c.day
+        AND c.day < p4.next_change
+        AND p4.token = b.token
+        WHERE b.token != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
+    ),
+
+    weighted_pool_liquidity_estimates AS (
+        SELECT
+            b.day,
+            b.pool_id,
+            q.name,
+            pool_type,
+            ROW_NUMBER() OVER (partition by b.day, b.pool_id ORDER BY SUM(b.protocol_liquidity_usd) ASC) AS pricing_count, --to avoid double count in pools with multiple pricing assets
+            SUM(b.protocol_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS protocol_liquidity
+        FROM cumulative_usd_balance b
+        LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id
+        AND b.token = w.token_address
+        AND b.protocol_liquidity_usd > 0
+        LEFT JOIN {{ source('balancer','token_whitelist') }} q ON b.token = q.address
+        AND b.blockchain = q.chain
+        LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
+        WHERE q.name IS NOT NULL
+        AND p.pool_type IN ('weighted') -- filters for weighted pools with pricing assets
+        AND w.blockchain = '{{blockchain}}'
+        AND w.version = '{{version}}'
+        GROUP BY 1, 2, 3, 4
+    ),
+
+    weighted_pool_liquidity_estimates_2 AS(
+    SELECT  e.day,
+            e.pool_id,
+            SUM(e.protocol_liquidity) / MAX(e.pricing_count) AS protocol_liquidity
+    FROM weighted_pool_liquidity_estimates e
+    GROUP BY 1,2
+    ),
+
+    tvl AS(
+    SELECT
+        c.day,
+        BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) AS pool_address,
+        '{{version}}' AS version,
+        '{{blockchain}}' AS blockchain,
+        SUM(COALESCE(b.protocol_liquidity * w.normalized_weight, c.protocol_liquidity_usd)) AS liquidity
+    FROM cumulative_usd_balance c
+    FULL OUTER JOIN weighted_pool_liquidity_estimates_2 b ON c.day = b.day
+    AND c.pool_id = b.pool_id
+    LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id
+    AND w.blockchain = '{{blockchain}}'
+    AND w.version = '{{version}}'
+    AND w.token_address = c.token
+    LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(c.pool_id, 1, 20)
+    GROUP BY 1, 2, 3, 4
+    )
+
+    SELECT
+        l.day,
+        l.blockchain,
+        l.version,
+        18 AS decimals,
+        l.pool_address AS contract_address,
+        pl.pool_type,
+        l.liquidity / s.supply AS bpt_price
+    FROM tvl l
+    LEFT JOIN {{ ref(base_spells_namespace + '_bpt_supply') }} s ON l.pool_address = s.token_address
+    AND l.blockchain = s.blockchain
+    AND l.version = s.version
+    AND l.day = s.day
+    LEFT JOIN pool_labels pl ON pl.pool_id = l.pool_address
+    WHERE supply > 0
+
+    {% endmacro %}