feature - Add serverless tasks and spend (#37)

* feature - Add serverless tasks and spend

* fix - groupby

* Update monthly_spend_fixture.csv

* fix-ref

* changie

Co-authored-by: Niall Woodward <[email protected]>

.changes/unreleased/Features-20221206-163014.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add serverless tasks history and spend
+time: 2022-12-06T16:30:14.851362Z
+custom:
+  Author: gthesheep
+  PR: "37"

README.md

@@ -63,7 +63,7 @@ We use changie to generate CHANGELOG entries. Note: Do not edit the CHANGELOG.md
 
 Follow the steps to [install changie](https://changie.dev/guide/installation/) for your system.
 
-Once changie is installed and your PR is created, simply run changie new and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete!
+Once changie is installed and your PR is created, simply run `changie new` and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete!
 
 ### Developing the package
 

integration_test_project/seeds/monthly_spend_fixture.csv

@@ -9,3 +9,4 @@ MONTH,SERVICE,SPEND
 2022-04-01,Search Optimization,0
 2022-04-01,Automatic Clustering,0.051113362
 2022-04-01,Snowpipe,0
+2022-04-01,Serverless Tasks,0

models/daily_spend.sql

@@ -99,6 +99,33 @@ compute_spend_daily as (
     group by 1, 2, 3, 4
 ),
 
+serverless_task_rates as (
+    select
+        date,
+        max(effective_rate) as effective_rate
+    from {{ ref('daily_rates') }}
+    where
+        service_type = 'COMPUTE'
+        and usage_type in ('serverless tasks')
+    group by 1
+),
+
+serverless_task_spend_daily as (
+    select
+        dates.date,
+        'Serverless Tasks' as service,
+        null as storage_type,
+        null as warehouse_name,
+        stg_serverless_task_history.database_name,
+        coalesce(sum(stg_serverless_task_history.credits_used * serverless_task_rates.effective_rate), 0) as spend
+    from dates
+    left join {{ ref('stg_serverless_task_history') }} on
+        dates.date = convert_timezone('UTC', stg_serverless_task_history.start_time)::date
+    left join serverless_task_rates on
+        dates.date = serverless_task_rates.date
+    group by 1, 2, 3, 4, 5
+),
+
 cloud_services_rates as (
     select
         date,
@@ -264,3 +291,5 @@ union all
 select * from replication_spend_daily
 union all
 select * from search_optimization_spend_daily
+union all
+select * from serverless_task_spend_daily

models/daily_spend.yml

@@ -13,6 +13,6 @@ models:
     - name: warehouse_name
       description: Subcategories where service = "Compute" or "Cloud Services".
     - name: database_name
-      description: Subcategories where service = "Storage" and storage_type = "Table and Time Travel" or "Failsafe".
+      description: Subcategories where service = "Serverless Tasks" or service = "Storage" and storage_type = "Table and Time Travel" or "Failsafe".
     - name: spend
       description: Spend in the account's currency.

models/staging/sources.yml

@@ -11,6 +11,7 @@ sources:
       - name: warehouse_metering_history
       - name: metering_history
       - name: metering_daily_history
+      - name: serverless_task_history
 
   - name: snowflake_organization_usage
     database: snowflake

models/staging/stg_serverless_task_history.sql

@@ -0,0 +1,16 @@
+{{ config(materialized='incremental') }}
+
+select
+    start_time,
+    end_time,
+    task_id,
+    task_name,
+    database_name,
+    credits_used
+from {{ source('snowflake_account_usage', 'serverless_task_history') }}
+
+{% if is_incremental() %}
+    where end_time > (select max(end_time) from {{ this }})
+{% endif %}
+
+order by start_time

models/staging/stg_serverless_task_history.yml

@@ -0,0 +1,18 @@
+version: 2
+
+models:
+  - name: stg_serverless_task_history
+    description: An incrementally materialized copy of the snowflake.account_usage.warehouse_metering_history view. See https://docs.snowflake.com/en/sql-reference/account-usage/serverless_task_history.html
+    columns:
+      - name: start_time
+        description: The date and beginning of the hour (in the UTC time zone) in which the serverless task took place.
+      - name: end_time
+        description: The date and end of the hour (in the UTC time zone) in which the serverless task took place.
+      - name: task_id
+        description: Internal/system-generated identifier for the task.
+      - name: task_name
+        description: Name of the task.
+      - name: database_name
+        description: Name of the database in which the task is located.
+      - name: credits_used
+        description: Total number of credits used for the task.