diff --git a/.changes/unreleased/Features-20221206-163014.yaml b/.changes/unreleased/Features-20221206-163014.yaml new file mode 100644 index 0000000..adf5982 --- /dev/null +++ b/.changes/unreleased/Features-20221206-163014.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add serverless tasks history and spend +time: 2022-12-06T16:30:14.851362Z +custom: + Author: gthesheep + PR: "37" diff --git a/README.md b/README.md index bde39f9..e0cb798 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ We use changie to generate CHANGELOG entries. Note: Do not edit the CHANGELOG.md Follow the steps to [install changie](https://changie.dev/guide/installation/) for your system. -Once changie is installed and your PR is created, simply run changie new and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete! +Once changie is installed and your PR is created, simply run `changie new` and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete! ### Developing the package diff --git a/integration_test_project/seeds/monthly_spend_fixture.csv b/integration_test_project/seeds/monthly_spend_fixture.csv index 69cdfec..17d00a6 100644 --- a/integration_test_project/seeds/monthly_spend_fixture.csv +++ b/integration_test_project/seeds/monthly_spend_fixture.csv @@ -9,3 +9,4 @@ MONTH,SERVICE,SPEND 2022-04-01,Search Optimization,0 2022-04-01,Automatic Clustering,0.051113362 2022-04-01,Snowpipe,0 +2022-04-01,Serverless Tasks,0 diff --git a/models/daily_spend.sql b/models/daily_spend.sql index 4920c86..ef7a8ab 100644 --- a/models/daily_spend.sql +++ b/models/daily_spend.sql @@ -99,6 +99,33 @@ compute_spend_daily as ( group by 1, 2, 3, 4 ), +serverless_task_rates as ( + select + date, + max(effective_rate) as effective_rate + from {{ ref('daily_rates') }} + where + service_type = 'COMPUTE' + and usage_type in ('serverless tasks') + group by 1 +), + +serverless_task_spend_daily as ( + select + dates.date, + 'Serverless Tasks' as service, + null as storage_type, + null as warehouse_name, + stg_serverless_task_history.database_name, + coalesce(sum(stg_serverless_task_history.credits_used * serverless_task_rates.effective_rate), 0) as spend + from dates + left join {{ ref('stg_serverless_task_history') }} on + dates.date = convert_timezone('UTC', stg_serverless_task_history.start_time)::date + left join serverless_task_rates on + dates.date = serverless_task_rates.date + group by 1, 2, 3, 4, 5 +), + cloud_services_rates as ( select date, @@ -264,3 +291,5 @@ union all select * from replication_spend_daily union all select * from search_optimization_spend_daily +union all +select * from serverless_task_spend_daily diff --git a/models/daily_spend.yml b/models/daily_spend.yml index 4d53442..76b4d39 100644 --- a/models/daily_spend.yml +++ b/models/daily_spend.yml @@ -13,6 +13,6 @@ models: - name: warehouse_name description: Subcategories where service = "Compute" or "Cloud Services". - name: database_name - description: Subcategories where service = "Storage" and storage_type = "Table and Time Travel" or "Failsafe". + description: Subcategories where service = "Serverless Tasks" or service = "Storage" and storage_type = "Table and Time Travel" or "Failsafe". - name: spend description: Spend in the account's currency. diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 3973b72..7bc483d 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -11,6 +11,7 @@ sources: - name: warehouse_metering_history - name: metering_history - name: metering_daily_history + - name: serverless_task_history - name: snowflake_organization_usage database: snowflake diff --git a/models/staging/stg_serverless_task_history.sql b/models/staging/stg_serverless_task_history.sql new file mode 100644 index 0000000..9de688e --- /dev/null +++ b/models/staging/stg_serverless_task_history.sql @@ -0,0 +1,16 @@ +{{ config(materialized='incremental') }} + +select + start_time, + end_time, + task_id, + task_name, + database_name, + credits_used +from {{ source('snowflake_account_usage', 'serverless_task_history') }} + +{% if is_incremental() %} + where end_time > (select max(end_time) from {{ this }}) +{% endif %} + +order by start_time diff --git a/models/staging/stg_serverless_task_history.yml b/models/staging/stg_serverless_task_history.yml new file mode 100644 index 0000000..aa3eedc --- /dev/null +++ b/models/staging/stg_serverless_task_history.yml @@ -0,0 +1,18 @@ +version: 2 + +models: + - name: stg_serverless_task_history + description: An incrementally materialized copy of the snowflake.account_usage.warehouse_metering_history view. See https://docs.snowflake.com/en/sql-reference/account-usage/serverless_task_history.html + columns: + - name: start_time + description: The date and beginning of the hour (in the UTC time zone) in which the serverless task took place. + - name: end_time + description: The date and end of the hour (in the UTC time zone) in which the serverless task took place. + - name: task_id + description: Internal/system-generated identifier for the task. + - name: task_name + description: Name of the task. + - name: database_name + description: Name of the database in which the task is located. + - name: credits_used + description: Total number of credits used for the task.