diff --git a/.gitignore b/.gitignore index 96e0d5c..7fcd3d0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ logs/ profiles.yml target/ *.log +dbt_internal_packages/ # IDE files .idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d56f76e..dc20e43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# dbt_recurly v1.3.1 + +[PR #43](https://github.com/fivetran/dbt_recurly/pull/43) includes the following updates: + +## Schema/Data Change +**2 total changes • 0 possible breaking changes** + +| Data Model(s) | Change type | Old | New | Notes | +| ---------- | ----------- | -------- | -------- | ----- | +| `recurly__monthly_recurring_revenue` | Logic update | `mrr_type` calculated by comparing unrounded MRR values | `mrr_type` calculated by rounding monetary values to two decimal points | Adds rounding with numeric casting to ensure proper classification of `mrr_type` | +| `recurly__line_item_enhanced` | Logic update | `line_item_index` window function ordered by `created_at` only | `line_item_index` window function now orders by `created_at`, then `line_item_id` | Ensures deterministic ordering when multiple line items have the same `created_at` timestamp | + +## Under the Hood +- Adds consistency tests for `recurly__account_daily_overview` and `recurly__monthly_recurring_revenue` to compare field amounts with rounding discrepancies. Updates existing consistency tests to remove above amount fields. + +## Documentation Update +- Fixes link to `streamlit_fivetran_billing_model` [repo](https://github.com/fivetran/streamlit_fivetran_billing_model) in the README. + # dbt_recurly v1.3.0 [PR #42](https://github.com/fivetran/dbt_recurly/pull/42) includes the following updates: diff --git a/README.md b/README.md index c7965d7..fdd0e2d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ By default, this package materializes the following final tables: --- ## Visualizations -Many of the above reports are now configurable for [visualization via Streamlit](https://github.com/fivetran/streamlit_recurly). Check out some [sample reports here](https://fivetran-billing-model.streamlit.app/). +Many of the above reports are now configurable for [visualization via Streamlit](https://github.com/fivetran/streamlit_fivetran_billing_model). Check out some [sample reports here](https://fivetran-billing-model.streamlit.app/).

diff --git a/dbt_project.yml b/dbt_project.yml index dcb1ec0..60035e3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'recurly' -version: '1.3.0' +version: '1.3.1' config-version: 2 require-dbt-version: [">=1.3.0", "<3.0.0"] diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index b07ff58..4c625dc 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -12,13 +12,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: recurly_integrations_tests_1 + schema: recurly_integration_tests threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: recurly_integrations_tests_1 + schema: recurly_integration_tests threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -29,7 +29,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: recurly_integrations_tests_1 + schema: recurly_integration_tests threads: 8 postgres: type: postgres @@ -38,13 +38,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: recurly_integrations_tests_1 + schema: recurly_integration_tests threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: recurly_integrations_tests_1 + schema: recurly_integration_tests threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index eeec3ce..8773988 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,10 +1,10 @@ name: 'recurly_integration_tests' -version: '1.3.0' +version: '1.3.1' profile: 'integration_tests' config-version: 2 vars: - recurly_schema: recurly_integrations_tests_1 + recurly_schema: recurly_integration_tests recurly: account_balance_history_identifier: "account_balance_history_data" account_history_identifier: "account_history_data" diff --git a/integration_tests/tests/consistency/consistency_account_daily_amounts_match.sql b/integration_tests/tests/consistency/consistency_account_daily_amounts_match.sql new file mode 100644 index 0000000..a80cc93 --- /dev/null +++ b/integration_tests/tests/consistency/consistency_account_daily_amounts_match.sql @@ -0,0 +1,88 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- this test ensures the recurly__monthly_recurring_revenue end model matches the prior version + +with prod as ( + + select + account_daily_id, + source_relation, + daily_charges, + daily_credits, + daily_discounts, + daily_net_change, + daily_taxes, + rolling_account_balance, + rolling_charge_balance, + rolling_credit_balance, + rolling_discount_balance, + rolling_tax_balance + from {{ target.schema }}_recurly_prod.recurly__account_daily_overview +), + +dev as ( + + select + account_daily_id, + source_relation, + daily_charges, + daily_credits, + daily_discounts, + daily_net_change, + daily_taxes, + rolling_account_balance, + rolling_charge_balance, + rolling_credit_balance, + rolling_discount_balance, + rolling_tax_balance + from {{ target.schema }}_recurly_dev.recurly__account_daily_overview +), + + +final as ( + + select + prod.account_daily_id, + prod.source_relation, + prod.daily_charges as prod_daily_charges, + dev.daily_charges as dev_daily_charges, + prod.daily_credits as prod_daily_credits, + dev.daily_credits as dev_daily_credits, + prod.daily_discounts as prod_daily_discounts, + dev.daily_discounts as dev_daily_discounts, + prod.daily_net_change as prod_daily_change, + dev.daily_net_change as dev_daily_change, + prod.daily_taxes as prod_daily_taxes, + dev.daily_taxes as dev_daily_taxes, + prod.rolling_account_balance as prod_account_balance, + dev.rolling_account_balance as dev_account_balance, + prod.rolling_charge_balance as prod_charge_balance, + dev.rolling_charge_balance as dev_charge_balance, + prod.rolling_credit_balance as prod_credit_balance, + dev.rolling_credit_balance as dev_credit_balance, + prod.rolling_discount_balance as prod_discount_balance, + dev.rolling_discount_balance as dev_discount_balance, + prod.rolling_tax_balance as prod_tax_balance, + dev.rolling_tax_balance as dev_tax_balance + + from prod + full outer join dev + on dev.account_daily_id = prod.account_daily_id + and dev.source_relation = prod.source_relation +) + +select * +from final +where abs(prod_account_balance - dev_account_balance) >= 0.01 +or abs(prod_charge_balance - dev_charge_balance) >= 0.01 +or abs(prod_credit_balance - dev_credit_balance) >= 0.01 +or abs(prod_discount_balance - dev_discount_balance) >= 0.01 +or abs(prod_tax_balance - dev_tax_balance) >= 0.01 +or abs(prod_daily_charges - dev_daily_charges) >= 0.01 +or abs(prod_daily_credits - dev_daily_credits) >= 0.01 +or abs(prod_daily_discounts - dev_daily_discounts) >= 0.01 +or abs(prod_daily_taxes - dev_daily_taxes) >= 0.01 +or abs(prod_daily_change - dev_daily_change) >= 0.01 \ No newline at end of file diff --git a/integration_tests/tests/consistency/consistency_account_daily_overview.sql b/integration_tests/tests/consistency/consistency_account_daily_overview.sql index 6285c72..b90dd0f 100644 --- a/integration_tests/tests/consistency/consistency_account_daily_overview.sql +++ b/integration_tests/tests/consistency/consistency_account_daily_overview.sql @@ -3,7 +3,7 @@ enabled=var('fivetran_validation_tests_enabled', false) ) }} -{% set exclude_cols = var('consistency_test_exclude_metrics', []) %} +{% set exclude_cols = ['daily_charges', 'daily_credits', 'daily_discounts', 'daily_net_change', 'daily_taxes', 'rolling_account_balance', 'rolling_charge_balance', 'rolling_credit_balance', 'rolling_discount_balance', 'rolling_tax_balance'] + var('consistency_test_exclude_metrics', []) %} -- this test ensures the recurly__account_daily_overview end model matches the prior version with prod as ( diff --git a/integration_tests/tests/consistency/consistency_monthly_recurring_revenue.sql b/integration_tests/tests/consistency/consistency_monthly_recurring_revenue.sql index 89746a0..bae5fc7 100644 --- a/integration_tests/tests/consistency/consistency_monthly_recurring_revenue.sql +++ b/integration_tests/tests/consistency/consistency_monthly_recurring_revenue.sql @@ -3,7 +3,7 @@ enabled=var('fivetran_validation_tests_enabled', false) ) }} -{% set exclude_cols = var('consistency_test_exclude_metrics', []) %} +{% set exclude_cols = ['current_month_mrr', 'previous_month_mrr'] + var('consistency_test_exclude_metrics', []) %} -- this test ensures the recurly__monthly_recurring_revenue end model matches the prior version with prod as ( @@ -45,4 +45,4 @@ final as ( ) select * -from final +from final \ No newline at end of file diff --git a/integration_tests/tests/consistency/consistency_mrr_amounts_match.sql b/integration_tests/tests/consistency/consistency_mrr_amounts_match.sql new file mode 100644 index 0000000..b877880 --- /dev/null +++ b/integration_tests/tests/consistency/consistency_mrr_amounts_match.sql @@ -0,0 +1,47 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- this test ensures the recurly__monthly_recurring_revenue end model matches the prior version + +with prod as ( + + select + account_monthly_id, + source_relation, + current_month_mrr, + previous_month_mrr + from {{ target.schema }}_recurly_prod.recurly__monthly_recurring_revenue +), + +dev as ( + + select + account_monthly_id, + source_relation, + current_month_mrr, + previous_month_mrr + from {{ target.schema }}_recurly_dev.recurly__monthly_recurring_revenue +), + + +final as ( + + select + prod.account_monthly_id, + prod.source_relation, + prod.current_month_mrr as prod_current_month_mrr, + dev.current_month_mrr as dev_current_month_mrr, + prod.previous_month_mrr as prod_previous_month_mrr, + dev.previous_month_mrr as dev_previous_month_mrr + from prod + full outer join dev + on dev.account_monthly_id = prod.account_monthly_id + and dev.source_relation = prod.source_relation +) + +select * +from final +where abs(prod_current_month_mrr - dev_current_month_mrr) >= 0.01 +or abs(prod_previous_month_mrr - dev_previous_month_mrr) >= 0.01 \ No newline at end of file diff --git a/models/recurly__monthly_recurring_revenue.sql b/models/recurly__monthly_recurring_revenue.sql index 643c214..d46cfa3 100644 --- a/models/recurly__monthly_recurring_revenue.sql +++ b/models/recurly__monthly_recurring_revenue.sql @@ -53,17 +53,17 @@ current_vs_previous_mrr as ( mrr_type_enhanced as ( - select + select *, - case when current_month_mrr > previous_month_mrr then 'expansion' - when current_month_mrr < previous_month_mrr then 'contraction' - when current_month_mrr = previous_month_mrr then 'unchanged' + case when round(cast(current_month_mrr as {{ dbt.type_numeric() }}), 2) > round(cast(previous_month_mrr as {{ dbt.type_numeric() }}), 2) then 'expansion' + when round(cast(current_month_mrr as {{ dbt.type_numeric() }}), 2) < round(cast(previous_month_mrr as {{ dbt.type_numeric() }}), 2) then 'contraction' + when round(cast(current_month_mrr as {{ dbt.type_numeric() }}), 2) = round(cast(previous_month_mrr as {{ dbt.type_numeric() }}), 2) then 'unchanged' when previous_month_mrr is null then 'new' when (current_month_mrr = 0.0 or current_month_mrr is null) and (previous_month_mrr != 0.0) then 'churned' - when (previous_month_mrr = 0.0 and current_month_mrr > 0.0 - and account_month_number >= 3) + when (previous_month_mrr = 0.0 and current_month_mrr > 0.0 + and account_month_number >= 3) then 'reactivation' end as mrr_type from current_vs_previous_mrr diff --git a/models/staging/stg_recurly__line_item_history.sql b/models/staging/stg_recurly__line_item_history.sql index e553aef..b398f1e 100644 --- a/models/staging/stg_recurly__line_item_history.sql +++ b/models/staging/stg_recurly__line_item_history.sql @@ -26,7 +26,7 @@ final as ( account_id, add_on_code, add_on_id, - cast(amount as {{ dbt.type_float() }}) as amount, + cast(amount as {{ dbt.type_numeric() }}) as amount, cast(created_at as {{ dbt.type_timestamp() }}) as created_at, credit_applied, currency, diff --git a/models/standardized_models/recurly__line_item_enhanced.sql b/models/standardized_models/recurly__line_item_enhanced.sql index 86bec7e..45fe495 100644 --- a/models/standardized_models/recurly__line_item_enhanced.sql +++ b/models/standardized_models/recurly__line_item_enhanced.sql @@ -58,7 +58,7 @@ enhanced as ( line_items.source_relation, line_items.invoice_id as header_id, line_items.line_item_id, - cast(row_number() over (partition by line_items.invoice_id {{ recurly.partition_by_source_relation(alias='line_items') }} order by line_items.created_at) as {{ dbt.type_int() }}) as line_item_index, + cast(row_number() over (partition by line_items.invoice_id {{ recurly.partition_by_source_relation(alias='line_items') }} order by line_items.created_at, line_items.line_item_id) as {{ dbt.type_int() }}) as line_item_index, line_items.created_at, line_items.currency, line_items.state as line_item_status,