[Bug fix] MRR type updates, deterministic line item indices, validation tests and streamlit link fixes

.gitignore

@@ -8,6 +8,7 @@ logs/
 profiles.yml
 target/
 *.log
+dbt_internal_packages/
 
 # IDE files
 .idea/

CHANGELOG.md

@@ -1,3 +1,21 @@
+# dbt_recurly v1.3.1
+
+[PR #43](https://github.com/fivetran/dbt_recurly/pull/43) includes the following updates:
+
+## Schema/Data Change
+**2 total changes • 0 possible breaking changes**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ---------- | ----------- | -------- | -------- | ----- |
+| `recurly__monthly_recurring_revenue` | Logic update | `mrr_type` calculated by comparing unrounded MRR values | `mrr_type` calculated by rounding monetary values to two decimal points | Adds rounding with numeric casting to ensure proper classification of `mrr_type` |
+| `recurly__line_item_enhanced` | Logic update | `line_item_index` window function ordered by `created_at` only | `line_item_index` window function now orders by `created_at`, then `line_item_id` | Ensures deterministic ordering when multiple line items have the same `created_at` timestamp |
+
+## Under the Hood
+- Adds consistency tests for `recurly__account_daily_overview` and `recurly__monthly_recurring_revenue` to compare field amounts with rounding discrepancies. Updates existing consistency tests to remove above amount fields. 
+
+## Documentation Update
+- Fixes link to `streamlit_fivetran_billing_model` [repo](https://github.com/fivetran/streamlit_fivetran_billing_model) in the README. 
+
 # dbt_recurly v1.3.0
 
 [PR #42](https://github.com/fivetran/dbt_recurly/pull/42) includes the following updates:

README.md

@@ -59,7 +59,7 @@ By default, this package materializes the following final tables:
 ---
 
 ## Visualizations
-Many of the above reports are now configurable for [visualization via Streamlit](https://github.com/fivetran/streamlit_recurly). Check out some [sample reports here](https://fivetran-billing-model.streamlit.app/).
+Many of the above reports are now configurable for [visualization via Streamlit](https://github.com/fivetran/streamlit_fivetran_billing_model). Check out some [sample reports here](https://fivetran-billing-model.streamlit.app/).
 
 <p align="center">
 <a href="https://fivetran-billing-model.streamlit.app/">

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'recurly'
-version: '1.3.0'
+version: '1.3.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 

integration_tests/ci/sample.profiles.yml

@@ -12,13 +12,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: recurly_integrations_tests_1
+      schema: recurly_integration_tests
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: recurly_integrations_tests_1
+      schema: recurly_integration_tests
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -29,7 +29,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: recurly_integrations_tests_1
+      schema: recurly_integration_tests
       threads: 8
     postgres:
       type: postgres
@@ -38,13 +38,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: recurly_integrations_tests_1
+      schema: recurly_integration_tests
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: recurly_integrations_tests_1
+      schema: recurly_integration_tests
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'recurly_integration_tests'
-version: '1.3.0'
+version: '1.3.1'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  recurly_schema: recurly_integrations_tests_1
+  recurly_schema: recurly_integration_tests
   recurly:
     account_balance_history_identifier: "account_balance_history_data"
     account_history_identifier: "account_history_data"

integration_tests/tests/consistency/consistency_account_daily_amounts_match.sql

@@ -0,0 +1,88 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the recurly__monthly_recurring_revenue end model matches the prior version
+
+with prod as (
+
+    select 
+        account_daily_id,
+        source_relation,
+        daily_charges,
+        daily_credits,
+        daily_discounts, 
+        daily_net_change, 
+        daily_taxes,
+        rolling_account_balance,
+        rolling_charge_balance,
+        rolling_credit_balance,
+        rolling_discount_balance,
+        rolling_tax_balance
+    from {{ target.schema }}_recurly_prod.recurly__account_daily_overview
+),
+
+dev as (
+
+    select
+        account_daily_id,
+        source_relation,
+        daily_charges,
+        daily_credits,
+        daily_discounts, 
+        daily_net_change, 
+        daily_taxes,
+        rolling_account_balance,
+        rolling_charge_balance,
+        rolling_credit_balance,
+        rolling_discount_balance,
+        rolling_tax_balance
+    from {{ target.schema }}_recurly_dev.recurly__account_daily_overview
+),
+
+
+final as (
+
+    select
+        prod.account_daily_id,
+        prod.source_relation,
+        prod.daily_charges as prod_daily_charges,
+        dev.daily_charges as dev_daily_charges,
+        prod.daily_credits as prod_daily_credits,
+        dev.daily_credits as dev_daily_credits,
+        prod.daily_discounts as prod_daily_discounts, 
+        dev.daily_discounts as dev_daily_discounts, 
+        prod.daily_net_change as prod_daily_change, 
+        dev.daily_net_change as dev_daily_change, 
+        prod.daily_taxes as prod_daily_taxes,
+        dev.daily_taxes as dev_daily_taxes,
+        prod.rolling_account_balance as prod_account_balance,
+        dev.rolling_account_balance as dev_account_balance,
+        prod.rolling_charge_balance as prod_charge_balance,
+        dev.rolling_charge_balance as dev_charge_balance,
+        prod.rolling_credit_balance as prod_credit_balance,
+        dev.rolling_credit_balance as dev_credit_balance,
+        prod.rolling_discount_balance as prod_discount_balance,
+        dev.rolling_discount_balance as dev_discount_balance, 
+        prod.rolling_tax_balance as prod_tax_balance,
+        dev.rolling_tax_balance as dev_tax_balance
+
+    from prod   
+    full outer join dev
+        on dev.account_daily_id = prod.account_daily_id 
+        and dev.source_relation = prod.source_relation
+)
+
+select * 
+from final
+where abs(prod_account_balance - dev_account_balance) >= 0.01
+or abs(prod_charge_balance - dev_charge_balance) >= 0.01
+or abs(prod_credit_balance - dev_credit_balance) >= 0.01
+or abs(prod_discount_balance - dev_discount_balance) >= 0.01
+or abs(prod_tax_balance - dev_tax_balance) >= 0.01
+or abs(prod_daily_charges - dev_daily_charges) >= 0.01
+or abs(prod_daily_credits - dev_daily_credits) >= 0.01
+or abs(prod_daily_discounts - dev_daily_discounts) >= 0.01
+or abs(prod_daily_taxes - dev_daily_taxes) >= 0.01
+or abs(prod_daily_change - dev_daily_change) >= 0.01

integration_tests/tests/consistency/consistency_account_daily_overview.sql

@@ -3,7 +3,7 @@
     enabled=var('fivetran_validation_tests_enabled', false)
 ) }}
 
-{% set exclude_cols = var('consistency_test_exclude_metrics', []) %}
+{% set exclude_cols = ['daily_charges', 'daily_credits', 'daily_discounts', 'daily_net_change', 'daily_taxes', 'rolling_account_balance', 'rolling_charge_balance', 'rolling_credit_balance', 'rolling_discount_balance', 'rolling_tax_balance'] + var('consistency_test_exclude_metrics', []) %}
 
 -- this test ensures the recurly__account_daily_overview end model matches the prior version
 with prod as (

integration_tests/tests/consistency/consistency_monthly_recurring_revenue.sql

@@ -3,7 +3,7 @@
     enabled=var('fivetran_validation_tests_enabled', false)
 ) }}
 
-{% set exclude_cols = var('consistency_test_exclude_metrics', []) %}
+{% set exclude_cols = ['current_month_mrr', 'previous_month_mrr'] + var('consistency_test_exclude_metrics', []) %}
 
 -- this test ensures the recurly__monthly_recurring_revenue end model matches the prior version
 with prod as (
@@ -45,4 +45,4 @@ final as (
 )
 
 select *
-from final
+from final

integration_tests/tests/consistency/consistency_mrr_amounts_match.sql

@@ -0,0 +1,47 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the recurly__monthly_recurring_revenue end model matches the prior version
+
+with prod as (
+
+    select 
+        account_monthly_id,
+        source_relation,
+        current_month_mrr,
+        previous_month_mrr
+    from {{ target.schema }}_recurly_prod.recurly__monthly_recurring_revenue
+),
+
+dev as (
+
+    select
+        account_monthly_id,
+        source_relation,
+        current_month_mrr,
+        previous_month_mrr 
+    from {{ target.schema }}_recurly_dev.recurly__monthly_recurring_revenue
+),
+
+
+final as (
+
+    select
+        prod.account_monthly_id, 
+        prod.source_relation, 
+        prod.current_month_mrr as prod_current_month_mrr,
+        dev.current_month_mrr as dev_current_month_mrr,
+        prod.previous_month_mrr as prod_previous_month_mrr,
+        dev.previous_month_mrr as dev_previous_month_mrr
+    from prod   
+    full outer join dev
+        on dev.account_monthly_id = prod.account_monthly_id 
+        and dev.source_relation = prod.source_relation
+)
+
+select * 
+from final
+where abs(prod_current_month_mrr - dev_current_month_mrr) >= 0.01
+or abs(prod_previous_month_mrr - dev_previous_month_mrr) >= 0.01

models/recurly__monthly_recurring_revenue.sql

@@ -53,17 +53,17 @@ current_vs_previous_mrr as (
 
 mrr_type_enhanced as (
 
-    select 
+    select
         *,
-        case when current_month_mrr > previous_month_mrr then 'expansion'
-            when current_month_mrr < previous_month_mrr then 'contraction'
-            when current_month_mrr = previous_month_mrr then 'unchanged'
+        case when round(cast(current_month_mrr as {{ dbt.type_numeric() }}), 2) > round(cast(previous_month_mrr as {{ dbt.type_numeric() }}), 2) then 'expansion'
+            when round(cast(current_month_mrr as {{ dbt.type_numeric() }}), 2) < round(cast(previous_month_mrr as {{ dbt.type_numeric() }}), 2) then 'contraction'
+            when round(cast(current_month_mrr as {{ dbt.type_numeric() }}), 2) = round(cast(previous_month_mrr as {{ dbt.type_numeric() }}), 2) then 'unchanged'
             when previous_month_mrr is null then 'new'
             when (current_month_mrr = 0.0 or current_month_mrr is null)
                 and (previous_month_mrr != 0.0)
                 then 'churned'
-            when (previous_month_mrr = 0.0 and current_month_mrr > 0.0 
-                and account_month_number >= 3) 
+            when (previous_month_mrr = 0.0 and current_month_mrr > 0.0
+                and account_month_number >= 3)
                 then 'reactivation'
             end as mrr_type
     from current_vs_previous_mrr

models/staging/stg_recurly__line_item_history.sql

@@ -26,7 +26,7 @@ final as (
         account_id,
         add_on_code,
         add_on_id,
-        cast(amount as {{ dbt.type_float() }}) as amount,
+        cast(amount as {{ dbt.type_numeric() }}) as amount,
         cast(created_at as {{ dbt.type_timestamp() }}) as created_at,
         credit_applied,
         currency,

models/standardized_models/recurly__line_item_enhanced.sql

@@ -58,7 +58,7 @@ enhanced as (
         line_items.source_relation,
         line_items.invoice_id as header_id,
         line_items.line_item_id,
-        cast(row_number() over (partition by line_items.invoice_id {{ recurly.partition_by_source_relation(alias='line_items') }} order by line_items.created_at) as {{ dbt.type_int() }}) as line_item_index,
+        cast(row_number() over (partition by line_items.invoice_id {{ recurly.partition_by_source_relation(alias='line_items') }} order by line_items.created_at, line_items.line_item_id) as {{ dbt.type_int() }}) as line_item_index,
         line_items.created_at,
         line_items.currency,
         line_items.state as line_item_status,