From c171d02dc8f280490629b19913fab1d4e8512fcd Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 2 Jul 2025 14:26:47 -0500 Subject: [PATCH 1/2] test/json-bigquery-compatibility --- .buildkite/scripts/run_models.sh | 4 +++ integration_tests/dbt_project.yml | 31 +++++++++++++++++++ .../seeds/shopify_collection_bq_json_data.csv | 4 +++ .../seeds/shopify_order_bq_json_data.csv | 9 ++++++ .../shopify_transaction_bq_json_data.csv | 6 ++++ packages.yml | 7 +++-- 6 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 integration_tests/seeds/shopify_collection_bq_json_data.csv create mode 100644 integration_tests/seeds/shopify_order_bq_json_data.csv create mode 100644 integration_tests/seeds/shopify_transaction_bq_json_data.csv diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 15842d05..6d0588db 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -31,5 +31,9 @@ dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false, shopify_using_metafield: false, shopify_using_discount_code_app: true, shopify_using_product_variant_media: true}' --target "$db" --full-refresh dbt test --target "$db" +if [ "$db" = "bigquery" ]; then +dbt run --vars '{shopify_collection_identifier: shopify_collection_data_bq_json, shopify_order_identifier: shopify_order_bq_json_data, shopify_transaction_identifier: shopify_transaction_bq_json_data, shopify_using_all_metafields: true}' --target "$db" --full-refresh +dbt test --target "$db" +fi dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" fi \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index fc24665b..68b67ed1 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -92,6 +92,21 @@ seeds: customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + total_shipping_price_set: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + shopify_order_bq_json_data: + +enabled: "{{ target.type == 'bigquery' }}" + +column_types: + created_at: timestamp + updated_at: timestamp + processed_at: timestamp + cancelled_at: timestamp + _fivetran_synced: timestamp + total_tax: float + id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + total_shipping_price_set: json shopify_order_line_data: +column_types: _fivetran_synced: timestamp @@ -133,6 +148,14 @@ seeds: id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + receipt: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + shopify_transaction_bq_json_data: + +enabled: "{{ target.type == 'bigquery' }}" + +column_types: + id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" + receipt: json shopify_refund_data: +column_types: id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}" @@ -176,6 +199,14 @@ seeds: disjunctive: boolean published_at: timestamp updated_at: timestamp + rules: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + shopify_collection_bq_json_data: + +enabled: "{{ target.type == 'bigquery' }}" + +column_types: + disjunctive: boolean + published_at: timestamp + updated_at: timestamp + rules: json shopify_metafield_data: +column_types: value_type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" diff --git a/integration_tests/seeds/shopify_collection_bq_json_data.csv b/integration_tests/seeds/shopify_collection_bq_json_data.csv new file mode 100644 index 00000000..04c9cabd --- /dev/null +++ b/integration_tests/seeds/shopify_collection_bq_json_data.csv @@ -0,0 +1,4 @@ +id,_fivetran_deleted,_fivetran_synced,handle,published_at,published_scope,title,updated_at,disjunctive,rules,sort_order,template_suffix,body_html +997355,true,2021-09-01 05:53:25.838000,,,,title_1,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""TEST""}]",,, +9930779,true,2021-09-01 05:53:26.673000,,,,title_2,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""FOO""}]",,, +99967,true,2022-04-08 06:52:19.524000,,,,title_3,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""BAR""}]",,, \ No newline at end of file diff --git a/integration_tests/seeds/shopify_order_bq_json_data.csv b/integration_tests/seeds/shopify_order_bq_json_data.csv new file mode 100644 index 00000000..2a4c1a01 --- /dev/null +++ b/integration_tests/seeds/shopify_order_bq_json_data.csv @@ -0,0 +1,9 @@ +id,note,email,taxes_included,currency,subtotal_price,total_tax,total_price,created_at,updated_at,name,shipping_address_name,shipping_address_first_name,shipping_address_last_name,shipping_address_company,shipping_address_phone,shipping_address_address_1,shipping_address_address_2,shipping_address_city,shipping_address_country,shipping_address_country_code,shipping_address_province,shipping_address_province_code,shipping_address_zip,shipping_address_latitude,shipping_address_longitude,billing_address_name,billing_address_first_name,billing_address_last_name,billing_address_company,billing_address_phone,billing_address_address_1,billing_address_address_2,billing_address_city,billing_address_country,billing_address_country_code,billing_address_province,billing_address_province_code,billing_address_zip,billing_address_latitude,billing_address_longitude,customer_id,location_id,user_id,number,order_number,financial_status,fulfillment_status,processed_at,processing_method,referring_site,cancel_reason,cancelled_at,closed_at,total_discounts,total_line_items_price,total_weight,source_name,browser_ip,buyer_accepts_marketing,token,cart_token,checkout_token,test,landing_site_base_url,_fivetran_synced,total_shipping_price_set +2674098602081,71509c29301d2cc14e37ecb53f735608,021cb20b5c78751fc7ddc091b6b69b3e,TRUE,GBP,2.8,0.00,2.8,2020-09-11 19:35:42.000,2020-09-11 19:35:46.000,d1743fc58a1e4d78769eaac49994a994,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,3589760876641,,,4135,5135,paid,,2020-09-11 19:35:42.000,,,,,,2.8,5.6,0,app,,TRUE,0f9c2880de17f71511eee5542c29b999,,,FALSE,,2020-09-12 00:15:10.199 ,"{""shop_money"":{""amount"":""3.00"",""currency_code"":""USD""}}" +2669516488801,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,2.8,0.00,3.79,2020-09-09 23:01:54.000,2020-09-10 15:38:26.000,4fcb884b5b46413bae526a6e7e49d706,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4066,5066,paid,fulfilled,2020-09-09 23:01:53.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:26.000,0,2.8,0,web,109.249.185.68,FALSE,fb489b3ccc0ae36ce47744d7595e9746,b1ff04883dfeab658cd5211050476729,7bdb994e1196de3e4f34586e357613f9,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.536 ,"{""shop_money"":{""amount"":""4.00"",""currency_code"":""USD""}}" +2669509541985,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,4.4,0.00,5.39,2020-09-09 22:57:51.000,2020-09-10 15:38:25.000,9e346f2e912c60e16679f4a4c8d29422,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4065,5065,paid,fulfilled,2020-09-09 22:57:50.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:25.000,0,4.4,0,web,109.249.185.68,FALSE,e44b7f04610a8f4032530cc7f12663de,9600543f4d4613db59ac58a1009ecbb9,cf0a9fe2c7c606b86559007dbb890a62,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.037 ,"{""shop_money"":{""amount"":""5.00"",""currency_code"":""USD""}}" +1000001,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,100.00,0.01,100.01,2025-05-01 08:00:00.000,2025-05-01 08:10:00.000,9e346f2e912c60e16679f4a4c8d29423,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10001,20001,30001,1001,1001,paid,,2025-05-01 08:00:00.000,direct,,,,,15.00,100.00,500,web,192.168.1.1,FALSE,t0ken1001,cart1001,checkout1001,FALSE,,2025-05-01 08:15:00.000, +1000002,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,80.00,0.02,100.02,2025-05-02 09:00:00.000,2025-05-02 09:10:00.000,9e346f2e912c60e16679f4a4c8d29424,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10002,20002,30002,1002,1002,paid,,2025-05-02 09:00:00.000,direct,,,,,20.00,80.00,450,web,192.168.1.2,FALSE,t0ken1002,cart1002,checkout1002,FALSE,,2025-05-02 09:15:00.000,"{""shop_money"":{""amount"":""6.00"",""currency_code"":""USD""}}" +1000003,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,50.00,0.00,50.00,2025-05-03 10:15:00.000,2025-05-03 10:20:00.000,9e346f2e912c60e16679f4a4c8d29425,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10003,20003,30003,1003,1003,paid,,2025-05-03 10:15:00.000,direct,,,,,0.00,50.00,250,web,192.168.1.3,FALSE,t0ken1003,cart1003,checkout1003,FALSE,,2025-05-03 10:30:00.000, +1000004,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,200.00,0.00,205.00,2025-05-04 11:30:00.000,2025-05-04 11:35:00.000,9e346f2e912c60e16679f4a4c8d29426,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10004,20004,30004,1004,1004,paid,,2025-05-04 11:30:00.000,direct,,,,,5.00,200.00,750,web,192.168.1.4,FALSE,t0ken1004,cart1004,checkout1004,FALSE,,2025-05-04 11:45:00.000,"{""shop_money"":{""amount"":""7.00"",""currency_code"":""USD""}}" +1000005,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,150.00,0.00,160.00,2025-05-05 14:00:00.000,2025-05-05 14:10:00.000,9e346f2e912c60e16679f4a4c8d29427,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10005,20005,30005,1005,1005,paid,,2025-05-05 14:00:00.000,direct,,,,,10.00,150.00,600,web,192.168.1.5,FALSE,t0ken1005,cart1005,checkout1005,FALSE,,2025-05-05 14:20:00.000,"{""shop_money"":{""amount"":""8.00"",""currency_code"":""USD""}}" \ No newline at end of file diff --git a/integration_tests/seeds/shopify_transaction_bq_json_data.csv b/integration_tests/seeds/shopify_transaction_bq_json_data.csv new file mode 100644 index 00000000..9647a9a3 --- /dev/null +++ b/integration_tests/seeds/shopify_transaction_bq_json_data.csv @@ -0,0 +1,6 @@ +id,order_id,refund_id,amount,authorization,created_at,processed_at,device_id,gateway,source_name,message,currency,location_id,parent_id,payment_avs_result_code,kind,currency_exchange_id,currency_exchange_adjustment,currency_exchange_original_amount,currency_exchange_final_amount,currency_exchange_currency,error_code,status,test,user_id,_fivetran_synced,payment_credit_card_bin,payment_cvv_result_code,payment_credit_card_number,payment_credit_card_company,receipt +2667417567303,2181743870023,,415,abcd999999,2020-02-27 16:05:37.000,2020-02-27 16:05:37.000,,gateway_here,source_name,message_here,USD,,,Z,sale,,,,,,,success,FALSE,,2020-10-28 20:33:09.797,,,,,"{ ""charges"": { ""data"": [ { ""balance_transaction"": { ""exchange_rate"": null } }] }}" +2572210896967,2089104834631,,415,abcd888888,2020-01-12 20:06:37.000,2020-01-12 20:06:37.000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 17:05:27.756,,,,, +2664325611591,2179107356743,,415,abcd77777,2020-02-26 00:12:37.000,2020-02-26 00:12:37.000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 20:23:50.344,,,,,"{ ""charges"": { ""data"": [ { ""balance_transaction"": { ""exchange_rate"": ""0.523"" } }] }}" +2595729735751,2114590769223,,15.95,abcd66666,2020-01-26 11:04:41.000,2020-01-26 11:04:41.000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 18:10:27.604,,,,, +2705030512711,2214516916295,,212.12,abcd5555,2020-03-18 00:17:24.000,2020-03-18 00:17:24.000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 22:14:02.944,,,,,"{ ""charges"": { ""data"": [ { ""balance_transaction"": { ""exchange_rate"": ""0.96581"" } }] }}" \ No newline at end of file diff --git a/packages.yml b/packages.yml index a404eb4a..c5b78802 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,6 @@ packages: - - package: fivetran/shopify_source - version: [">=0.18.0", "<0.19.0"] +# - package: fivetran/shopify_source +# version: [">=0.18.0", "<0.19.0"] + - git: https://github.com/fivetran/dbt_shopify_source.git + revision: patch/json-bigquery-compatibility + warn-unpinned: false \ No newline at end of file From 7143aba3c4298eeebfb873657c4e1d2303141de0 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:29:40 -0500 Subject: [PATCH 2/2] Update run_models.sh --- .buildkite/scripts/run_models.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 6d0588db..8d38acd4 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -32,8 +32,8 @@ dbt test --target "$db" dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false, shopify_using_metafield: false, shopify_using_discount_code_app: true, shopify_using_product_variant_media: true}' --target "$db" --full-refresh dbt test --target "$db" if [ "$db" = "bigquery" ]; then -dbt run --vars '{shopify_collection_identifier: shopify_collection_data_bq_json, shopify_order_identifier: shopify_order_bq_json_data, shopify_transaction_identifier: shopify_transaction_bq_json_data, shopify_using_all_metafields: true}' --target "$db" --full-refresh +dbt run --vars '{shopify_collection_identifier: shopify_collection_bq_json_data, shopify_order_identifier: shopify_order_bq_json_data, shopify_transaction_identifier: shopify_transaction_bq_json_data, shopify_using_all_metafields: true}' --target "$db" --full-refresh dbt test --target "$db" fi dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" -fi \ No newline at end of file +fi