test/json-bigquery-compatibility

.buildkite/scripts/run_models.sh

@@ -31,5 +31,9 @@ dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false, shopify_using_metafield: false, shopify_using_discount_code_app: true, shopify_using_product_variant_media: true}' --target "$db" --full-refresh
 dbt test --target "$db"
+if [ "$db" = "bigquery" ]; then
+dbt run --vars '{shopify_collection_identifier: shopify_collection_bq_json_data, shopify_order_identifier: shopify_order_bq_json_data, shopify_transaction_identifier: shopify_transaction_bq_json_data, shopify_using_all_metafields: true}' --target "$db" --full-refresh
+dbt test --target "$db"
+fi
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
-fi
+fi

integration_tests/dbt_project.yml

@@ -92,6 +92,21 @@ seeds:
         customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        total_shipping_price_set: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_order_bq_json_data:
+      +enabled: "{{ target.type == 'bigquery' }}"
+      +column_types:
+        created_at: timestamp
+        updated_at: timestamp
+        processed_at: timestamp
+        cancelled_at: timestamp
+        _fivetran_synced: timestamp
+        total_tax: float
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        user_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        total_shipping_price_set: json
     shopify_order_line_data:
       +column_types:
         _fivetran_synced: timestamp
@@ -133,6 +148,14 @@ seeds:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        receipt: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_transaction_bq_json_data:
+      +enabled: "{{ target.type == 'bigquery' }}"
+      +column_types:
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        receipt: json
     shopify_refund_data:
       +column_types:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
@@ -176,6 +199,14 @@ seeds:
         disjunctive: boolean
         published_at: timestamp
         updated_at: timestamp
+        rules: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_collection_bq_json_data:
+      +enabled: "{{ target.type == 'bigquery' }}"
+      +column_types:
+        disjunctive: boolean
+        published_at: timestamp
+        updated_at: timestamp
+        rules: json
     shopify_metafield_data:
       +column_types:
         value_type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"

integration_tests/seeds/shopify_collection_bq_json_data.csv

@@ -0,0 +1,4 @@
+id,_fivetran_deleted,_fivetran_synced,handle,published_at,published_scope,title,updated_at,disjunctive,rules,sort_order,template_suffix,body_html
+997355,true,2021-09-01 05:53:25.838000,,,,title_1,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""TEST""}]",,,
+9930779,true,2021-09-01 05:53:26.673000,,,,title_2,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""FOO""}]",,,
+99967,true,2022-04-08 06:52:19.524000,,,,title_3,1970-01-01 00:00:00.000000,,"[{""column"":""name"",""relation"":""contains"",""condition"":""BAR""}]",,,

integration_tests/seeds/shopify_order_bq_json_data.csv

@@ -0,0 +1,9 @@
+id,note,email,taxes_included,currency,subtotal_price,total_tax,total_price,created_at,updated_at,name,shipping_address_name,shipping_address_first_name,shipping_address_last_name,shipping_address_company,shipping_address_phone,shipping_address_address_1,shipping_address_address_2,shipping_address_city,shipping_address_country,shipping_address_country_code,shipping_address_province,shipping_address_province_code,shipping_address_zip,shipping_address_latitude,shipping_address_longitude,billing_address_name,billing_address_first_name,billing_address_last_name,billing_address_company,billing_address_phone,billing_address_address_1,billing_address_address_2,billing_address_city,billing_address_country,billing_address_country_code,billing_address_province,billing_address_province_code,billing_address_zip,billing_address_latitude,billing_address_longitude,customer_id,location_id,user_id,number,order_number,financial_status,fulfillment_status,processed_at,processing_method,referring_site,cancel_reason,cancelled_at,closed_at,total_discounts,total_line_items_price,total_weight,source_name,browser_ip,buyer_accepts_marketing,token,cart_token,checkout_token,test,landing_site_base_url,_fivetran_synced,total_shipping_price_set
+2674098602081,71509c29301d2cc14e37ecb53f735608,021cb20b5c78751fc7ddc091b6b69b3e,TRUE,GBP,2.8,0.00,2.8,2020-09-11 19:35:42.000,2020-09-11 19:35:46.000,d1743fc58a1e4d78769eaac49994a994,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,8b121314a4d97bc9dc15bfba8518ec88,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,d6f4a399883df85d9d4b3a02bf6e738a,bc9b8576178dcd886639ba718f1d45c8,ac08c606d455cde42980f980524a8038,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,d41d8cd98f00b204e9800998ecf8427e,,00079ce435afddc28205639142773870,d97319f64674c02595f2989019970fc8,c08dae474c5d4d3326fd6764d2a0ebe6,3589760876641,,,4135,5135,paid,,2020-09-11 19:35:42.000,,,,,,2.8,5.6,0,app,,TRUE,0f9c2880de17f71511eee5542c29b999,,,FALSE,,2020-09-12 00:15:10.199 ,"{""shop_money"":{""amount"":""3.00"",""currency_code"":""USD""}}"
+2669516488801,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,2.8,0.00,3.79,2020-09-09 23:01:54.000,2020-09-10 15:38:26.000,4fcb884b5b46413bae526a6e7e49d706,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4066,5066,paid,fulfilled,2020-09-09 23:01:53.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:26.000,0,2.8,0,web,109.249.185.68,FALSE,fb489b3ccc0ae36ce47744d7595e9746,b1ff04883dfeab658cd5211050476729,7bdb994e1196de3e4f34586e357613f9,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.536 ,"{""shop_money"":{""amount"":""4.00"",""currency_code"":""USD""}}"
+2669509541985,,dce90c7b4e52e045e5975836aff49cf1,TRUE,GBP,4.4,0.00,5.39,2020-09-09 22:57:51.000,2020-09-10 15:38:25.000,9e346f2e912c60e16679f4a4c8d29422,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,c8189c7add9755e66391b58ecc12b3e2,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,d41d8cd98f00b204e9800998ecf8427e,d41d8cd98f00b204e9800998ecf8427e,1ff1de774005f8da13f42943881c655f,70111f8840ccbd8b1007cc3f387ced6b,1ac412baeba98370017c73df41c98a07,89f9c9f489be2a83cf57e53b9197d288,79cba1185463850dedba31f172f1dc5b,,,2357e65b582faa0a2da3603b16fa4a7f,75c29d6dd29594a652fcbd7c4c279a29,75468fbebc28e02ec5d4f54f4cbd4099,3584045351009,,,4065,5065,paid,fulfilled,2020-09-09 22:57:50.000,direct,2cc983716a820bc713b793a6e8e73f42,,,2020-09-10 15:38:25.000,0,4.4,0,web,109.249.185.68,FALSE,e44b7f04610a8f4032530cc7f12663de,9600543f4d4613db59ac58a1009ecbb9,cf0a9fe2c7c606b86559007dbb890a62,FALSE,8584e97b29b0802fb393fa453a8b6a7a,2020-09-11 00:14:33.037 ,"{""shop_money"":{""amount"":""5.00"",""currency_code"":""USD""}}"
+1000001,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,100.00,0.01,100.01,2025-05-01 08:00:00.000,2025-05-01 08:10:00.000,9e346f2e912c60e16679f4a4c8d29423,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10001,20001,30001,1001,1001,paid,,2025-05-01 08:00:00.000,direct,,,,,15.00,100.00,500,web,192.168.1.1,FALSE,t0ken1001,cart1001,checkout1001,FALSE,,2025-05-01 08:15:00.000,
+1000002,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,80.00,0.02,100.02,2025-05-02 09:00:00.000,2025-05-02 09:10:00.000,9e346f2e912c60e16679f4a4c8d29424,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10002,20002,30002,1002,1002,paid,,2025-05-02 09:00:00.000,direct,,,,,20.00,80.00,450,web,192.168.1.2,FALSE,t0ken1002,cart1002,checkout1002,FALSE,,2025-05-02 09:15:00.000,"{""shop_money"":{""amount"":""6.00"",""currency_code"":""USD""}}"
+1000003,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,50.00,0.00,50.00,2025-05-03 10:15:00.000,2025-05-03 10:20:00.000,9e346f2e912c60e16679f4a4c8d29425,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10003,20003,30003,1003,1003,paid,,2025-05-03 10:15:00.000,direct,,,,,0.00,50.00,250,web,192.168.1.3,FALSE,t0ken1003,cart1003,checkout1003,FALSE,,2025-05-03 10:30:00.000,
+1000004,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,200.00,0.00,205.00,2025-05-04 11:30:00.000,2025-05-04 11:35:00.000,9e346f2e912c60e16679f4a4c8d29426,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10004,20004,30004,1004,1004,paid,,2025-05-04 11:30:00.000,direct,,,,,5.00,200.00,750,web,192.168.1.4,FALSE,t0ken1004,cart1004,checkout1004,FALSE,,2025-05-04 11:45:00.000,"{""shop_money"":{""amount"":""7.00"",""currency_code"":""USD""}}"
+1000005,,dce90c7b4e52e045e5975836aff49cf1,TRUE,USD,150.00,0.00,160.00,2025-05-05 14:00:00.000,2025-05-05 14:10:00.000,9e346f2e912c60e16679f4a4c8d29427,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10005,20005,30005,1005,1005,paid,,2025-05-05 14:00:00.000,direct,,,,,10.00,150.00,600,web,192.168.1.5,FALSE,t0ken1005,cart1005,checkout1005,FALSE,,2025-05-05 14:20:00.000,"{""shop_money"":{""amount"":""8.00"",""currency_code"":""USD""}}"

integration_tests/seeds/shopify_transaction_bq_json_data.csv

@@ -0,0 +1,6 @@
+id,order_id,refund_id,amount,authorization,created_at,processed_at,device_id,gateway,source_name,message,currency,location_id,parent_id,payment_avs_result_code,kind,currency_exchange_id,currency_exchange_adjustment,currency_exchange_original_amount,currency_exchange_final_amount,currency_exchange_currency,error_code,status,test,user_id,_fivetran_synced,payment_credit_card_bin,payment_cvv_result_code,payment_credit_card_number,payment_credit_card_company,receipt
+2667417567303,2181743870023,,415,abcd999999,2020-02-27 16:05:37.000,2020-02-27 16:05:37.000,,gateway_here,source_name,message_here,USD,,,Z,sale,,,,,,,success,FALSE,,2020-10-28 20:33:09.797,,,,,"{ ""charges"": { ""data"": [ { ""balance_transaction"": { ""exchange_rate"": null }  }] }}"
+2572210896967,2089104834631,,415,abcd888888,2020-01-12 20:06:37.000,2020-01-12 20:06:37.000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 17:05:27.756,,,,,
+2664325611591,2179107356743,,415,abcd77777,2020-02-26 00:12:37.000,2020-02-26 00:12:37.000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 20:23:50.344,,,,,"{ ""charges"": { ""data"": [ { ""balance_transaction"": { ""exchange_rate"": ""0.523"" }  }] }}"
+2595729735751,2114590769223,,15.95,abcd66666,2020-01-26 11:04:41.000,2020-01-26 11:04:41.000,,gateway_here,source_name,message_here,USD,,,Y,sale,,,,,,,success,FALSE,,2020-10-28 18:10:27.604,,,,,
+2705030512711,2214516916295,,212.12,abcd5555,2020-03-18 00:17:24.000,2020-03-18 00:17:24.000,,gateway_here,source_name,message_here,USD,,,,sale,,,,,,,success,FALSE,,2020-10-28 22:14:02.944,,,,,"{ ""charges"": { ""data"": [ { ""balance_transaction"": { ""exchange_rate"": ""0.96581"" }  }] }}"

packages.yml

@@ -1,3 +1,6 @@
 packages:
- - package: fivetran/shopify_source
-   version: [">=0.18.0", "<0.19.0"]
+#  - package: fivetran/shopify_source
+#    version: [">=0.18.0", "<0.19.0"]
+  - git: https://github.com/fivetran/dbt_shopify_source.git
+    revision: patch/json-bigquery-compatibility
+    warn-unpinned: false