From 397c698eb6402d70320c8fd17f26ababaf0ebe64 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 10 Feb 2025 00:23:54 +0800 Subject: [PATCH 01/15] fsc-evm udtfs --- README.md | 8 +- data/live__verified_api_endpoints.csv | 12 - dbt_project.yml | 34 +- macros/evm/evm.yaml.sql | 92 ++++- macros/evm/evm_live_views.sql | 322 ++++++++++++++++++ macros/utils/logging.sql | 36 ++ macros/utils/udtf_utils/render.sql | 65 ++++ macros/utils/udtf_utils/udtf_return_type.sql | 72 ++++ .../overrides/core/bronze/bronze__blocks.sql | 13 + .../core/bronze/bronze__blocks_fr.sql | 13 + .../core/bronze/bronze__receipts.sql | 13 + .../core/bronze/bronze__receipts_fr.sql | 13 + .../overrides/core/bronze/bronze__traces.sql | 13 + .../core/bronze/bronze__traces_fr.sql | 13 + .../core/bronze/bronze__transactions.sql | 17 + .../core/bronze/bronze__transactions_fr.sql | 17 + .../overrides/core/gold/evm__fact_blocks.sql | 5 + .../core/gold/evm__fact_event_logs.sql | 13 + .../overrides/core/gold/evm__fact_traces.sql | 16 + .../core/gold/evm__fact_transactions.sql | 12 + .../prices/price__ez_prices_hourly.sql | 3 + package-lock.yml | 18 + packages.yml | 2 + requirements.txt | 2 +- 24 files changed, 797 insertions(+), 27 deletions(-) delete mode 100644 data/live__verified_api_endpoints.csv create mode 100644 macros/evm/evm_live_views.sql create mode 100644 macros/utils/logging.sql create mode 100644 macros/utils/udtf_utils/render.sql create mode 100644 macros/utils/udtf_utils/udtf_return_type.sql create mode 100644 models/overrides/core/bronze/bronze__blocks.sql create mode 100644 models/overrides/core/bronze/bronze__blocks_fr.sql create mode 100644 models/overrides/core/bronze/bronze__receipts.sql create mode 100644 models/overrides/core/bronze/bronze__receipts_fr.sql create mode 100644 models/overrides/core/bronze/bronze__traces.sql create mode 100644 models/overrides/core/bronze/bronze__traces_fr.sql create mode 100644 models/overrides/core/bronze/bronze__transactions.sql create mode 100644 models/overrides/core/bronze/bronze__transactions_fr.sql create mode 100644 models/overrides/core/gold/evm__fact_blocks.sql create mode 100644 models/overrides/core/gold/evm__fact_event_logs.sql create mode 100644 models/overrides/core/gold/evm__fact_traces.sql create mode 100644 models/overrides/core/gold/evm__fact_transactions.sql create mode 100644 models/overrides/prices/price__ez_prices_hourly.sql create mode 100644 package-lock.yml diff --git a/README.md b/README.md index 4e64cd98..6306ed40 100644 --- a/README.md +++ b/README.md @@ -462,20 +462,20 @@ livequery: To control the creation of UDF or SP macros with dbt run: -* UPDATE_UDFS_AND_SPS +* LQ_UPDATE_UDFS_AND_SPS When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal When False, none of the on-run-start macros are executed on model run Default values are False * Usage: -`dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ...` +`dbt run --var '{"LQ_UPDATE_UDFS_AND_SPS":True}' -m ...` Dropping and creating udfs can also be done without running a model: ```sh -dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}' -dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}' +dbt run-operation create_udfs --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}' +dbt run-operation create_udfs --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}' ``` ## Resources diff --git a/data/live__verified_api_endpoints.csv b/data/live__verified_api_endpoints.csv deleted file mode 100644 index 00db711f..00000000 --- a/data/live__verified_api_endpoints.csv +++ /dev/null @@ -1,12 +0,0 @@ -Project,Endpoint,Type,Documentation -Snapshot,https://hub.snapshot.org/graphql,GraphQL,https://docs.snapshot.org/graphql-api -Defillama,https://api.llama.fi/,REST,https://defillama.com/docs/api -Defillama,https://yields.llama.fi/,REST,https://defillama.com/docs/api -Defillama,https://stablecoins.llama.fi/,REST,https://defillama.com/docs/api -Defillama,https://bridges.llama.fi/,REST,https://defillama.com/docs/api -Defillama,https://coins.llama.fi/,REST,https://defillama.com/docs/api -zkSync,https://api.zksync.io/api/v0.2/,REST,https://docs.zksync.io/apiv02-docs/ -DeepNFT Value,https://api.deepnftvalue.com/v1,REST,https://deepnftvalue.readme.io/reference/getting-started-with-deepnftvalue-api -Zapper,https://api.zapper.fi/v2/,REST,https://api.zapper.fi/api/static/index.html#/Apps/AppsController_getApps -Helius,https://api.helius.xyz,REST,https://docs.helius.xyz/introduction/why-helius -Stargaze Name Service,https://rest.stargaze-apis.com,REST,https://github.com/public-awesome/names/blob/main/API.md \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 2946a8fe..df9a3b6f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,6 +36,33 @@ models: +tags: evm marketplace: +tags: marketplace + overrides: + +materialized: ephemeral + fsc_evm: + +enabled: false # disable package by default + +materialized: ephemeral + main_package: + core: + gold: + core__fact_blocks: + +enabled: true + core__fact_transactions: + +enabled: true + core__fact_event_logs: + +enabled: true + core__fact_traces: + +enabled: true + core__ez_native_transfers: + +enabled: true + silver: + silver__blocks: + +enabled: true + silver__transactions: + +enabled: true + silver__receipts: + +enabled: true + silver__traces: + +enabled: true tests: +store_failures: true # all tests @@ -44,7 +71,7 @@ tests: marketplace: blockpour: # TODO: enable tests for blockpour once we get an API key - +enabled: false + +enabled: false on-run-start: - "{{ create_sps() }}" @@ -61,7 +88,7 @@ on-run-end: vars: "dbt_date:time_zone": GMT - UPDATE_UDFS_AND_SPS: false + LQ_UPDATE_UDFS_AND_SPS: false DROP_UDFS_AND_SPS: false UPDATE_SNOWFLAKE_TAGS: true STREAMLINE_INVOKE_STREAMS: False @@ -71,9 +98,10 @@ vars: API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}' ROLES: '{{ var("config")[target.name]["ROLES"] }}' + GLOBAL_PROD_DB_NAME: 'LIVEQUERY_DEV' config: - # The keys correspond to dbt profiles and are case sensitive + # The keys correspond to dbt profiles and are case sensitive dev: API_INTEGRATION: AWS_LIVE_QUERY_STG EXTERNAL_FUNCTION_URI: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/ diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 32bd2eb8..1aa462a5 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -24,7 +24,7 @@ COMMENT = $$Returns the native asset balance at the latest block for a given address.$$ sql: | {{ evm_latest_native_balance_string(schema, blockchain, network) | indent(4) -}} - + - name: {{ schema -}}.tf_latest_native_balance signature: - [wallets, ARRAY, An array of addresses string to get the balance of at the latest block] @@ -41,7 +41,7 @@ - name: {{ schema -}}.tf_latest_token_balance signature: - [wallet, STRING, The address to get the balance of at the latest block] - - [token, STRING, The address of the token to get the balance of] + - [token, STRING, The address of the token to get the balance of] return_type: - "TABLE(status STRING, blockchain STRING, network STRING, wallet_address STRING, token_address STRING, symbol STRING, raw_balance STRING, balance FLOAT)" options: | @@ -55,7 +55,7 @@ - name: {{ schema -}}.tf_latest_token_balance signature: - [wallet, STRING, The address to get the balance of at the latest block] - - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] + - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] return_type: - "TABLE(status STRING, blockchain STRING, network STRING, wallet_address STRING, token_address STRING, symbol STRING, raw_balance STRING, balance FLOAT)" options: | @@ -83,7 +83,7 @@ - name: {{ schema -}}.tf_latest_token_balance signature: - [wallets, ARRAY, An array of addresses string to get the balance of at the latest block] - - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] + - [tokens, ARRAY, An array of address strings of the tokens to get the balance of] return_type: - "TABLE(status STRING, blockchain STRING, network STRING, wallet_address STRING, token_address STRING, symbol STRING, raw_balance STRING, balance FLOAT)" options: | @@ -281,7 +281,7 @@ COMMENT = $$Returns the latest events emitted by a contract within the last `lookback` blocks. *Please note there are RPC limitations on this method.*$$ sql: | {{ evm_latest_contract_events_si(schema, blockchain, network) | indent(4) -}} - + - name: {{ schema -}}.tf_latest_contract_events signature: - [addresses, ARRAY, The addresses of the contracts to get the events of] @@ -362,6 +362,84 @@ COMMENT = $$Returns the latest decoded events emitted by multiple contracts within the last `lookback` blocks. Submit missing ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/). *Please note there are RPC limitations on this method.* $$ sql: | {{ evm_latest_contract_events_decoded_ai(schema, blockchain, network) | indent(4) -}} + +- name: {{ schema -}}.tf_fact_blocks + signature: + - [block_height, INTEGER, The start block height to get the blocks from] + - [to_latest, BOOLEAN, Whether to continue fetching blocks until the latest block or not] + return_type: + - "{{ generate_udtf_return_type(blockchain, get_fact_blocks_columns()) }}" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the block data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ + sql: | + {{ evm_fact_blocks(schema, blockchain, network) | indent(4) -}} + +- name: {{ schema -}}.tf_fact_transactions + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + return_type: + - "{{ generate_udtf_return_type(blockchain, get_fact_transactions_columns()) }}" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the transactions for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ + sql: | + {{ evm_fact_transactions(schema, blockchain, network) | indent(4) -}} + +- name: {{ schema -}}.tf_fact_event_logs + signature: + - [block_height, INTEGER, The start block height to get the events from] + - [to_latest, BOOLEAN, Whether to continue fetching events until the latest block or not] + return_type: + - "TABLE( + block_number INTEGER, + block_timestamp TIMESTAMP_NTZ, + tx_hash STRING, + tx_position INTEGER, + event_index INTEGER, + contract_address STRING, + topics VARIANT, + topic_0 STRING, + topic_1 STRING, + topic_2 STRING, + topic_3 STRING, + DATA STRING, + event_removed BOOLEAN, + origin_from_address STRING, + origin_to_address STRING, + origin_function_signature STRING, + tx_succeeded BOOLEAN, + fact_event_logs_id STRING, + inserted_timestamp TIMESTAMP_NTZ, + modified_timestamp TIMESTAMP_NTZ + )" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the event logs for a given block height. If to_latest is true, it will continue fetching events until the latest block. Otherwise, it will fetch events until the block height is reached.$$ + sql: | + {{ evm_fact_event_logs(schema, blockchain, network) | indent(4) -}} + +- name: {{ schema -}}.tf_fact_traces + signature: + - [block_height, INTEGER, The start block height to get the traces from] + - [to_latest, BOOLEAN, Whether to continue fetching traces until the latest block or not] + return_type: + - "{{ generate_udtf_return_type(blockchain, get_fact_traces_columns()) }}" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the traces for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ + sql: | + {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} + {%- endmacro -%} {% macro config_eth_high_level_abstractions(blockchain, network) -%} @@ -393,7 +471,7 @@ NOT NULL RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns the decoded events emitted by a contract from a specific block to the latest block. Submit missing ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/).$$ + COMMENT = $$Returns the decoded events emitted by a contract from a specific block to the latest block. Submit missing ABIs [here](https://science.flipsidecrypto.xyz/abi-requestor/).$$ sql: | {{ evm_contract_events_decoded(schema, blockchain, network) | indent(4) -}} -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql new file mode 100644 index 00000000..35aafc10 --- /dev/null +++ b/macros/evm/evm_live_views.sql @@ -0,0 +1,322 @@ +{% macro evm_live_view_latest_block_height(schema, blockchain, network) %} + SELECT + {{ schema }}.udf_rpc('eth_blockNumber', []) as result, + utils.udf_hex_to_int(result)::integer AS latest_block_height, + COALESCE( + block_height, + latest_block_height + ) AS min_height, + iff( + COALESCE(to_latest, false), + latest_block_height, + min_height + ) AS max_height +{% endmacro %} + +{% macro evm_live_view_target_blocks(schema, blockchain, network, batch_size=10) %} + WITH heights AS ( + {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} + ), + block_spine AS ( + SELECT + ROW_NUMBER() OVER ( + ORDER BY + NULL + ) - 1 + COALESCE(block_height, latest_block_height)::integer AS block_number, + min_height, + IFF( + COALESCE(to_latest, false), + block_height, + min_height + ) AS max_height, + latest_block_height + FROM + TABLE(generator(ROWCOUNT => 1000)), + heights qualify block_number BETWEEN min_height + AND max_height + ) + + SELECT + CEIL(ROW_NUMBER() OVER (ORDER BY block_number) / {{ batch_size }}) AS batch_id, + block_number, + latest_block_height + FROM block_spine +{% endmacro %} + +{% macro evm_batch_udf_api(blockchain, network) %} + live.udf_api( + '{endpoint}' + ,params + ,concat_ws('/', 'integration', _utils.udf_provider(), '{{ blockchain }}', '{{ network }}') + )::VARIANT:data::ARRAY AS data +{% endmacro %} + +-- Get Raw EVM chain data +{% macro evm_live_view_bronze_blocks(schema, blockchain, network, table_name) %} +WITH blocks_agg AS ( + SELECT + batch_id, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'eth_getBlockByNumber', + [utils.udf_int_to_hex(block_number), true] + ) + ) AS params + FROM + {{ table_name }} + GROUP BY batch_id +), result as ( + SELECT + {{ evm_batch_udf_api(blockchain, network) }} + FROM blocks_agg +) +, flattened as ( + SELECT + COALESCE(value:result, {'error':value:error}) AS result + FROM result, LATERAL FLATTEN(input => result.data) v +) + +SELECT + 0 AS partition_key, + utils.udf_hex_to_int(result:number::STRING)::INT AS block_number, + result as data, + SYSDATE() AS _inserted_timestamp +FROM flattened +{% endmacro %} + +{% macro evm_live_view_bronze_receipts(schema, blockchain, network, table_name) %} +WITH blocks_agg AS ( + SELECT + batch_id, + latest_block_height, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'eth_getBlockReceipts', + [utils.udf_int_to_hex(block_number)] + ) + ) AS params + FROM + {{ table_name }} + GROUP BY 1,2 +), + +get_batch_result AS ( + SELECT + latest_block_height, + {{ evm_batch_udf_api(blockchain, network) }} + FROM blocks_agg +) + +SELECT + 0 AS partition_key, + latest_block_height, + utils.udf_hex_to_int(w.value:blockNumber::STRING)::INT AS block_number, + w.index AS array_index, + w.value AS DATA, + SYSDATE() AS _inserted_timestamp +FROM + (SELECT + latest_block_height, + v.value:result AS DATA + FROM get_batch_result, + LATERAL FLATTEN(data) v), LATERAL FLATTEN(data) w +{% endmacro %} + +{% macro evm_live_view_bronze_transactions(table_name) %} + SELECT + 0 AS partition_key, + block_number, + v.index::INT AS tx_position, -- mimic's streamline's logic to add tx_position + OBJECT_CONSTRUCT( + 'array_index', v.index::INT + ) AS VALUE, + v.value as DATA, + SYSDATE() AS _inserted_timestamp + FROM + {{ table_name }} AS r, + lateral flatten(r.data:transactions) v +{% endmacro %} + +{% macro evm_live_view_bronze_traces(schema, blockchain, network, table_name)%} +WITH blocks_agg AS ( + SELECT + batch_id, + ARRAY_AGG( + utils.udf_json_rpc_call( + 'debug_traceBlockByNumber', + [utils.udf_int_to_hex(s.block_number), {'tracer': 'callTracer'}], + s.block_number -- to put block_number in the id to retrieve the block numberlater + ) + ) AS params + FROM + {{ table_name }} s + GROUP BY batch_id +), result as ( + SELECT + {{ evm_batch_udf_api(blockchain, network) }} + FROM blocks_agg +), flattened as ( + SELECT + value:id::INT AS block_number, + COALESCE(value:result, {'error':value:error}) AS result + FROM result, LATERAL FLATTEN(input => result.data) v +) + +SELECT + 0 AS partition_key, + s.block_number, + v.index::INT AS tx_position, -- mimic's streamline's logic to add tx_position + OBJECT_CONSTRUCT( + 'array_index', v.index::INT + ) AS VALUE, + v.value AS DATA, + SYSDATE() AS _inserted_timestamp +FROM flattened s, +LATERAL FLATTEN(input => result) v +{% endmacro %} + +{% macro evm_fact_blocks(schema, blockchain, network) %} + {%- set evm__fact_blocks = get_rendered_model('livequery_models', 'evm__fact_blocks', schema, blockchain, network) -%} + {{ evm__fact_blocks }} +{% endmacro %} + +{% macro evm_fact_transactions(schema, blockchain, network) %} + {%- set evm__fact_transactions = get_rendered_model('livequery_models', 'evm__fact_transactions', schema, blockchain, network) -%} + {{ evm__fact_transactions }} +{% endmacro %} + +{% macro evm_fact_event_logs(schema, blockchain, network) %} + {%- set evm__fact_event_logs = get_rendered_model('livequery_models', 'evm__fact_event_logs', schema, blockchain, network) -%} + {{ evm__fact_event_logs }} +{% endmacro %} + +{% macro evm_fact_traces(schema, blockchain, network) %} + {%- set evm__fact_traces = get_rendered_model('livequery_models', 'evm__fact_traces', schema, blockchain, network) -%} + {{ evm__fact_traces }} +{% endmacro %} + +-- UDTF Return Columns +{% macro get_fact_blocks_columns() %} + {% set columns = [ + {'name': 'block_number', 'type': 'INTEGER'}, + {'name': 'block_hash', 'type': 'STRING'}, + {'name': 'block_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'network', 'type': 'STRING'}, + {'name': 'tx_count', 'type': 'INTEGER'}, + {'name': 'size', 'type': 'INTEGER'}, + {'name': 'miner', 'type': 'STRING'}, + {'name': 'mix_hash', 'type': 'STRING', 'flag': 'uses_mix_hash'}, + {'name': 'extra_data', 'type': 'STRING'}, + {'name': 'parent_hash', 'type': 'STRING'}, + {'name': 'gas_used', 'type': 'INTEGER'}, + {'name': 'gas_limit', 'type': 'INTEGER'}, + {'name': 'base_fee_per_gas', 'type': 'INTEGER', 'flag': 'uses_base_fee'}, + {'name': 'difficulty', 'type': 'INTEGER'}, + {'name': 'total_difficulty', 'type': 'INTEGER', 'flag': 'uses_total_difficulty'}, + {'name': 'sha3_uncles', 'type': 'STRING'}, + {'name': 'uncle_blocks', 'type': 'VARIANT'}, + {'name': 'nonce', 'type': 'INTEGER'}, + {'name': 'receipts_root', 'type': 'STRING'}, + {'name': 'state_root', 'type': 'STRING'}, + {'name': 'transactions_root', 'type': 'STRING'}, + {'name': 'logs_bloom', 'type': 'STRING'}, + {'name': 'blob_gas_used', 'type': 'INTEGER', 'flag': 'uses_blob_gas_used'}, + {'name': 'excess_blob_gas', 'type': 'INTEGER', 'flag': 'uses_blob_gas_used'}, + {'name': 'parent_beacon_block_root', 'type': 'STRING', 'flag': 'uses_parent_beacon_block_root'}, + {'name': 'withdrawals', 'type': 'VARIANT', 'flag': 'uses_withdrawals'}, + {'name': 'withdrawals_root', 'type': 'STRING', 'flag': 'uses_withdrawals'}, + {'name': 'fact_blocks_id', 'type': 'STRING'}, + {'name': 'inserted_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'modified_timestamp', 'type': 'TIMESTAMP_NTZ'} + ] %} + {{ return(columns) }} +{% endmacro %} + +{% macro get_fact_transactions_columns() %} + {% set columns = [ + {'name': 'block_number', 'type': 'NUMBER'}, + {'name': 'block_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'tx_hash', 'type': 'STRING'}, + {'name': 'from_address', 'type': 'STRING'}, + {'name': 'to_address', 'type': 'STRING'}, + {'name': 'origin_function_signature', 'type': 'STRING'}, + {'name': 'value', 'type': 'FLOAT'}, + {'name': 'value_precise_raw', 'type': 'STRING'}, + {'name': 'value_precise', 'type': 'STRING'}, + {'name': 'tx_fee', 'type': 'FLOAT'}, + {'name': 'tx_fee_precise', 'type': 'STRING'}, + {'name': 'tx_succeeded', 'type': 'BOOLEAN'}, + {'name': 'tx_type', 'type': 'NUMBER'}, + {'name': 'nonce', 'type': 'NUMBER'}, + {'name': 'tx_position', 'type': 'NUMBER'}, + {'name': 'input_data', 'type': 'STRING'}, + {'name': 'gas_price', 'type': 'FLOAT'}, + {'name': 'gas_used', 'type': 'NUMBER'}, + {'name': 'gas_limit', 'type': 'NUMBER'}, + {'name': 'cumulative_gas_used', 'type': 'NUMBER'}, + {'name': 'effective_gas_price', 'type': 'NUMBER'}, + {'name': 'max_fee_per_gas', 'type': 'FLOAT', 'flag': 'uses_eip_1559'}, + {'name': 'max_priority_fee_per_gas', 'type': 'FLOAT', 'flag': 'uses_eip_1559'}, + {'name': 'l1_fee', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_fee_precise_raw', 'type': 'STRING', 'flag': 'uses_l1_columns'}, + {'name': 'l1_fee_precise', 'type': 'STRING', 'flag': 'uses_l1_columns'}, + {'name': 'l1_fee_scalar', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_gas_used', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_gas_price', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_base_fee_scalar', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_blob_base_fee', 'type': 'FLOAT', 'flag': 'uses_blob_base_fee'}, + {'name': 'l1_blob_base_fee_scalar', 'type': 'FLOAT', 'flag': 'uses_blob_base_fee'}, + {'name': 'mint', 'type': 'FLOAT', 'flag': 'uses_mint'}, + {'name': 'mint_precise_raw', 'type': 'STRING', 'flag': 'uses_mint'}, + {'name': 'mint_precise', 'type': 'STRING', 'flag': 'uses_mint'}, + {'name': 'eth_value', 'type': 'FLOAT', 'flag': 'uses_eth_value'}, + {'name': 'eth_value_precise_raw', 'type': 'STRING', 'flag': 'uses_eth_value'}, + {'name': 'eth_value_precise', 'type': 'STRING', 'flag': 'uses_eth_value'}, + {'name': 'y_parity', 'type': 'FLOAT', 'flag': 'uses_y_parity'}, + {'name': 'access_list', 'type': 'VARIANT', 'flag': 'uses_access_list'}, + {'name': 'r', 'type': 'STRING'}, + {'name': 's', 'type': 'STRING'}, + {'name': 'v', 'type': 'NUMBER'}, + {'name': 'source_hash', 'type': 'STRING', 'flag': 'uses_source_hash'}, + {'name': 'fact_transactions_id', 'type': 'STRING'}, + {'name': 'inserted_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'modified_timestamp', 'type': 'TIMESTAMP_NTZ'} + ] %} + {{ return(columns) }} +{% endmacro %} + +{% macro get_fact_traces_columns() %} + {% set columns = [ + {'name': 'block_number', 'type': 'INTEGER'}, + {'name': 'block_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'tx_hash', 'type': 'STRING'}, + {'name': 'tx_position', 'type': 'INTEGER'}, + {'name': 'trace_index', 'type': 'INTEGER'}, + {'name': 'from_address', 'type': 'STRING'}, + {'name': 'to_address', 'type': 'STRING'}, + {'name': 'input', 'type': 'STRING'}, + {'name': 'output', 'type': 'STRING'}, + {'name': 'type', 'type': 'STRING'}, + {'name': 'trace_address', 'type': 'STRING'}, + {'name': 'sub_traces', 'type': 'INTEGER'}, + {'name': 'value', 'type': 'FLOAT'}, + {'name': 'value_precise_raw', 'type': 'STRING'}, + {'name': 'value_precise', 'type': 'STRING'}, + {'name': 'value_hex', 'type': 'STRING'}, + {'name': 'gas', 'type': 'INTEGER'}, + {'name': 'gas_used', 'type': 'INTEGER'}, + {'name': 'origin_from_address', 'type': 'STRING'}, + {'name': 'origin_to_address', 'type': 'STRING'}, + {'name': 'origin_function_signature', 'type': 'STRING'}, + {'name': 'before_evm_transfers', 'type': 'VARIANT', 'flag': 'uses_traces_arb_mode'}, + {'name': 'after_evm_transfers', 'type': 'VARIANT', 'flag': 'uses_traces_arb_mode'}, + {'name': 'trace_succeeded', 'type': 'BOOLEAN'}, + {'name': 'error_reason', 'type': 'STRING'}, + {'name': 'revert_reason', 'type': 'STRING'}, + {'name': 'tx_succeeded', 'type': 'BOOLEAN'}, + {'name': 'fact_traces_id', 'type': 'STRING'}, + {'name': 'inserted_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'modified_timestamp', 'type': 'TIMESTAMP_NTZ'} + ] %} + {{ return(columns) }} +{% endmacro %} diff --git a/macros/utils/logging.sql b/macros/utils/logging.sql new file mode 100644 index 00000000..ccb10200 --- /dev/null +++ b/macros/utils/logging.sql @@ -0,0 +1,36 @@ +{% macro log_model_details(vars=false, params=false) %} + +{%- if execute -%} +/* +DBT Model Config: +{{ model.config | tojson(indent=2) }} +*/ + +{% if vars is not false %} + +{% if var('LOG_MODEL_DETAILS', false) %} +{{ log( vars | tojson(indent=2), info=True) }} +{% endif %} +/* +Variables: +{{ vars | tojson(indent=2) }} +*/ +{% endif %} + +{% if params is not false %} + +{% if var('LOG_MODEL_DETAILS', false) %} +{{ log( params | tojson(indent=2), info=True) }} +{% endif %} +/* +Parameters: +{{ params | tojson(indent=2) }} +*/ +{% endif %} + +/* +Raw Code: +{{ model.raw_code }} +*/ +{%- endif -%} +{% endmacro %} diff --git a/macros/utils/udtf_utils/render.sql b/macros/utils/udtf_utils/render.sql new file mode 100644 index 00000000..980c84b4 --- /dev/null +++ b/macros/utils/udtf_utils/render.sql @@ -0,0 +1,65 @@ +{% macro get_rendered_model(package_name, model_name, schema, blockchain, network) %} + {# + This macro retrieves and renders a specified model from the graph. + + Args: + package_name (str): The name of the package containing the model. + model_name (str): The name of the model to be rendered. + schema (str): The schema to be used. + blockchain (str): The blockchain to be used. + network (str): The network to be used. + + Returns: + str: The rendered SQL of the specified model. + #} + {% if execute %} + {{ log("=== Starting get_rendered_model ===", info=True) }} + {# Use a list to store the node to avoid scope issues #} + {%- set nodes = [] -%} + {{ log("Looking for node: " ~ package_name ~ "." ~ model_name, info=True) }} + {%- for node in graph.nodes.values() -%} + {%- if node.package_name == package_name and node.name == model_name -%} + {{ log("Found target node: " ~ node.unique_id, info=True) }} + {%- do nodes.append(node) -%} + {%- endif -%} + {%- endfor -%} + + {%- if nodes | length == 0 -%} + {{ log("No target node found!", info=True) }} + {{ return('') }} + {%- endif -%} + + {%- set target_node = nodes[0] -%} + {{ log("Processing node: " ~ target_node.unique_id, info=True) }} + {{ log("Dependencies: " ~ target_node.depends_on.nodes | join(", "), info=True) }} + + {# First render all dependency CTEs #} + {%- set ctes = [] -%} + {%- for dep_id in target_node.depends_on.nodes -%} + {%- set dep_node = graph.nodes[dep_id] -%} + {%- set rendered_sql = render(dep_node.raw_code) | trim -%} + + {%- if rendered_sql -%} + {%- set cte_sql -%} +__dbt__cte__{{ dep_node.name }} AS ( + {{ rendered_sql }} +) + {%- endset -%} + {%- do ctes.append(cte_sql) -%} + {%- endif -%} + {%- endfor -%} + + {{ log("Number of CTEs generated: " ~ ctes | length, info=True) }} + + {# Combine CTEs with main query #} + {%- set final_sql -%} +WITH {{ ctes | join(',\n\n') }} + +{{ render(target_node.raw_code) }} + {%- endset -%} + + {{ log("=== End get_rendered_model ===", info=True) }} + + {{ return(final_sql) }} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/udtf_utils/udtf_return_type.sql b/macros/utils/udtf_utils/udtf_return_type.sql new file mode 100644 index 00000000..18e069f1 --- /dev/null +++ b/macros/utils/udtf_utils/udtf_return_type.sql @@ -0,0 +1,72 @@ +{# Macro to determine which columns to include based on feature flags #} +{% macro generate_udtf_return_type(blockchain, columns) %} + + {# Set Global Variables #} + {% set GLOBAL_PROD_DB_NAME = blockchain %} + + {# Columns included by default, with specific exclusions #} + {% set excludes_eip_1559 = ['CORE','RONIN'] %} + {% set excludes_base_fee = ['CORE'] %} + {% set excludes_total_difficulty = ['INK','SWELL'] %} + + {# Columns excluded by default, with explicit inclusion #} + {% set includes_l1_columns = ['INK', 'MANTLE', 'SWELL'] %} + {% set includes_l1_tx_fee_calc = ['INK', 'MANTLE', 'SWELL'] %} + {% set includes_eth_value = ['MANTLE'] %} + {% set includes_mint = ['INK', 'MANTLE', 'SWELL'] %} + {% set includes_y_parity = ['INK', 'SWELL'] %} + {% set includes_access_list = ['INK', 'SWELL'] %} + {% set includes_source_hash = ['INK','MANTLE','SWELL'] %} + {% set includes_blob_base_fee = ['INK','SWELL'] %} + {% set includes_mix_hash = ['INK', 'MANTLE', 'SWELL', 'RONIN'] %} + {% set includes_blob_gas_used = ['INK', 'SWELL'] %} + {% set includes_parent_beacon_block_root = ['INK', 'SWELL'] %} + {% set includes_withdrawals = ['INK', 'SWELL'] %} + + {# Set Variables using inclusions and exclusions #} + {% set current_db = blockchain.upper() %} + + {# Transaction feature flags #} + {% set uses_eip_1559 = current_db not in excludes_eip_1559 %} + {% set uses_l1_columns = current_db in includes_l1_columns %} + {% set uses_l1_tx_fee_calc = current_db in includes_l1_tx_fee_calc %} + {% set uses_eth_value = current_db in includes_eth_value %} + {% set uses_mint = current_db in includes_mint %} + {% set uses_y_parity = current_db in includes_y_parity %} + {% set uses_access_list = current_db in includes_access_list %} + {% set uses_source_hash = current_db in includes_source_hash %} + {% set uses_blob_base_fee = current_db in includes_blob_base_fee %} + {% set uses_traces_arb_mode = var('TRACES_ARB_MODE', false) %} + + {# Block feature flags #} + {% set uses_base_fee = current_db not in excludes_base_fee %} + {% set uses_total_difficulty = current_db not in excludes_total_difficulty %} + {% set uses_mix_hash = current_db in includes_mix_hash %} + {% set uses_blob_gas_used = current_db in includes_blob_gas_used %} + {% set uses_parent_beacon_block_root = current_db in includes_parent_beacon_block_root %} + {% set uses_withdrawals = current_db in includes_withdrawals %} + + TABLE( + {% for col in columns %} + {% if col.flag is not defined + or (col.flag == 'uses_eip_1559' and uses_eip_1559) + or (col.flag == 'uses_l1_columns' and uses_l1_columns) + or (col.flag == 'uses_eth_value' and uses_eth_value) + or (col.flag == 'uses_mint' and uses_mint) + or (col.flag == 'uses_y_parity' and uses_y_parity) + or (col.flag == 'uses_access_list' and uses_access_list) + or (col.flag == 'uses_source_hash' and uses_source_hash) + or (col.flag == 'uses_blob_base_fee' and uses_blob_base_fee) + or (col.flag == 'uses_traces_arb_mode' and uses_traces_arb_mode) + or (col.flag == 'uses_mix_hash' and uses_mix_hash) + or (col.flag == 'uses_base_fee' and uses_base_fee) + or (col.flag == 'uses_total_difficulty' and uses_total_difficulty) + or (col.flag == 'uses_blob_gas_used' and uses_blob_gas_used) + or (col.flag == 'uses_parent_beacon_block_root' and uses_parent_beacon_block_root) + or (col.flag == 'uses_withdrawals' and uses_withdrawals) + %} + {{ col.name }} {{ col.type }}{% if not loop.last %},{% endif %} + {% endif %} + {% endfor %} + ) +{% endmacro %} diff --git a/models/overrides/core/bronze/bronze__blocks.sql b/models/overrides/core/bronze/bronze__blocks.sql new file mode 100644 index 00000000..65d52f02 --- /dev/null +++ b/models/overrides/core/bronze/bronze__blocks.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_blocks AS ( + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_blocks diff --git a/models/overrides/core/bronze/bronze__blocks_fr.sql b/models/overrides/core/bronze/bronze__blocks_fr.sql new file mode 100644 index 00000000..65d52f02 --- /dev/null +++ b/models/overrides/core/bronze/bronze__blocks_fr.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_blocks AS ( + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_blocks diff --git a/models/overrides/core/bronze/bronze__receipts.sql b/models/overrides/core/bronze/bronze__receipts.sql new file mode 100644 index 00000000..c3804b86 --- /dev/null +++ b/models/overrides/core/bronze/bronze__receipts.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_receipts diff --git a/models/overrides/core/bronze/bronze__receipts_fr.sql b/models/overrides/core/bronze/bronze__receipts_fr.sql new file mode 100644 index 00000000..c3804b86 --- /dev/null +++ b/models/overrides/core/bronze/bronze__receipts_fr.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_receipts AS ( + {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_receipts diff --git a/models/overrides/core/bronze/bronze__traces.sql b/models/overrides/core/bronze/bronze__traces.sql new file mode 100644 index 00000000..71a19b8a --- /dev/null +++ b/models/overrides/core/bronze/bronze__traces.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_traces AS ( + {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_traces diff --git a/models/overrides/core/bronze/bronze__traces_fr.sql b/models/overrides/core/bronze/bronze__traces_fr.sql new file mode 100644 index 00000000..71a19b8a --- /dev/null +++ b/models/overrides/core/bronze/bronze__traces_fr.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_traces AS ( + {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_traces diff --git a/models/overrides/core/bronze/bronze__transactions.sql b/models/overrides/core/bronze/bronze__transactions.sql new file mode 100644 index 00000000..ae3a491f --- /dev/null +++ b/models/overrides/core/bronze/bronze__transactions.sql @@ -0,0 +1,17 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +), + +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} +) + +SELECT * FROM raw_transactions diff --git a/models/overrides/core/bronze/bronze__transactions_fr.sql b/models/overrides/core/bronze/bronze__transactions_fr.sql new file mode 100644 index 00000000..ae3a491f --- /dev/null +++ b/models/overrides/core/bronze/bronze__transactions_fr.sql @@ -0,0 +1,17 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_block_txs AS ( + {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +), + +raw_transactions AS ( + {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} +) + +SELECT * FROM raw_transactions diff --git a/models/overrides/core/gold/evm__fact_blocks.sql b/models/overrides/core/gold/evm__fact_blocks.sql new file mode 100644 index 00000000..73be526f --- /dev/null +++ b/models/overrides/core/gold/evm__fact_blocks.sql @@ -0,0 +1,5 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_blocks') }} diff --git a/models/overrides/core/gold/evm__fact_event_logs.sql b/models/overrides/core/gold/evm__fact_event_logs.sql new file mode 100644 index 00000000..72caac05 --- /dev/null +++ b/models/overrides/core/gold/evm__fact_event_logs.sql @@ -0,0 +1,13 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_event_logs') }} diff --git a/models/overrides/core/gold/evm__fact_traces.sql b/models/overrides/core/gold/evm__fact_traces.sql new file mode 100644 index 00000000..d7a21f01 --- /dev/null +++ b/models/overrides/core/gold/evm__fact_traces.sql @@ -0,0 +1,16 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('bronze__traces') }} +-- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_traces') }} diff --git a/models/overrides/core/gold/evm__fact_transactions.sql b/models/overrides/core/gold/evm__fact_transactions.sql new file mode 100644 index 00000000..fae0fa25 --- /dev/null +++ b/models/overrides/core/gold/evm__fact_transactions.sql @@ -0,0 +1,12 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_transactions') }} diff --git a/models/overrides/prices/price__ez_prices_hourly.sql b/models/overrides/prices/price__ez_prices_hourly.sql new file mode 100644 index 00000000..2f13307a --- /dev/null +++ b/models/overrides/prices/price__ez_prices_hourly.sql @@ -0,0 +1,3 @@ +{%- set blockchain = this.schema -%} + +SELECT * FROM {{ blockchain.upper() }}.PRICE.EZ_PRICES_HOURLY diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 00000000..f0391526 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,18 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.2 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-evm.git + revision: 0898439658da5a0d830c7dc40d9f4859a4f1012d +- package: calogica/dbt_date + version: 0.7.2 +- package: dbt-labs/dbt_external_tables + version: 0.8.2 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: 6096046f00c5c27fa7b8d60ae6ec1dfe31d7fa10 +- package: get-select/dbt_snowflake_query_tags + version: 2.5.0 +- git: https://github.com/FlipsideCrypto/livequery-base.git + revision: 006980d40fbbc48c2ad0f0b74efa37ec6fa1c74b +sha1_hash: d8ed2190282b052a4e8372ca9c4d08f7fa0d7327 diff --git a/packages.yml b/packages.yml index ee9544fe..24ae7f72 100644 --- a/packages.yml +++ b/packages.yml @@ -3,3 +3,5 @@ packages: version: [">=0.8.0", "<0.9.0"] - package: dbt-labs/dbt_utils version: [">=1.0.0", "<1.1.0"] + - git: "https://github.com/FlipsideCrypto/fsc-evm.git" + revision: "STREAM-1049/add-package-name-for-livequery" diff --git a/requirements.txt b/requirements.txt index ff8e7ca3..54b64e9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt-snowflake~=1.5.0 \ No newline at end of file +dbt-snowflake~=1.7.0 From 35e16e52fda9617636a70c82506d6486cc455c9e Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 10 Feb 2025 00:31:02 +0800 Subject: [PATCH 02/15] update changes from shah's branch --- macros/create_sps.sql | 2 +- macros/livequery/manage_udfs.sql | 6 +- models/deploy/core/_live.sql | 2 - models/deploy/core/_utils.sql | 2 - models/deploy/core/_utils.yml | 11 - models/deploy/core/live.sql | 5 - models/deploy/core/live.yml | 27 -- models/deploy/core/secrets.sql | 4 - models/deploy/core/secrets.yml | 45 --- models/deploy/core/utils.sql | 3 - models/deploy/core/utils.yml | 268 ------------------ models/deploy/evm/ethereum__mainnet.sql | 18 ++ .../integration/__overview__.md | 4 +- models/live__allow_list.sql | 11 - 14 files changed, 24 insertions(+), 384 deletions(-) delete mode 100644 models/deploy/core/_live.sql delete mode 100644 models/deploy/core/_utils.sql delete mode 100644 models/deploy/core/_utils.yml delete mode 100644 models/deploy/core/live.sql delete mode 100644 models/deploy/core/live.yml delete mode 100644 models/deploy/core/secrets.sql delete mode 100644 models/deploy/core/secrets.yml delete mode 100644 models/deploy/core/utils.sql delete mode 100644 models/deploy/core/utils.yml delete mode 100644 models/live__allow_list.sql diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 24045461..f4536c61 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,5 +1,5 @@ {% macro create_sps() %} - {% if var("UPDATE_UDFS_AND_SPS") %} + {% if var("LQ_UPDATE_UDFS_AND_SPS") %} {% if target.database == 'LIVEQUERY' %} CREATE schema IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; diff --git a/macros/livequery/manage_udfs.sql b/macros/livequery/manage_udfs.sql index b5b656f4..fedd21b5 100644 --- a/macros/livequery/manage_udfs.sql +++ b/macros/livequery/manage_udfs.sql @@ -152,7 +152,7 @@ This macro is used to deploy functions using ephemeral models. It should only be used within an ephemeral model. #} - {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} {% set sql %} {{- crud_udfs(config, this.schema, var("DROP_UDFS_AND_SPS")) -}} {%- endset -%} @@ -174,7 +174,7 @@ {%- set blockchain = this.schema -%} {%- set network = this.identifier -%} {% set schema = blockchain ~ "_" ~ network %} - {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} {% set sql %} {% for config in configs %} {{- crud_udfs_by_chain(config, blockchain, network, var("DROP_UDFS_AND_SPS")) -}} @@ -197,7 +197,7 @@ #} {%- set schema = this.schema -%} {%- set utility_schema = this.identifier -%} - {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} {% set sql %} {% for config in configs %} {{- crud_udfs_by_marketplace(config, schema, utility_schema, var("DROP_UDFS_AND_SPS")) -}} diff --git a/models/deploy/core/_live.sql b/models/deploy/core/_live.sql deleted file mode 100644 index f37331e3..00000000 --- a/models/deploy/core/_live.sql +++ /dev/null @@ -1,2 +0,0 @@ -{% set config = config_core__live %} -{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/_utils.sql b/models/deploy/core/_utils.sql deleted file mode 100644 index ac40ddfb..00000000 --- a/models/deploy/core/_utils.sql +++ /dev/null @@ -1,2 +0,0 @@ -{% set config = config_core__utils %} -{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/_utils.yml b/models/deploy/core/_utils.yml deleted file mode 100644 index 80fcb8ba..00000000 --- a/models/deploy/core/_utils.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -models: - - name: _utils - columns: - - name: udf_introspect - tests: - - test_udf: - name: test___utils_udf_introspect - args: "'hello'" - assertions: - - result = 'hello' \ No newline at end of file diff --git a/models/deploy/core/live.sql b/models/deploy/core/live.sql deleted file mode 100644 index 9e088f23..00000000 --- a/models/deploy/core/live.sql +++ /dev/null @@ -1,5 +0,0 @@ --- depends_on: {{ ref('_utils') }} --- depends_on: {{ ref('utils') }} --- depends_on: {{ ref('_live') }} -{% set config = config_core_live %} -{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/live.yml b/models/deploy/core/live.yml deleted file mode 100644 index cb4bbf1a..00000000 --- a/models/deploy/core/live.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: 2 -models: - - name: live - columns: - - name: udf_api - tests: - - test_udf: - name: test__live_udf_api_post_data_object - args: | - 'https://httpbin.org/post', {'foo': 'bar'} - assertions: - - result:data.json is not null - - result:data.json = OBJECT_CONSTRUCT('foo', 'bar') - - test_udf: - name: test__live_udf_api_post_data_array - args: | - 'https://httpbin.org/post', ['foo', 'bar'] - assertions: - - result:data.json is not null - - result:data.json = ARRAY_CONSTRUCT('foo', 'bar') - - test_udf: - name: test__live_udf_api_post_data_string - args: | - 'https://httpbin.org/post', 'foo'::VARIANT - assertions: - - result:data.json is not null - - result:data.json = 'foo' \ No newline at end of file diff --git a/models/deploy/core/secrets.sql b/models/deploy/core/secrets.sql deleted file mode 100644 index 668ca142..00000000 --- a/models/deploy/core/secrets.sql +++ /dev/null @@ -1,4 +0,0 @@ --- depends_on: {{ ref('_utils') }} --- depends_on: {{ ref('live') }} -{% set config = config_core_secrets %} -{{ ephemeral_deploy_core(config) }} \ No newline at end of file diff --git a/models/deploy/core/secrets.yml b/models/deploy/core/secrets.yml deleted file mode 100644 index 7d54d746..00000000 --- a/models/deploy/core/secrets.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: 2 -models: - - name: secrets - columns: - - name: udf_register_secret - tests: - - test_udf: - name: test_secrets__udf_register_secret - args: > - 'test', 'test' - assertions: - - result = {'request_id':'test','success':false} - - name: udf_get_secret - tests: - - test_udf: - name: test_secrets__udf_get_secret - args: > - 'test' - assertions: - - > - result::OBJECT = {'key': 'value'} - - name: udf_get_secrets - tests: - - test_udf: - name: test_secrets__udf_get_secrets - args: '' - assertions: - - > - result = {'test': {'key': 'value'}} - - name: udf_create_secret - tests: - - test_udf: - name: test_secrets__udf_create_secret - args: > - 'test', {'key': 'value'} - assertions: - - result = 200 - - name: udf_delete_secret - tests: - - test_udf: - name: test_secrets__udf_delete_secret - args: > - 'test_delete' - assertions: - - result = 200 \ No newline at end of file diff --git a/models/deploy/core/utils.sql b/models/deploy/core/utils.sql deleted file mode 100644 index fbabfe17..00000000 --- a/models/deploy/core/utils.sql +++ /dev/null @@ -1,3 +0,0 @@ - -- depends_on: {{ ref('_utils') }} -{% set config = config_core_utils %} -{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/utils.yml b/models/deploy/core/utils.yml deleted file mode 100644 index 659b025e..00000000 --- a/models/deploy/core/utils.yml +++ /dev/null @@ -1,268 +0,0 @@ -version: 2 -models: - - name: utils - columns: - - name: udf_json_rpc_call - tests: - - test_udf: - args: "'foo', [], 1" - assertions: - - > - result = { - 'jsonrpc': '2.0', - 'method': 'foo', - 'params': [], - 'id': '1' - } - - name: udf_urlencode - tests: - - test_udf: - name: test_utils__udf_urlencode_dict_true_1 - args: > - {'a':'b'}, TRUE - assertions: - - result = 'a=b' - - test_udf: - name: test_utils__udf_urlencode_dict_true_2 - args: > - {'a':'a', 'b':'b'}, TRUE - assertions: - - result = 'a=a&b=b' - - test_udf: - name: test_utils__udf_urlencode_dict_true_space - args: > - {'space': 'abc 123'}, TRUE - assertions: - - result = 'space=abc+123' - - test_udf: - name: test_utils__udf_urlencode_dict_true_special - args: > - {'special!': ' !@#$,+"'}, TRUE - assertions: - - result = 'special%21=+%21%40%23%24%2C%2B%22' - - test_udf: - name: test_utils__udf_urlencode_dict_true_array - args: > - {'array': [0, 1, 2]}, TRUE - assertions: - - result = 'array=0&array=1&array=2' - - test_udf: - name: test_utils__udf_urlencode_dict_false_1 - args: > - {'a':'b'}, FALSE - assertions: - - result = 'a=b' - - test_udf: - name: test_utils__udf_urlencode_dict_false_2 - args: > - {'a':'b', 'b':'b'}, FALSE - assertions: - - result = 'a=b&b=b' - - test_udf: - name: test_utils__udf_urlencode_dict_false_space - args: > - {'space': 'abc 123'}, FALSE - assertions: - - result = 'space=abc+123' - - test_udf: - name: test_utils__udf_urlencode_dict_false_special - args: > - {'special!': ' !@#$,+"'}, FALSE - assertions: - - result = 'special%21=+%21%40%23%24%2C%2B%22' - - test_udf: - name: test_utils__udf_urlencode_dict_false_array - args: > - {'array': [0, 1, 2]}, FALSE - assertions: - - result = 'array=%5B0%2C+1%2C+2%5D' - - test_udf: - name: test_utils__udf_urlencode_dict_1 - args: > - {'a':'b'} - assertions: - - result = 'a=b' - - test_udf: - name: test_utils__udf_urlencode_dict_2 - args: > - {'a':'b', 'b':'b'} - assertions: - - result = 'a=b&b=b' - - test_udf: - name: test_utils__udf_urlencode_dict_space - args: > - {'space': 'abc 123'} - assertions: - - result = 'space=abc+123' - - test_udf: - name: test_utils__udf_urlencode_dict_special - args: > - {'special!': ' !@#$,+"'} - assertions: - - result = 'special%21=+%21%40%23%24%2C%2B%22' - - test_udf: - name: test_utils__udf_urlencode_dict_array - args: > - {'array': [0, 1, 2]} - assertions: - - result = 'array=%5B0%2C+1%2C+2%5D' - # write tests but use arrays of arrays instead of dictionaries - - test_udf: - name: test_utils__udf_urlencode_array_true_1 - args: > - [['a', 'b']], TRUE - assertions: - - result = 'a=b' - - test_udf: - name: test_utils__udf_urlencode_array_true_2 - args: > - [['a', 'a'], ['b', 'b']], TRUE - assertions: - - result = 'a=a&b=b' - - test_udf: - name: test_utils__udf_urlencode_array_true_space - args: > - [['space', 'abc 123']], TRUE - assertions: - - result = 'space=abc+123' - - test_udf: - name: test_utils__udf_urlencode_array_true_special - args: > - [['special!', ' !@#$,+"']], TRUE - assertions: - - result = 'special%21=+%21%40%23%24%2C%2B%22' - - test_udf: - name: test_utils__udf_urlencode_array_true_array - args: > - [['array', [0, 1, 2]]], TRUE - assertions: - - result = 'array=0&array=1&array=2' - - test_udf: - name: test_utils__udf_urlencode_array_false_1 - args: > - [['a', 'b']], FALSE - assertions: - - result = 'a=b' - - test_udf: - name: test_utils__udf_urlencode_array_false_2 - args: > - [['a', 'a'], ['b', 'b']], FALSE - assertions: - - result = 'a=a&b=b' - - test_udf: - name: test_utils__udf_urlencode_array_false_space - args: > - [['space', 'abc 123']], FALSE - assertions: - - result = 'space=abc+123' - - test_udf: - name: test_utils__udf_urlencode_array_false_special - args: > - [['special!', ' !@#$,+"']], FALSE - assertions: - - result = 'special%21=+%21%40%23%24%2C%2B%22' - - test_udf: - name: test_utils__udf_urlencode_array_false_array - args: > - [['array', [0, 1, 2]]], FALSE - assertions: - - result = 'array=%5B0%2C+1%2C+2%5D' - - test_udf: - name: test_utils__udf_urlencode_array_1 - args: > - [['a', 'b']] - assertions: - - result = 'a=b' - - test_udf: - name: test_utils__udf_urlencode_array_2 - args: > - [['a', 'a'], ['b', 'b']] - assertions: - - result = 'a=a&b=b' - - test_udf: - name: test_utils__udf_urlencode_array_space - args: > - [['space', 'abc 123']] - assertions: - - result = 'space=abc+123' - - test_udf: - name: test_utils__udf_urlencode_array_special - args: > - [['special!', ' !@#$,+"']] - assertions: - - result = 'special%21=+%21%40%23%24%2C%2B%22' - - test_udf: - name: test_utils__udf_urlencode_array_array - args: > - [['array', [0, 1, 2]]] - assertions: - - result = 'array=%5B0%2C+1%2C+2%5D' - - name: udf_int_to_binary - tests: - - test_udf: - name: test_utils__udf_int_to_binary - args: 123456789 - assertions: - - result = '111010110111100110100010101' - - name: udf_int_to_binary - tests: - - test_udf: - name: test_utils__udf_int_to_binary_large_number - args: "'123456789123456789123456789123456789123456789'" - assertions: - - result = '101100010010011011011100101001111010001001110011010111111101111101010111011001001101000001111110001010100001011011010000100000001000101111100010101' - - name: udf_binary_to_int - tests: - - test_udf: - name: test_utils__udf_binary_to_int - args: '111010110111100110100010101' - assertions: - - result = '123456789' - - name: udf_binary_to_int - tests: - - test_udf: - name: test_utils__udf_binary_to_int_large_number - args: "'110110110100110110100101110101100110100000101111100010101'" - assertions: - - result = '123456789123456789' - - name: udf_evm_decode_log - tests: - - test_udf: - name: test_utils__udf_evm_decode_log - args: > - { - 'anonymous': false, - 'inputs': [ - { - 'indexed': false, - 'name': 'nonce', - 'type': 'uint256' - } - ], - 'name': 'NonceChanged', - 'type': 'event' - }::variant, - { - 'address': '0x55032650b14df07b85bf18a3a3ec8e0af2e028d5', - 'data': '0x000000000000000000000000000000000000000000000000000000000000279c', - 'topics': [ - '0x7220970e1f1f12864ecccd8942690a837c7a8dd45d158cb891eb45a8a69134aa' - ] - }::variant - assertions: - - > - result = [ - { - 'address': '0x55032650b14df07b85bF18A3a3eC8E0Af2e028d5', - 'data': [ - { - 'decoded': true, - 'name': 'nonce', - 'type': 'uint256', - 'value': 10140 - } - ], - 'decoded': true, - 'name': 'NonceChanged' - } - ] \ No newline at end of file diff --git a/models/deploy/evm/ethereum__mainnet.sql b/models/deploy/evm/ethereum__mainnet.sql index 6f3fadc2..a2b9be5d 100644 --- a/models/deploy/evm/ethereum__mainnet.sql +++ b/models/deploy/evm/ethereum__mainnet.sql @@ -5,6 +5,24 @@ -- depends_on: {{ ref('_eth__decoded_logs') }} -- depends_on: {{ ref('live') }} -- depends_on: {{ ref('utils') }} +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('bronze__traces') }} +-- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('evm__fact_blocks') }} +-- depends_on: {{ ref('evm__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} {%- set configs = [ config_evm_rpc_primitives, config_evm_high_level_abstractions, diff --git a/models/doc_descriptions/integration/__overview__.md b/models/doc_descriptions/integration/__overview__.md index 9c0c0a12..d2002c7f 100644 --- a/models/doc_descriptions/integration/__overview__.md +++ b/models/doc_descriptions/integration/__overview__.md @@ -46,9 +46,9 @@ Within the same directory (`models/deploy/marketplace/`), create a new folder fo Deploy your model following the standard deployment procedures. -- `dbt run -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"UPDATE_UDFS_AND_SPS":True}'` +- `dbt run -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}'` -- `dbt test -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"UPDATE_UDFS_AND_SPS":True}'` +- `dbt test -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}'` #### Additional Tips: diff --git a/models/live__allow_list.sql b/models/live__allow_list.sql deleted file mode 100644 index 611fda3c..00000000 --- a/models/live__allow_list.sql +++ /dev/null @@ -1,11 +0,0 @@ --- depends_on: {{ ref('live') }} -{{ config( - materialized = 'view', - grants = {'+select': fromyaml(var('ROLES'))} -) }} -SELECT '*.' || t.value AS allowed_domains -FROM table(flatten(input => {{ this.database }}.live.udf_allow_list())) AS t -ORDER BY - split_part(allowed_domains, '.', -1), - split_part(allowed_domains, '.', -2), - split_part(allowed_domains, '.', -3) From d8f4af4968829bc495ffcc92c1a64fb52b519dfc Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 10 Feb 2025 15:06:12 +0800 Subject: [PATCH 03/15] update row_count variable --- macros/evm/evm_live_views.sql | 4 ++-- .../overrides/core/bronze/bronze__blocks.sql | 15 ++------------- .../core/bronze/bronze__blocks_fr.sql | 2 +- .../core/bronze/bronze__receipts.sql | 15 ++------------- .../core/bronze/bronze__receipts_fr.sql | 2 +- .../overrides/core/bronze/bronze__traces.sql | 15 ++------------- .../core/bronze/bronze__traces_fr.sql | 2 +- .../core/bronze/bronze__transactions.sql | 19 ++----------------- .../core/bronze/bronze__transactions_fr.sql | 2 +- 9 files changed, 14 insertions(+), 62 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 35aafc10..de783459 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -13,7 +13,7 @@ ) AS max_height {% endmacro %} -{% macro evm_live_view_target_blocks(schema, blockchain, network, batch_size=10) %} +{% macro evm_live_view_target_blocks(schema, blockchain, network, row_count=1000, batch_size=10) %} WITH heights AS ( {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} ), @@ -31,7 +31,7 @@ ) AS max_height, latest_block_height FROM - TABLE(generator(ROWCOUNT => 1000)), + TABLE(generator(ROWCOUNT => {{ row_count }})), heights qualify block_number BETWEEN min_height AND max_height ) diff --git a/models/overrides/core/bronze/bronze__blocks.sql b/models/overrides/core/bronze/bronze__blocks.sql index 65d52f02..3b26fd95 100644 --- a/models/overrides/core/bronze/bronze__blocks.sql +++ b/models/overrides/core/bronze/bronze__blocks.sql @@ -1,13 +1,2 @@ -{%- set blockchain = this.schema -%} -{%- set network = this.identifier -%} -{%- set schema = blockchain ~ "_" ~ network -%} - -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} -), - -raw_blocks AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -) - -SELECT * FROM raw_blocks +-- unused in livequery +SELECT 1 diff --git a/models/overrides/core/bronze/bronze__blocks_fr.sql b/models/overrides/core/bronze/bronze__blocks_fr.sql index 65d52f02..a4549206 100644 --- a/models/overrides/core/bronze/bronze__blocks_fr.sql +++ b/models/overrides/core/bronze/bronze__blocks_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 1000, 10) | indent(4) -}} ), raw_blocks AS ( diff --git a/models/overrides/core/bronze/bronze__receipts.sql b/models/overrides/core/bronze/bronze__receipts.sql index c3804b86..3b26fd95 100644 --- a/models/overrides/core/bronze/bronze__receipts.sql +++ b/models/overrides/core/bronze/bronze__receipts.sql @@ -1,13 +1,2 @@ -{%- set blockchain = this.schema -%} -{%- set network = this.identifier -%} -{%- set schema = blockchain ~ "_" ~ network -%} - -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} -), - -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} -) - -SELECT * FROM raw_receipts +-- unused in livequery +SELECT 1 diff --git a/models/overrides/core/bronze/bronze__receipts_fr.sql b/models/overrides/core/bronze/bronze__receipts_fr.sql index c3804b86..adb1c1fa 100644 --- a/models/overrides/core/bronze/bronze__receipts_fr.sql +++ b/models/overrides/core/bronze/bronze__receipts_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 100, 10) | indent(4) -}} ), raw_receipts AS ( diff --git a/models/overrides/core/bronze/bronze__traces.sql b/models/overrides/core/bronze/bronze__traces.sql index 71a19b8a..3b26fd95 100644 --- a/models/overrides/core/bronze/bronze__traces.sql +++ b/models/overrides/core/bronze/bronze__traces.sql @@ -1,13 +1,2 @@ -{%- set blockchain = this.schema -%} -{%- set network = this.identifier -%} -{%- set schema = blockchain ~ "_" ~ network -%} - -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} -), - -raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} -) - -SELECT * FROM raw_traces +-- unused in livequery +SELECT 1 diff --git a/models/overrides/core/bronze/bronze__traces_fr.sql b/models/overrides/core/bronze/bronze__traces_fr.sql index 71a19b8a..f0036226 100644 --- a/models/overrides/core/bronze/bronze__traces_fr.sql +++ b/models/overrides/core/bronze/bronze__traces_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 100, 10) | indent(4) -}} ), raw_traces AS ( diff --git a/models/overrides/core/bronze/bronze__transactions.sql b/models/overrides/core/bronze/bronze__transactions.sql index ae3a491f..3b26fd95 100644 --- a/models/overrides/core/bronze/bronze__transactions.sql +++ b/models/overrides/core/bronze/bronze__transactions.sql @@ -1,17 +1,2 @@ -{%- set blockchain = this.schema -%} -{%- set network = this.identifier -%} -{%- set schema = blockchain ~ "_" ~ network -%} - -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} -), - -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), - -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -) - -SELECT * FROM raw_transactions +-- unused in livequery +SELECT 1 diff --git a/models/overrides/core/bronze/bronze__transactions_fr.sql b/models/overrides/core/bronze/bronze__transactions_fr.sql index ae3a491f..c5a85219 100644 --- a/models/overrides/core/bronze/bronze__transactions_fr.sql +++ b/models/overrides/core/bronze/bronze__transactions_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 1000, 10) | indent(4) -}} ), raw_block_txs AS ( From 2a7308b846520aaaa5e6d32fc1820d723461cfa2 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 10 Feb 2025 16:26:31 +0800 Subject: [PATCH 04/15] add row_count as signature --- macros/evm/evm.yaml.sql | 4 ++++ macros/evm/evm_live_views.sql | 4 ++-- models/overrides/core/bronze/bronze__blocks_fr.sql | 2 +- models/overrides/core/bronze/bronze__receipts_fr.sql | 2 +- models/overrides/core/bronze/bronze__traces_fr.sql | 2 +- models/overrides/core/bronze/bronze__transactions_fr.sql | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 1aa462a5..fcc67897 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -367,6 +367,7 @@ signature: - [block_height, INTEGER, The start block height to get the blocks from] - [to_latest, BOOLEAN, Whether to continue fetching blocks until the latest block or not] + - [row_count, INTEGER, The number of blocks to fetch] return_type: - "{{ generate_udtf_return_type(blockchain, get_fact_blocks_columns()) }}" options: | @@ -381,6 +382,7 @@ signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [row_count, INTEGER, The number of blocks to fetch] return_type: - "{{ generate_udtf_return_type(blockchain, get_fact_transactions_columns()) }}" options: | @@ -395,6 +397,7 @@ signature: - [block_height, INTEGER, The start block height to get the events from] - [to_latest, BOOLEAN, Whether to continue fetching events until the latest block or not] + - [row_count, INTEGER, The number of blocks to fetch] return_type: - "TABLE( block_number INTEGER, @@ -430,6 +433,7 @@ signature: - [block_height, INTEGER, The start block height to get the traces from] - [to_latest, BOOLEAN, Whether to continue fetching traces until the latest block or not] + - [row_count, INTEGER, The number of blocks to fetch] return_type: - "{{ generate_udtf_return_type(blockchain, get_fact_traces_columns()) }}" options: | diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index de783459..b7983823 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -13,7 +13,7 @@ ) AS max_height {% endmacro %} -{% macro evm_live_view_target_blocks(schema, blockchain, network, row_count=1000, batch_size=10) %} +{% macro evm_live_view_target_blocks(schema, blockchain, network, batch_size=10) %} WITH heights AS ( {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} ), @@ -31,7 +31,7 @@ ) AS max_height, latest_block_height FROM - TABLE(generator(ROWCOUNT => {{ row_count }})), + TABLE(generator(ROWCOUNT => COALESCE(row_count, 100))), heights qualify block_number BETWEEN min_height AND max_height ) diff --git a/models/overrides/core/bronze/bronze__blocks_fr.sql b/models/overrides/core/bronze/bronze__blocks_fr.sql index a4549206..65d52f02 100644 --- a/models/overrides/core/bronze/bronze__blocks_fr.sql +++ b/models/overrides/core/bronze/bronze__blocks_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 1000, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_blocks AS ( diff --git a/models/overrides/core/bronze/bronze__receipts_fr.sql b/models/overrides/core/bronze/bronze__receipts_fr.sql index adb1c1fa..c3804b86 100644 --- a/models/overrides/core/bronze/bronze__receipts_fr.sql +++ b/models/overrides/core/bronze/bronze__receipts_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 100, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_receipts AS ( diff --git a/models/overrides/core/bronze/bronze__traces_fr.sql b/models/overrides/core/bronze/bronze__traces_fr.sql index f0036226..71a19b8a 100644 --- a/models/overrides/core/bronze/bronze__traces_fr.sql +++ b/models/overrides/core/bronze/bronze__traces_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 100, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_traces AS ( diff --git a/models/overrides/core/bronze/bronze__transactions_fr.sql b/models/overrides/core/bronze/bronze__transactions_fr.sql index c5a85219..ae3a491f 100644 --- a/models/overrides/core/bronze/bronze__transactions_fr.sql +++ b/models/overrides/core/bronze/bronze__transactions_fr.sql @@ -3,7 +3,7 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 1000, 10) | indent(4) -}} + {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_block_txs AS ( From 5584885dc0af5f0d936db686718d75aa427a5523 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 10 Feb 2025 22:59:30 +0800 Subject: [PATCH 05/15] add ez_native_transfers udtf --- macros/evm/evm.yaml.sql | 34 +++++++++++++++++++ macros/evm/evm_live_views.sql | 7 +++- models/deploy/evm/ethereum__mainnet.sql | 2 ++ .../core/gold/evm__ez_native_transfers.sql | 18 ++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 models/overrides/core/gold/evm__ez_native_transfers.sql diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index fcc67897..e83179e5 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -444,6 +444,40 @@ sql: | {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_ez_native_transfers + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [row_count, INTEGER, The number of blocks to fetch] + return_type: + - "TABLE( + tx_hash STRING, + block_number NUMBER, + block_timestamp TIMESTAMP_NTZ, + tx_position NUMBER, + trace_index NUMBER, + origin_from_address STRING, + origin_to_address STRING, + origin_function_signature STRING, + from_address STRING, + to_address STRING, + amount FLOAT, + amount_precise_raw STRING, + amount_precise STRING, + amount_usd FLOAT, + ez_native_transfers_id STRING, + inserted_timestamp TIMESTAMP_NTZ, + modified_timestamp TIMESTAMP_NTZ, + identifier STRING + )" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the native transfers for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ + sql: | + {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} + {%- endmacro -%} {% macro config_eth_high_level_abstractions(blockchain, network) -%} diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index b7983823..caebfd8e 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -31,7 +31,7 @@ ) AS max_height, latest_block_height FROM - TABLE(generator(ROWCOUNT => COALESCE(row_count, 100))), + TABLE(generator(ROWCOUNT => COALESCE(row_count, 1))), heights qualify block_number BETWEEN min_height AND max_height ) @@ -195,6 +195,11 @@ LATERAL FLATTEN(input => result) v {{ evm__fact_traces }} {% endmacro %} +{% macro evm_ez_native_transfers(schema, blockchain, network) %} + {%- set evm__ez_native_transfers = get_rendered_model('livequery_models', 'evm__ez_native_transfers', schema, blockchain, network) -%} + {{ evm__ez_native_transfers }} +{% endmacro %} + -- UDTF Return Columns {% macro get_fact_blocks_columns() %} {% set columns = [ diff --git a/models/deploy/evm/ethereum__mainnet.sql b/models/deploy/evm/ethereum__mainnet.sql index a2b9be5d..270e04b7 100644 --- a/models/deploy/evm/ethereum__mainnet.sql +++ b/models/deploy/evm/ethereum__mainnet.sql @@ -5,6 +5,7 @@ -- depends_on: {{ ref('_eth__decoded_logs') }} -- depends_on: {{ ref('live') }} -- depends_on: {{ ref('utils') }} +-- depends_on: {{ ref('price__ez_prices_hourly') }} -- depends_on: {{ ref('bronze__blocks') }} -- depends_on: {{ ref('bronze__blocks_fr') }} -- depends_on: {{ ref('bronze__transactions') }} @@ -23,6 +24,7 @@ -- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} -- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} -- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} +-- depends_on: {{ ref('fsc_evm', 'core__ez_native_transfers') }} {%- set configs = [ config_evm_rpc_primitives, config_evm_high_level_abstractions, diff --git a/models/overrides/core/gold/evm__ez_native_transfers.sql b/models/overrides/core/gold/evm__ez_native_transfers.sql new file mode 100644 index 00000000..0364b67e --- /dev/null +++ b/models/overrides/core/gold/evm__ez_native_transfers.sql @@ -0,0 +1,18 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('bronze__traces') }} +-- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} +-- depends_on: {{ ref('price__ez_prices_hourly') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__ez_native_transfers') }} From 0f31a43eec128772b58f27ec9e7cea5e003a01b8 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 11 Feb 2025 16:41:42 +0800 Subject: [PATCH 06/15] update --- dbt_project.yml | 8 ++ macros/evm/evm.yaml.sql | 57 ++++++++++++-- macros/evm/evm_live_views.sql | 49 ++++++++---- macros/utils/udtf_utils/render.sql | 15 ++-- macros/utils/udtf_utils/udtf_return_type.sql | 74 +++++++++---------- models/deploy/evm/ethereum__mainnet.sql | 5 ++ .../core/bronze/bronze__transactions_fr.sql | 17 ----- .../bronze/bronze__decoded_logs.sql | 1 + .../bronze/bronze__decoded_logs_fr.sql | 18 +++++ .../gold/evm__ez_decoded_event_logs.sql | 6 ++ .../core/bronze/bronze__blocks.sql | 0 .../core/bronze/bronze__blocks_fr.sql | 4 +- .../core/bronze/bronze__receipts.sql | 0 .../core/bronze/bronze__receipts_fr.sql | 4 +- .../core/bronze/bronze__traces.sql | 0 .../core/bronze/bronze__traces_fr.sql | 4 +- .../core/bronze/bronze__transactions.sql | 0 .../core/bronze/bronze__transactions_fr.sql | 17 +++++ .../core/gold/core__dim_contracts.sql | 3 + .../core/gold/evm__ez_native_transfers.sql | 0 .../core/gold/evm__fact_blocks.sql | 0 .../core/gold/evm__fact_event_logs.sql | 0 .../core/gold/evm__fact_traces.sql | 0 .../core/gold/evm__fact_transactions.sql | 4 - .../prices/price__ez_prices_hourly.sql | 0 25 files changed, 197 insertions(+), 89 deletions(-) delete mode 100644 models/overrides/core/bronze/bronze__transactions_fr.sql create mode 100644 models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql create mode 100644 models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql create mode 100644 models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql rename models/overrides/{ => evm/main_package}/core/bronze/bronze__blocks.sql (100%) rename models/overrides/{ => evm/main_package}/core/bronze/bronze__blocks_fr.sql (52%) rename models/overrides/{ => evm/main_package}/core/bronze/bronze__receipts.sql (100%) rename models/overrides/{ => evm/main_package}/core/bronze/bronze__receipts_fr.sql (52%) rename models/overrides/{ => evm/main_package}/core/bronze/bronze__traces.sql (100%) rename models/overrides/{ => evm/main_package}/core/bronze/bronze__traces_fr.sql (52%) rename models/overrides/{ => evm/main_package}/core/bronze/bronze__transactions.sql (100%) create mode 100644 models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql create mode 100644 models/overrides/evm/main_package/core/gold/core__dim_contracts.sql rename models/overrides/{ => evm/main_package}/core/gold/evm__ez_native_transfers.sql (100%) rename models/overrides/{ => evm/main_package}/core/gold/evm__fact_blocks.sql (100%) rename models/overrides/{ => evm/main_package}/core/gold/evm__fact_event_logs.sql (100%) rename models/overrides/{ => evm/main_package}/core/gold/evm__fact_traces.sql (100%) rename models/overrides/{ => evm/main_package}/core/gold/evm__fact_transactions.sql (65%) rename models/overrides/{ => evm/main_package}/prices/price__ez_prices_hourly.sql (100%) diff --git a/dbt_project.yml b/dbt_project.yml index df9a3b6f..e2f6e0b4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -41,6 +41,14 @@ models: fsc_evm: +enabled: false # disable package by default +materialized: ephemeral + decoder_package: + decoded_logs: + gold: + core__ez_decoded_event_logs: + +enabled: true + silver: + silver__decoded_logs: + +enabled: true main_package: core: gold: diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index e83179e5..d6470e3e 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -367,7 +367,7 @@ signature: - [block_height, INTEGER, The start block height to get the blocks from] - [to_latest, BOOLEAN, Whether to continue fetching blocks until the latest block or not] - - [row_count, INTEGER, The number of blocks to fetch] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - "{{ generate_udtf_return_type(blockchain, get_fact_blocks_columns()) }}" options: | @@ -382,7 +382,7 @@ signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - - [row_count, INTEGER, The number of blocks to fetch] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - "{{ generate_udtf_return_type(blockchain, get_fact_transactions_columns()) }}" options: | @@ -397,7 +397,7 @@ signature: - [block_height, INTEGER, The start block height to get the events from] - [to_latest, BOOLEAN, Whether to continue fetching events until the latest block or not] - - [row_count, INTEGER, The number of blocks to fetch] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - "TABLE( block_number INTEGER, @@ -429,11 +429,52 @@ sql: | {{ evm_fact_event_logs(schema, blockchain, network) | indent(4) -}} +- name: {{ schema -}}.tf_ez_decoded_event_logs + signature: + - [block_height, INTEGER, The start block height to get the logs from] + - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] + return_type: + - "TABLE( + block_number INTEGER, + block_timestamp TIMESTAMP_NTZ, + tx_hash STRING, + tx_position INTEGER, + event_index INTEGER, + contract_address STRING, + topics VARIANT, + topic_0 STRING, + topic_1 STRING, + topic_2 STRING, + topic_3 STRING, + DATA STRING, + event_removed BOOLEAN, + origin_from_address STRING, + origin_to_address STRING, + origin_function_signature STRING, + tx_succeeded BOOLEAN, + event_name STRING, + full_decoded_log VARIANT, + decoded_log OBJECT, + contract_name STRING, + ez_decoded_event_logs_id STRING, + inserted_timestamp TIMESTAMP_NTZ, + modified_timestamp TIMESTAMP_NTZ)" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the ez decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ + sql: | + {{ evm_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} + + +{# - name: {{ schema -}}.tf_fact_traces signature: - [block_height, INTEGER, The start block height to get the traces from] - [to_latest, BOOLEAN, Whether to continue fetching traces until the latest block or not] - - [row_count, INTEGER, The number of blocks to fetch] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - "{{ generate_udtf_return_type(blockchain, get_fact_traces_columns()) }}" options: | @@ -442,13 +483,15 @@ VOLATILE COMMENT = $$Returns the traces for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} + {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} #} + +{# - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - - [row_count, INTEGER, The number of blocks to fetch] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - "TABLE( tx_hash STRING, @@ -476,7 +519,7 @@ VOLATILE COMMENT = $$Returns the native transfers for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} + {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} #} {%- endmacro -%} diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index caebfd8e..d4469179 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1,4 +1,4 @@ -{% macro evm_live_view_latest_block_height(schema, blockchain, network) %} +{% macro evm_latest_block_height(schema, blockchain, network) %} SELECT {{ schema }}.udf_rpc('eth_blockNumber', []) as result, utils.udf_hex_to_int(result)::integer AS latest_block_height, @@ -13,9 +13,9 @@ ) AS max_height {% endmacro %} -{% macro evm_live_view_target_blocks(schema, blockchain, network, batch_size=10) %} +{% macro evm_target_blocks(schema, blockchain, network, batch_size=10) %} WITH heights AS ( - {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} + {{ evm_latest_block_height(schema, blockchain, network) | indent(4) -}} ), block_spine AS ( SELECT @@ -31,7 +31,7 @@ ) AS max_height, latest_block_height FROM - TABLE(generator(ROWCOUNT => COALESCE(row_count, 1))), + TABLE(generator(ROWCOUNT => COALESCE(block_size, 1))), heights qualify block_number BETWEEN min_height AND max_height ) @@ -52,7 +52,7 @@ {% endmacro %} -- Get Raw EVM chain data -{% macro evm_live_view_bronze_blocks(schema, blockchain, network, table_name) %} +{% macro evm_bronze_blocks(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( SELECT batch_id, @@ -84,7 +84,7 @@ SELECT FROM flattened {% endmacro %} -{% macro evm_live_view_bronze_receipts(schema, blockchain, network, table_name) %} +{% macro evm_bronze_receipts(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( SELECT batch_id, @@ -122,7 +122,7 @@ FROM LATERAL FLATTEN(data) v), LATERAL FLATTEN(data) w {% endmacro %} -{% macro evm_live_view_bronze_transactions(table_name) %} +{% macro evm_bronze_transactions(table_name) %} SELECT 0 AS partition_key, block_number, @@ -137,7 +137,7 @@ FROM lateral flatten(r.data:transactions) v {% endmacro %} -{% macro evm_live_view_bronze_traces(schema, blockchain, network, table_name)%} +{% macro evm_bronze_traces(schema, blockchain, network, table_name)%} WITH blocks_agg AS ( SELECT batch_id, @@ -176,30 +176,53 @@ LATERAL FLATTEN(input => result) v {% endmacro %} {% macro evm_fact_blocks(schema, blockchain, network) %} - {%- set evm__fact_blocks = get_rendered_model('livequery_models', 'evm__fact_blocks', schema, blockchain, network) -%} + {%- set evm__fact_blocks = get_rendered_model('livequery_models', 'evm__fact_blocks', schema, blockchain, network, 'True') -%} {{ evm__fact_blocks }} {% endmacro %} {% macro evm_fact_transactions(schema, blockchain, network) %} - {%- set evm__fact_transactions = get_rendered_model('livequery_models', 'evm__fact_transactions', schema, blockchain, network) -%} + {%- set evm__fact_transactions = get_rendered_model('livequery_models', 'evm__fact_transactions', schema, blockchain, network, 'False') -%} + + WITH __dbt__cte__core__fact_blocks AS ( + SELECT * FROM table({{ schema }}.tf_fact_blocks(block_height, to_latest, block_size)) + ), + {{ evm__fact_transactions }} {% endmacro %} {% macro evm_fact_event_logs(schema, blockchain, network) %} - {%- set evm__fact_event_logs = get_rendered_model('livequery_models', 'evm__fact_event_logs', schema, blockchain, network) -%} + {%- set evm__fact_event_logs = get_rendered_model('livequery_models', 'evm__fact_event_logs', schema, blockchain, network, 'True') -%} {{ evm__fact_event_logs }} {% endmacro %} {% macro evm_fact_traces(schema, blockchain, network) %} - {%- set evm__fact_traces = get_rendered_model('livequery_models', 'evm__fact_traces', schema, blockchain, network) -%} + {%- set evm__fact_traces = get_rendered_model('livequery_models', 'evm__fact_traces', schema, blockchain, network, 'True') -%} {{ evm__fact_traces }} {% endmacro %} {% macro evm_ez_native_transfers(schema, blockchain, network) %} - {%- set evm__ez_native_transfers = get_rendered_model('livequery_models', 'evm__ez_native_transfers', schema, blockchain, network) -%} + {%- set evm__ez_native_transfers = get_rendered_model('livequery_models', 'evm__ez_native_transfers', schema, blockchain, network, 'True') -%} {{ evm__ez_native_transfers }} {% endmacro %} +{% macro evm_ez_decoded_event_logs(schema, blockchain, network) %} + {%- set evm__ez_decoded_event_logs = get_rendered_model('livequery_models', 'evm__ez_decoded_event_logs', schema, blockchain, network, 'False') -%} + + WITH __dbt__cte__core__fact_blocks AS ( + SELECT * FROM table({{ schema }}.tf_fact_blocks(block_height, to_latest, block_size)) + ) + + , __dbt_cte__core__fact_transactions AS ( + SELECT * FROM table({{ schema }}.tf_fact_transactions(block_height, to_latest, block_size)) + ) + + , __dbt_cte__core__fact_event_logs AS ( + SELECT * FROM table({{ schema }}.tf_fact_event_logs(block_height, to_latest, block_size)) + ), + + {{ evm__ez_decoded_event_logs }} +{% endmacro %} + -- UDTF Return Columns {% macro get_fact_blocks_columns() %} {% set columns = [ diff --git a/macros/utils/udtf_utils/render.sql b/macros/utils/udtf_utils/render.sql index 980c84b4..54a371ba 100644 --- a/macros/utils/udtf_utils/render.sql +++ b/macros/utils/udtf_utils/render.sql @@ -1,14 +1,15 @@ -{% macro get_rendered_model(package_name, model_name, schema, blockchain, network) %} - {# +{% macro get_rendered_model(package_name, model_name, schema, blockchain, network, start_sql_with_clause) %} + {# This macro retrieves and renders a specified model from the graph. - + Args: package_name (str): The name of the package containing the model. model_name (str): The name of the model to be rendered. schema (str): The schema to be used. blockchain (str): The blockchain to be used. network (str): The network to be used. - + start_sql_with_clause (str): Whether to start the SQL with a WITH clause. + Returns: str: The rendered SQL of the specified model. #} @@ -53,7 +54,11 @@ __dbt__cte__{{ dep_node.name }} AS ( {# Combine CTEs with main query #} {%- set final_sql -%} +{%- if start_sql_with_clause.upper() == 'TRUE' -%} WITH {{ ctes | join(',\n\n') }} +{%- else -%} + {{ ctes | join(',\n\n') }} +{%- endif -%} {{ render(target_node.raw_code) }} {%- endset -%} @@ -62,4 +67,4 @@ WITH {{ ctes | join(',\n\n') }} {{ return(final_sql) }} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/utils/udtf_utils/udtf_return_type.sql b/macros/utils/udtf_utils/udtf_return_type.sql index 18e069f1..a2e30997 100644 --- a/macros/utils/udtf_utils/udtf_return_type.sql +++ b/macros/utils/udtf_utils/udtf_return_type.sql @@ -2,53 +2,53 @@ {% macro generate_udtf_return_type(blockchain, columns) %} {# Set Global Variables #} - {% set GLOBAL_PROD_DB_NAME = blockchain %} + {%- set GLOBAL_PROD_DB_NAME = blockchain -%} {# Columns included by default, with specific exclusions #} - {% set excludes_eip_1559 = ['CORE','RONIN'] %} - {% set excludes_base_fee = ['CORE'] %} - {% set excludes_total_difficulty = ['INK','SWELL'] %} + {%- set excludes_eip_1559 = ['CORE','RONIN'] -%} + {%- set excludes_base_fee = ['CORE'] -%} + {%- set excludes_total_difficulty = ['INK','SWELL'] -%} {# Columns excluded by default, with explicit inclusion #} - {% set includes_l1_columns = ['INK', 'MANTLE', 'SWELL'] %} - {% set includes_l1_tx_fee_calc = ['INK', 'MANTLE', 'SWELL'] %} - {% set includes_eth_value = ['MANTLE'] %} - {% set includes_mint = ['INK', 'MANTLE', 'SWELL'] %} - {% set includes_y_parity = ['INK', 'SWELL'] %} - {% set includes_access_list = ['INK', 'SWELL'] %} - {% set includes_source_hash = ['INK','MANTLE','SWELL'] %} - {% set includes_blob_base_fee = ['INK','SWELL'] %} - {% set includes_mix_hash = ['INK', 'MANTLE', 'SWELL', 'RONIN'] %} - {% set includes_blob_gas_used = ['INK', 'SWELL'] %} - {% set includes_parent_beacon_block_root = ['INK', 'SWELL'] %} - {% set includes_withdrawals = ['INK', 'SWELL'] %} + {%- set includes_l1_columns = ['INK', 'MANTLE', 'SWELL'] -%} + {%- set includes_l1_tx_fee_calc = ['INK', 'MANTLE', 'SWELL'] -%} + {%- set includes_eth_value = ['MANTLE'] -%} + {%- set includes_mint = ['INK', 'MANTLE', 'SWELL'] -%} + {%- set includes_y_parity = ['INK', 'SWELL'] -%} + {%- set includes_access_list = ['INK', 'SWELL'] -%} + {%- set includes_source_hash = ['INK','MANTLE','SWELL'] -%} + {%- set includes_blob_base_fee = ['INK','SWELL'] -%} + {%- set includes_mix_hash = ['INK', 'MANTLE', 'SWELL', 'RONIN'] -%} + {%- set includes_blob_gas_used = ['INK', 'SWELL'] -%} + {%- set includes_parent_beacon_block_root = ['INK', 'SWELL'] -%} + {%- set includes_withdrawals = ['INK', 'SWELL'] -%} {# Set Variables using inclusions and exclusions #} - {% set current_db = blockchain.upper() %} + {%- set current_db = blockchain.upper() -%} {# Transaction feature flags #} - {% set uses_eip_1559 = current_db not in excludes_eip_1559 %} - {% set uses_l1_columns = current_db in includes_l1_columns %} - {% set uses_l1_tx_fee_calc = current_db in includes_l1_tx_fee_calc %} - {% set uses_eth_value = current_db in includes_eth_value %} - {% set uses_mint = current_db in includes_mint %} - {% set uses_y_parity = current_db in includes_y_parity %} - {% set uses_access_list = current_db in includes_access_list %} - {% set uses_source_hash = current_db in includes_source_hash %} - {% set uses_blob_base_fee = current_db in includes_blob_base_fee %} - {% set uses_traces_arb_mode = var('TRACES_ARB_MODE', false) %} + {%- set uses_eip_1559 = current_db not in excludes_eip_1559 -%} + {%- set uses_l1_columns = current_db in includes_l1_columns -%} + {%- set uses_l1_tx_fee_calc = current_db in includes_l1_tx_fee_calc -%} + {%- set uses_eth_value = current_db in includes_eth_value -%} + {%- set uses_mint = current_db in includes_mint -%} + {%- set uses_y_parity = current_db in includes_y_parity -%} + {%- set uses_access_list = current_db in includes_access_list -%} + {%- set uses_source_hash = current_db in includes_source_hash -%} + {%- set uses_blob_base_fee = current_db in includes_blob_base_fee -%} + {%- set uses_traces_arb_mode = var('TRACES_ARB_MODE', false) -%} {# Block feature flags #} - {% set uses_base_fee = current_db not in excludes_base_fee %} - {% set uses_total_difficulty = current_db not in excludes_total_difficulty %} - {% set uses_mix_hash = current_db in includes_mix_hash %} - {% set uses_blob_gas_used = current_db in includes_blob_gas_used %} - {% set uses_parent_beacon_block_root = current_db in includes_parent_beacon_block_root %} - {% set uses_withdrawals = current_db in includes_withdrawals %} + {%- set uses_base_fee = current_db not in excludes_base_fee -%} + {%- set uses_total_difficulty = current_db not in excludes_total_difficulty -%} + {%- set uses_mix_hash = current_db in includes_mix_hash -%} + {%- set uses_blob_gas_used = current_db in includes_blob_gas_used -%} + {%- set uses_parent_beacon_block_root = current_db in includes_parent_beacon_block_root -%} + {%- set uses_withdrawals = current_db in includes_withdrawals -%} TABLE( - {% for col in columns %} - {% if col.flag is not defined + {%- for col in columns %} + {%- if col.flag is not defined or (col.flag == 'uses_eip_1559' and uses_eip_1559) or (col.flag == 'uses_l1_columns' and uses_l1_columns) or (col.flag == 'uses_eth_value' and uses_eth_value) @@ -66,7 +66,7 @@ or (col.flag == 'uses_withdrawals' and uses_withdrawals) %} {{ col.name }} {{ col.type }}{% if not loop.last %},{% endif %} - {% endif %} - {% endfor %} + {%- endif -%} + {%- endfor -%} ) {% endmacro %} diff --git a/models/deploy/evm/ethereum__mainnet.sql b/models/deploy/evm/ethereum__mainnet.sql index 270e04b7..371e8d06 100644 --- a/models/deploy/evm/ethereum__mainnet.sql +++ b/models/deploy/evm/ethereum__mainnet.sql @@ -6,6 +6,7 @@ -- depends_on: {{ ref('live') }} -- depends_on: {{ ref('utils') }} -- depends_on: {{ ref('price__ez_prices_hourly') }} +-- depends_on: {{ ref('core__dim_contracts')}} -- depends_on: {{ ref('bronze__blocks') }} -- depends_on: {{ ref('bronze__blocks_fr') }} -- depends_on: {{ ref('bronze__transactions') }} @@ -14,10 +15,13 @@ -- depends_on: {{ ref('bronze__receipts_fr') }} -- depends_on: {{ ref('bronze__traces') }} -- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('bronze__decoded_logs') }} +-- depends_on: {{ ref('bronze__decoded_logs_fr') }} -- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} -- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} -- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} -- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('fsc_evm', 'silver__decoded_logs') }} -- depends_on: {{ ref('evm__fact_blocks') }} -- depends_on: {{ ref('evm__fact_transactions') }} -- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} @@ -25,6 +29,7 @@ -- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} -- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} -- depends_on: {{ ref('fsc_evm', 'core__ez_native_transfers') }} +-- depends_on: {{ ref('fsc_evm', 'core__ez_decoded_event_logs') }} {%- set configs = [ config_evm_rpc_primitives, config_evm_high_level_abstractions, diff --git a/models/overrides/core/bronze/bronze__transactions_fr.sql b/models/overrides/core/bronze/bronze__transactions_fr.sql deleted file mode 100644 index ae3a491f..00000000 --- a/models/overrides/core/bronze/bronze__transactions_fr.sql +++ /dev/null @@ -1,17 +0,0 @@ -{%- set blockchain = this.schema -%} -{%- set network = this.identifier -%} -{%- set schema = blockchain ~ "_" ~ network -%} - -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} -), - -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), - -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -) - -SELECT * FROM raw_transactions diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql new file mode 100644 index 00000000..2e3761f7 --- /dev/null +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql @@ -0,0 +1 @@ +SELECT 1 diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql new file mode 100644 index 00000000..bad3e0f9 --- /dev/null +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql @@ -0,0 +1,18 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + + SELECT + block_number, + fact_event_logs_id AS id, + utils.udf_evm_decode_log(abi, event_data)[0] AS DATA, + TO_TIMESTAMP_NTZ(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('fsc_evm', 'core__fact_event_logs')}} + JOIN + {{ blockchain }}.core.dim_contract_abis + USING + (contract_address) + WHERE + tx_status = 'SUCCESS' + diff --git a/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql b/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql new file mode 100644 index 00000000..d190e4c9 --- /dev/null +++ b/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql @@ -0,0 +1,6 @@ +-- depends_on: {{ ref('bronze__decoded_logs') }} +-- depends_on: {{ ref('bronze__decoded_logs_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__decoded_logs') }} +-- depends_on: {{ ref('core__dim_contracts')}} + +SELECT * FROM {{ ref('fsc_evm', 'core__ez_decoded_event_logs') }} diff --git a/models/overrides/core/bronze/bronze__blocks.sql b/models/overrides/evm/main_package/core/bronze/bronze__blocks.sql similarity index 100% rename from models/overrides/core/bronze/bronze__blocks.sql rename to models/overrides/evm/main_package/core/bronze/bronze__blocks.sql diff --git a/models/overrides/core/bronze/bronze__blocks_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__blocks_fr.sql similarity index 52% rename from models/overrides/core/bronze/bronze__blocks_fr.sql rename to models/overrides/evm/main_package/core/bronze/bronze__blocks_fr.sql index 65d52f02..81e31ccb 100644 --- a/models/overrides/core/bronze/bronze__blocks_fr.sql +++ b/models/overrides/evm/main_package/core/bronze/bronze__blocks_fr.sql @@ -3,11 +3,11 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_blocks AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} + {{ evm_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} ) SELECT * FROM raw_blocks diff --git a/models/overrides/core/bronze/bronze__receipts.sql b/models/overrides/evm/main_package/core/bronze/bronze__receipts.sql similarity index 100% rename from models/overrides/core/bronze/bronze__receipts.sql rename to models/overrides/evm/main_package/core/bronze/bronze__receipts.sql diff --git a/models/overrides/core/bronze/bronze__receipts_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__receipts_fr.sql similarity index 52% rename from models/overrides/core/bronze/bronze__receipts_fr.sql rename to models/overrides/evm/main_package/core/bronze/bronze__receipts_fr.sql index c3804b86..80b72ea9 100644 --- a/models/overrides/core/bronze/bronze__receipts_fr.sql +++ b/models/overrides/evm/main_package/core/bronze/bronze__receipts_fr.sql @@ -3,11 +3,11 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} + {{ evm_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} ) SELECT * FROM raw_receipts diff --git a/models/overrides/core/bronze/bronze__traces.sql b/models/overrides/evm/main_package/core/bronze/bronze__traces.sql similarity index 100% rename from models/overrides/core/bronze/bronze__traces.sql rename to models/overrides/evm/main_package/core/bronze/bronze__traces.sql diff --git a/models/overrides/core/bronze/bronze__traces_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__traces_fr.sql similarity index 52% rename from models/overrides/core/bronze/bronze__traces_fr.sql rename to models/overrides/evm/main_package/core/bronze/bronze__traces_fr.sql index 71a19b8a..ab4d8397 100644 --- a/models/overrides/core/bronze/bronze__traces_fr.sql +++ b/models/overrides/evm/main_package/core/bronze/bronze__traces_fr.sql @@ -3,11 +3,11 @@ {%- set schema = blockchain ~ "_" ~ network -%} WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 10) | indent(4) -}} + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} ), raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} + {{ evm_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} ) SELECT * FROM raw_traces diff --git a/models/overrides/core/bronze/bronze__transactions.sql b/models/overrides/evm/main_package/core/bronze/bronze__transactions.sql similarity index 100% rename from models/overrides/core/bronze/bronze__transactions.sql rename to models/overrides/evm/main_package/core/bronze/bronze__transactions.sql diff --git a/models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql new file mode 100644 index 00000000..bd8c9805 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql @@ -0,0 +1,17 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_block_txs AS ( + {{ evm_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +), + +raw_transactions AS ( + {{ evm_bronze_transactions('raw_block_txs') | indent(4) -}} +) + +SELECT * FROM raw_transactions diff --git a/models/overrides/evm/main_package/core/gold/core__dim_contracts.sql b/models/overrides/evm/main_package/core/gold/core__dim_contracts.sql new file mode 100644 index 00000000..34927953 --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/core__dim_contracts.sql @@ -0,0 +1,3 @@ +{%- set blockchain = this.schema -%} + +SELECT * FROM {{ blockchain.upper() }}.CORE.DIM_CONTRACTS diff --git a/models/overrides/core/gold/evm__ez_native_transfers.sql b/models/overrides/evm/main_package/core/gold/evm__ez_native_transfers.sql similarity index 100% rename from models/overrides/core/gold/evm__ez_native_transfers.sql rename to models/overrides/evm/main_package/core/gold/evm__ez_native_transfers.sql diff --git a/models/overrides/core/gold/evm__fact_blocks.sql b/models/overrides/evm/main_package/core/gold/evm__fact_blocks.sql similarity index 100% rename from models/overrides/core/gold/evm__fact_blocks.sql rename to models/overrides/evm/main_package/core/gold/evm__fact_blocks.sql diff --git a/models/overrides/core/gold/evm__fact_event_logs.sql b/models/overrides/evm/main_package/core/gold/evm__fact_event_logs.sql similarity index 100% rename from models/overrides/core/gold/evm__fact_event_logs.sql rename to models/overrides/evm/main_package/core/gold/evm__fact_event_logs.sql diff --git a/models/overrides/core/gold/evm__fact_traces.sql b/models/overrides/evm/main_package/core/gold/evm__fact_traces.sql similarity index 100% rename from models/overrides/core/gold/evm__fact_traces.sql rename to models/overrides/evm/main_package/core/gold/evm__fact_traces.sql diff --git a/models/overrides/core/gold/evm__fact_transactions.sql b/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql similarity index 65% rename from models/overrides/core/gold/evm__fact_transactions.sql rename to models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql index fae0fa25..5af414a6 100644 --- a/models/overrides/core/gold/evm__fact_transactions.sql +++ b/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql @@ -1,12 +1,8 @@ --- depends_on: {{ ref('bronze__blocks') }} --- depends_on: {{ ref('bronze__blocks_fr') }} -- depends_on: {{ ref('bronze__transactions') }} -- depends_on: {{ ref('bronze__transactions_fr') }} -- depends_on: {{ ref('bronze__receipts') }} -- depends_on: {{ ref('bronze__receipts_fr') }} --- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} -- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} --- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} -- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} SELECT * FROM {{ ref('fsc_evm', 'core__fact_transactions') }} diff --git a/models/overrides/prices/price__ez_prices_hourly.sql b/models/overrides/evm/main_package/prices/price__ez_prices_hourly.sql similarity index 100% rename from models/overrides/prices/price__ez_prices_hourly.sql rename to models/overrides/evm/main_package/prices/price__ez_prices_hourly.sql From 106998095f18d8dbf16da0f3053c49a4a894a61b Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 11 Feb 2025 21:33:35 +0800 Subject: [PATCH 07/15] update temp --- macros/evm/evm_live_views.sql | 23 ++++--------------- macros/utils/udtf_utils/render.sql | 7 +----- .../core/gold/evm__fact_transactions.sql | 4 ++++ 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index d4469179..81eb1ca2 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -176,7 +176,7 @@ LATERAL FLATTEN(input => result) v {% endmacro %} {% macro evm_fact_blocks(schema, blockchain, network) %} - {%- set evm__fact_blocks = get_rendered_model('livequery_models', 'evm__fact_blocks', schema, blockchain, network, 'True') -%} + {%- set evm__fact_blocks = get_rendered_model('livequery_models', 'evm__fact_blocks', schema, blockchain, network) -%} {{ evm__fact_blocks }} {% endmacro %} @@ -191,35 +191,22 @@ LATERAL FLATTEN(input => result) v {% endmacro %} {% macro evm_fact_event_logs(schema, blockchain, network) %} - {%- set evm__fact_event_logs = get_rendered_model('livequery_models', 'evm__fact_event_logs', schema, blockchain, network, 'True') -%} + {%- set evm__fact_event_logs = get_rendered_model('livequery_models', 'evm__fact_event_logs', schema, blockchain, network) -%} {{ evm__fact_event_logs }} {% endmacro %} {% macro evm_fact_traces(schema, blockchain, network) %} - {%- set evm__fact_traces = get_rendered_model('livequery_models', 'evm__fact_traces', schema, blockchain, network, 'True') -%} + {%- set evm__fact_traces = get_rendered_model('livequery_models', 'evm__fact_traces', schema, blockchain, network) -%} {{ evm__fact_traces }} {% endmacro %} {% macro evm_ez_native_transfers(schema, blockchain, network) %} - {%- set evm__ez_native_transfers = get_rendered_model('livequery_models', 'evm__ez_native_transfers', schema, blockchain, network, 'True') -%} + {%- set evm__ez_native_transfers = get_rendered_model('livequery_models', 'evm__ez_native_transfers', schema, blockchain, network) -%} {{ evm__ez_native_transfers }} {% endmacro %} {% macro evm_ez_decoded_event_logs(schema, blockchain, network) %} - {%- set evm__ez_decoded_event_logs = get_rendered_model('livequery_models', 'evm__ez_decoded_event_logs', schema, blockchain, network, 'False') -%} - - WITH __dbt__cte__core__fact_blocks AS ( - SELECT * FROM table({{ schema }}.tf_fact_blocks(block_height, to_latest, block_size)) - ) - - , __dbt_cte__core__fact_transactions AS ( - SELECT * FROM table({{ schema }}.tf_fact_transactions(block_height, to_latest, block_size)) - ) - - , __dbt_cte__core__fact_event_logs AS ( - SELECT * FROM table({{ schema }}.tf_fact_event_logs(block_height, to_latest, block_size)) - ), - + {%- set evm__ez_decoded_event_logs = get_rendered_model('livequery_models', 'evm__ez_decoded_event_logs', schema, blockchain, network) -%} {{ evm__ez_decoded_event_logs }} {% endmacro %} diff --git a/macros/utils/udtf_utils/render.sql b/macros/utils/udtf_utils/render.sql index 54a371ba..ad0f3f09 100644 --- a/macros/utils/udtf_utils/render.sql +++ b/macros/utils/udtf_utils/render.sql @@ -1,4 +1,4 @@ -{% macro get_rendered_model(package_name, model_name, schema, blockchain, network, start_sql_with_clause) %} +{% macro get_rendered_model(package_name, model_name, schema, blockchain, network) %} {# This macro retrieves and renders a specified model from the graph. @@ -8,7 +8,6 @@ schema (str): The schema to be used. blockchain (str): The blockchain to be used. network (str): The network to be used. - start_sql_with_clause (str): Whether to start the SQL with a WITH clause. Returns: str: The rendered SQL of the specified model. @@ -54,11 +53,7 @@ __dbt__cte__{{ dep_node.name }} AS ( {# Combine CTEs with main query #} {%- set final_sql -%} -{%- if start_sql_with_clause.upper() == 'TRUE' -%} WITH {{ ctes | join(',\n\n') }} -{%- else -%} - {{ ctes | join(',\n\n') }} -{%- endif -%} {{ render(target_node.raw_code) }} {%- endset -%} diff --git a/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql b/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql index 5af414a6..0bd78a4d 100644 --- a/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql +++ b/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql @@ -1,3 +1,7 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} -- depends_on: {{ ref('bronze__transactions') }} -- depends_on: {{ ref('bronze__transactions_fr') }} -- depends_on: {{ ref('bronze__receipts') }} From 8e29a5aefd47f4b70178118bcdb0653436bb0f33 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 11 Feb 2025 21:44:47 +0800 Subject: [PATCH 08/15] update tf transactions --- macros/evm/evm_live_views.sql | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 81eb1ca2..b39379f0 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -181,12 +181,7 @@ LATERAL FLATTEN(input => result) v {% endmacro %} {% macro evm_fact_transactions(schema, blockchain, network) %} - {%- set evm__fact_transactions = get_rendered_model('livequery_models', 'evm__fact_transactions', schema, blockchain, network, 'False') -%} - - WITH __dbt__cte__core__fact_blocks AS ( - SELECT * FROM table({{ schema }}.tf_fact_blocks(block_height, to_latest, block_size)) - ), - + {%- set evm__fact_transactions = get_rendered_model('livequery_models', 'evm__fact_transactions', schema, blockchain, network) -%} {{ evm__fact_transactions }} {% endmacro %} From dce15137c5b705eb47264562f503a1d372877b75 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Tue, 11 Feb 2025 21:58:22 +0800 Subject: [PATCH 09/15] update decoded event logs --- .../decoded_logs/gold/evm__ez_decoded_event_logs.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql b/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql index d190e4c9..6f17667e 100644 --- a/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql +++ b/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql @@ -1,3 +1,15 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} -- depends_on: {{ ref('bronze__decoded_logs') }} -- depends_on: {{ ref('bronze__decoded_logs_fr') }} -- depends_on: {{ ref('fsc_evm', 'silver__decoded_logs') }} From 41579c927064ccc784429053f3f8c1b63cbad610 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Wed, 12 Feb 2025 16:29:07 +0800 Subject: [PATCH 10/15] update decoded log bronze logic --- .../decoded_logs/bronze/bronze__decoded_logs_fr.sql | 5 +++-- package-lock.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql index bad3e0f9..f6ea5ccf 100644 --- a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql @@ -4,7 +4,8 @@ SELECT block_number, - fact_event_logs_id AS id, + tx_hash :: STRING || '-' || event_index :: STRING AS id, + OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, utils.udf_evm_decode_log(abi, event_data)[0] AS DATA, TO_TIMESTAMP_NTZ(_inserted_timestamp) AS _inserted_timestamp FROM @@ -14,5 +15,5 @@ USING (contract_address) WHERE - tx_status = 'SUCCESS' + tx_succeeded = TRUE diff --git a/package-lock.yml b/package-lock.yml index f0391526..5b6a7e68 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -4,7 +4,7 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: 0898439658da5a0d830c7dc40d9f4859a4f1012d + revision: 31a7c51d6486abdbf9e8c6f2f1a2e7079413df43 - package: calogica/dbt_date version: 0.7.2 - package: dbt-labs/dbt_external_tables From a6c441d3241226173503dfaa1f7f8320a396db8d Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Wed, 12 Feb 2025 21:48:01 +0800 Subject: [PATCH 11/15] add evm ez token transfers --- dbt_project.yml | 2 + macros/evm/evm.yaml.sql | 90 +++++++++---------- macros/evm/evm_live_views.sql | 5 ++ .../core/gold/evm__ez_token_transfers.sql | 15 ++++ 4 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql diff --git a/dbt_project.yml b/dbt_project.yml index e2f6e0b4..17c8a287 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -62,6 +62,8 @@ models: +enabled: true core__ez_native_transfers: +enabled: true + core__ez_token_transfers: + +enabled: true silver: silver__blocks: +enabled: true diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index d6470e3e..4adf80c3 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -399,28 +399,7 @@ - [to_latest, BOOLEAN, Whether to continue fetching events until the latest block or not] - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE( - block_number INTEGER, - block_timestamp TIMESTAMP_NTZ, - tx_hash STRING, - tx_position INTEGER, - event_index INTEGER, - contract_address STRING, - topics VARIANT, - topic_0 STRING, - topic_1 STRING, - topic_2 STRING, - topic_3 STRING, - DATA STRING, - event_removed BOOLEAN, - origin_from_address STRING, - origin_to_address STRING, - origin_function_signature STRING, - tx_succeeded BOOLEAN, - fact_event_logs_id STRING, - inserted_timestamp TIMESTAMP_NTZ, - modified_timestamp TIMESTAMP_NTZ - )" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, tx_position INTEGER, event_index INTEGER, contract_address STRING, topics VARIANT, topic_0 STRING, topic_1 STRING, topic_2 STRING, topic_3 STRING, DATA STRING, event_removed BOOLEAN, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, tx_succeeded BOOLEAN, fact_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -435,31 +414,7 @@ - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE( - block_number INTEGER, - block_timestamp TIMESTAMP_NTZ, - tx_hash STRING, - tx_position INTEGER, - event_index INTEGER, - contract_address STRING, - topics VARIANT, - topic_0 STRING, - topic_1 STRING, - topic_2 STRING, - topic_3 STRING, - DATA STRING, - event_removed BOOLEAN, - origin_from_address STRING, - origin_to_address STRING, - origin_function_signature STRING, - tx_succeeded BOOLEAN, - event_name STRING, - full_decoded_log VARIANT, - decoded_log OBJECT, - contract_name STRING, - ez_decoded_event_logs_id STRING, - inserted_timestamp TIMESTAMP_NTZ, - modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, tx_position INTEGER, event_index INTEGER, contract_address STRING, topics VARIANT, topic_0 STRING, topic_1 STRING, topic_2 STRING, topic_3 STRING, DATA STRING, event_removed BOOLEAN, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, tx_succeeded BOOLEAN, event_name STRING, full_decoded_log VARIANT, decoded_log OBJECT, contract_name STRING, ez_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -521,6 +476,47 @@ sql: | {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} #} +{# +- name: {{ schema -}}.tf_ez_token_transfers + signature: + - [block_height, INTEGER, The start block height to get the transfers from] + - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] + return_type: + - "TABLE( + block_number NUMBER, + block_timestamp TIMESTAMP_NTZ, + tx_hash STRING, + tx_position NUMBER, + event_index NUMBER, + from_address STRING, + to_address STRING, + contract_address STRING, + token_standard STRING, + NAME STRING, + symbol STRING, + decimals NUMBER, + raw_amount_precise STRING, + raw_amount STRING, + amount_precise STRING, + amount FLOAT, + amount_usd FLOAT, + origin_function_signature STRING, + origin_from_address STRING, + origin_to_address STRING, + ez_token_transfers_id STRING, + inserted_timestamp TIMESTAMP_NTZ, + modified_timestamp TIMESTAMP_NTZ + )" + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns the token transfers for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ + sql: | + {{ evm_ez_token_transfers(schema, blockchain, network) | indent(4) -}} #} + + {%- endmacro -%} {% macro config_eth_high_level_abstractions(blockchain, network) -%} diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index b39379f0..bbe5177b 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -200,6 +200,11 @@ LATERAL FLATTEN(input => result) v {{ evm__ez_native_transfers }} {% endmacro %} +{% macro evm_ez_token_transfers(schema, blockchain, network) %} + {%- set evm__ez_token_transfers = get_rendered_model('livequery_models', 'evm__ez_token_transfers', schema, blockchain, network) -%} + {{ evm__ez_token_transfers }} +{% endmacro %} + {% macro evm_ez_decoded_event_logs(schema, blockchain, network) %} {%- set evm__ez_decoded_event_logs = get_rendered_model('livequery_models', 'evm__ez_decoded_event_logs', schema, blockchain, network) -%} {{ evm__ez_decoded_event_logs }} diff --git a/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql b/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql new file mode 100644 index 00000000..28cf9dc6 --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql @@ -0,0 +1,15 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} +-- depends_on: {{ ref('price__ez_prices_hourly') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__ez_token_transfers') }} From 57ba812e4bf4b4b90af2c75ae6aab06b0c4fa430 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 13 Feb 2025 10:59:40 +0800 Subject: [PATCH 12/15] add stored procedure macro --- macros/livequery/manage_stored_procedures.sql | 63 +++++++++++++++++++ .../bronze/bronze__decoded_logs_fr.sql | 2 +- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 macros/livequery/manage_stored_procedures.sql diff --git a/macros/livequery/manage_stored_procedures.sql b/macros/livequery/manage_stored_procedures.sql new file mode 100644 index 00000000..cb222b03 --- /dev/null +++ b/macros/livequery/manage_stored_procedures.sql @@ -0,0 +1,63 @@ +{% macro create_stored_procedure( + name_, + signature, + return_type, + sql_, + language='sql', + execute_as='caller', + options=none +) %} + CREATE OR REPLACE PROCEDURE {{ name_ }}( + {{- compile_signature(signature) }} + ) + RETURNS {{ return_type }} + LANGUAGE {{ language }} + EXECUTE AS {{ execute_as }} + {% if options %} + {{ options }} + {% endif %} + AS + $$ + {{ sql_ }} + $$; +{% endmacro %} + +{% macro ephemeral_deploy_procedures(configs) %} + {%- set blockchain = this.schema -%} + {%- set network = this.identifier -%} + {% set schema = blockchain ~ "_" ~ network %} + + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% set sql %} + {% for config in configs %} + CREATE SCHEMA IF NOT EXISTS {{ schema }}; + + {% for sp in fromyaml(config(blockchain, network)) %} + {% if var("DROP_UDFS_AND_SPS") %} + DROP PROCEDURE IF EXISTS {{ sp.name }}({{ compile_signature(sp.signature, drop_=True) }}); + {% else %} + {{ create_stored_procedure( + name_=sp.name, + signature=sp.signature, + return_type=sp.return_type, + sql_=sp.sql, + language=sp.language | default('sql'), + execute_as=sp.execute_as | default('caller'), + options=sp.options + ) }} + {% endif %} + {% endfor %} + {% endfor %} + {% endset %} + + {% if var("DROP_UDFS_AND_SPS") %} + {% do log("Drop Stored Procedures: " ~ this.database ~ "." ~ schema, true) %} + {% else %} + {% do log("Deploy Stored Procedures: " ~ this.database ~ "." ~ schema, true) %} + {% endif %} + + {% do run_query(sql ~ apply_grants_by_schema(schema)) %} + {% endif %} + + SELECT '{{ model.schema }}' as schema_ +{% endmacro %} diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql index f6ea5ccf..62fe28d5 100644 --- a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql @@ -7,7 +7,7 @@ tx_hash :: STRING || '-' || event_index :: STRING AS id, OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, utils.udf_evm_decode_log(abi, event_data)[0] AS DATA, - TO_TIMESTAMP_NTZ(_inserted_timestamp) AS _inserted_timestamp + TO_TIMESTAMP_NTZ(inserted_timestamp) AS _inserted_timestamp FROM {{ ref('fsc_evm', 'core__fact_event_logs')}} JOIN From f48d5a38771246396a9d0bd49bad4b061fac65b1 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Mon, 24 Feb 2025 15:20:14 +0900 Subject: [PATCH 13/15] temp, fix token transfers --- macros/evm/evm.yaml.sql | 50 +++++++++---------- models/deploy/evm/ethereum__mainnet.sql | 2 +- .../bronze/bronze__decoded_logs_fr.sql | 3 +- .../core/gold/evm__ez_token_transfers.sql | 1 + 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index 4adf80c3..afe22e01 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -408,7 +408,7 @@ sql: | {{ evm_fact_event_logs(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_ez_decoded_event_logs +{# - name: {{ schema -}}.tf_ez_decoded_event_logs signature: - [block_height, INTEGER, The start block height to get the logs from] - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] @@ -421,10 +421,8 @@ VOLATILE COMMENT = $$Returns the ez decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ sql: | - {{ evm_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} + {{ evm_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} #} - -{# - name: {{ schema -}}.tf_fact_traces signature: - [block_height, INTEGER, The start block height to get the traces from] @@ -438,10 +436,8 @@ VOLATILE COMMENT = $$Returns the traces for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} #} - + {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} -{# - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] @@ -449,24 +445,25 @@ - [block_size, INTEGER, The number of blocks to fetch] return_type: - "TABLE( - tx_hash STRING, - block_number NUMBER, + block_number INTEGER, block_timestamp TIMESTAMP_NTZ, - tx_position NUMBER, - trace_index NUMBER, - origin_from_address STRING, - origin_to_address STRING, - origin_function_signature STRING, + tx_hash STRING, + tx_position INTEGER, + trace_index INTEGER, + trace_address STRING, + TYPE STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw STRING, amount_precise STRING, amount_usd FLOAT, + origin_from_address STRING, + origin_to_address STRING, + origin_function_signature STRING, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ, - modified_timestamp TIMESTAMP_NTZ, - identifier STRING + modified_timestamp TIMESTAMP_NTZ )" options: | NOT NULL @@ -474,9 +471,8 @@ VOLATILE COMMENT = $$Returns the native transfers for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} #} + {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} -{# - name: {{ schema -}}.tf_ez_token_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] @@ -484,23 +480,23 @@ - [block_size, INTEGER, The number of blocks to fetch] return_type: - "TABLE( - block_number NUMBER, + block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, - tx_position NUMBER, - event_index NUMBER, + tx_position INTEGER, + event_index INTEGER, from_address STRING, to_address STRING, contract_address STRING, token_standard STRING, NAME STRING, symbol STRING, - decimals NUMBER, + decimals INTEGER, raw_amount_precise STRING, - raw_amount STRING, - amount_precise STRING, - amount FLOAT, - amount_usd FLOAT, + raw_amount FLOAT, + amount_precise_heal STRING, + amount_heal FLOAT, + amount_usd_heal FLOAT, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, @@ -514,7 +510,7 @@ VOLATILE COMMENT = $$Returns the token transfers for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_ez_token_transfers(schema, blockchain, network) | indent(4) -}} #} + {{ evm_ez_token_transfers(schema, blockchain, network) | indent(4) -}} {%- endmacro -%} diff --git a/models/deploy/evm/ethereum__mainnet.sql b/models/deploy/evm/ethereum__mainnet.sql index 371e8d06..43824196 100644 --- a/models/deploy/evm/ethereum__mainnet.sql +++ b/models/deploy/evm/ethereum__mainnet.sql @@ -29,10 +29,10 @@ -- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} -- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} -- depends_on: {{ ref('fsc_evm', 'core__ez_native_transfers') }} +-- depends_on: {{ ref('fsc_evm', 'core__ez_token_transfers') }} -- depends_on: {{ ref('fsc_evm', 'core__ez_decoded_event_logs') }} {%- set configs = [ config_evm_rpc_primitives, config_evm_high_level_abstractions, - config_eth_high_level_abstractions ] -%} {{- ephemeral_deploy(configs) -}} diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql index 62fe28d5..958dbca4 100644 --- a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql @@ -7,7 +7,8 @@ tx_hash :: STRING || '-' || event_index :: STRING AS id, OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, utils.udf_evm_decode_log(abi, event_data)[0] AS DATA, - TO_TIMESTAMP_NTZ(inserted_timestamp) AS _inserted_timestamp + TO_TIMESTAMP_NTZ(inserted_timestamp) AS _inserted_timestamp, + _inserted_timestamp AS _partition_by_created_date FROM {{ ref('fsc_evm', 'core__fact_event_logs')}} JOIN diff --git a/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql b/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql index 28cf9dc6..f872258e 100644 --- a/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql +++ b/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql @@ -11,5 +11,6 @@ -- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} -- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} -- depends_on: {{ ref('price__ez_prices_hourly') }} +-- depends_on: {{ ref('core__dim_contracts') }} SELECT * FROM {{ ref('fsc_evm', 'core__ez_token_transfers') }} From 54c94c27a39158fde7931d6a2842b0d84ce4760e Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 4 Apr 2025 16:11:52 +0900 Subject: [PATCH 14/15] uncomment ez decoded event logs --- macros/evm/evm.yaml.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index afe22e01..0efe4423 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -408,7 +408,7 @@ sql: | {{ evm_fact_event_logs(schema, blockchain, network) | indent(4) -}} -{# - name: {{ schema -}}.tf_ez_decoded_event_logs +- name: {{ schema -}}.tf_ez_decoded_event_logs signature: - [block_height, INTEGER, The start block height to get the logs from] - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] @@ -421,7 +421,7 @@ VOLATILE COMMENT = $$Returns the ez decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ sql: | - {{ evm_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} #} + {{ evm_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_fact_traces signature: From 7e48c8900639b59c70604fbdbb1ee1d0b37f210c Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Fri, 4 Apr 2025 22:47:54 +0900 Subject: [PATCH 15/15] fix --- macros/evm/evm.yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index d50a71c3..a9ddfce9 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -510,7 +510,7 @@ VOLATILE COMMENT = $$Returns the token transfers for a given block height. If to_latest is true, it will continue fetching transfers until the latest block. Otherwise, it will fetch transfers until the block height is reached.$$ sql: | - {{ evm_live_view_ez_token_transfers(schema, blockchain, network) | indent(4) -}} + {{ evm_ez_token_transfers(schema, blockchain, network) | indent(4) -}} {%- endmacro -%}