diff --git a/README.md b/README.md index 088e8034..6306ed40 100644 --- a/README.md +++ b/README.md @@ -462,20 +462,20 @@ livequery: To control the creation of UDF or SP macros with dbt run: -* UPDATE_UDFS_AND_SPS +* LQ_UPDATE_UDFS_AND_SPS When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal When False, none of the on-run-start macros are executed on model run Default values are False * Usage: -`dbt run --vars '{"UPDATE_UDFS_AND_SPS":True}' -m ...` +`dbt run --var '{"LQ_UPDATE_UDFS_AND_SPS":True}' -m ...` Dropping and creating udfs can also be done without running a model: ```sh -dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}' -dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}' +dbt run-operation create_udfs --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}' +dbt run-operation create_udfs --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}' ``` ## Resources diff --git a/dbt_project.yml b/dbt_project.yml index 3d209be8..a2191c8c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -32,6 +32,43 @@ models: +tags: evm marketplace: +tags: marketplace + overrides: + +materialized: ephemeral + fsc_evm: + +enabled: false # disable package by default + +materialized: ephemeral + decoder_package: + decoded_logs: + gold: + core__ez_decoded_event_logs: + +enabled: true + silver: + silver__decoded_logs: + +enabled: true + main_package: + core: + gold: + core__fact_blocks: + +enabled: true + core__fact_transactions: + +enabled: true + core__fact_event_logs: + +enabled: true + core__fact_traces: + +enabled: true + core__ez_native_transfers: + +enabled: true + core__ez_token_transfers: + +enabled: true + silver: + silver__blocks: + +enabled: true + silver__transactions: + +enabled: true + silver__receipts: + +enabled: true + silver__traces: + +enabled: true tests: +store_failures: true # all tests @@ -40,7 +77,7 @@ tests: marketplace: blockpour: # TODO: enable tests for blockpour once we get an API key - +enabled: false + +enabled: false on-run-start: - "{{ create_sps() }}" @@ -57,7 +94,7 @@ on-run-end: vars: "dbt_date:time_zone": GMT - UPDATE_UDFS_AND_SPS: false + LQ_UPDATE_UDFS_AND_SPS: false DROP_UDFS_AND_SPS: false UPDATE_SNOWFLAKE_TAGS: true STREAMLINE_INVOKE_STREAMS: False @@ -67,9 +104,10 @@ vars: API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}' ROLES: '{{ var("config")[target.name]["ROLES"] }}' + GLOBAL_PROD_DB_NAME: 'LIVEQUERY_DEV' config: - # The keys correspond to dbt profiles and are case sensitive + # The keys correspond to dbt profiles and are case sensitive dev: API_INTEGRATION: AWS_LIVE_QUERY_STG EXTERNAL_FUNCTION_URI: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/ diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 24045461..f4536c61 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,5 +1,5 @@ {% macro create_sps() %} - {% if var("UPDATE_UDFS_AND_SPS") %} + {% if var("LQ_UPDATE_UDFS_AND_SPS") %} {% if target.database == 'LIVEQUERY' %} CREATE schema IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; diff --git a/macros/evm/evm.yaml.sql b/macros/evm/evm.yaml.sql index df40296d..a9ddfce9 100644 --- a/macros/evm/evm.yaml.sql +++ b/macros/evm/evm.yaml.sql @@ -367,159 +367,150 @@ signature: - [block_height, INTEGER, The start block height to get the blocks from] - [to_latest, BOOLEAN, Whether to continue fetching blocks until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, network STRING, blockchain STRING, tx_count INTEGER, difficulty INTEGER, total_difficulty INTEGER, extra_data STRING, gas_limit INTEGER, gas_used INTEGER, hash STRING, parent_hash STRING, miner STRING, nonce INTEGER, receipts_root STRING, sha3_uncles STRING, size INTEGER, uncle_blocks VARIANT, block_header_json OBJECT, excess_blob_gas INTEGER, blob_gas_used INTEGER, fact_blocks_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ, withdrawals VARIANT, withdrawals_root STRING)" + - "{{ generate_udtf_return_type(blockchain, get_fact_blocks_columns()) }}" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE COMMENT = $$Returns the block data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ sql: | - {{ evm_live_view_fact_blocks(schema, blockchain, network) | indent(4) -}} - -- name: {{ schema -}}.tf_fact_event_logs - signature: - - [block_height, INTEGER, The start block height to get the logs from] - - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] - return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, event_index INTEGER, contract_address STRING, topics VARIANT, data STRING, event_removed BOOLEAN, tx_status STRING, _log_id STRING, fact_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" - options: | - NOT NULL - RETURNS NULL ON NULL INPUT - VOLATILE - COMMENT = $$Returns the logs for a given block height. If to_latest is true, it will continue fetching logs until the latest block. Otherwise, it will fetch logs until the block height is reached.$$ - sql: | - {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} - -- name: {{ schema -}}.tf_fact_decoded_event_logs - signature: - - [block_height, INTEGER, The start block height to get the logs from] - - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] - return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, fact_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" - options: | - NOT NULL - RETURNS NULL ON NULL INPUT - VOLATILE - COMMENT = $$Returns the decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ - sql: | - {{ evm_live_view_fact_decoded_event_logs(schema, blockchain, network) | indent(4) -}} - -- name: {{ schema -}}.tf_fact_decoded_traces - signature: - - [block_height, INTEGER, The start block height to get the logs from] - - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] - return_type: - - "TABLE(block_number INTEGER, tx_hash STRING, block_timestamp TIMESTAMP_NTZ, tx_status STRING, tx_position INTEGER, trace_index INTEGER, from_address STRING, to_address STRING, VALUE FLOAT, value_precise_raw STRING, value_precise STRING, gas INTEGER, gas_used INTEGER, TYPE STRING, identifier STRING, sub_traces INTEGER, error_reason STRING, trace_status STRING, input STRING, output STRING, function_name STRING, decoded_input_data VARIANT, decoded_output_data VARIANT, fact_decoded_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" - options: | - NOT NULL - RETURNS NULL ON NULL INPUT - VOLATILE - COMMENT = $$Returns the decoded traces data for a given block height. If to_latest is true, - it will continue fetching blocks until the latest block. Otherwise, - it will fetch blocks until the block height is reached.$$ - sql: | - {{ evm_live_view_fact_decoded_traces(schema, - blockchain, network) | indent(4) -}} - -- name: {{ schema -}}.tf_fact_traces - signature: - - [block_height, INTEGER, The start block height to get the transfers from] - - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - return_type: - - "TABLE(tx_hash STRING, block_number NUMBER, block_timestamp TIMESTAMP_NTZ(9), from_address STRING, to_address STRING, value FLOAT, value_precise_raw STRING, value_precise STRING, gas NUMBER, gas_used NUMBER, input STRING, output STRING, TYPE STRING, identifier STRING, DATA OBJECT, tx_status STRING, sub_traces NUMBER, trace_status STRING, error_reason STRING, trace_index NUMBER, fact_traces_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" - options: | - NOT NULL - RETURNS NULL ON NULL INPUT - VOLATILE - COMMENT = $$Returns the traces for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ - sql: | - {{ evm_live_view_fact_traces(schema, blockchain, network) | indent(4) -}} + {{ evm_fact_blocks(schema, blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_fact_transactions signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, block_hash STRING, tx_hash STRING, nonce NUMBER, POSITION NUMBER, origin_function_signature STRING, from_address STRING, to_address STRING, VALUE FLOAT, value_precise_raw STRING, value_precise STRING, tx_fee FLOAT, tx_fee_precise STRING, gas_price FLOAT, gas_limit NUMBER, gas_used NUMBER, cumulative_gas_used NUMBER, input_data STRING, status STRING, effective_gas_price FLOAT, max_fee_per_gas FLOAT, max_priority_fee_per_gas FLOAT, r STRING, s STRING, v STRING, tx_type NUMBER, chain_id NUMBER, blob_versioned_hashes ARRAY, max_fee_per_blob_gas NUMBER, blob_gas_used NUMBER, blob_gas_price NUMBER, fact_transactions_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "{{ generate_udtf_return_type(blockchain, get_fact_transactions_columns()) }}" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE COMMENT = $$Returns the transactions for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ sql: | - {{ evm_live_view_fact_transactions(schema, blockchain, network) | indent(4) -}} + {{ evm_fact_transactions(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_fact_token_balances +- name: {{ schema -}}.tf_fact_event_logs signature: - - [block_height, INTEGER, The start block height to get the transfers from] - - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [block_height, INTEGER, The start block height to get the events from] + - [to_latest, BOOLEAN, Whether to continue fetching events until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, address STRING, contract_address STRING, balance NUMBER(38,0), fact_token_balances_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, tx_position INTEGER, event_index INTEGER, contract_address STRING, topics VARIANT, topic_0 STRING, topic_1 STRING, topic_2 STRING, topic_3 STRING, DATA STRING, event_removed BOOLEAN, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, tx_succeeded BOOLEAN, fact_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns the token balances for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ + COMMENT = $$Returns the event logs for a given block height. If to_latest is true, it will continue fetching events until the latest block. Otherwise, it will fetch events until the block height is reached.$$ sql: | - {{ evm_live_view_fact_token_balances(schema, blockchain, network) | indent(4) -}} + {{ evm_fact_event_logs(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_fact_eth_balances +- name: {{ schema -}}.tf_ez_decoded_event_logs signature: - - [block_height, INTEGER, The start block height to get the transfers from] - - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [block_height, INTEGER, The start block height to get the logs from] + - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE(block_number NUMBER, block_timestamp TIMESTAMP_NTZ, address STRING, balance NUMBER(38,0), fact_eth_balances_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, tx_position INTEGER, event_index INTEGER, contract_address STRING, topics VARIANT, topic_0 STRING, topic_1 STRING, topic_2 STRING, topic_3 STRING, DATA STRING, event_removed BOOLEAN, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, tx_succeeded BOOLEAN, event_name STRING, full_decoded_log VARIANT, decoded_log OBJECT, contract_name STRING, ez_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns the eth balances for a given block height. If to_latest is true, it will continue fetching transactions until the latest block. Otherwise, it will fetch transactions until the block height is reached.$$ + COMMENT = $$Returns the ez decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ sql: | - {{ evm_live_view_fact_eth_balances(schema, blockchain, network) | indent(4) -}} - + {{ evm_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} -- name: {{ schema -}}.tf_ez_decoded_event_logs +- name: {{ schema -}}.tf_fact_traces signature: - - [block_height, INTEGER, The start block height to get the logs from] - - [to_latest, BOOLEAN, Whether to continue fetching logs until the latest block or not] + - [block_height, INTEGER, The start block height to get the traces from] + - [to_latest, BOOLEAN, Whether to continue fetching traces until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, contract_address STRING, contract_name STRING, event_name STRING, decoded_log OBJECT, full_decoded_log VARIANT, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, topics VARIANT, data STRING, event_removed BOOLEAN, tx_status STRING, ez_decoded_event_logs_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - "{{ generate_udtf_return_type(blockchain, get_fact_traces_columns()) }}" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns the ez decoded event logs data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block height is reached.$$ + COMMENT = $$Returns the traces for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_live_view_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} + {{ evm_fact_traces(schema, blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_ez_native_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] + - [block_size, INTEGER, The number of blocks to fetch] return_type: - - "TABLE(tx_hash STRING, block_number NUMBER(38,0), block_timestamp TIMESTAMP_NTZ(9), tx_position NUMBER(38,0), trace_index NUMBER(19,0), identifier STRING, origin_from_address STRING, origin_to_address STRING, origin_function_signature STRING, from_address STRING, to_address STRING, amount FLOAT, amount_precise_raw STRING, amount_precise STRING, amount_usd FLOAT, ez_native_transfers_id STRING, inserted_timestamp TIMESTAMP_NTZ(9), modified_timestamp TIMESTAMP_NTZ(9))" + - "TABLE( + block_number INTEGER, + block_timestamp TIMESTAMP_NTZ, + tx_hash STRING, + tx_position INTEGER, + trace_index INTEGER, + trace_address STRING, + TYPE STRING, + from_address STRING, + to_address STRING, + amount FLOAT, + amount_precise_raw STRING, + amount_precise STRING, + amount_usd FLOAT, + origin_from_address STRING, + origin_to_address STRING, + origin_function_signature STRING, + ez_native_transfers_id STRING, + inserted_timestamp TIMESTAMP_NTZ, + modified_timestamp TIMESTAMP_NTZ + )" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns the native transfers for a given block height. If to_latest is true, it will continue fetching transfers until the latest block. Otherwise, it will fetch transfers until the block height is reached.$$ + COMMENT = $$Returns the native transfers for a given block height. If to_latest is true, it will continue fetching traces until the latest block. Otherwise, it will fetch traces until the block height is reached.$$ sql: | - {{ evm_live_view_ez_native_transfers(schema, blockchain, network) | indent(4) -}} + {{ evm_ez_native_transfers(schema, blockchain, network) | indent(4) -}} - name: {{ schema -}}.tf_ez_token_transfers signature: - [block_height, INTEGER, The start block height to get the transfers from] - [to_latest, BOOLEAN, Whether to continue fetching transfers until the latest block or not] - return_type: - - "TABLE(block_number INTEGER, block_timestamp TIMESTAMP_NTZ, tx_hash STRING, event_index INTEGER, origin_function_signature STRING, origin_from_address STRING, origin_to_address STRING, contract_address STRING, from_address STRING, to_address STRING, raw_amount_precise STRING, raw_amount FLOAT, amount_precise FLOAT, amount FLOAT, amount_usd FLOAT, decimals INTEGER, symbol STRING, token_price FLOAT, has_decimal STRING, has_price STRING, _log_id STRING, ez_token_transfers_id STRING, _inserted_timestamp TIMESTAMP_NTZ, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)" + - [block_size, INTEGER, The number of blocks to fetch] + return_type: + - "TABLE( + block_number INTEGER, + block_timestamp TIMESTAMP_NTZ, + tx_hash STRING, + tx_position INTEGER, + event_index INTEGER, + from_address STRING, + to_address STRING, + contract_address STRING, + token_standard STRING, + NAME STRING, + symbol STRING, + decimals INTEGER, + raw_amount_precise STRING, + raw_amount FLOAT, + amount_precise_heal STRING, + amount_heal FLOAT, + amount_usd_heal FLOAT, + origin_function_signature STRING, + origin_from_address STRING, + origin_to_address STRING, + ez_token_transfers_id STRING, + inserted_timestamp TIMESTAMP_NTZ, + modified_timestamp TIMESTAMP_NTZ + )" options: | NOT NULL RETURNS NULL ON NULL INPUT VOLATILE COMMENT = $$Returns the token transfers for a given block height. If to_latest is true, it will continue fetching transfers until the latest block. Otherwise, it will fetch transfers until the block height is reached.$$ sql: | - {{ evm_live_view_ez_token_transfers(schema, blockchain, network) | indent(4) -}} + {{ evm_ez_token_transfers(schema, blockchain, network) | indent(4) -}} {%- endmacro -%} diff --git a/macros/evm/evm_live_views.sql b/macros/evm/evm_live_views.sql index 542410e1..7228d474 100644 --- a/macros/evm/evm_live_views.sql +++ b/macros/evm/evm_live_views.sql @@ -1,4 +1,4 @@ -{% macro evm_live_view_latest_block_height(schema, blockchain, network) %} +{% macro evm_latest_block_height(schema, blockchain, network) %} SELECT {{ schema }}.udf_rpc('eth_blockNumber', []) as result, utils.udf_hex_to_int(result)::integer AS latest_block_height, @@ -13,9 +13,9 @@ ) AS max_height {% endmacro %} -{% macro evm_live_view_target_blocks(schema, blockchain, network, batch_size=10) %} +{% macro evm_target_blocks(schema, blockchain, network, batch_size=10) %} WITH heights AS ( - {{ evm_live_view_latest_block_height(schema, blockchain, network) | indent(4) -}} + {{ evm_latest_block_height(schema, blockchain, network) | indent(4) -}} ), block_spine AS ( SELECT @@ -31,7 +31,7 @@ ) AS max_height, latest_block_height FROM - TABLE(generator(ROWCOUNT => 1000)), + TABLE(generator(ROWCOUNT => COALESCE(block_size, 1))), heights qualify block_number BETWEEN min_height AND max_height ) @@ -52,7 +52,7 @@ {% endmacro %} -- Get Raw EVM chain data -{% macro evm_live_view_bronze_blocks(schema, blockchain, network, table_name) %} +{% macro evm_bronze_blocks(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( SELECT batch_id, @@ -77,12 +77,14 @@ WITH blocks_agg AS ( ) SELECT + 0 AS partition_key, utils.udf_hex_to_int(result:number::STRING)::INT AS block_number, - result as data + result as data, + SYSDATE() AS _inserted_timestamp FROM flattened {% endmacro %} -{% macro evm_live_view_bronze_receipts(schema, blockchain, network, table_name) %} +{% macro evm_bronze_receipts(schema, blockchain, network, table_name) %} WITH blocks_agg AS ( SELECT batch_id, @@ -106,9 +108,12 @@ get_batch_result AS ( ) SELECT + 0 AS partition_key, latest_block_height, utils.udf_hex_to_int(w.value:blockNumber::STRING)::INT AS block_number, - w.value AS DATA + w.index AS array_index, + w.value AS DATA, + SYSDATE() AS _inserted_timestamp FROM (SELECT latest_block_height, @@ -117,25 +122,22 @@ FROM LATERAL FLATTEN(data) v), LATERAL FLATTEN(data) w {% endmacro %} -{% macro evm_live_view_bronze_logs(table_name) %} -SELECT - r.block_number, - v.value -FROM - {{ table_name }} AS r, - lateral flatten(r.data:logs) v -{% endmacro %} - -{% macro evm_live_view_bronze_transactions(table_name) %} -SELECT - block_number, - v.value as DATA -FROM - {{ table_name }} AS r, - lateral flatten(r.data:transactions) v +{% macro evm_bronze_transactions(table_name) %} + SELECT + 0 AS partition_key, + block_number, + v.index::INT AS tx_position, -- mimic's streamline's logic to add tx_position + OBJECT_CONSTRUCT( + 'array_index', v.index::INT + ) AS VALUE, + v.value as DATA, + SYSDATE() AS _inserted_timestamp + FROM + {{ table_name }} AS r, + lateral flatten(r.data:transactions) v {% endmacro %} -{% macro evm_live_view_bronze_traces(schema, blockchain, network, table_name)%} +{% macro evm_bronze_traces(schema, blockchain, network, table_name)%} WITH blocks_agg AS ( SELECT batch_id, @@ -161,1478 +163,176 @@ WITH blocks_agg AS ( ) SELECT + 0 AS partition_key, s.block_number, v.index::INT AS tx_position, -- mimic's streamline's logic to add tx_position - v.value:result AS full_traces, + OBJECT_CONSTRUCT( + 'array_index', v.index::INT + ) AS VALUE, + v.value AS DATA, SYSDATE() AS _inserted_timestamp FROM flattened s, LATERAL FLATTEN(input => result) v {% endmacro %} -{% macro evm_live_view_bronze_token_balances(schema, blockchain, network, table_name) %} -WITH block_spine AS ( - SELECT - CEIL(ROW_NUMBER() OVER (ORDER BY block_number, address, contract_address) / 10) AS batch_id, - block_number, - address, - contract_address - FROM - {{ table_name }} -), -blocks_agg AS ( - SELECT - batch_id, - ARRAY_AGG( - utils.udf_json_rpc_call( - 'eth_call', - ARRAY_CONSTRUCT( - OBJECT_CONSTRUCT( - 'to', - contract_address, - 'data', - CONCAT( - '0x70a08231000000000000000000000000', - SUBSTR( - address, - 3 - ) - ) - ), - utils.udf_int_to_hex(block_number) - ), - CONCAT( - block_number, - '-', - address, - '-', - contract_address - ) - ) - ) AS params - FROM - block_spine - GROUP BY batch_id -), result as ( - SELECT - {{ evm_batch_udf_api(blockchain, network) }} - FROM blocks_agg -) - -SELECT - SPLIT(value:id::STRING, '-')[0]::INT AS block_number, - SPLIT(value:id::STRING, '-')[1]::STRING AS address, - SPLIT(value:id::STRING, '-')[2]::STRING AS contract_address, - COALESCE(value:result, {'error':value:error}) AS DATA -FROM result, LATERAL FLATTEN(input => result.data) v -{% endmacro %} - -{% macro evm_live_view_bronze_eth_balances(schema, blockchain, network, table_name) %} -WITH block_spine AS ( - SELECT - CEIL(ROW_NUMBER() OVER (ORDER BY block_number, address) / 10) AS batch_id, - block_number, - address - FROM - {{ table_name }} -), -blocks_agg AS ( - SELECT - batch_id, - ARRAY_AGG( - utils.udf_json_rpc_call( - 'eth_getBalance', - ARRAY_CONSTRUCT(address, utils.udf_int_to_hex(block_number)), - CONCAT( - block_number, - '-', - address - ) - ) - ) AS params - FROM - block_spine - GROUP BY batch_id -), result as ( - SELECT - {{ evm_batch_udf_api(blockchain, network) }} - FROM blocks_agg -) - -SELECT - SPLIT(value:id::STRING, '-')[0]::INT AS block_number, - SPLIT(value:id::STRING, '-')[1]::STRING AS address, - COALESCE(value:result, {'error':value:error}) AS DATA -FROM result, LATERAL FLATTEN(input => result.data) v -{% endmacro %} - --- Transformation macro for EVM chains -{% macro evm_live_view_silver_blocks(table_name) %} -SELECT - block_number, - utils.udf_hex_to_int(DATA:timestamp::STRING)::TIMESTAMP AS block_timestamp, - utils.udf_hex_to_int(DATA:baseFeePerGas::STRING)::INT AS base_fee_per_gas, - utils.udf_hex_to_int(DATA:difficulty::STRING)::INT AS difficulty, - DATA:extraData::STRING AS extra_data, - utils.udf_hex_to_int(DATA:gasLimit::STRING)::INT AS gas_limit, - utils.udf_hex_to_int(DATA:gasUsed::STRING)::INT AS gas_used, - DATA:hash::STRING AS HASH, - DATA:logsBloom::STRING AS logs_bloom, - DATA:miner::STRING AS miner, - utils.udf_hex_to_int(DATA:nonce::STRING)::INT AS nonce, - utils.udf_hex_to_int(DATA:number::STRING)::INT AS NUMBER, - DATA:parentHash::STRING AS parent_hash, - DATA:receiptsRoot::STRING AS receipts_root, - DATA:sha3Uncles::STRING AS sha3_uncles, - utils.udf_hex_to_int(DATA:size::STRING)::INT AS SIZE, - DATA:stateRoot::STRING AS state_root, - utils.udf_hex_to_int(DATA:totalDifficulty::STRING)::INT AS total_difficulty, - ARRAY_SIZE(DATA:transactions) AS tx_count, - DATA:transactionsRoot::STRING AS transactions_root, - DATA:uncles AS uncles, - DATA:withdrawals AS withdrawals, - DATA:withdrawalsRoot::STRING AS withdrawals_root, - md5( - CAST( - COALESCE( - CAST(block_number AS TEXT), - '_dbt_utils_surrogate_key_null_' - ) AS TEXT - ) - ) AS blocks_id, - utils.udf_hex_to_int(DATA:blobGasUsed::STRING)::INT AS blob_gas_used, - utils.udf_hex_to_int(DATA:excessBlobGas::STRING)::INT AS excess_blob_gas -FROM - {{ table_name }} -{% endmacro %} - -{% macro evm_live_view_silver_receipts(table_name) %} -SELECT - latest_block_height, - block_number, - DATA :blockHash::STRING AS block_hash, - utils.udf_hex_to_int(DATA :blockNumber::STRING)::INT AS blockNumber, - utils.udf_hex_to_int(DATA :cumulativeGasUsed::STRING)::INT AS cumulative_gas_used, - utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT / pow(10, 9) AS effective_gas_price, - DATA :from::STRING AS from_address, - utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS gas_used, - DATA :logs AS logs, - DATA :logsBloom::STRING AS logs_bloom, - utils.udf_hex_to_int(DATA :status::STRING)::INT AS status, - CASE - WHEN status = 1 THEN TRUE - ELSE FALSE - END AS tx_success, - CASE - WHEN status = 1 THEN 'SUCCESS' - ELSE 'FAIL' - END AS tx_status, - DATA :to::STRING AS to_address1, - CASE - WHEN to_address1 = '' THEN NULL - ELSE to_address1 - END AS to_address, - DATA :transactionHash::STRING AS tx_hash, - utils.udf_hex_to_int(DATA :transactionIndex::STRING)::INT AS POSITION, - utils.udf_hex_to_int(DATA :type::STRING)::INT AS TYPE, - utils.udf_hex_to_int(DATA :effectiveGasPrice::STRING)::INT AS blob_gas_price, - utils.udf_hex_to_int(DATA :gasUsed::STRING)::INT AS blob_gas_used -FROM - {{ table_name }} -{% endmacro %} - -{% macro evm_live_view_silver_logs(silver_receipts, silver_transactions) %} -SELECT - r.block_number, - txs.block_timestamp, - r.tx_hash, - txs.origin_function_signature, - r.from_address AS origin_from_address, - r.to_address AS origin_to_address, - utils.udf_hex_to_int(v.VALUE :logIndex::STRING)::INT AS event_index, - v.VALUE :address::STRING AS contract_address, - v.VALUE :blockHash::STRING AS block_hash, - v.VALUE :data::STRING AS DATA, - v.VALUE :removed::BOOLEAN AS event_removed, - v.VALUE :topics AS topics, - r.tx_status, - CASE - WHEN txs.block_timestamp IS NULL - OR txs.origin_function_signature IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, -FROM - {{ silver_receipts }} AS r - LEFT JOIN {{ silver_transactions }} AS txs on txs.tx_hash = r.tx_hash, - lateral flatten(r.logs) v -{% endmacro %} - -{% macro evm_live_view_silver_transactions(bronze_transactions, silver_blocks, silver_receipts) %} -SELECT - A.block_number AS block_number, - A.data :blockHash::STRING AS block_hash, - utils.udf_hex_to_int(A.data :blockNumber::STRING)::INT AS blockNumber, - utils.udf_hex_to_int(A.data :chainId::STRING)::INT AS chain_id, - A.data :from::STRING AS from_address, - utils.udf_hex_to_int(A.data :gas::STRING)::INT AS gas, - utils.udf_hex_to_int(A.data :gasPrice::STRING)::INT / pow(10, 9) AS gas_price, - A.data :hash::STRING AS tx_hash, - A.data :input::STRING AS input_data, - SUBSTR(input_data, 1, 10) AS origin_function_signature, - utils.udf_hex_to_int(A.data :maxFeePerGas::STRING)::INT / pow(10, 9) AS max_fee_per_gas, - utils.udf_hex_to_int( - A.data :maxPriorityFeePerGas::STRING - )::INT / pow(10, 9) AS max_priority_fee_per_gas, - utils.udf_hex_to_int(A.data :nonce::STRING)::INT AS nonce, - A.data :r::STRING AS r, - A.data :s::STRING AS s, - A.data :to::STRING AS to_address1, - utils.udf_hex_to_int(A.data :transactionIndex::STRING)::INT AS POSITION, - A.data :type::STRING AS TYPE, - A.data :v::STRING AS v, - utils.udf_hex_to_int(A.data :value::STRING) AS value_precise_raw, - value_precise_raw * power(10, -18) AS value_precise, - value_precise::FLOAT AS VALUE, - A.data :accessList AS access_list, - A.data, - A.data: blobVersionedHashes::ARRAY AS blob_versioned_hashes, - utils.udf_hex_to_int(A.data: maxFeePerGas::STRING)::INT AS max_fee_per_blob_gas, - block_timestamp, - CASE - WHEN block_timestamp IS NULL - OR tx_status IS NULL THEN TRUE - ELSE FALSE - END AS is_pending, - r.gas_used, - tx_success, - tx_status, - cumulative_gas_used, - effective_gas_price, - utils.udf_hex_to_int(A.data :gasPrice) * power(10, -18) * r.gas_used AS tx_fee_precise, - COALESCE(tx_fee_precise::FLOAT, 0) AS tx_fee, - r.type as tx_type, - r.blob_gas_used, - r.blob_gas_price, -FROM - {{ bronze_transactions }} AS A - LEFT JOIN {{ silver_blocks }} AS b on b.block_number = A.block_number - LEFT JOIN {{ silver_receipts }} AS r on r.tx_hash = A.data :hash::STRING -{% endmacro %} - -{% macro evm_live_view_silver_traces(raw_traces) %} -flatten_traces AS ( - SELECT - block_number, - tx_position, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - _inserted_timestamp, - OBJECT_AGG( - key, - VALUE - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS str_array - FROM - {{ raw_traces }}, - TABLE( - FLATTEN( - input => PARSE_JSON(full_traces), - recursive => TRUE - ) - ) f - WHERE - f.index IS NULL - AND f.key != 'calls' - AND f.path != 'result' - GROUP BY - block_number, - tx_position, - trace_address, - _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS eth_value_precise_raw, - ethereum.utils.udf_decimal_adjust( - eth_value_precise_raw, - 18 - ) AS eth_value_precise, - eth_value_precise :: FLOAT AS eth_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces -), -new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.eth_value_precise_raw, - f.eth_value_precise, - f.eth_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN transactions t - ON f.tx_position = t.position - AND f.block_number = t.block_number -), -traces_final AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records -) -{% endmacro %} - -{% macro evm_live_view_silver_token_balances(schema, blockchain, network) %} -WITH silver_logs AS ( - SELECT - CONCAT('0x', SUBSTR(l.topics [1] :: STRING, 27, 42)) AS address1, - CONCAT('0x', SUBSTR(l.topics [2] :: STRING, 27, 42)) AS address2, - l.contract_address, - l.block_timestamp, - l.block_number - FROM - ( - {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} - ) l - WHERE - ( - l.topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - OR ( - l.topics [0] :: STRING = '0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65' - AND l.contract_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' - ) - OR ( - l.topics [0] :: STRING = '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c' - AND l.contract_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' - ) - ) -), - -transfers AS ( - SELECT - DISTINCT block_number, - block_timestamp, - contract_address, - address1 AS address - FROM - silver_logs - WHERE - address1 IS NOT NULL - AND address1 <> '0x0000000000000000000000000000000000000000' - UNION - SELECT - DISTINCT block_number, - block_timestamp, - contract_address, - address2 AS address - FROM - silver_logs - WHERE - address2 IS NOT NULL - AND address2 <> '0x0000000000000000000000000000000000000000' -), - -balances AS ( - {{ evm_live_view_bronze_token_balances(schema, blockchain, network, 'transfers') | indent(4) -}} -) - -SELECT - b.block_number, - block_timestamp, - address, - contract_address, - IFF(DATA :: STRING = '{}', NULL, DATA :: STRING) AS casted_data, - CASE - WHEN - LENGTH( - casted_data - ) <= 4300 - AND casted_data IS NOT NULL THEN LEFT(casted_data, 66) - ELSE NULL - END - AS hex_balance, - TRY_TO_NUMBER( - CASE - WHEN LENGTH( - hex_balance - ) <= 4300 - AND hex_balance IS NOT NULL THEN utils.udf_hex_to_int(hex_balance) - ELSE NULL - END - ) AS balance, - SYSDATE() AS _inserted_timestamp, - cast( - coalesce( - cast(block_number as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || - coalesce( - cast(address as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || - coalesce( - cast(contract_address as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) AS id, - id AS token_balances_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM balances b -LEFT JOIN ( - SELECT DISTINCT block_number, block_timestamp FROM transfers -) USING (block_number) -{% endmacro %} - -{% macro evm_live_view_silver_eth_balances(schema, blockchain, network) %} -WITH silver_traces AS ( - SELECT - block_timestamp, - block_number, - from_address, - to_address - FROM - ( - {{ evm_live_view_fact_traces(schema, blockchain, network) | indent(4) -}} - ) l - WHERE - VALUE > 0 -- VALUE is the amount of ETH transferred - AND trace_status = 'SUCCESS' - AND tx_status = 'SUCCESS' -), - -stacked AS ( - SELECT - DISTINCT block_number, - block_timestamp, - from_address AS address - FROM - silver_traces - WHERE - from_address IS NOT NULL - AND from_address <> '0x0000000000000000000000000000000000000000' - UNION - SELECT - DISTINCT block_number, - block_timestamp, - to_address AS address - FROM - silver_traces - WHERE - to_address IS NOT NULL - AND to_address <> '0x0000000000000000000000000000000000000000' -), - -eth_balances AS ( - {{ evm_live_view_bronze_eth_balances(schema, blockchain, network, 'stacked') | indent(4) -}} -) - -SELECT - block_number, - block_timestamp, - address, - IFF(DATA :: STRING = '{}', NULL, DATA :: STRING) AS casted_data, - CASE - WHEN casted_data IS NOT NULL THEN casted_data - ELSE NULL - END - AS hex_balance, - TRY_TO_NUMBER( - CASE - WHEN hex_balance IS NOT NULL THEN utils.udf_hex_to_int(hex_balance) - ELSE NULL - END - ) AS balance, - SYSDATE() AS _inserted_timestamp, - cast( - coalesce( - cast(block_number as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || - coalesce( - cast(address as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) AS id, - id AS eth_balances_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM eth_balances -LEFT JOIN ( - SELECT DISTINCT block_number, block_timestamp FROM stacked -) USING (block_number) +{% macro evm_fact_blocks(schema, blockchain, network) %} + {%- set evm__fact_blocks = get_rendered_model('livequery_models', 'evm__fact_blocks', schema, blockchain, network) -%} + {{ evm__fact_blocks }} {% endmacro %} --- Get EVM chain fact data -{% macro evm_live_view_fact_blocks(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} - ), - raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} - ), - silver_blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} - ) - select - block_number, - block_timestamp, - '{{ network }}' AS network, - '{{ blockchain }}' AS blockchain, - tx_count, - difficulty, - total_difficulty, - extra_data, - gas_limit, - gas_used, - HASH, - parent_hash, - miner, - nonce, - receipts_root, - sha3_uncles, - SIZE, - uncles AS uncle_blocks, - OBJECT_CONSTRUCT( - 'baseFeePerGas', - base_fee_per_gas, - 'difficulty', - difficulty, - 'extraData', - extra_data, - 'gasLimit', - gas_limit, - 'gasUsed', - gas_used, - 'hash', - HASH, - 'logsBloom', - logs_bloom, - 'miner', - miner, - 'nonce', - nonce, - 'number', - NUMBER, - 'parentHash', - parent_hash, - 'receiptsRoot', - receipts_root, - 'sha3Uncles', - sha3_uncles, - 'size', - SIZE, - 'stateRoot', - state_root, - 'timestamp', - block_timestamp, - 'totalDifficulty', - total_difficulty, - 'transactionsRoot', - transactions_root, - 'uncles', - uncles, - 'excessBlobGas', - excess_blob_gas, - 'blobGasUsed', - blob_gas_used - ) AS block_header_json, - excess_blob_gas, - blob_gas_used, - block_number::STRING AS fact_blocks_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - withdrawals, - withdrawals_root - from silver_blocks +{% macro evm_fact_transactions(schema, blockchain, network) %} + {%- set evm__fact_transactions = get_rendered_model('livequery_models', 'evm__fact_transactions', schema, blockchain, network) -%} + {{ evm__fact_transactions }} {% endmacro %} -{% macro evm_live_view_fact_event_logs(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 5) | indent(4) -}} -), -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_logs AS ( - {{ evm_live_view_bronze_logs('raw_receipts') | indent(4) -}} -), -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -), -blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} -), -receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} -), -transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} -), -logs AS ( - {{ evm_live_view_silver_logs('receipts', 'transactions') | indent(4) -}} -) -SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - event_index, - contract_address, - topics, - DATA, - event_removed, - tx_status, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || coalesce( - cast(event_index as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS fact_event_logs_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM logs +{% macro evm_fact_event_logs(schema, blockchain, network) %} + {%- set evm__fact_event_logs = get_rendered_model('livequery_models', 'evm__fact_event_logs', schema, blockchain, network) -%} + {{ evm__fact_event_logs }} {% endmacro %} -{% macro evm_live_view_fact_decoded_event_logs(schema, blockchain, network) %} -WITH _ez_decoded_event_logs AS ( - {{ evm_live_view_ez_decoded_event_logs(schema, blockchain, network) | indent(4) -}} -) - -SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - contract_address, - event_name, - decoded_log, - full_decoded_log, - fact_decoded_event_logs_id, - inserted_timestamp, - modified_timestamp -FROM _ez_decoded_event_logs +{% macro evm_fact_traces(schema, blockchain, network) %} + {%- set evm__fact_traces = get_rendered_model('livequery_models', 'evm__fact_traces', schema, blockchain, network) -%} + {{ evm__fact_traces }} {% endmacro %} -{% macro evm_live_view_fact_transactions(schema, blockchain, network) %} - -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 5) | indent(4) -}} -), -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -), -blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} -), -receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} -), -transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} -) -SELECT - block_number, - block_timestamp, - block_hash, - tx_hash, - nonce, - POSITION, - origin_function_signature, - from_address, - to_address1 as to_address, - VALUE, - value_precise_raw, - value_precise::STRING as value_precise, - tx_fee, - tx_fee_precise::STRING as tx_fee_precise, - gas_price, - gas AS gas_limit, - gas_used, - cumulative_gas_used, - input_data, - tx_status AS status, - effective_gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, - r, - s, - v, - tx_type, - chain_id, - blob_versioned_hashes, - max_fee_per_blob_gas, - blob_gas_used, - blob_gas_price, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS fact_transactions_id, - SYSDATE() inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - transactions - +{% macro evm_ez_native_transfers(schema, blockchain, network) %} + {%- set evm__ez_native_transfers = get_rendered_model('livequery_models', 'evm__ez_native_transfers', schema, blockchain, network) -%} + {{ evm__ez_native_transfers }} {% endmacro %} -{% macro evm_live_view_fact_traces(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} -), -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -), -blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} -), -receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} -), -transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} -), -raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} -), - -{{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} - -SELECT - tx_hash, - block_number, - block_timestamp, - from_address, - to_address, - eth_value AS VALUE, - eth_value_precise_raw AS value_precise_raw, - eth_value_precise AS value_precise, - gas, - gas_used, - input, - output, - TYPE, - identifier, - DATA, - tx_status, - sub_traces, - trace_status, - error_reason, - trace_index, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || coalesce( - cast(trace_index as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) as fact_traces_id, - COALESCE( - _inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM traces_final +{% macro evm_ez_token_transfers(schema, blockchain, network) %} + {%- set evm__ez_token_transfers = get_rendered_model('livequery_models', 'evm__ez_token_transfers', schema, blockchain, network) -%} + {{ evm__ez_token_transfers }} {% endmacro %} -{% macro evm_live_view_fact_decoded_traces(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network, 5) | indent(4) -}} -), -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -), -blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} -), -receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} -), -transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} -), -raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} -), - -{{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} -, - -decoded_traces AS ( - SELECT - t.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.eth_value AS VALUE, - t.eth_value_precise_raw AS value_precise_raw, - t.eth_value_precise AS value_precise, - t.gas, - t.gas_used, - t.TYPE AS TYPE, - t.identifier, - t.sub_traces, - t.error_reason, - t.trace_status, - A.abi AS abi, - A.function_name AS function_name, - CASE - WHEN TYPE = 'DELEGATECALL' THEN from_address - ELSE to_address - END AS abi_address, - t.input AS input, - COALESCE( - t.output, - '0x' - ) AS output, - OBJECT_CONSTRUCT('input', input, 'output', output, 'function_name', function_name) AS function_data, - utils.udf_evm_decode_trace(abi, function_data)[0] AS decoded_data - FROM traces_final t - INNER JOIN {{ blockchain }}.SILVER.COMPLETE_FUNCTION_ABIS A - ON A.parent_contract_address = abi_address - AND LEFT( - t.input, - 10 - ) = LEFT( - A.function_signature, - 10 - ) - AND t.block_number BETWEEN A.start_block - AND A.end_block - AND t.block_number IS NOT NULL - -) - -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - VALUE, - value_precise_raw, - value_precise, - gas, - gas_used, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - input, - output, - decoded_data :function_name :: STRING AS function_name, - decoded_data :decoded_input_data AS decoded_input_data, - decoded_data :decoded_output_data AS decoded_output_data, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || coalesce( - cast(trace_index as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) AS fact_decoded_traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM decoded_traces -{% endmacro %} - -{% macro evm_live_view_fact_token_balances(schema, blockchain, network) %} -WITH silver_token_balances AS ( - {{ evm_live_view_silver_token_balances(schema, blockchain, network) | indent(4) -}} -) - -SELECT - block_number, - block_timestamp, - address AS user_address, - contract_address, - balance, - token_balances_id AS fact_token_balances_id, - inserted_timestamp, - modified_timestamp -FROM - silver_token_balances +{% macro evm_ez_decoded_event_logs(schema, blockchain, network) %} + {%- set evm__ez_decoded_event_logs = get_rendered_model('livequery_models', 'evm__ez_decoded_event_logs', schema, blockchain, network) -%} + {{ evm__ez_decoded_event_logs }} {% endmacro %} -{% macro evm_live_view_fact_eth_balances(schema, blockchain, network) %} -WITH silver_eth_balances AS ( - {{ evm_live_view_silver_eth_balances(schema, blockchain, network) | indent(4) -}} -) - -SELECT - block_number, - block_timestamp, - address AS user_address, - balance, - eth_balances_id AS fact_eth_balances_id, - inserted_timestamp, - modified_timestamp -FROM silver_eth_balances +-- UDTF Return Columns +{% macro get_fact_blocks_columns() %} + {% set columns = [ + {'name': 'block_number', 'type': 'INTEGER'}, + {'name': 'block_hash', 'type': 'STRING'}, + {'name': 'block_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'network', 'type': 'STRING'}, + {'name': 'tx_count', 'type': 'INTEGER'}, + {'name': 'size', 'type': 'INTEGER'}, + {'name': 'miner', 'type': 'STRING'}, + {'name': 'mix_hash', 'type': 'STRING', 'flag': 'uses_mix_hash'}, + {'name': 'extra_data', 'type': 'STRING'}, + {'name': 'parent_hash', 'type': 'STRING'}, + {'name': 'gas_used', 'type': 'INTEGER'}, + {'name': 'gas_limit', 'type': 'INTEGER'}, + {'name': 'base_fee_per_gas', 'type': 'INTEGER', 'flag': 'uses_base_fee'}, + {'name': 'difficulty', 'type': 'INTEGER'}, + {'name': 'total_difficulty', 'type': 'INTEGER', 'flag': 'uses_total_difficulty'}, + {'name': 'sha3_uncles', 'type': 'STRING'}, + {'name': 'uncle_blocks', 'type': 'VARIANT'}, + {'name': 'nonce', 'type': 'INTEGER'}, + {'name': 'receipts_root', 'type': 'STRING'}, + {'name': 'state_root', 'type': 'STRING'}, + {'name': 'transactions_root', 'type': 'STRING'}, + {'name': 'logs_bloom', 'type': 'STRING'}, + {'name': 'blob_gas_used', 'type': 'INTEGER', 'flag': 'uses_blob_gas_used'}, + {'name': 'excess_blob_gas', 'type': 'INTEGER', 'flag': 'uses_blob_gas_used'}, + {'name': 'parent_beacon_block_root', 'type': 'STRING', 'flag': 'uses_parent_beacon_block_root'}, + {'name': 'withdrawals', 'type': 'VARIANT', 'flag': 'uses_withdrawals'}, + {'name': 'withdrawals_root', 'type': 'STRING', 'flag': 'uses_withdrawals'}, + {'name': 'fact_blocks_id', 'type': 'STRING'}, + {'name': 'inserted_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'modified_timestamp', 'type': 'TIMESTAMP_NTZ'} + ] %} + {{ return(columns) }} {% endmacro %} --- Get EVM chain ez data -{% macro evm_live_view_ez_decoded_event_logs(schema, blockchain, network) %} -WITH _fact_event_logs AS ( - {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} -), - -_silver_decoded_logs AS ( - SELECT - block_number, - block_timestamp, - tx_hash, - origin_function_signature, - origin_from_address, - origin_to_address, - event_index, - topics, - DATA, - contract_address, - OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, - abi, - utils.udf_evm_decode_log(abi, event_data)[0] AS decoded_data, - event_removed, - decoded_data:name::string AS event_name, - {{ blockchain }}.utils.udf_transform_logs(decoded_data) AS transformed, - _log_id, - inserted_timestamp, - tx_status - FROM - _fact_event_logs - JOIN - {{ blockchain }}.core.dim_contract_abis - USING - (contract_address) - WHERE - tx_status = 'SUCCESS' -), - -_flatten_logs AS ( - SELECT - b.tx_hash, - b.block_number, - b.event_index, - b.event_name, - b.contract_address, - b.decoded_data, - b.transformed, - b._log_id, - b.inserted_timestamp, - OBJECT_AGG( - DISTINCT CASE - WHEN v.value :name = '' THEN CONCAT( - 'anonymous_', - v.index - ) - ELSE v.value :name - END, - v.value :value - ) AS decoded_flat - FROM - _silver_decoded_logs b, - LATERAL FLATTEN( - input => b.transformed :data - ) v - GROUP BY - b.tx_hash, - b.block_number, - b.event_index, - b.event_name, - b.contract_address, - b.decoded_data, - b.transformed, - b._log_id, - b.inserted_timestamp -) - -SELECT - block_number, - C.block_timestamp, - B.tx_hash, - B.event_index, - B.contract_address, - D.name AS contract_name, - B.event_name, - B.decoded_flat AS decoded_log, - B.decoded_data AS full_decoded_log, - C.origin_function_signature, - C.origin_from_address, - C.origin_to_address, - C.topics, - C.DATA, - C.event_removed, - C.tx_status, - md5(_log_id) AS fact_decoded_event_logs_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM _flatten_logs AS B -LEFT JOIN _silver_decoded_logs AS C USING (block_number, _log_id) -LEFT JOIN {{ blockchain }}.core.dim_contracts AS D - ON B.contract_address = D.address +{% macro get_fact_transactions_columns() %} + {% set columns = [ + {'name': 'block_number', 'type': 'NUMBER'}, + {'name': 'block_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'tx_hash', 'type': 'STRING'}, + {'name': 'from_address', 'type': 'STRING'}, + {'name': 'to_address', 'type': 'STRING'}, + {'name': 'origin_function_signature', 'type': 'STRING'}, + {'name': 'value', 'type': 'FLOAT'}, + {'name': 'value_precise_raw', 'type': 'STRING'}, + {'name': 'value_precise', 'type': 'STRING'}, + {'name': 'tx_fee', 'type': 'FLOAT'}, + {'name': 'tx_fee_precise', 'type': 'STRING'}, + {'name': 'tx_succeeded', 'type': 'BOOLEAN'}, + {'name': 'tx_type', 'type': 'NUMBER'}, + {'name': 'nonce', 'type': 'NUMBER'}, + {'name': 'tx_position', 'type': 'NUMBER'}, + {'name': 'input_data', 'type': 'STRING'}, + {'name': 'gas_price', 'type': 'FLOAT'}, + {'name': 'gas_used', 'type': 'NUMBER'}, + {'name': 'gas_limit', 'type': 'NUMBER'}, + {'name': 'cumulative_gas_used', 'type': 'NUMBER'}, + {'name': 'effective_gas_price', 'type': 'NUMBER'}, + {'name': 'max_fee_per_gas', 'type': 'FLOAT', 'flag': 'uses_eip_1559'}, + {'name': 'max_priority_fee_per_gas', 'type': 'FLOAT', 'flag': 'uses_eip_1559'}, + {'name': 'l1_fee', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_fee_precise_raw', 'type': 'STRING', 'flag': 'uses_l1_columns'}, + {'name': 'l1_fee_precise', 'type': 'STRING', 'flag': 'uses_l1_columns'}, + {'name': 'l1_fee_scalar', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_gas_used', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_gas_price', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_base_fee_scalar', 'type': 'FLOAT', 'flag': 'uses_l1_columns'}, + {'name': 'l1_blob_base_fee', 'type': 'FLOAT', 'flag': 'uses_blob_base_fee'}, + {'name': 'l1_blob_base_fee_scalar', 'type': 'FLOAT', 'flag': 'uses_blob_base_fee'}, + {'name': 'mint', 'type': 'FLOAT', 'flag': 'uses_mint'}, + {'name': 'mint_precise_raw', 'type': 'STRING', 'flag': 'uses_mint'}, + {'name': 'mint_precise', 'type': 'STRING', 'flag': 'uses_mint'}, + {'name': 'eth_value', 'type': 'FLOAT', 'flag': 'uses_eth_value'}, + {'name': 'eth_value_precise_raw', 'type': 'STRING', 'flag': 'uses_eth_value'}, + {'name': 'eth_value_precise', 'type': 'STRING', 'flag': 'uses_eth_value'}, + {'name': 'y_parity', 'type': 'FLOAT', 'flag': 'uses_y_parity'}, + {'name': 'access_list', 'type': 'VARIANT', 'flag': 'uses_access_list'}, + {'name': 'r', 'type': 'STRING'}, + {'name': 's', 'type': 'STRING'}, + {'name': 'v', 'type': 'NUMBER'}, + {'name': 'source_hash', 'type': 'STRING', 'flag': 'uses_source_hash'}, + {'name': 'fact_transactions_id', 'type': 'STRING'}, + {'name': 'inserted_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'modified_timestamp', 'type': 'TIMESTAMP_NTZ'} + ] %} + {{ return(columns) }} {% endmacro %} -{% macro evm_live_view_ez_token_transfers(schema, blockchain, network) %} -WITH fact_logs AS ( - {{ evm_live_view_fact_event_logs(schema, blockchain, network) | indent(4) -}} -) - -SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - origin_function_signature, - origin_from_address, - origin_to_address, - contract_address::STRING AS contract_address, - CONCAT('0x', SUBSTR(topics [1], 27, 40))::STRING AS from_address, - CONCAT('0x', SUBSTR(topics [2], 27, 40))::STRING AS to_address, - utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) AS raw_amount_precise, - raw_amount_precise::FLOAT AS raw_amount, - IFF( - C.decimals IS NOT NULL, - raw_amount_precise * power(10, C.decimals * -1), - NULL - ) AS amount_precise, - amount_precise::FLOAT AS amount, - IFF( - C.decimals IS NOT NULL - AND price IS NOT NULL, - amount * price, - NULL - ) AS amount_usd, - C.decimals AS decimals, - C.symbol AS symbol, - price AS token_price, - CASE - WHEN C.decimals IS NULL THEN 'false' - ELSE 'true' - END AS has_decimal, - CASE - WHEN price IS NULL THEN 'false' - ELSE 'true' - END AS has_price, - _log_id, - md5( - cast( - coalesce( - cast(tx_hash as TEXT), - '_dbt_utils_surrogate_key_null_' - ) || '-' || coalesce( - cast(event_index as TEXT), - '_dbt_utils_surrogate_key_null_' - ) as TEXT - ) - ) as ez_token_transfers_id, - SYSDATE() AS _inserted_timestamp, - sysdate() as inserted_timestamp, - sysdate() as modified_timestamp -FROM - fact_logs l - LEFT JOIN {{ blockchain }}.price.EZ_PRICES_HOURLY p ON l.contract_address = p.token_address - AND DATE_TRUNC('hour', l.block_timestamp) = HOUR - LEFT JOIN {{ blockchain }}.core.DIM_CONTRACTS C ON l.contract_address = C.address -WHERE - topics [0]::STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - AND tx_status = 'SUCCESS' - and raw_amount IS NOT NULL - AND to_address IS NOT NULL -AND from_address IS NOT NULL -{% endmacro %} +{% macro get_fact_traces_columns() %} + {% set columns = [ + {'name': 'block_number', 'type': 'INTEGER'}, + {'name': 'block_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'tx_hash', 'type': 'STRING'}, + {'name': 'tx_position', 'type': 'INTEGER'}, + {'name': 'trace_index', 'type': 'INTEGER'}, + {'name': 'from_address', 'type': 'STRING'}, + {'name': 'to_address', 'type': 'STRING'}, + {'name': 'input', 'type': 'STRING'}, + {'name': 'output', 'type': 'STRING'}, + {'name': 'type', 'type': 'STRING'}, + {'name': 'trace_address', 'type': 'STRING'}, + {'name': 'sub_traces', 'type': 'INTEGER'}, + {'name': 'value', 'type': 'FLOAT'}, + {'name': 'value_precise_raw', 'type': 'STRING'}, + {'name': 'value_precise', 'type': 'STRING'}, + {'name': 'value_hex', 'type': 'STRING'}, + {'name': 'gas', 'type': 'INTEGER'}, + {'name': 'gas_used', 'type': 'INTEGER'}, + {'name': 'origin_from_address', 'type': 'STRING'}, + {'name': 'origin_to_address', 'type': 'STRING'}, + {'name': 'origin_function_signature', 'type': 'STRING'}, + {'name': 'before_evm_transfers', 'type': 'VARIANT', 'flag': 'uses_traces_arb_mode'}, + {'name': 'after_evm_transfers', 'type': 'VARIANT', 'flag': 'uses_traces_arb_mode'}, + {'name': 'trace_succeeded', 'type': 'BOOLEAN'}, + {'name': 'error_reason', 'type': 'STRING'}, + {'name': 'revert_reason', 'type': 'STRING'}, + {'name': 'tx_succeeded', 'type': 'BOOLEAN'}, + {'name': 'fact_traces_id', 'type': 'STRING'}, + {'name': 'inserted_timestamp', 'type': 'TIMESTAMP_NTZ'}, + {'name': 'modified_timestamp', 'type': 'TIMESTAMP_NTZ'} + ] %} + {{ return(columns) }} -{% macro evm_live_view_ez_native_transfers(schema, blockchain, network) %} -WITH spine AS ( - {{ evm_live_view_target_blocks(schema, blockchain, network) | indent(4) -}} -), -raw_receipts AS ( - {{ evm_live_view_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_block_txs AS ( - {{ evm_live_view_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} -), -raw_transactions AS ( - {{ evm_live_view_bronze_transactions('raw_block_txs') | indent(4) -}} -), -blocks AS ( - {{ evm_live_view_silver_blocks('raw_block_txs') | indent(4) -}} -), -receipts AS ( - {{ evm_live_view_silver_receipts('raw_receipts') | indent(4) -}} -), -transactions AS ( - {{ evm_live_view_silver_transactions('raw_transactions', 'blocks', 'receipts') | indent(4) -}} -), -raw_traces AS ( - {{ evm_live_view_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} -), -{{ evm_live_view_silver_traces('raw_traces') | indent(4) -}} -, -eth_base AS ( - SELECT - tx_hash, - block_number, - block_timestamp, - identifier, - from_address, - to_address, - eth_value AS amount, - _call_id, - _inserted_timestamp, - eth_value_precise_raw AS amount_precise_raw, - eth_value_precise AS amount_precise, - tx_position, - trace_index - FROM - traces_final - WHERE - eth_value > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' - AND TYPE NOT IN ( - 'DELEGATECALL', - 'STATICCALL' - ) -), -tx_table AS ( - SELECT - block_number, - tx_hash, - from_address AS origin_from_address, - to_address1 AS origin_to_address, - origin_function_signature - FROM - transactions - WHERE - tx_hash IN ( - SELECT - DISTINCT tx_hash - FROM - eth_base - ) -), -native_transfers AS ( - SELECT - e.tx_hash, - e.block_number, - e.block_timestamp, - e.identifier, - t.origin_from_address, - t.origin_to_address, - t.origin_function_signature, - e.from_address, - e.to_address, - e.amount, - e.amount_precise_raw, - e.amount_precise, - ROUND( - e.amount * p.price, - 2 - ) AS amount_usd, - e._call_id, - e._inserted_timestamp, - e.tx_position, - e.trace_index, - md5( - cast( - coalesce(cast(e.tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') - || '-' || coalesce(cast(e.trace_index as TEXT), '_dbt_utils_surrogate_key_null_') - as TEXT - ) - ) as native_transfers_id, - SYSDATE() as inserted_timestamp, - SYSDATE() as modified_timestamp - FROM - eth_base e - JOIN tx_table t ON e.tx_hash = t.tx_hash AND e.block_number = t.block_number - LEFT JOIN {{ blockchain }}.PRICE.EZ_PRICES_HOURLY p - ON DATE_TRUNC('hour', e.block_timestamp) = p.HOUR - AND p.token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' -- WETH address -) -SELECT - tx_hash, - block_number, - block_timestamp, - tx_position, - trace_index, - identifier AS trace_type, - origin_from_address, - origin_to_address, - origin_function_signature, - from_address AS trace_from_address, - to_address AS trace_to_address, - amount, - amount_precise_raw, - amount_precise, - amount_usd, - COALESCE( - native_transfers_id, - md5( - cast( - coalesce(cast(tx_hash as TEXT), '_dbt_utils_surrogate_key_null_') - || '-' || coalesce(cast(trace_index as TEXT), '_dbt_utils_surrogate_key_null_') - as TEXT - ) - ) - ) AS ez_native_transfers_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - native_transfers -QUALIFY (ROW_NUMBER() OVER ( - PARTITION BY block_number, tx_position, trace_index - ORDER BY _inserted_timestamp DESC -)) = 1 {% endmacro %} diff --git a/macros/livequery/manage_stored_procedures.sql b/macros/livequery/manage_stored_procedures.sql new file mode 100644 index 00000000..cb222b03 --- /dev/null +++ b/macros/livequery/manage_stored_procedures.sql @@ -0,0 +1,63 @@ +{% macro create_stored_procedure( + name_, + signature, + return_type, + sql_, + language='sql', + execute_as='caller', + options=none +) %} + CREATE OR REPLACE PROCEDURE {{ name_ }}( + {{- compile_signature(signature) }} + ) + RETURNS {{ return_type }} + LANGUAGE {{ language }} + EXECUTE AS {{ execute_as }} + {% if options %} + {{ options }} + {% endif %} + AS + $$ + {{ sql_ }} + $$; +{% endmacro %} + +{% macro ephemeral_deploy_procedures(configs) %} + {%- set blockchain = this.schema -%} + {%- set network = this.identifier -%} + {% set schema = blockchain ~ "_" ~ network %} + + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% set sql %} + {% for config in configs %} + CREATE SCHEMA IF NOT EXISTS {{ schema }}; + + {% for sp in fromyaml(config(blockchain, network)) %} + {% if var("DROP_UDFS_AND_SPS") %} + DROP PROCEDURE IF EXISTS {{ sp.name }}({{ compile_signature(sp.signature, drop_=True) }}); + {% else %} + {{ create_stored_procedure( + name_=sp.name, + signature=sp.signature, + return_type=sp.return_type, + sql_=sp.sql, + language=sp.language | default('sql'), + execute_as=sp.execute_as | default('caller'), + options=sp.options + ) }} + {% endif %} + {% endfor %} + {% endfor %} + {% endset %} + + {% if var("DROP_UDFS_AND_SPS") %} + {% do log("Drop Stored Procedures: " ~ this.database ~ "." ~ schema, true) %} + {% else %} + {% do log("Deploy Stored Procedures: " ~ this.database ~ "." ~ schema, true) %} + {% endif %} + + {% do run_query(sql ~ apply_grants_by_schema(schema)) %} + {% endif %} + + SELECT '{{ model.schema }}' as schema_ +{% endmacro %} diff --git a/macros/livequery/manage_udfs.sql b/macros/livequery/manage_udfs.sql index b5b656f4..fedd21b5 100644 --- a/macros/livequery/manage_udfs.sql +++ b/macros/livequery/manage_udfs.sql @@ -152,7 +152,7 @@ This macro is used to deploy functions using ephemeral models. It should only be used within an ephemeral model. #} - {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} {% set sql %} {{- crud_udfs(config, this.schema, var("DROP_UDFS_AND_SPS")) -}} {%- endset -%} @@ -174,7 +174,7 @@ {%- set blockchain = this.schema -%} {%- set network = this.identifier -%} {% set schema = blockchain ~ "_" ~ network %} - {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} {% set sql %} {% for config in configs %} {{- crud_udfs_by_chain(config, blockchain, network, var("DROP_UDFS_AND_SPS")) -}} @@ -197,7 +197,7 @@ #} {%- set schema = this.schema -%} {%- set utility_schema = this.identifier -%} - {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} {% set sql %} {% for config in configs %} {{- crud_udfs_by_marketplace(config, schema, utility_schema, var("DROP_UDFS_AND_SPS")) -}} diff --git a/macros/utils/logging.sql b/macros/utils/logging.sql new file mode 100644 index 00000000..ccb10200 --- /dev/null +++ b/macros/utils/logging.sql @@ -0,0 +1,36 @@ +{% macro log_model_details(vars=false, params=false) %} + +{%- if execute -%} +/* +DBT Model Config: +{{ model.config | tojson(indent=2) }} +*/ + +{% if vars is not false %} + +{% if var('LOG_MODEL_DETAILS', false) %} +{{ log( vars | tojson(indent=2), info=True) }} +{% endif %} +/* +Variables: +{{ vars | tojson(indent=2) }} +*/ +{% endif %} + +{% if params is not false %} + +{% if var('LOG_MODEL_DETAILS', false) %} +{{ log( params | tojson(indent=2), info=True) }} +{% endif %} +/* +Parameters: +{{ params | tojson(indent=2) }} +*/ +{% endif %} + +/* +Raw Code: +{{ model.raw_code }} +*/ +{%- endif -%} +{% endmacro %} diff --git a/macros/utils/udtf_utils/render.sql b/macros/utils/udtf_utils/render.sql new file mode 100644 index 00000000..ad0f3f09 --- /dev/null +++ b/macros/utils/udtf_utils/render.sql @@ -0,0 +1,65 @@ +{% macro get_rendered_model(package_name, model_name, schema, blockchain, network) %} + {# + This macro retrieves and renders a specified model from the graph. + + Args: + package_name (str): The name of the package containing the model. + model_name (str): The name of the model to be rendered. + schema (str): The schema to be used. + blockchain (str): The blockchain to be used. + network (str): The network to be used. + + Returns: + str: The rendered SQL of the specified model. + #} + {% if execute %} + {{ log("=== Starting get_rendered_model ===", info=True) }} + {# Use a list to store the node to avoid scope issues #} + {%- set nodes = [] -%} + {{ log("Looking for node: " ~ package_name ~ "." ~ model_name, info=True) }} + {%- for node in graph.nodes.values() -%} + {%- if node.package_name == package_name and node.name == model_name -%} + {{ log("Found target node: " ~ node.unique_id, info=True) }} + {%- do nodes.append(node) -%} + {%- endif -%} + {%- endfor -%} + + {%- if nodes | length == 0 -%} + {{ log("No target node found!", info=True) }} + {{ return('') }} + {%- endif -%} + + {%- set target_node = nodes[0] -%} + {{ log("Processing node: " ~ target_node.unique_id, info=True) }} + {{ log("Dependencies: " ~ target_node.depends_on.nodes | join(", "), info=True) }} + + {# First render all dependency CTEs #} + {%- set ctes = [] -%} + {%- for dep_id in target_node.depends_on.nodes -%} + {%- set dep_node = graph.nodes[dep_id] -%} + {%- set rendered_sql = render(dep_node.raw_code) | trim -%} + + {%- if rendered_sql -%} + {%- set cte_sql -%} +__dbt__cte__{{ dep_node.name }} AS ( + {{ rendered_sql }} +) + {%- endset -%} + {%- do ctes.append(cte_sql) -%} + {%- endif -%} + {%- endfor -%} + + {{ log("Number of CTEs generated: " ~ ctes | length, info=True) }} + + {# Combine CTEs with main query #} + {%- set final_sql -%} +WITH {{ ctes | join(',\n\n') }} + +{{ render(target_node.raw_code) }} + {%- endset -%} + + {{ log("=== End get_rendered_model ===", info=True) }} + + {{ return(final_sql) }} + {% endif %} +{% endmacro %} diff --git a/macros/utils/udtf_utils/udtf_return_type.sql b/macros/utils/udtf_utils/udtf_return_type.sql new file mode 100644 index 00000000..a2e30997 --- /dev/null +++ b/macros/utils/udtf_utils/udtf_return_type.sql @@ -0,0 +1,72 @@ +{# Macro to determine which columns to include based on feature flags #} +{% macro generate_udtf_return_type(blockchain, columns) %} + + {# Set Global Variables #} + {%- set GLOBAL_PROD_DB_NAME = blockchain -%} + + {# Columns included by default, with specific exclusions #} + {%- set excludes_eip_1559 = ['CORE','RONIN'] -%} + {%- set excludes_base_fee = ['CORE'] -%} + {%- set excludes_total_difficulty = ['INK','SWELL'] -%} + + {# Columns excluded by default, with explicit inclusion #} + {%- set includes_l1_columns = ['INK', 'MANTLE', 'SWELL'] -%} + {%- set includes_l1_tx_fee_calc = ['INK', 'MANTLE', 'SWELL'] -%} + {%- set includes_eth_value = ['MANTLE'] -%} + {%- set includes_mint = ['INK', 'MANTLE', 'SWELL'] -%} + {%- set includes_y_parity = ['INK', 'SWELL'] -%} + {%- set includes_access_list = ['INK', 'SWELL'] -%} + {%- set includes_source_hash = ['INK','MANTLE','SWELL'] -%} + {%- set includes_blob_base_fee = ['INK','SWELL'] -%} + {%- set includes_mix_hash = ['INK', 'MANTLE', 'SWELL', 'RONIN'] -%} + {%- set includes_blob_gas_used = ['INK', 'SWELL'] -%} + {%- set includes_parent_beacon_block_root = ['INK', 'SWELL'] -%} + {%- set includes_withdrawals = ['INK', 'SWELL'] -%} + + {# Set Variables using inclusions and exclusions #} + {%- set current_db = blockchain.upper() -%} + + {# Transaction feature flags #} + {%- set uses_eip_1559 = current_db not in excludes_eip_1559 -%} + {%- set uses_l1_columns = current_db in includes_l1_columns -%} + {%- set uses_l1_tx_fee_calc = current_db in includes_l1_tx_fee_calc -%} + {%- set uses_eth_value = current_db in includes_eth_value -%} + {%- set uses_mint = current_db in includes_mint -%} + {%- set uses_y_parity = current_db in includes_y_parity -%} + {%- set uses_access_list = current_db in includes_access_list -%} + {%- set uses_source_hash = current_db in includes_source_hash -%} + {%- set uses_blob_base_fee = current_db in includes_blob_base_fee -%} + {%- set uses_traces_arb_mode = var('TRACES_ARB_MODE', false) -%} + + {# Block feature flags #} + {%- set uses_base_fee = current_db not in excludes_base_fee -%} + {%- set uses_total_difficulty = current_db not in excludes_total_difficulty -%} + {%- set uses_mix_hash = current_db in includes_mix_hash -%} + {%- set uses_blob_gas_used = current_db in includes_blob_gas_used -%} + {%- set uses_parent_beacon_block_root = current_db in includes_parent_beacon_block_root -%} + {%- set uses_withdrawals = current_db in includes_withdrawals -%} + + TABLE( + {%- for col in columns %} + {%- if col.flag is not defined + or (col.flag == 'uses_eip_1559' and uses_eip_1559) + or (col.flag == 'uses_l1_columns' and uses_l1_columns) + or (col.flag == 'uses_eth_value' and uses_eth_value) + or (col.flag == 'uses_mint' and uses_mint) + or (col.flag == 'uses_y_parity' and uses_y_parity) + or (col.flag == 'uses_access_list' and uses_access_list) + or (col.flag == 'uses_source_hash' and uses_source_hash) + or (col.flag == 'uses_blob_base_fee' and uses_blob_base_fee) + or (col.flag == 'uses_traces_arb_mode' and uses_traces_arb_mode) + or (col.flag == 'uses_mix_hash' and uses_mix_hash) + or (col.flag == 'uses_base_fee' and uses_base_fee) + or (col.flag == 'uses_total_difficulty' and uses_total_difficulty) + or (col.flag == 'uses_blob_gas_used' and uses_blob_gas_used) + or (col.flag == 'uses_parent_beacon_block_root' and uses_parent_beacon_block_root) + or (col.flag == 'uses_withdrawals' and uses_withdrawals) + %} + {{ col.name }} {{ col.type }}{% if not loop.last %},{% endif %} + {%- endif -%} + {%- endfor -%} + ) +{% endmacro %} diff --git a/models/deploy/evm/ethereum__mainnet.sql b/models/deploy/evm/ethereum__mainnet.sql index ce18a8ba..ea342a1a 100644 --- a/models/deploy/evm/ethereum__mainnet.sql +++ b/models/deploy/evm/ethereum__mainnet.sql @@ -3,11 +3,37 @@ -- depends_on: {{ ref('_evm__native_symbol_map') }} -- depends_on: {{ ref('_eth__logs') }} -- depends_on: {{ ref('_eth__decoded_logs') }} - +-- depends_on: {{ ref('live') }} +-- depends_on: {{ ref('utils') }} +-- depends_on: {{ ref('price__ez_prices_hourly') }} +-- depends_on: {{ ref('core__dim_contracts')}} +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('bronze__traces') }} +-- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('bronze__decoded_logs') }} +-- depends_on: {{ ref('bronze__decoded_logs_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('fsc_evm', 'silver__decoded_logs') }} +-- depends_on: {{ ref('evm__fact_blocks') }} +-- depends_on: {{ ref('evm__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} +-- depends_on: {{ ref('fsc_evm', 'core__ez_native_transfers') }} +-- depends_on: {{ ref('fsc_evm', 'core__ez_token_transfers') }} +-- depends_on: {{ ref('fsc_evm', 'core__ez_decoded_event_logs') }} {%- set configs = [ config_evm_rpc_primitives, config_evm_high_level_abstractions, - config_eth_high_level_abstractions ] -%} {{- ephemeral_deploy(configs) -}} diff --git a/models/doc_descriptions/integration/__overview__.md b/models/doc_descriptions/integration/__overview__.md index 9c0c0a12..d2002c7f 100644 --- a/models/doc_descriptions/integration/__overview__.md +++ b/models/doc_descriptions/integration/__overview__.md @@ -46,9 +46,9 @@ Within the same directory (`models/deploy/marketplace/`), create a new folder fo Deploy your model following the standard deployment procedures. -- `dbt run -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"UPDATE_UDFS_AND_SPS":True}'` +- `dbt run -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}'` -- `dbt test -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"UPDATE_UDFS_AND_SPS":True}'` +- `dbt test -s models/deploy/marketplace/your_model/your_model__.sql -t dev --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}'` #### Additional Tips: diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql new file mode 100644 index 00000000..2e3761f7 --- /dev/null +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs.sql @@ -0,0 +1 @@ +SELECT 1 diff --git a/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql new file mode 100644 index 00000000..958dbca4 --- /dev/null +++ b/models/overrides/evm/decoder_package/decoded_logs/bronze/bronze__decoded_logs_fr.sql @@ -0,0 +1,20 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + + SELECT + block_number, + tx_hash :: STRING || '-' || event_index :: STRING AS id, + OBJECT_CONSTRUCT('topics', topics, 'data', data, 'address', contract_address) AS event_data, + utils.udf_evm_decode_log(abi, event_data)[0] AS DATA, + TO_TIMESTAMP_NTZ(inserted_timestamp) AS _inserted_timestamp, + _inserted_timestamp AS _partition_by_created_date + FROM + {{ ref('fsc_evm', 'core__fact_event_logs')}} + JOIN + {{ blockchain }}.core.dim_contract_abis + USING + (contract_address) + WHERE + tx_succeeded = TRUE + diff --git a/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql b/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql new file mode 100644 index 00000000..6f17667e --- /dev/null +++ b/models/overrides/evm/decoder_package/decoded_logs/gold/evm__ez_decoded_event_logs.sql @@ -0,0 +1,18 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} +-- depends_on: {{ ref('bronze__decoded_logs') }} +-- depends_on: {{ ref('bronze__decoded_logs_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__decoded_logs') }} +-- depends_on: {{ ref('core__dim_contracts')}} + +SELECT * FROM {{ ref('fsc_evm', 'core__ez_decoded_event_logs') }} diff --git a/models/overrides/evm/main_package/core/bronze/bronze__blocks.sql b/models/overrides/evm/main_package/core/bronze/bronze__blocks.sql new file mode 100644 index 00000000..3b26fd95 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__blocks.sql @@ -0,0 +1,2 @@ +-- unused in livequery +SELECT 1 diff --git a/models/overrides/evm/main_package/core/bronze/bronze__blocks_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__blocks_fr.sql new file mode 100644 index 00000000..81e31ccb --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__blocks_fr.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_blocks AS ( + {{ evm_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_blocks diff --git a/models/overrides/evm/main_package/core/bronze/bronze__receipts.sql b/models/overrides/evm/main_package/core/bronze/bronze__receipts.sql new file mode 100644 index 00000000..3b26fd95 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__receipts.sql @@ -0,0 +1,2 @@ +-- unused in livequery +SELECT 1 diff --git a/models/overrides/evm/main_package/core/bronze/bronze__receipts_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__receipts_fr.sql new file mode 100644 index 00000000..80b72ea9 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__receipts_fr.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_receipts AS ( + {{ evm_bronze_receipts(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_receipts diff --git a/models/overrides/evm/main_package/core/bronze/bronze__traces.sql b/models/overrides/evm/main_package/core/bronze/bronze__traces.sql new file mode 100644 index 00000000..3b26fd95 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__traces.sql @@ -0,0 +1,2 @@ +-- unused in livequery +SELECT 1 diff --git a/models/overrides/evm/main_package/core/bronze/bronze__traces_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__traces_fr.sql new file mode 100644 index 00000000..ab4d8397 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__traces_fr.sql @@ -0,0 +1,13 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_traces AS ( + {{ evm_bronze_traces(schema, blockchain, network, 'spine') | indent(4) -}} +) + +SELECT * FROM raw_traces diff --git a/models/overrides/evm/main_package/core/bronze/bronze__transactions.sql b/models/overrides/evm/main_package/core/bronze/bronze__transactions.sql new file mode 100644 index 00000000..3b26fd95 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__transactions.sql @@ -0,0 +1,2 @@ +-- unused in livequery +SELECT 1 diff --git a/models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql b/models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql new file mode 100644 index 00000000..bd8c9805 --- /dev/null +++ b/models/overrides/evm/main_package/core/bronze/bronze__transactions_fr.sql @@ -0,0 +1,17 @@ +{%- set blockchain = this.schema -%} +{%- set network = this.identifier -%} +{%- set schema = blockchain ~ "_" ~ network -%} + +WITH spine AS ( + {{ evm_target_blocks(schema, blockchain, network, 10) | indent(4) -}} +), + +raw_block_txs AS ( + {{ evm_bronze_blocks(schema, blockchain, network, 'spine') | indent(4) -}} +), + +raw_transactions AS ( + {{ evm_bronze_transactions('raw_block_txs') | indent(4) -}} +) + +SELECT * FROM raw_transactions diff --git a/models/overrides/evm/main_package/core/gold/core__dim_contracts.sql b/models/overrides/evm/main_package/core/gold/core__dim_contracts.sql new file mode 100644 index 00000000..34927953 --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/core__dim_contracts.sql @@ -0,0 +1,3 @@ +{%- set blockchain = this.schema -%} + +SELECT * FROM {{ blockchain.upper() }}.CORE.DIM_CONTRACTS diff --git a/models/overrides/evm/main_package/core/gold/evm__ez_native_transfers.sql b/models/overrides/evm/main_package/core/gold/evm__ez_native_transfers.sql new file mode 100644 index 00000000..0364b67e --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__ez_native_transfers.sql @@ -0,0 +1,18 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('bronze__traces') }} +-- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_traces') }} +-- depends_on: {{ ref('price__ez_prices_hourly') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__ez_native_transfers') }} diff --git a/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql b/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql new file mode 100644 index 00000000..f872258e --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__ez_token_transfers.sql @@ -0,0 +1,16 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_event_logs') }} +-- depends_on: {{ ref('price__ez_prices_hourly') }} +-- depends_on: {{ ref('core__dim_contracts') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__ez_token_transfers') }} diff --git a/models/overrides/evm/main_package/core/gold/evm__fact_blocks.sql b/models/overrides/evm/main_package/core/gold/evm__fact_blocks.sql new file mode 100644 index 00000000..73be526f --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__fact_blocks.sql @@ -0,0 +1,5 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_blocks') }} diff --git a/models/overrides/evm/main_package/core/gold/evm__fact_event_logs.sql b/models/overrides/evm/main_package/core/gold/evm__fact_event_logs.sql new file mode 100644 index 00000000..72caac05 --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__fact_event_logs.sql @@ -0,0 +1,13 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_event_logs') }} diff --git a/models/overrides/evm/main_package/core/gold/evm__fact_traces.sql b/models/overrides/evm/main_package/core/gold/evm__fact_traces.sql new file mode 100644 index 00000000..d7a21f01 --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__fact_traces.sql @@ -0,0 +1,16 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('bronze__traces') }} +-- depends_on: {{ ref('bronze__traces_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__traces') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_transactions') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_traces') }} diff --git a/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql b/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql new file mode 100644 index 00000000..0bd78a4d --- /dev/null +++ b/models/overrides/evm/main_package/core/gold/evm__fact_transactions.sql @@ -0,0 +1,12 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__blocks_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__blocks') }} +-- depends_on: {{ ref('fsc_evm', 'core__fact_blocks') }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} +-- depends_on: {{ ref('bronze__receipts') }} +-- depends_on: {{ ref('bronze__receipts_fr') }} +-- depends_on: {{ ref('fsc_evm', 'silver__transactions') }} +-- depends_on: {{ ref('fsc_evm', 'silver__receipts') }} + +SELECT * FROM {{ ref('fsc_evm', 'core__fact_transactions') }} diff --git a/models/overrides/evm/main_package/prices/price__ez_prices_hourly.sql b/models/overrides/evm/main_package/prices/price__ez_prices_hourly.sql new file mode 100644 index 00000000..2f13307a --- /dev/null +++ b/models/overrides/evm/main_package/prices/price__ez_prices_hourly.sql @@ -0,0 +1,3 @@ +{%- set blockchain = this.schema -%} + +SELECT * FROM {{ blockchain.upper() }}.PRICE.EZ_PRICES_HOURLY diff --git a/package-lock.yml b/package-lock.yml index 84537d91..5b6a7e68 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -1,8 +1,18 @@ packages: - package: calogica/dbt_expectations - version: 0.8.5 + version: 0.8.2 - package: dbt-labs/dbt_utils version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-evm.git + revision: 31a7c51d6486abdbf9e8c6f2f1a2e7079413df43 - package: calogica/dbt_date version: 0.7.2 -sha1_hash: beca0ae13045be0399683f2d9a36c07e1674880d +- package: dbt-labs/dbt_external_tables + version: 0.8.2 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: 6096046f00c5c27fa7b8d60ae6ec1dfe31d7fa10 +- package: get-select/dbt_snowflake_query_tags + version: 2.5.0 +- git: https://github.com/FlipsideCrypto/livequery-base.git + revision: 006980d40fbbc48c2ad0f0b74efa37ec6fa1c74b +sha1_hash: d8ed2190282b052a4e8372ca9c4d08f7fa0d7327 diff --git a/packages.yml b/packages.yml index 172201ae..280c810c 100644 --- a/packages.yml +++ b/packages.yml @@ -3,5 +3,7 @@ packages: version: [">=0.8.0", "<0.9.0"] - package: dbt-labs/dbt_utils version: [">=1.0.0", "<1.1.0"] + - git: "https://github.com/FlipsideCrypto/fsc-evm.git" + revision: "STREAM-1049/add-package-name-for-livequery" - git: https://github.com/FlipsideCrypto/livequery-base.git - revision: v1.0.1 \ No newline at end of file + revision: v1.1.0 diff --git a/requirements.txt b/requirements.txt index f32ca688..54b64e9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt-snowflake~=1.5.0 +dbt-snowflake~=1.7.0