Merge pull request #2872 from blockstack/feat/fee-rate-rpc

Fee estimation: RPC interface for estimating transaction costs and fees

Author: Aaron Blankstein

.github/workflows/bitcoin-tests.yml

@@ -52,7 +52,7 @@ jobs:
           - tests::neon_integrations::mining_transactions_is_fair
     steps:
       - uses: actions/checkout@v2
-      - name: Download math result for job 1
+      - name: Download docker image
         uses: actions/download-artifact@v2
         with:
           name: integration-image.tar

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 - FeeEstimator and CostEstimator interfaces. These can be controlled
   via node configuration options. See the `README.md` for more
   information on the configuration.
+- New fee rate estimation endpoint `/v2/fees/transaction` (#2872). See
+  `docs/rpc/openapi.yaml` for more information.
 
 ## Changed
 

docs/rpc/api/core-node/post-fee-transaction-response.example.json

@@ -0,0 +1,25 @@
+{
+  "cost_scalar_change_by_byte": 0.00476837158203125,
+  "estimated_cost": {
+    "read_count": 19,
+    "read_length": 4814,
+    "runtime": 7175000,
+    "write_count": 2,
+    "write_length": 1020
+  },
+  "estimated_cost_scalar": 14,
+  "estimations": [
+    {
+      "fee": 17,
+      "fee_rate": 1.2410714285714286
+    },
+    {
+      "fee": 125,
+      "fee_rate": 8.958333333333332
+    },
+    {
+      "fee": 140,
+      "fee_rate": 10
+    }
+  ]
+}

docs/rpc/api/core-node/post-fee-transaction-response.schema.json

@@ -0,0 +1,42 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "POST response for estimated fee",
+  "title": "TransactionFeeEstimateResponse",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["estimated_cost", "estimated_cost_scalar", "estimated_fee_rates", "estimated_fees"],
+  "properties": {
+    "estimated_cost_scalar": {
+      "type": "integer"
+    },
+    "cost_scalar_change_by_byte": {
+      "type": "number"
+    },
+    "estimated_cost": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["read_count", "write_count", "read_length", "write_length", "runtime"],
+      "properties": {
+        "read_count": { "type": "integer" },
+        "read_length": { "type": "integer" },
+        "runtime": { "type": "integer" },
+        "write_count": { "type": "integer" },
+        "write_length": { "type": "integer" }
+      }
+    },
+    "estimations": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "fee_rate": {
+            "type": "number"
+          },
+          "fee": {
+            "type": "number"
+          }
+        }
+      }
+    }
+  }
+}

docs/rpc/api/core-node/post-fee-transaction.example.json

@@ -0,0 +1,4 @@
+{
+  "estimated_len": 350,
+  "transaction_payload": "021af942874ce525e87f21bbe8c121b12fac831d02f4086765742d696e666f0b7570646174652d696e666f00000000"
+}

docs/rpc/api/core-node/post-fee-transaction.schema.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "POST request for estimated fee",
+  "title": "TransactionFeeEstimateRequest",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["transaction_payload"],
+  "properties": {
+    "transaction_payload": {
+      "type": "string"
+    },
+    "estimated_len": {
+      "type": "integer"
+    }
+  }
+}

docs/rpc/openapi.yaml

@@ -276,6 +276,96 @@ paths:
               example:
                 $ref: ./api/core-node/get-account-data.example.json
 
+  /v2/fees/transaction:
+    post:
+      summary: Get approximate fees for the given transaction
+      tags:
+        - Fees
+      description: |
+        Get an estimated fee for the supplied transaction.  This
+        estimates the execution cost of the transaction, the current
+        fee rate of the network, and returns estimates for fee
+        amounts.
+
+        * `transaction_payload` is a hex-encoded serialization of
+          the TransactionPayload for the transaction.
+        * `estimated_len` is an optional argument that provides the
+          endpoint with an estimation of the final length (in bytes)
+          of the transaction, including any post-conditions and
+          signatures
+
+        If the node cannot provide an estimate for the transaction
+        (e.g., if the node has never seen a contract-call for the
+        given contract and function) or if estimation is not
+        configured on this node, a 400 response is returned.
+        The 400 response will be a JSON error containing a `reason`
+        field which can be one of the following:
+
+        * `DatabaseError` - this Stacks node has had an internal
+          database error while trying to estimate the costs of the
+          supplied transaction.
+        * `NoEstimateAvailable` - this Stacks node has not seen this
+          kind of contract-call before, and it cannot provide an
+          estimate yet.
+        * `CostEstimationDisabled` - this Stacks node does not perform
+          fee or cost estimation, and it cannot respond on this
+          endpoint.
+
+        The 200 response contains the following data:
+
+        * `estimated_cost` - the estimated multi-dimensional cost of
+          executing the Clarity VM on the provided transaction.
+        * `estimated_cost_scalar` - a unitless integer that the Stacks
+          node uses to compare how much of the block limit is consumed
+          by different transactions. This value incorporates the
+          estimated length of the transaction and the estimated
+          execution cost of the transaction. The range of this integer
+          may vary between different Stacks nodes. In order to compute
+          an estimate of total fee amount for the transaction, this
+          value is multiplied by the same Stacks node's estimated fee
+          rate.
+        * `cost_scalar_change_by_byte` - a float value that indicates how
+          much the `estimated_cost_scalar` value would increase for every
+          additional byte in the final transaction.
+        * `estimations` - an array of estimated fee rates and total fees to
+          pay in microSTX for the transaction. This array provides a range of
+          estimates (default: 3) that may be used. Each element of the array
+          contains the following fields:
+            * `fee_rate` - the estimated value for the current fee
+              rates in the network
+            * `fee` - the estimated value for the total fee in
+              microSTX that the given transaction should pay. These
+              values are the result of computing:
+              `fee_rate` x `estimated_cost_scalar`.
+              If the estimated fees are less than the minimum relay
+              fee `(1 ustx x estimated_len)`, then that minimum relay
+              fee will be returned here instead.
+
+
+        Note: If the final transaction's byte size is larger than
+        supplied to `estimated_len`, then applications should increase
+        this fee amount by:
+
+          `fee_rate` x `cost_scalar_change_by_byte` x (`final_size` - `estimated_size`)
+
+      operationId: post_fee_transaction
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: ./api/core-node/post-fee-transaction.schema.json
+            example:
+              $ref: ./api/core-node/post-fee-transaction.example.json
+      responses:
+        200:
+          description: Estimated fees for the transaction
+          content:
+            application/json:
+              schema:
+                $ref: ./api/core-node/post-fee-transaction-response.schema.json
+              example:
+                $ref: ./api/core-node/post-fee-transaction-response.example.json
+
   /v2/fees/transfer:
     get:
       summary: Get estimated fee
@@ -378,4 +468,4 @@ paths:
         in: query
         schema:
           type: string
-        description: The Stacks chain tip to query from
+        description: The Stacks chain tip to query from

src/cost_estimates/fee_scalar.rs

@@ -20,6 +20,7 @@ use core::BLOCK_LIMIT_MAINNET;
 use chainstate::stacks::db::StacksEpochReceipt;
 use chainstate::stacks::events::TransactionOrigin;
 
+use crate::util::db::set_wal_mode;
 use crate::util::db::sql_pragma;
 use crate::util::db::table_exists;
 
@@ -31,15 +32,15 @@ const SINGLETON_ROW_ID: i64 = 1;
 const CREATE_TABLE: &'static str = "
 CREATE TABLE scalar_fee_estimator (
     estimate_key NUMBER PRIMARY KEY,
-    fast NUMBER NOT NULL,
-    medium NUMBER NOT NULL,
-    slow NUMBER NOT NULL
+    high NUMBER NOT NULL,
+    middle NUMBER NOT NULL,
+    low NUMBER NOT NULL
 )";
 
 /// This struct estimates fee rates by translating a transaction's `ExecutionCost`
 /// into a scalar using `ExecutionCost::proportion_dot_product` and computing
 /// the subsequent fee rate using the actual paid fee. The 5th, 50th and 95th
-/// percentile fee rates for each block are used as the slow, medium, and fast
+/// percentile fee rates for each block are used as the low, middle, and high
 /// estimates. Estimates are updated via exponential decay windowing.
 pub struct ScalarFeeRateEstimator<M: CostMetric> {
     db: Connection,
@@ -53,11 +54,16 @@ pub struct ScalarFeeRateEstimator<M: CostMetric> {
 impl<M: CostMetric> ScalarFeeRateEstimator<M> {
     /// Open a fee rate estimator at the given db path. Creates if not existent.
     pub fn open(p: &Path, metric: M) -> Result<Self, SqliteError> {
-        let db = Connection::open_with_flags(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE)
-            .or_else(|e| {
+        let db = match Connection::open_with_flags(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE) {
+            Ok(db) => {
+                set_wal_mode(&db)?;
+                Ok(db)
+            }
+            Err(e) => {
                 if let SqliteError::SqliteFailure(ref internal, _) = e {
                     if let rusqlite::ErrorCode::CannotOpen = internal.code {
                         let mut db = Connection::open(p)?;
+                        set_wal_mode(&db)?;
                         let tx = tx_begin_immediate_sqlite(&mut db)?;
                         Self::instantiate_db(&tx)?;
                         tx.commit()?;
@@ -68,7 +74,9 @@ impl<M: CostMetric> ScalarFeeRateEstimator<M> {
                 } else {
                     Err(e)
                 }
-            })?;
+            }
+        }?;
+
         Ok(Self {
             db,
             metric,
@@ -84,7 +92,6 @@ impl<M: CostMetric> ScalarFeeRateEstimator<M> {
 
     fn instantiate_db(tx: &SqlTransaction) -> Result<(), SqliteError> {
         if !Self::db_already_instantiated(tx)? {
-            tx.pragma_update(None, "journal_mode", &"WAL".to_string())?;
             tx.execute(CREATE_TABLE, rusqlite::NO_PARAMS)?;
         }
 
@@ -103,18 +110,18 @@ impl<M: CostMetric> ScalarFeeRateEstimator<M> {
                 // because of integer math, we can end up with some edge effects
                 // when the estimate is < decay_rate_fraction.1, so just saturate
                 // on the low end at a rate of "1"
-                next_computed.fast = if next_computed.fast >= 1f64 {
-                    next_computed.fast
+                next_computed.high = if next_computed.high >= 1f64 {
+                    next_computed.high
                 } else {
                     1f64
                 };
-                next_computed.medium = if next_computed.medium >= 1f64 {
-                    next_computed.medium
+                next_computed.middle = if next_computed.middle >= 1f64 {
+                    next_computed.middle
                 } else {
                     1f64
                 };
-                next_computed.slow = if next_computed.slow >= 1f64 {
-                    next_computed.slow
+                next_computed.low = if next_computed.low >= 1f64 {
+                    next_computed.low
                 } else {
                     1f64
                 };
@@ -129,25 +136,25 @@ impl<M: CostMetric> ScalarFeeRateEstimator<M> {
         };
 
         debug!("Updating fee rate estimate for new block";
-               "new_measure_fast" => new_measure.fast,
-               "new_measure_medium" => new_measure.medium,
-               "new_measure_slow" => new_measure.slow,
-               "new_estimate_fast" => next_estimate.fast,
-               "new_estimate_medium" => next_estimate.medium,
-               "new_estimate_slow" => next_estimate.slow);
+               "new_measure_high" => new_measure.high,
+               "new_measure_middle" => new_measure.middle,
+               "new_measure_low" => new_measure.low,
+               "new_estimate_high" => next_estimate.high,
+               "new_estimate_middle" => next_estimate.middle,
+               "new_estimate_low" => next_estimate.low);
 
         let sql = "INSERT OR REPLACE INTO scalar_fee_estimator
-                     (estimate_key, fast, medium, slow) VALUES (?, ?, ?, ?)";
+                     (estimate_key, high, middle, low) VALUES (?, ?, ?, ?)";
 
         let tx = tx_begin_immediate_sqlite(&mut self.db).expect("SQLite failure");
 
         tx.execute(
             sql,
             rusqlite::params![
                 SINGLETON_ROW_ID,
-                next_estimate.fast,
-                next_estimate.medium,
-                next_estimate.slow,
+                next_estimate.high,
+                next_estimate.middle,
+                next_estimate.low,
             ],
         )
         .expect("SQLite failure");
@@ -207,13 +214,13 @@ impl<M: CostMetric> FeeEstimator for ScalarFeeRateEstimator<M> {
         let measures_len = all_fee_rates.len();
         if measures_len > 0 {
             // use 5th, 50th, and 95th percentiles from block
-            let fastest_index = measures_len - cmp::max(1, measures_len / 20);
+            let highest_index = measures_len - cmp::max(1, measures_len / 20);
             let median_index = measures_len / 2;
-            let slowest_index = measures_len / 20;
+            let lowest_index = measures_len / 20;
             let block_estimate = FeeRateEstimate {
-                fast: all_fee_rates[fastest_index],
-                medium: all_fee_rates[median_index],
-                slow: all_fee_rates[slowest_index],
+                high: all_fee_rates[highest_index],
+                middle: all_fee_rates[median_index],
+                low: all_fee_rates[lowest_index],
             };
 
             self.update_estimate(block_estimate);
@@ -223,17 +230,17 @@ impl<M: CostMetric> FeeEstimator for ScalarFeeRateEstimator<M> {
     }
 
     fn get_rate_estimates(&self) -> Result<FeeRateEstimate, EstimatorError> {
-        let sql = "SELECT fast, medium, slow FROM scalar_fee_estimator WHERE estimate_key = ?";
+        let sql = "SELECT high, middle, low FROM scalar_fee_estimator WHERE estimate_key = ?";
         self.db
             .query_row(sql, &[SINGLETON_ROW_ID], |row| {
-                let fast: f64 = row.get(0)?;
-                let medium: f64 = row.get(1)?;
-                let slow: f64 = row.get(2)?;
-                Ok((fast, medium, slow))
+                let high: f64 = row.get(0)?;
+                let middle: f64 = row.get(1)?;
+                let low: f64 = row.get(2)?;
+                Ok((high, middle, low))
             })
             .optional()
             .expect("SQLite failure")
-            .map(|(fast, medium, slow)| FeeRateEstimate { fast, medium, slow })
+            .map(|(high, middle, low)| FeeRateEstimate { high, middle, low })
             .ok_or_else(|| EstimatorError::NoEstimateAvailable)
     }
 }