From efbf76613baf65e3a8061a41078aab60d0bc0e09 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 00:24:54 +0200 Subject: [PATCH 01/25] new skill from original docs --- .claude-plugin/marketplace.json | 6 + nautilus-docs/REBUILD.md | 233 ++ nautilus-docs/SKILL.md | 293 +++ nautilus-docs/agents/openai.yaml | 4 + nautilus-docs/battle_tested.md | 221 ++ .../docs/api_reference/_static/custom.css | 14 + .../docs/api_reference/accounting.md | 45 + .../docs/api_reference/adapters/betfair.md | 109 + .../docs/api_reference/adapters/binance.md | 143 ++ .../docs/api_reference/adapters/bybit.md | 69 + .../docs/api_reference/adapters/databento.md | 79 + .../docs/api_reference/adapters/dydx.md | 59 + .../docs/api_reference/adapters/index.md | 23 + .../adapters/interactive_brokers.md | 117 + .../docs/api_reference/adapters/okx.md | 59 + .../docs/api_reference/adapters/polymarket.md | 69 + .../docs/api_reference/adapters/tardis.md | 59 + nautilus-docs/docs/api_reference/analysis.md | 29 + nautilus-docs/docs/api_reference/backtest.md | 61 + nautilus-docs/docs/api_reference/cache.md | 29 + nautilus-docs/docs/api_reference/common.md | 59 + nautilus-docs/docs/api_reference/conf.py | 122 + nautilus-docs/docs/api_reference/config.md | 101 + nautilus-docs/docs/api_reference/core.md | 55 + nautilus-docs/docs/api_reference/data.md | 45 + nautilus-docs/docs/api_reference/execution.md | 75 + nautilus-docs/docs/api_reference/index.md | 8 + .../docs/api_reference/indicators.md | 61 + nautilus-docs/docs/api_reference/live.md | 61 + .../docs/api_reference/model/book.md | 9 + .../docs/api_reference/model/data.md | 9 + .../docs/api_reference/model/events.md | 29 + .../docs/api_reference/model/identifiers.md | 9 + .../docs/api_reference/model/index.md | 23 + .../docs/api_reference/model/instruments.md | 141 ++ .../docs/api_reference/model/objects.md | 9 + .../docs/api_reference/model/orders.md | 93 + .../docs/api_reference/model/position.md | 9 + .../docs/api_reference/model/tick_scheme.md | 29 + .../docs/api_reference/persistence.md | 37 + nautilus-docs/docs/api_reference/portfolio.md | 21 + nautilus-docs/docs/api_reference/risk.md | 21 + .../docs/api_reference/serialization.md | 21 + nautilus-docs/docs/api_reference/system.md | 13 + nautilus-docs/docs/api_reference/trading.md | 37 + nautilus-docs/docs/concepts/actors.md | 342 +++ nautilus-docs/docs/concepts/adapters.md | 194 ++ nautilus-docs/docs/concepts/architecture.md | 641 +++++ nautilus-docs/docs/concepts/backtesting.md | 1636 +++++++++++++ nautilus-docs/docs/concepts/cache.md | 539 +++++ nautilus-docs/docs/concepts/custom_data.md | 396 +++ nautilus-docs/docs/concepts/data.md | 1892 +++++++++++++++ nautilus-docs/docs/concepts/execution.md | 504 ++++ nautilus-docs/docs/concepts/greeks.md | 321 +++ nautilus-docs/docs/concepts/index.md | 113 + nautilus-docs/docs/concepts/instruments.md | 541 +++++ nautilus-docs/docs/concepts/live.md | 597 +++++ nautilus-docs/docs/concepts/logging.md | 470 ++++ nautilus-docs/docs/concepts/message_bus.md | 473 ++++ nautilus-docs/docs/concepts/options.md | 291 +++ nautilus-docs/docs/concepts/order_book.md | 246 ++ nautilus-docs/docs/concepts/orders.md | 838 +++++++ nautilus-docs/docs/concepts/overview.md | 255 ++ nautilus-docs/docs/concepts/portfolio.md | 169 ++ nautilus-docs/docs/concepts/positions.md | 460 ++++ nautilus-docs/docs/concepts/reports.md | 435 ++++ nautilus-docs/docs/concepts/strategies.md | 692 ++++++ nautilus-docs/docs/concepts/value_types.md | 303 +++ nautilus-docs/docs/concepts/visualization.md | 571 +++++ .../docs/dev_templates/criterion_template.rs | 52 + .../docs/dev_templates/iai_template.rs | 33 + .../docs/developer_guide/adapters.md | 2131 +++++++++++++++++ .../docs/developer_guide/benchmarking.md | 180 ++ .../docs/developer_guide/coding_standards.md | 141 ++ nautilus-docs/docs/developer_guide/docs.md | 134 ++ .../docs/developer_guide/environment_setup.md | 399 +++ nautilus-docs/docs/developer_guide/ffi.md | 142 ++ nautilus-docs/docs/developer_guide/index.md | 27 + nautilus-docs/docs/developer_guide/python.md | 98 + .../docs/developer_guide/releases.md | 271 +++ nautilus-docs/docs/developer_guide/rust.md | 1265 ++++++++++ .../docs/developer_guide/spec_data_testing.md | 854 +++++++ .../docs/developer_guide/spec_exec_testing.md | 1690 +++++++++++++ .../docs/developer_guide/test_datasets.md | 282 +++ nautilus-docs/docs/developer_guide/testing.md | 289 +++ .../getting_started/backtest_high_level.py | 251 ++ .../getting_started/backtest_low_level.py | 220 ++ nautilus-docs/docs/getting_started/index.md | 75 + .../docs/getting_started/installation.md | 361 +++ .../docs/getting_started/quickstart.py | 211 ++ .../docs/integrations/architect_ax.md | 400 ++++ nautilus-docs/docs/integrations/betfair.md | 574 +++++ nautilus-docs/docs/integrations/binance.md | 857 +++++++ nautilus-docs/docs/integrations/bitmex.md | 871 +++++++ nautilus-docs/docs/integrations/blockchain.md | 92 + nautilus-docs/docs/integrations/bybit.md | 673 ++++++ nautilus-docs/docs/integrations/databento.md | 881 +++++++ nautilus-docs/docs/integrations/deribit.md | 648 +++++ nautilus-docs/docs/integrations/dydx.md | 803 +++++++ .../docs/integrations/hyperliquid.md | 494 ++++ nautilus-docs/docs/integrations/ib.md | 2058 ++++++++++++++++ nautilus-docs/docs/integrations/index.md | 60 + nautilus-docs/docs/integrations/kraken.md | 555 +++++ nautilus-docs/docs/integrations/okx.md | 675 ++++++ nautilus-docs/docs/integrations/polymarket.md | 801 +++++++ nautilus-docs/docs/integrations/tardis.md | 695 ++++++ .../docs/tutorials/ax_fx_mean_reversion.md | 279 +++ .../docs/tutorials/ax_gold_book_imbalance.md | 306 +++ .../tutorials/backtest_binance_orderbook.py | 190 ++ .../tutorials/backtest_bybit_orderbook.py | 184 ++ .../docs/tutorials/backtest_fx_bars.py | 164 ++ .../tutorials/bitmex_grid_market_maker.md | 541 +++++ .../docs/tutorials/databento_data_catalog.py | 221 ++ .../docs/tutorials/dydx_grid_market_maker.md | 597 +++++ nautilus-docs/docs/tutorials/index.md | 15 + .../docs/tutorials/loading_external_data.py | 123 + 116 files changed, 38600 insertions(+) create mode 100644 nautilus-docs/REBUILD.md create mode 100644 nautilus-docs/SKILL.md create mode 100644 nautilus-docs/agents/openai.yaml create mode 100644 nautilus-docs/battle_tested.md create mode 100644 nautilus-docs/docs/api_reference/_static/custom.css create mode 100644 nautilus-docs/docs/api_reference/accounting.md create mode 100644 nautilus-docs/docs/api_reference/adapters/betfair.md create mode 100644 nautilus-docs/docs/api_reference/adapters/binance.md create mode 100644 nautilus-docs/docs/api_reference/adapters/bybit.md create mode 100644 nautilus-docs/docs/api_reference/adapters/databento.md create mode 100644 nautilus-docs/docs/api_reference/adapters/dydx.md create mode 100644 nautilus-docs/docs/api_reference/adapters/index.md create mode 100644 nautilus-docs/docs/api_reference/adapters/interactive_brokers.md create mode 100644 nautilus-docs/docs/api_reference/adapters/okx.md create mode 100644 nautilus-docs/docs/api_reference/adapters/polymarket.md create mode 100644 nautilus-docs/docs/api_reference/adapters/tardis.md create mode 100644 nautilus-docs/docs/api_reference/analysis.md create mode 100644 nautilus-docs/docs/api_reference/backtest.md create mode 100644 nautilus-docs/docs/api_reference/cache.md create mode 100644 nautilus-docs/docs/api_reference/common.md create mode 100644 nautilus-docs/docs/api_reference/conf.py create mode 100644 nautilus-docs/docs/api_reference/config.md create mode 100644 nautilus-docs/docs/api_reference/core.md create mode 100644 nautilus-docs/docs/api_reference/data.md create mode 100644 nautilus-docs/docs/api_reference/execution.md create mode 100644 nautilus-docs/docs/api_reference/index.md create mode 100644 nautilus-docs/docs/api_reference/indicators.md create mode 100644 nautilus-docs/docs/api_reference/live.md create mode 100644 nautilus-docs/docs/api_reference/model/book.md create mode 100644 nautilus-docs/docs/api_reference/model/data.md create mode 100644 nautilus-docs/docs/api_reference/model/events.md create mode 100644 nautilus-docs/docs/api_reference/model/identifiers.md create mode 100644 nautilus-docs/docs/api_reference/model/index.md create mode 100644 nautilus-docs/docs/api_reference/model/instruments.md create mode 100644 nautilus-docs/docs/api_reference/model/objects.md create mode 100644 nautilus-docs/docs/api_reference/model/orders.md create mode 100644 nautilus-docs/docs/api_reference/model/position.md create mode 100644 nautilus-docs/docs/api_reference/model/tick_scheme.md create mode 100644 nautilus-docs/docs/api_reference/persistence.md create mode 100644 nautilus-docs/docs/api_reference/portfolio.md create mode 100644 nautilus-docs/docs/api_reference/risk.md create mode 100644 nautilus-docs/docs/api_reference/serialization.md create mode 100644 nautilus-docs/docs/api_reference/system.md create mode 100644 nautilus-docs/docs/api_reference/trading.md create mode 100644 nautilus-docs/docs/concepts/actors.md create mode 100644 nautilus-docs/docs/concepts/adapters.md create mode 100644 nautilus-docs/docs/concepts/architecture.md create mode 100644 nautilus-docs/docs/concepts/backtesting.md create mode 100644 nautilus-docs/docs/concepts/cache.md create mode 100644 nautilus-docs/docs/concepts/custom_data.md create mode 100644 nautilus-docs/docs/concepts/data.md create mode 100644 nautilus-docs/docs/concepts/execution.md create mode 100644 nautilus-docs/docs/concepts/greeks.md create mode 100644 nautilus-docs/docs/concepts/index.md create mode 100644 nautilus-docs/docs/concepts/instruments.md create mode 100644 nautilus-docs/docs/concepts/live.md create mode 100644 nautilus-docs/docs/concepts/logging.md create mode 100644 nautilus-docs/docs/concepts/message_bus.md create mode 100644 nautilus-docs/docs/concepts/options.md create mode 100644 nautilus-docs/docs/concepts/order_book.md create mode 100644 nautilus-docs/docs/concepts/orders.md create mode 100644 nautilus-docs/docs/concepts/overview.md create mode 100644 nautilus-docs/docs/concepts/portfolio.md create mode 100644 nautilus-docs/docs/concepts/positions.md create mode 100644 nautilus-docs/docs/concepts/reports.md create mode 100644 nautilus-docs/docs/concepts/strategies.md create mode 100644 nautilus-docs/docs/concepts/value_types.md create mode 100644 nautilus-docs/docs/concepts/visualization.md create mode 100644 nautilus-docs/docs/dev_templates/criterion_template.rs create mode 100644 nautilus-docs/docs/dev_templates/iai_template.rs create mode 100644 nautilus-docs/docs/developer_guide/adapters.md create mode 100644 nautilus-docs/docs/developer_guide/benchmarking.md create mode 100644 nautilus-docs/docs/developer_guide/coding_standards.md create mode 100644 nautilus-docs/docs/developer_guide/docs.md create mode 100644 nautilus-docs/docs/developer_guide/environment_setup.md create mode 100644 nautilus-docs/docs/developer_guide/ffi.md create mode 100644 nautilus-docs/docs/developer_guide/index.md create mode 100644 nautilus-docs/docs/developer_guide/python.md create mode 100644 nautilus-docs/docs/developer_guide/releases.md create mode 100644 nautilus-docs/docs/developer_guide/rust.md create mode 100644 nautilus-docs/docs/developer_guide/spec_data_testing.md create mode 100644 nautilus-docs/docs/developer_guide/spec_exec_testing.md create mode 100644 nautilus-docs/docs/developer_guide/test_datasets.md create mode 100644 nautilus-docs/docs/developer_guide/testing.md create mode 100644 nautilus-docs/docs/getting_started/backtest_high_level.py create mode 100644 nautilus-docs/docs/getting_started/backtest_low_level.py create mode 100644 nautilus-docs/docs/getting_started/index.md create mode 100644 nautilus-docs/docs/getting_started/installation.md create mode 100644 nautilus-docs/docs/getting_started/quickstart.py create mode 100644 nautilus-docs/docs/integrations/architect_ax.md create mode 100644 nautilus-docs/docs/integrations/betfair.md create mode 100644 nautilus-docs/docs/integrations/binance.md create mode 100644 nautilus-docs/docs/integrations/bitmex.md create mode 100644 nautilus-docs/docs/integrations/blockchain.md create mode 100644 nautilus-docs/docs/integrations/bybit.md create mode 100644 nautilus-docs/docs/integrations/databento.md create mode 100644 nautilus-docs/docs/integrations/deribit.md create mode 100644 nautilus-docs/docs/integrations/dydx.md create mode 100644 nautilus-docs/docs/integrations/hyperliquid.md create mode 100644 nautilus-docs/docs/integrations/ib.md create mode 100644 nautilus-docs/docs/integrations/index.md create mode 100644 nautilus-docs/docs/integrations/kraken.md create mode 100644 nautilus-docs/docs/integrations/okx.md create mode 100644 nautilus-docs/docs/integrations/polymarket.md create mode 100644 nautilus-docs/docs/integrations/tardis.md create mode 100644 nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md create mode 100644 nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md create mode 100644 nautilus-docs/docs/tutorials/backtest_binance_orderbook.py create mode 100644 nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py create mode 100644 nautilus-docs/docs/tutorials/backtest_fx_bars.py create mode 100644 nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md create mode 100644 nautilus-docs/docs/tutorials/databento_data_catalog.py create mode 100644 nautilus-docs/docs/tutorials/dydx_grid_market_maker.md create mode 100644 nautilus-docs/docs/tutorials/index.md create mode 100644 nautilus-docs/docs/tutorials/loading_external_data.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 33c37d0..7d048f9 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -105,6 +105,12 @@ "description": "Hierarchical microstructure knowledge base for trading venues: exchange mechanics, order books, auctions, Reg NMS, Chinese futures (CTP), feed handlers, execution models", "source": "./venue-expert", "skills": ["./"] + }, + { + "name": "nautilus-docs", + "description": "Official NautilusTrader documentation — concepts, API reference, integration guides, tutorials", + "source": "./nautilus-docs", + "skills": ["./"] } ] } diff --git a/nautilus-docs/REBUILD.md b/nautilus-docs/REBUILD.md new file mode 100644 index 0000000..58c5dcc --- /dev/null +++ b/nautilus-docs/REBUILD.md @@ -0,0 +1,233 @@ +# Rebuilding the nautilus-docs Skill + +Prompt and checklist for regenerating SKILL.md when NautilusTrader's API changes. + +## When to Rebuild + +- NautilusTrader version bump (check `nautilus_trader.__version__` or workspace `Cargo.toml`) +- New adapter added or removed +- Rust crate renamed, split, or merged +- Breaking API changes (trait signatures, config structs, builder patterns) +- New docs added to the official `docs/` folder + +## Step 1: Update the Docs + +```bash +# Pull latest official docs from nautilus_trader repo +# Replace docs/ contents with the updated versions +# Keep the directory structure identical +``` + +Diff against previous version to identify: +- New files (new concepts, new adapters, new tutorials) +- Removed files (deprecated features) +- Changed files (API changes, renamed types) + +```bash +diff -rq docs/ /path/to/new/docs/ | head -50 +``` + +## Step 2: Rebuild the Doc Navigator + +For every new `.md` or `.py` file in `docs/`, add a row to the appropriate navigator table in SKILL.md (Concepts, Venue Integrations, Dev/Setup, or Tutorials). + +For removed files, delete the row. For renamed files, update the path. + +## Step 3: Rebuild the Rust Crate Map + +The crate map is the most fragile part — it breaks on every workspace restructure. + +**Source of truth**: The workspace `Cargo.toml` at the root of the nautilus_trader repo. + +```bash +# After pulling the latest nautilus_trader git dep: +CHECKOUT=$(find ~/.cargo/git/checkouts/nautilus_trader-* -maxdepth 1 -type d | sort | tail -1) + +# List all crate package names +find "$CHECKOUT/crates" -name "Cargo.toml" -maxdepth 3 -exec grep -H '^name' {} \; + +# List all adapter crates +ls "$CHECKOUT/crates/adapters/" + +# Check MSRV +grep 'rust-version' "$CHECKOUT/Cargo.toml" +``` + +For each crate, verify: +- Package name matches SKILL.md crate map +- Key re-exports haven't moved (especially `DataActor`, `Strategy`, `InstrumentId`) +- Feature flags still work as documented + +**Critical re-exports to verify**: + +```bash +# DataActor location +grep -r "pub use.*DataActor" "$CHECKOUT/crates/common/src/" + +# Strategy location +grep -r "pub use.*Strategy" "$CHECKOUT/crates/trading/src/" + +# InstrumentId location +grep -r "pub struct InstrumentId" "$CHECKOUT/crates/model/src/" + +# LiveNode builder +grep -r "pub fn builder" "$CHECKOUT/crates/live/src/" + +# BinanceDataClientConfig fields +grep -A 20 "pub struct BinanceDataClientConfig" "$CHECKOUT/crates/adapters/binance/src/config.rs" +``` + +## Step 4: Rebuild Anti-Hallucination Tables + +This is the highest-value part of the skill. Each row prevents a specific mistake Claude makes repeatedly. + +**How to maintain**: + +1. **Test existing rows** — for each hallucination, verify the "Reality" column is still correct: + ```python + # Python hallucinations — test in a venv with nautilus_trader installed + from nautilus_trader.common.actor import Actor # still correct? + from nautilus_trader.indicators import ExponentialMovingAverage # still correct? + ``` + +2. **Discover new hallucinations** — ask Claude to write code for common tasks WITHOUT the skill, then diff against what actually compiles/runs. Common sources: + - New config struct fields that Claude will guess wrong + - Renamed enums or moved types + - Changed method signatures (added/removed args) + - New features that Claude will confuse with old patterns + +3. **Remove stale rows** — if a hallucinated API now exists (e.g., a missing method was added), remove the row. + +4. **Rust hallucinations** — rebuild by attempting to compile a standalone binary: + ```bash + cd vpin_rust && cargo build 2>&1 | grep "error\[E" + ``` + Every compilation error that comes from a wrong import path, missing trait, or wrong struct field is a hallucination row candidate. + +## Step 5: Verify Working Patterns + +The DataActor pattern and LiveNode wiring in SKILL.md must compile. Test: + +```bash +cd vpin_rust && cargo build --release 2>&1 +``` + +If it fails, the Rust section needs updating. Common breakage: +- `DataActor` trait method signatures changed +- `LiveNode::builder()` API changed +- `BinanceDataClientConfig` fields renamed +- New required fields added to configs (no more `..Default::default()`) + +--- + +## Fundamental Questions the Skill Must Always Answer + +These are the enduring questions any trading infrastructure user will ask. The skill must provide correct answers for ALL of these regardless of API version. If a section doesn't cover one, it's a gap. + +### Data Ingestion +- How do I subscribe to trades, quotes, order book, bars? +- What's the difference between INTERNAL and EXTERNAL bars? +- How do I handle custom data types? +- What happens when I subscribe to data that doesn't exist? (silent failure) +- How does the DataEngine buffer and dispatch data? + +### Order Lifecycle +- How do I submit, modify, cancel orders? +- What order types are available? (Market, Limit, StopMarket, StopLimit, TrailingStop) +- How do bracket orders work? +- What happens when modify_order isn't supported by the venue? +- How do I track order state changes? (callbacks: on_order_accepted, on_order_filled, etc.) + +### Position Management +- How do I query open positions? +- NETTING vs HEDGING — when to use which? +- How do I compute unrealized PnL? +- How does signed_qty work? (float, not Decimal) + +### Instrument Discovery +- How do I load instruments? (load_ids is REQUIRED) +- What's the InstrumentId format? ("SYMBOL.VENUE") +- What instrument types exist? +- How do venue-specific instrument IDs map? + +### Actor / Strategy Architecture +- When to use Actor vs Strategy? +- What's the component lifecycle FSM? +- How do I publish/subscribe signals between actors? +- How do timers and alerts work? +- What's the correct on_start() ordering? (cache instrument → register indicators → subscribe) + +### Backtesting +- How do I set up a BacktestEngine? +- How do I load historical data? (wranglers, catalog) +- How does the FillModel work? +- What's the difference between BacktestEngine (low-level) and BacktestRunConfig (high-level)? + +### Live Deployment +- How do I configure a TradingNode (Python) / LiveNode (Rust)? +- What are the adapter config patterns for each venue? +- How do I handle shutdown gracefully? +- What are the expected warnings/errors at startup and shutdown? + +### Venue Connectivity +- Which adapters exist and what markets do they support? +- What data subscriptions does each adapter support? +- What execution features does each adapter support? (modify_order, etc.) +- How do API keys and authentication work per venue? + +### Rust-Specific +- Which crates do I need for my use case? +- How do I implement DataActor / Strategy traits? +- How do I build a standalone binary outside the workspace? +- What are the correct import paths? (nautilus_model vs nautilus_core) +- How does LiveNode builder work? +- What feature flags do I need? + +### Performance & Operations +- Why must callbacks return fast? (single-threaded event loop) +- How do I avoid blocking the event loop? +- How do I use the clock for scheduling? +- What logging/monitoring is available? +- How do I handle degraded/faulted states? + +--- + +## Meta-Prompt for Full Rebuild + +Use this prompt to have Claude regenerate the entire SKILL.md from scratch: + +``` +You are rebuilding the nautilus-docs skill for NautilusTrader. + +The official docs are in docs/ (read the full directory tree). +The current SKILL.md is your starting point — keep the structure but update all content. + +Steps: +1. Read every file in docs/concepts/, docs/integrations/, docs/developer_guide/, docs/tutorials/, docs/getting_started/ +2. Build the Doc Navigator tables from the actual files present +3. Check the Rust workspace Cargo.toml for current crate names and versions +4. Verify all anti-hallucination table entries by checking actual source code: + - For Python: test imports in the installed nautilus_trader package + - For Rust: check actual struct/trait definitions in the crate source +5. Update the Rust crate map from workspace members +6. Update the DataActor pattern and LiveNode wiring from actual working examples +7. Test that the Rust example compiles: cd vpin_rust && cargo build +8. Verify word count is under 2,000 + +For anti-hallucination entries, try to write code for each of the "Fundamental Questions" +listed in REBUILD.md WITHOUT looking at the skill, then compare what you generated +against what actually works. Every mistake you made is a hallucination row. + +The skill must be self-sufficient — a user should be able to write correct +NautilusTrader code (Python or Rust) using ONLY this skill, without deepwiki or +context7. If they can't, the skill has gaps. +``` + +## Versioning + +After rebuild, update the version line in SKILL.md: +``` +**Tested against v1.XXX.0** — all code validated by running tests. +``` + +Track what version the anti-hallucination table was last verified against. Stale tables are worse than no table — they give false confidence. diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md new file mode 100644 index 0000000..84e3132 --- /dev/null +++ b/nautilus-docs/SKILL.md @@ -0,0 +1,293 @@ +--- +name: nautilus-docs +description: > + Official NautilusTrader documentation — concepts, API reference, integration guides, + tutorials. Use when needing detailed explanations of NautilusTrader architecture, + component behavior, adapter configuration, or API signatures beyond what the + nautilus-trader skill covers. +--- + +# NautilusTrader Official Docs + +Read the relevant doc before answering questions about platform behavior. All paths relative to `docs/`. + +## Doc Navigator + +### Concepts + +| Topic | Doc | +|-------|-----| +| Architecture / overview | [architecture.md](docs/concepts/architecture.md), [overview.md](docs/concepts/overview.md) | +| Actor development | [actors.md](docs/concepts/actors.md) | +| Strategy development | [strategies.md](docs/concepts/strategies.md) | +| Backtesting | [backtesting.md](docs/concepts/backtesting.md) | +| Live trading | [live.md](docs/concepts/live.md) | +| Order types & execution | [orders.md](docs/concepts/orders.md), [execution.md](docs/concepts/execution.md) | +| Order book | [order_book.md](docs/concepts/order_book.md) | +| Data types & custom data | [data.md](docs/concepts/data.md), [custom_data.md](docs/concepts/custom_data.md) | +| Instruments | [instruments.md](docs/concepts/instruments.md) | +| Value types (Price, Quantity, Money) | [value_types.md](docs/concepts/value_types.md) | +| Positions & PnL | [positions.md](docs/concepts/positions.md) | +| Cache | [cache.md](docs/concepts/cache.md) | +| MessageBus | [message_bus.md](docs/concepts/message_bus.md) | +| Portfolio | [portfolio.md](docs/concepts/portfolio.md) | +| Options & Greeks | [options.md](docs/concepts/options.md), [greeks.md](docs/concepts/greeks.md) | +| Logging | [logging.md](docs/concepts/logging.md) | +| Reports & analysis | [reports.md](docs/concepts/reports.md) | +| Visualization | [visualization.md](docs/concepts/visualization.md) | +| Adapter development | [concepts/adapters.md](docs/concepts/adapters.md), [developer_guide/adapters.md](docs/developer_guide/adapters.md) | + +### Venue Integrations + +| Venue | Doc | +|-------|-----| +| Binance | [binance.md](docs/integrations/binance.md) | +| Bybit | [bybit.md](docs/integrations/bybit.md) | +| OKX | [okx.md](docs/integrations/okx.md) | +| dYdX | [dydx.md](docs/integrations/dydx.md) | +| Deribit | [deribit.md](docs/integrations/deribit.md) | +| Hyperliquid | [hyperliquid.md](docs/integrations/hyperliquid.md) | +| Kraken | [kraken.md](docs/integrations/kraken.md) | +| Interactive Brokers | [ib.md](docs/integrations/ib.md) | +| Betfair | [betfair.md](docs/integrations/betfair.md) | +| Polymarket | [polymarket.md](docs/integrations/polymarket.md) | +| Databento | [databento.md](docs/integrations/databento.md) | +| Tardis | [tardis.md](docs/integrations/tardis.md) | +| BitMEX | [bitmex.md](docs/integrations/bitmex.md) | +| AX Exchange | [architect_ax.md](docs/integrations/architect_ax.md) | + +### Dev / Setup + +| Topic | Doc | +|-------|-----| +| Installation | [installation.md](docs/getting_started/installation.md) | +| Quickstart | [quickstart.py](docs/getting_started/quickstart.py) | +| Environment setup | [environment_setup.md](docs/developer_guide/environment_setup.md) | +| Rust development | [rust.md](docs/developer_guide/rust.md) | +| Testing | [testing.md](docs/developer_guide/testing.md) | +| Coding standards | [coding_standards.md](docs/developer_guide/coding_standards.md) | +| Benchmarking | [benchmarking.md](docs/developer_guide/benchmarking.md) | +| FFI | [ffi.md](docs/developer_guide/ffi.md) | + +### Tutorials + +| Tutorial | Doc | +|----------|-----| +| FX mean reversion (AX) | [ax_fx_mean_reversion.md](docs/tutorials/ax_fx_mean_reversion.md) | +| Gold book imbalance (AX) | [ax_gold_book_imbalance.md](docs/tutorials/ax_gold_book_imbalance.md) | +| dYdX grid market maker | [dydx_grid_market_maker.md](docs/tutorials/dydx_grid_market_maker.md) | +| BitMEX grid market maker | [bitmex_grid_market_maker.md](docs/tutorials/bitmex_grid_market_maker.md) | +| Backtest Binance orderbook | [backtest_binance_orderbook.py](docs/tutorials/backtest_binance_orderbook.py) | +| Backtest Bybit orderbook | [backtest_bybit_orderbook.py](docs/tutorials/backtest_bybit_orderbook.py) | +| Backtest FX bars | [backtest_fx_bars.py](docs/tutorials/backtest_fx_bars.py) | +| Databento data catalog | [databento_data_catalog.py](docs/tutorials/databento_data_catalog.py) | +| Loading external data | [loading_external_data.py](docs/tutorials/loading_external_data.py) | + +## Battle-Tested Patterns + +Non-obvious tricks from live testing — NOT in official docs: [battle_tested.md](battle_tested.md). Covers: on_start() ordering, microprice, inventory skew, signal pipeline, timer polling, FillModel, frozen_account inversion, F_LAST rule, venue gotchas, performance checklist. + +## Rust Standalone Binary + +External Rust binaries work via git dependencies. No need to clone the nautilus workspace. Requires Rust 1.94+, edition 2024. + +All crates: `{ git = "https://github.com/nautechsystems/nautilus_trader.git" }`. Add `default-features = false` if no Python headers. Naming: `nautilus-{name}` in Cargo.toml → `nautilus_{name}` in Rust. + +### Crate Map + +| Layer | Crate | Provides | +|-------|-------|----------| +| Foundation | `nautilus-core` | Primitives, FFI | +| Foundation | `nautilus-model` | `InstrumentId`, `TraderId`, `Price`, `Quantity`, orders, positions, instruments | +| Foundation | `nautilus-common` | `DataActor`, `DataActorCore`, `DataActorConfig`, timers, message bus, component lifecycle | +| Trading | `nautilus-trading` | `Strategy` trait (extends DataActor), order/position management | +| Runtime | `nautilus-live` | `LiveNode`, `LiveNodeBuilder` | +| Runtime | `nautilus-backtest` | `BacktestNode`, `BacktestEngine` — needs `features = ["streaming"]` | +| Infra | `nautilus-indicators` | EMA, SMA, RSI, MACD, Bollinger — enable via `nautilus-trading` `features = ["examples"]` | +| Infra | `nautilus-persistence` | Parquet catalog, DataFusion | +| Infra | `nautilus-serialization` | Arrow, MessagePack encoding | +| Infra | `nautilus-network` | HTTP/WebSocket clients | +| Infra | `nautilus-cryptography` | HMAC, Ed25519 signing | +| Infra | `nautilus-infrastructure` | Redis, logging, monitoring | +| Infra | `nautilus-analysis` | Performance stats | +| Engine | `nautilus-data` | Data engine internals | +| Engine | `nautilus-execution` | Order routing internals | +| Engine | `nautilus-portfolio` | Account/position aggregation | +| Engine | `nautilus-risk` | Pre-trade checks | +| System | `nautilus-system` | Kernel orchestration | +| Test | `nautilus-testkit` | Test fixtures, mock instruments | + +### Adapter Crates + +All follow `nautilus-{venue}` naming: + +| Crate | Markets | +|-------|---------| +| `nautilus-binance` | Spot, USDT-M, COIN-M | +| `nautilus-bybit` | Spot, Perpetuals | +| `nautilus-okx` | Spot, Futures, Options | +| `nautilus-kraken` | Spot, Futures | +| `nautilus-bitmex` | Perpetuals | +| `nautilus-deribit` | Options | +| `nautilus-dydx` | Perpetuals (DEX) | +| `nautilus-hyperliquid` | Perpetuals (DEX) | +| `nautilus-databento` | Historical data provider | +| `nautilus-tardis` | Historical data provider | +| `nautilus-betfair` | Betting exchange | +| `nautilus-polymarket` | Prediction markets | +| `nautilus-architect-ax` | FX, commodities | +| `nautilus-sandbox` | Simulated exchange | + +### Which crates for your task + +| Use case | Crates | +|----------|--------| +| Data-only live actor | `nautilus-common`, `nautilus-model`, `nautilus-live`, `nautilus-{venue}` | +| Live strategy (orders) | above + `nautilus-trading` | +| Backtest | `nautilus-common`, `nautilus-model`, `nautilus-backtest` (+ `streaming`), `nautilus-trading` | +| With indicators | add `nautilus-trading` with `features = ["examples"]` | +| With Parquet catalog | add `nautilus-persistence` | +| `high-precision` (128-bit) | Default on most crates — `default-features = false` disables it, re-add explicitly | + +### DataActor pattern (the only way to build custom Rust actors) + +```rust +use std::ops::{Deref, DerefMut}; +use nautilus_common::actor::{DataActor, DataActorConfig, DataActorCore}; +use nautilus_model::data::TradeTick; +use nautilus_model::identifiers::InstrumentId; + +#[derive(Debug)] +struct MyActor { + core: DataActorCore, // REQUIRED — blanket impls derive Actor+Component from this + instrument_id: InstrumentId, +} + +impl Deref for MyActor { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { &self.core } +} +impl DerefMut for MyActor { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.core } +} + +impl DataActor for MyActor { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_trades(self.instrument_id, None, None); // 3 args + Ok(()) + } + fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { + let px = trade.price.as_f64(); + let qty = trade.size.as_f64(); + // logic here + Ok(()) + } + fn on_stop(&mut self) -> anyhow::Result<()> { Ok(()) } // MUST implement — warns if missing +} + +// Constructor: +impl MyActor { + fn new(instrument_id: InstrumentId) -> Self { + Self { + core: DataActorCore::new(DataActorConfig { actor_id: Some("MyActor-001".into()), ..Default::default() }), + instrument_id, + } + } +} +``` + +### LiveNode + Binance wiring + +```rust +use nautilus_binance::common::enums::{BinanceEnvironment, BinanceProductType}; +use nautilus_binance::config::BinanceDataClientConfig; +use nautilus_binance::factories::BinanceDataClientFactory; +use nautilus_common::enums::Environment; +use nautilus_live::node::LiveNode; +use nautilus_model::identifiers::TraderId; + +let data_config = BinanceDataClientConfig { + product_types: vec![BinanceProductType::UsdM], // or Spot, CoinM + environment: BinanceEnvironment::Mainnet, // or Testnet + api_key: Some(key), + api_secret: Some(secret), // Ed25519: wrap in PEM headers + ..Default::default() +}; + +let mut node = LiveNode::builder(TraderId::from("MY-001"), Environment::Live)? + .with_name("MyNode") + .add_data_client(None, Box::new(BinanceDataClientFactory::new()), Box::new(data_config))? + .build()?; + +node.add_actor(my_actor)?; +node.run().await?; +``` + +**Do NOT use `env_logger`** — Nautilus registers its own `log` backend. `env_logger::init()` will crash with "logger already initialized". Use `log::info!()` directly. + +## Common Hallucinations + +These do NOT exist in v1.224.0: + +| Hallucination | Reality | +|--------------|---------| +| `cache.position_for_instrument(id)` | `cache.positions_open(instrument_id=id)` returns list | +| `engine.trader.cache` | `engine.cache` directly | +| `book.filtered_view()` | Use `cache.own_order_book()` | +| `book.get_avg_px_qty_for_exposure()` | Use `get_avg_px_for_quantity()` + `get_quantity_for_price()` | +| `level.count` / `book.count` | `book.update_count` | +| `BookOrder(price=, size=, side=)` | 4 positional args: `BookOrder(OrderSide.BUY, Price, Quantity, order_id=0)`. Import from `model.data`, NOT `model.book` | +| `GenericDataWrangler` | TradeTickDataWrangler, QuoteTickDataWrangler, OrderBookDeltaDataWrangler, BarDataWrangler | +| `catalog.data_types()` | `catalog.list_data_types()` | +| `BacktestEngineConfig` from `nautilus_trader.config` | from `nautilus_trader.backtest.engine` or `nautilus_trader.backtest.config` | +| `FillModel(prob_fill_on_stop=...)` | Only: prob_fill_on_limit, prob_slippage, random_seed | +| `LoggingConfig(log_file_path=)` | `log_directory=` | +| `cache.orders_filled()` | `cache.orders_closed()` | +| `pos.signed_qty / Decimal(...)` | TypeError: returns float — `Decimal(str(pos.signed_qty))` | +| `from nautilus_trader.trading.actor` | `from nautilus_trader.common.actor import Actor` | +| `from nautilus_trader.indicators.ema` | `from nautilus_trader.indicators import ExponentialMovingAverage` | +| `BollingerBands(20)` | `BollingerBands(20, 2.0)` — k mandatory | +| `MACD(12, 26, 9)` | 3rd param is `MovingAverageType`, not signal_period | +| `publish_signal(value=dict(...))` | KeyError — values must be int, float, or str | +| Encrypted Ed25519 private key | Must be unencrypted PKCS#8 | +| `on_timer()` as callback | `clock.set_timer(callback=handler)` | +| `order.order_side` | `order.side` — events use `event.order_side` | +| `request_bars(bar_type)` one arg | Requires `start`: `request_bars(bar_type, start=datetime(...))` | +| `GreeksCalculator(cache, clock, logger)` | Only 2 args: `GreeksCalculator(cache, clock)` | +| `SyntheticInstrument(sym, prec, comps, formula)` | 6 required args — also needs `ts_event`, `ts_init` | +| `from nautilus_trader.core.nautilus_pyo3` wrong path | `from nautilus_trader.core.nautilus_pyo3 import black_scholes_greeks` | +| `BacktestEngine.add_venue(venue=BacktestVenueConfig(...))` | Takes positional args. `BacktestVenueConfig` is for `BacktestRunConfig` only | +| `DYDXDataClientConfig` (uppercase) | `DydxDataClientConfig` (mixed case) | +| `DydxOraclePrice` custom data type | Does not exist in v1.224.0 | +| `from nautilus_trader.common.clock import TestClock` | Use `from nautilus_trader.common.component import LiveClock` | +| `from nautilus_trader.common.config import CacheConfig` | `from nautilus_trader.config import CacheConfig` or `nautilus_trader.cache.config` | +| `Equity(..., max_price=, min_price=)` | Constructor rejects these kwargs — properties exist but return None | +| `FuturesContract(..., size_precision=, size_increment=)` | Hardcoded to 0/1 in Cython | +| `RSI` value in [0, 100] | Value in [0, 1] — divide by 100 if comparing to standard | +| Indicator warmup returns NaN | Returns partial values (silently wrong, not NaN) — guard with `indicators_initialized()` | +| `BookType.L3_MBO` for crypto | L3 not available on crypto exchanges — L2 at best. L3 is for traditional exchanges only | +| `subscribe_funding_rates()` everywhere | Method exists on Strategy but not all adapters support the feed | +| `subscribe_instrument_status()` on Binance | Binance does NOT implement this — not all adapters support it | +| `MarketStatusAction.RESUME` | Does not exist — use `TRADING` to detect resumption | +| `BinanceAccountType.USDT_FUTURE` (no S) | Must be `USDT_FUTURES` (with S) | +| `load_ids=frozenset({"BTCUSDT-PERP"})` bare symbol | Must be full InstrumentId: `frozenset({"BTCUSDT-PERP.BINANCE"})` — crashes at connect with `missing '.' separator` | +| `modify_order` auto-fallback | Adapter errors if venue doesn't support — no auto cancel+replace fallback | +| `InstrumentStatus` stops order flow | Does NOT automatically stop orders — strategy must react manually | +| Omitting `load_ids` from `InstrumentProviderConfig` | REQUIRED — without it, 0 instruments load, all subscriptions silently produce 0 data | +| `frozen_account=True` means checks active | INVERTED: `True` = checks DISABLED, `False` = checks ACTIVE | +| `testnet=True` for Deribit/dYdX | `is_testnet=True` | +| `AccountType.CASH` for perps backtest | Use `AccountType.MARGIN` — CASH + perps = 0 fills silently | +| `book.best_bid_price()` returns float | Returns `Price` object — cast with `float()` for arithmetic | +| **Rust-specific** | | +| `LiveNode::new(config)` or `LiveNodeConfig::builder()` | `LiveNode::builder(trader_id, Environment::Live)?.build()?` | +| `BinanceDataClientConfig::builder()` | Struct literal: `BinanceDataClientConfig { product_types: vec![...], ..Default::default() }` | +| `BinanceAccountType::UsdtFutures` (Rust) | `BinanceProductType::UsdM` from `nautilus_binance::common::enums` | +| `nautilus_core::identifiers::InstrumentId` | `nautilus_model::identifiers::InstrumentId` | +| `DataActor` in `nautilus_trading` | `DataActor` in `nautilus_common::actor` | +| `env_logger` alongside Nautilus | Crashes — Nautilus registers its own `log` backend. Use `log::info!()` directly | +| Nautilus crates from crates.io | Must use git source — `nautilus-persistence-macros` not on crates.io | +| `nautilus_trader_model` crate name | `nautilus_model` | +| `on_book_delta` (singular) | `on_book_deltas` (plural — batch) | +| Skip `on_stop` in DataActor | Warns "on_stop handler was called when not overridden" — always implement | +| `cancel_all_orders` in `on_stop` | Trading channel closed before `on_stop` — call from `on_trade` or other live callbacks | +| `node.stop()` exits cleanly | WebSocket lingers ~10s producing `channel closed` errors — add `std::process::exit(0)` after stop | diff --git a/nautilus-docs/agents/openai.yaml b/nautilus-docs/agents/openai.yaml new file mode 100644 index 0000000..cae5d5e --- /dev/null +++ b/nautilus-docs/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "NautilusDocs" + short_description: "NautilusTrader library expert knowledge" + default_prompt: "Use $nautilus-docs to design/implement or fix trading systems with NautilusTrader." diff --git a/nautilus-docs/battle_tested.md b/nautilus-docs/battle_tested.md new file mode 100644 index 0000000..7eef1fb --- /dev/null +++ b/nautilus-docs/battle_tested.md @@ -0,0 +1,221 @@ +# Battle-Tested Patterns & Tricks + +Non-obvious knowledge from live testing NautilusTrader v1.224.0. These patterns are verified against real exchanges and won't be found in official docs. + +## Critical Ordering Rules + +### on_start() must follow this exact sequence + +```python +def on_start(self) -> None: + # 1. Cache instrument FIRST — None if not loaded, crashes later + self.instrument = self.cache.instrument(self.config.instrument_id) + if self.instrument is None: + self.log.error(f"Not found: {self.config.instrument_id}") + return + + # 2. Register indicators BEFORE subscribing + self.register_indicator_for_bars(bar_type, self.ema) + + # 3. Subscribe AFTER instrument + indicators ready + self.subscribe_bars(bar_type) +``` + +Wrong order = silent failures: indicators never receive data, subscriptions produce 0 callbacks. + +### Data loading order for backtest + +```python +# Load multiple instruments with deferred sorting (faster) +engine.add_data(ticks_btc, sort=False) +engine.add_data(ticks_eth, sort=False) +engine.sort_data() # single O(n log n) sort at the end +``` + +### Bar timestamp correction + +If bar data uses opening timestamps (common in CSV exports), set `ts_init_delta` to bar duration: + +```python +bars = BarDataWrangler(bar_type=bar_type, instrument=inst).process( + data=df, ts_init_delta=60_000_000_000 # 1 minute in nanoseconds +) +``` + +Without this: look-ahead bias — strategy sees bar data before the bar closes. + +## Execution Tricks + +### modify_order > cancel+replace + +- Single message, lower latency, less detectable +- But NOT supported everywhere: dYdX, Binance Spot, Polymarket don't support it +- Check venue support before using — adapter errors with no auto-fallback + +### Bracket order pattern (manual) + +```python +entry = self.order_factory.market(instrument_id, OrderSide.BUY, quantity) +self.submit_order(entry) + +# In on_order_filled: +sl = self.order_factory.stop_market(instrument_id, OrderSide.SELL, quantity, trigger_price=sl_price) +tp = self.order_factory.limit(instrument_id, OrderSide.SELL, quantity, price=tp_price) +self.submit_order(sl) +self.submit_order(tp) +``` + +For Rust backtest: `support_contingent_orders = Some(true)` required in `add_venue`. + +### Reconciliation delay + +`reconciliation_startup_delay_secs >= 10` — lower values cause false positive "missing order" resolutions on startup. + +## Actor Patterns + +### Timer-based REST polling + +```python +def on_start(self) -> None: + self.clock.set_timer("poll_oi", interval=timedelta(seconds=5), callback=self._on_poll) + +def _on_poll(self, event) -> None: + self.queue_for_executor(self._fetch_open_interest) + +async def _fetch_open_interest(self) -> None: + # async HTTP call here — non-blocking + pass +``` + +### Signal pipeline (Actor → Strategy) + +**Actor publishes:** +```python +self.publish_signal(name="momentum", value=score, ts_event=tick.ts_event) +``` + +Name `"momentum"` auto-generates class `SignalMomentum` (capitalized). Values must be int/float/str — dict causes KeyError. + +**Strategy receives:** +```python +def on_signal(self, signal) -> None: + if type(signal).__name__ == "SignalMomentum": + self.momentum = signal.value +``` + +### Custom data for structured payloads + +When signal values need to be richer than a single number: + +```python +class VPINData(Data): + def __init__(self, vpin: float, volume: float, ts_event: int, ts_init: int): + self.vpin = vpin + self.volume = volume + self._ts_event = ts_event + self._ts_init = ts_init + + @property + def ts_event(self) -> int: return self._ts_event + @property + def ts_init(self) -> int: return self._ts_init +``` + +Actor: `self.publish_data(data_type=DataType(VPINData), data=vpin_data)` +Strategy: subscribe in `on_start`, receive in `on_data` + +## Backtest Configuration + +### FillModel tuning + +```python +FillModel( + prob_fill_on_limit=0.3, # 30% fill probability on limit orders + prob_slippage=0.5, # 50% chance of 1-tick slippage + random_seed=42, # reproducibility +) +``` + +No `prob_fill_on_stop` parameter exists. + +### frozen_account naming is INVERTED + +- `frozen_account=False` → margin checks ARE active (not frozen) +- `frozen_account=True` → margin checks DISABLED (frozen) + +For realistic backtesting, use `frozen_account=False` (the confusing one). + +### Multi-currency accounts + +`base_currency=None` for multi-currency (standard for crypto). `Currency.from_str("USDT")` not bare constant. + +### Venue setup (low-level API) + +```python +engine.add_venue( + venue=Venue("BINANCE"), + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, # MUST be MARGIN for perps/futures + starting_balances=[Money(10_000, Currency.from_str("USDT"))], + fill_model=fill_model, +) +``` + +`AccountType.CASH` + perps = 0 fills silently. + +## Order Book + +### F_LAST flag is mandatory + +Every delta batch MUST end with `RecordFlag.F_LAST` or DataEngine buffers indefinitely — subscribers never receive data. + +- Single delta: always set F_LAST +- Batch: set F_LAST only on last delta + +### book.best_bid_price() returns Price, not float + +Cast with `float(book.best_bid_price())` for arithmetic. Same for `best_ask_price()`, `spread()`, `midpoint()`. + +### L2_MBP is the ceiling for crypto + +L3_MBO not available on any crypto exchange. L2 is sufficient for all HFT strategies. L3 is only for traditional exchanges (equities). + +### OrderBookDepth10 for lightweight signals + +Pre-aggregated top 10 levels — lower overhead than full book subscription for signal generation. + +## Venue-Specific Gotchas + +### Binance + +- `-PERP` suffix mandatory: `BTCUSDT-PERP.BINANCE` for futures, `BTCUSDT.BINANCE` for spot +- `BinanceAccountType.USDT_FUTURES` (with S) — `USDT_FUTURE` doesn't exist +- `BinanceFuturesMarkPriceUpdate` contains mark, index, AND funding_rate — don't subscribe separately +- Spot doesn't support `modify_order` +- High-supply tokens (SHIB, PEPE) need `high-precision` feature flag + +### dYdX + +- Market orders implemented as aggressive limit: buy at `oracle_price × 1.01`, sell at `× 0.99` +- Config: `is_testnet=True` (NOT `testnet=True`) +- No `modify_order` support + +### Deribit + +- Config: `is_testnet=True` (NOT `testnet=True`) +- `CryptoOption.underlying` is `Currency`, `OptionContract.underlying` is `str` — different types + +### Polymarket + +- No `modify_order` support +- Binary outcomes only — `YES`/`NO` tokens + +## Performance Checklist + +1. Cache `self.instrument` in `on_start()` — avoid repeated `cache.instrument()` lookups in hot path +2. Keep `on_order_book_deltas` tight — fires at tick frequency +3. Never `time.sleep()` in callbacks — single-threaded event loop blocks everything +4. Use `modify_order` over cancel+replace where supported — single message, less latency +5. `sort=False` + `engine.sort_data()` for multi-instrument backtest loading +6. Order books are Rust-native — all operations execute as native code, no Python overhead +7. `indicators_initialized()` guard prevents acting on partial warmup values diff --git a/nautilus-docs/docs/api_reference/_static/custom.css b/nautilus-docs/docs/api_reference/_static/custom.css new file mode 100644 index 0000000..3c75676 --- /dev/null +++ b/nautilus-docs/docs/api_reference/_static/custom.css @@ -0,0 +1,14 @@ +body[data-theme="dark"] h1, +body[data-theme="dark"] h2, +body[data-theme="dark"] h3, +body[data-theme="dark"] h4, +body[data-theme="dark"] h5, +body[data-theme="dark"] h6, +body[data-theme="dark"] .sidebar-title, +body[data-theme="dark"] .toctree-l1 > a { + color: #e6edf3; +} + +.bottom-of-page .related-information { + display: none; +} diff --git a/nautilus-docs/docs/api_reference/accounting.md b/nautilus-docs/docs/api_reference/accounting.md new file mode 100644 index 0000000..43b9777 --- /dev/null +++ b/nautilus-docs/docs/api_reference/accounting.md @@ -0,0 +1,45 @@ +# Accounting + +```{eval-rst} +.. automodule:: nautilus_trader.accounting +``` + +```{eval-rst} +.. automodule:: nautilus_trader.accounting.accounts.cash + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.accounting.accounts.margin + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.accounting.calculators + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.accounting.factory + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.accounting.manager + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/betfair.md b/nautilus-docs/docs/api_reference/adapters/betfair.md new file mode 100644 index 0000000..96394a3 --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/betfair.md @@ -0,0 +1,109 @@ +# Betfair + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Client + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Common + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.common + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data Types + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.data_types + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## OrderBook + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.orderbook + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Sockets + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.betfair.sockets + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/binance.md b/nautilus-docs/docs/api_reference/adapters/binance.md new file mode 100644 index 0000000..3f37fdc --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/binance.md @@ -0,0 +1,143 @@ +# Binance + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Enums + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.common.enums + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Types + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.common.types + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Futures + +### Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.futures.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Enums + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.futures.enums + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.futures.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.futures.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Types + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.futures.types + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Spot + +### Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.spot.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Enums + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.spot.enums + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.spot.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +### Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.binance.spot.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/bybit.md b/nautilus-docs/docs/api_reference/adapters/bybit.md new file mode 100644 index 0000000..4d5fccd --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/bybit.md @@ -0,0 +1,69 @@ +# Bybit + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Enums + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit.common.enums + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.bybit.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/databento.md b/nautilus-docs/docs/api_reference/adapters/databento.md new file mode 100644 index 0000000..67b8ead --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/databento.md @@ -0,0 +1,79 @@ +# Databento + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Enums + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.enums + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Types + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.types + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Loaders + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.loaders + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.databento.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/dydx.md b/nautilus-docs/docs/api_reference/adapters/dydx.md new file mode 100644 index 0000000..f079d1c --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/dydx.md @@ -0,0 +1,59 @@ +# dYdX + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.dydx + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.dydx.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.dydx.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.dydx.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.dydx.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.dydx.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/index.md b/nautilus-docs/docs/api_reference/adapters/index.md new file mode 100644 index 0000000..4ed2be4 --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/index.md @@ -0,0 +1,23 @@ +# Adapters + +```{eval-rst} +.. automodule:: nautilus_trader.adapters +``` + +```{eval-rst} +.. toctree:: + :maxdepth: 2 + :glob: + :titlesonly: + :hidden: + + betfair.md + binance.md + bybit.md + databento.md + dydx.md + interactive_brokers.md + okx.md + polymarket.md + tardis.md +``` diff --git a/nautilus-docs/docs/api_reference/adapters/interactive_brokers.md b/nautilus-docs/docs/api_reference/adapters/interactive_brokers.md new file mode 100644 index 0000000..aa7a1c3 --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/interactive_brokers.md @@ -0,0 +1,117 @@ +# Interactive Brokers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Client + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.client.client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Common + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.common + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Gateway + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.gateway + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Historical + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.historical.client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Parsing + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.parsing.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.parsing.instruments + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.interactive_brokers.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/okx.md b/nautilus-docs/docs/api_reference/adapters/okx.md new file mode 100644 index 0000000..d28a00a --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/okx.md @@ -0,0 +1,59 @@ +# OKX + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.okx + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.okx.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.okx.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.okx.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.okx.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.okx.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/polymarket.md b/nautilus-docs/docs/api_reference/adapters/polymarket.md new file mode 100644 index 0000000..cac4450 --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/polymarket.md @@ -0,0 +1,69 @@ +# Polymarket + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Enums + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket.common.enums + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.polymarket.execution + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/adapters/tardis.md b/nautilus-docs/docs/api_reference/adapters/tardis.md new file mode 100644 index 0000000..dd4b6b7 --- /dev/null +++ b/nautilus-docs/docs/api_reference/adapters/tardis.md @@ -0,0 +1,59 @@ +# Tardis + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.tardis + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Loaders + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.tardis.loaders + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Config + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.tardis.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Providers + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.tardis.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Factories + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.tardis.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.adapters.tardis.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/analysis.md b/nautilus-docs/docs/api_reference/analysis.md new file mode 100644 index 0000000..77cd33c --- /dev/null +++ b/nautilus-docs/docs/api_reference/analysis.md @@ -0,0 +1,29 @@ +# Analysis + +```{eval-rst} +.. automodule:: nautilus_trader.analysis +``` + +```{eval-rst} +.. automodule:: nautilus_trader.analysis.analyzer + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.analysis.reporter + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.analysis.statistic + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/backtest.md b/nautilus-docs/docs/api_reference/backtest.md new file mode 100644 index 0000000..2d7af89 --- /dev/null +++ b/nautilus-docs/docs/api_reference/backtest.md @@ -0,0 +1,61 @@ +# Backtest + +```{eval-rst} +.. automodule:: nautilus_trader.backtest +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.data_client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.execution_client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.models + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.modules + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.node + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.results + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/cache.md b/nautilus-docs/docs/api_reference/cache.md new file mode 100644 index 0000000..5a26d96 --- /dev/null +++ b/nautilus-docs/docs/api_reference/cache.md @@ -0,0 +1,29 @@ +# Cache + +```{eval-rst} +.. automodule:: nautilus_trader.cache +``` + +```{eval-rst} +.. automodule:: nautilus_trader.cache.cache + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.cache.database + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.cache.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/common.md b/nautilus-docs/docs/api_reference/common.md new file mode 100644 index 0000000..80ec5c2 --- /dev/null +++ b/nautilus-docs/docs/api_reference/common.md @@ -0,0 +1,59 @@ +# Common + +```{eval-rst} +.. automodule:: nautilus_trader.common +``` + +```{eval-rst} +.. automodule:: nautilus_trader.common.actor + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.common.factories + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Component + +```{eval-rst} +.. automodule:: nautilus_trader.common.component + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Executor + +```{eval-rst} +.. automodule:: nautilus_trader.common.executor + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Generators + +```{eval-rst} +.. automodule:: nautilus_trader.common.generators + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.common.providers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/conf.py b/nautilus-docs/docs/api_reference/conf.py new file mode 100644 index 0000000..61b1d6b --- /dev/null +++ b/nautilus-docs/docs/api_reference/conf.py @@ -0,0 +1,122 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# + +import nautilus_trader + + +# -- Project information ----------------------------------------------------- +project = "NautilusTrader" +author = "Nautech Systems Pty Ltd." +copyright = "2015-2026 Nautech Systems Pty Ltd" +version = nautilus_trader.__version__ + +# -- General configuration --------------------------------------------------- +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx_comments", +] + +html_theme = "furo" +html_title = "NautilusTrader Python API" +html_favicon = "https://nautilustrader.io/docs/img/shell.ico" + +html_theme_options = { + "dark_css_variables": { + "color-brand-primary": "#33bca4", + "color-brand-content": "#33bca4", + "color-background-primary": "#0d1117", + "color-background-secondary": "#0e1316", + "color-background-border": "#23282c", + "color-foreground-primary": "#b7b7b7", + "color-foreground-secondary": "#cdcdcd", + "color-highlight-on-target": "#171c20", + "color-api-name": "#33bca4", + "color-api-pre-name": "#33bca4", + }, + "light_css_variables": { + "color-brand-primary": "#007e68", + "color-brand-content": "#007e68", + }, + "footer_icons": [], + "source_repository": "https://github.com/nautechsystems/nautilus_trader", + "source_branch": "master", + "source_directory": "docs/api_reference/", +} + +html_show_sphinx = False +html_static_path = ["_static"] +html_css_files = ["custom.css"] + +comments_config = {"hypothesis": False, "utterances": False} +exclude_patterns = ["**.ipynb_checkpoints", ".DS_Store", "Thumbs.db", "_build"] +source_suffix = [".rst", ".md"] + +myst_enable_extensions = [ + "colon_fence", + "dollarmath", + "fieldlist", + "linkify", + "substitution", + "tasklist", +] +myst_url_schemes = ("mailto", "http", "https") +suppress_warnings = ["myst.domains"] + +add_module_names = False +todo_include_todos = False + +autosummary_generate = True +autodoc_member_order = "bysource" +autoclass_content = "class" +autodoc_class_signature = "separated" + +# -- Extension configuration ------------------------------------------------- +autodoc_default_options = { + "members": True, + "undoc-members": False, + "private-members": False, + "exclude-members": "__init__,__new__", + "show-inheritance": True, + "class-signature": "separated", +} + +# -- Napoleon settings ------------------------------------------------------- +napoleon_google_docstring = False +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = False +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = False +napoleon_use_admonition_for_examples = True +napoleon_use_admonition_for_notes = True +napoleon_use_admonition_for_references = True +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/nautilus-docs/docs/api_reference/config.md b/nautilus-docs/docs/api_reference/config.md new file mode 100644 index 0000000..f33ee8f --- /dev/null +++ b/nautilus-docs/docs/api_reference/config.md @@ -0,0 +1,101 @@ +# Config + +## Backtest + +```{eval-rst} +.. automodule:: nautilus_trader.backtest.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Cache + +```{eval-rst} +.. automodule:: nautilus_trader.cache.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Common + +```{eval-rst} +.. automodule:: nautilus_trader.common.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Data + +```{eval-rst} +.. automodule:: nautilus_trader.data.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Execution + +```{eval-rst} +.. automodule:: nautilus_trader.execution.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Live + +```{eval-rst} +.. automodule:: nautilus_trader.live.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Persistence + +```{eval-rst} +.. automodule:: nautilus_trader.persistence.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Risk + +```{eval-rst} +.. automodule:: nautilus_trader.risk.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## System + +```{eval-rst} +.. automodule:: nautilus_trader.system.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Trading + +```{eval-rst} +.. automodule:: nautilus_trader.trading.config + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/core.md b/nautilus-docs/docs/api_reference/core.md new file mode 100644 index 0000000..3a6a546 --- /dev/null +++ b/nautilus-docs/docs/api_reference/core.md @@ -0,0 +1,55 @@ +# Core + +```{eval-rst} +.. automodule:: nautilus_trader.core +``` + +## Datetime + +```{eval-rst} +.. automodule:: nautilus_trader.core.datetime + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Finite-State Machine (FSM) + +```{eval-rst} +.. automodule:: nautilus_trader.core.fsm + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Message + +```{eval-rst} +.. automodule:: nautilus_trader.core.message + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Stats + +```{eval-rst} +.. automodule:: nautilus_trader.core.stats + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## UUID + +```{eval-rst} +.. automodule:: nautilus_trader.core.uuid + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/data.md b/nautilus-docs/docs/api_reference/data.md new file mode 100644 index 0000000..03ef65f --- /dev/null +++ b/nautilus-docs/docs/api_reference/data.md @@ -0,0 +1,45 @@ +# Data + +```{eval-rst} +.. automodule:: nautilus_trader.data +``` + +## Aggregation + +```{eval-rst} +.. automodule:: nautilus_trader.data.aggregation + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Client + +```{eval-rst} +.. automodule:: nautilus_trader.data.client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Engine + +```{eval-rst} +.. automodule:: nautilus_trader.data.engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Messages + +```{eval-rst} +.. automodule:: nautilus_trader.data.messages + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/execution.md b/nautilus-docs/docs/api_reference/execution.md new file mode 100644 index 0000000..801daad --- /dev/null +++ b/nautilus-docs/docs/api_reference/execution.md @@ -0,0 +1,75 @@ +# Execution + +```{eval-rst} +.. automodule:: nautilus_trader.execution +``` + +## Components + +```{eval-rst} +.. automodule:: nautilus_trader.execution.algorithm + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.execution.client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.execution.emulator + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.execution.engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.execution.manager + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.execution.matching_core + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Messages + +```{eval-rst} +.. automodule:: nautilus_trader.execution.messages + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +## Reports + +```{eval-rst} +.. automodule:: nautilus_trader.execution.reports + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/index.md b/nautilus-docs/docs/api_reference/index.md new file mode 100644 index 0000000..4507125 --- /dev/null +++ b/nautilus-docs/docs/api_reference/index.md @@ -0,0 +1,8 @@ +# Python API + +Auto-generated from the latest source using [Sphinx](https://www.sphinx-doc.org/en/master/). + +- **Latest**: Built from the `develop` branch (latest stable release). +- **Nightly**: Built from the `nightly` branch (bleeding-edge features in development). + +Select the version from the **Versions** drop-down menu (top left). diff --git a/nautilus-docs/docs/api_reference/indicators.md b/nautilus-docs/docs/api_reference/indicators.md new file mode 100644 index 0000000..4962dfe --- /dev/null +++ b/nautilus-docs/docs/api_reference/indicators.md @@ -0,0 +1,61 @@ +# Indicators + +```{eval-rst} +.. automodule:: nautilus_trader.indicators +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.averages + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.fuzzy_candlesticks + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.momentum + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.spread_analyzer + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.trend + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.volatility + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.indicators.volume + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/live.md b/nautilus-docs/docs/api_reference/live.md new file mode 100644 index 0000000..b73736b --- /dev/null +++ b/nautilus-docs/docs/api_reference/live.md @@ -0,0 +1,61 @@ +# Live + +```{eval-rst} +.. automodule:: nautilus_trader.live +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.data_client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.data_engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.execution_client + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.execution_engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.risk_engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.node + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.live.node_builder + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/book.md b/nautilus-docs/docs/api_reference/model/book.md new file mode 100644 index 0000000..4ff3983 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/book.md @@ -0,0 +1,9 @@ +# Order Book + +```{eval-rst} +.. automodule:: nautilus_trader.model.book + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/data.md b/nautilus-docs/docs/api_reference/model/data.md new file mode 100644 index 0000000..8fd5065 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/data.md @@ -0,0 +1,9 @@ +# Data + +```{eval-rst} +.. automodule:: nautilus_trader.model.data + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/events.md b/nautilus-docs/docs/api_reference/model/events.md new file mode 100644 index 0000000..22d6705 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/events.md @@ -0,0 +1,29 @@ +# Events + +```{eval-rst} +.. automodule:: nautilus_trader.model.events +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.events.account + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.events.order + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.events.position + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/identifiers.md b/nautilus-docs/docs/api_reference/model/identifiers.md new file mode 100644 index 0000000..5bfb715 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/identifiers.md @@ -0,0 +1,9 @@ +# Identifiers + +```{eval-rst} +.. automodule:: nautilus_trader.model.identifiers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/index.md b/nautilus-docs/docs/api_reference/model/index.md new file mode 100644 index 0000000..70ef14a --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/index.md @@ -0,0 +1,23 @@ +# Model + +```{eval-rst} +.. automodule:: nautilus_trader.model +``` + +```{eval-rst} +.. toctree:: + :maxdepth: 2 + :glob: + :titlesonly: + :hidden: + + book.md + data.md + events.md + identifiers.md + instruments.md + objects.md + orders.md + position.md + tick_scheme.md +``` diff --git a/nautilus-docs/docs/api_reference/model/instruments.md b/nautilus-docs/docs/api_reference/model/instruments.md new file mode 100644 index 0000000..520c5d0 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/instruments.md @@ -0,0 +1,141 @@ +# Instruments + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.betting + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.binary_option + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.cfd + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.commodity + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.crypto_future + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.crypto_option + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.crypto_perpetual + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.currency_pair + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.equity + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.futures_contract + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.futures_spread + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.index + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.perpetual_contract + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.option_contract + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.option_spread + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.synthetic + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.instruments.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/objects.md b/nautilus-docs/docs/api_reference/model/objects.md new file mode 100644 index 0000000..42fa6ba --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/objects.md @@ -0,0 +1,9 @@ +# Objects + +```{eval-rst} +.. automodule:: nautilus_trader.model.objects + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/orders.md b/nautilus-docs/docs/api_reference/model/orders.md new file mode 100644 index 0000000..6906107 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/orders.md @@ -0,0 +1,93 @@ +# Orders + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.market + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.limit + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.stop_market + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.stop_limit + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.market_to_limit + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.market_if_touched + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.limit_if_touched + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.trailing_stop_market + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.trailing_stop_limit + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.list + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.orders.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/position.md b/nautilus-docs/docs/api_reference/model/position.md new file mode 100644 index 0000000..cfefed0 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/position.md @@ -0,0 +1,9 @@ +# Position + +```{eval-rst} +.. automodule:: nautilus_trader.model.position + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/model/tick_scheme.md b/nautilus-docs/docs/api_reference/model/tick_scheme.md new file mode 100644 index 0000000..345f040 --- /dev/null +++ b/nautilus-docs/docs/api_reference/model/tick_scheme.md @@ -0,0 +1,29 @@ +# Tick Scheme + +```{eval-rst} +.. automodule:: nautilus_trader.model.tick_scheme +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.tick_scheme.implementations.fixed + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.tick_scheme.implementations.tiered + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.model.tick_scheme.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/persistence.md b/nautilus-docs/docs/api_reference/persistence.md new file mode 100644 index 0000000..9b1109a --- /dev/null +++ b/nautilus-docs/docs/api_reference/persistence.md @@ -0,0 +1,37 @@ +# Persistence + +```{eval-rst} +.. automodule:: nautilus_trader.persistence +``` + +```{eval-rst} +.. automodule:: nautilus_trader.persistence.catalog.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.persistence.catalog.parquet + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.persistence.wranglers + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.persistence.writer + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/portfolio.md b/nautilus-docs/docs/api_reference/portfolio.md new file mode 100644 index 0000000..ada9683 --- /dev/null +++ b/nautilus-docs/docs/api_reference/portfolio.md @@ -0,0 +1,21 @@ +# Portfolio + +```{eval-rst} +.. automodule:: nautilus_trader.portfolio +``` + +```{eval-rst} +.. automodule:: nautilus_trader.portfolio.portfolio + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.portfolio.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/risk.md b/nautilus-docs/docs/api_reference/risk.md new file mode 100644 index 0000000..13b78f2 --- /dev/null +++ b/nautilus-docs/docs/api_reference/risk.md @@ -0,0 +1,21 @@ +# Risk + +```{eval-rst} +.. automodule:: nautilus_trader.risk +``` + +```{eval-rst} +.. automodule:: nautilus_trader.risk.engine + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.risk.sizing + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/serialization.md b/nautilus-docs/docs/api_reference/serialization.md new file mode 100644 index 0000000..11b3cf3 --- /dev/null +++ b/nautilus-docs/docs/api_reference/serialization.md @@ -0,0 +1,21 @@ +# Serialization + +```{eval-rst} +.. automodule:: nautilus_trader.serialization +``` + +```{eval-rst} +.. automodule:: nautilus_trader.serialization.serializer + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.serialization.base + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/system.md b/nautilus-docs/docs/api_reference/system.md new file mode 100644 index 0000000..4dc885e --- /dev/null +++ b/nautilus-docs/docs/api_reference/system.md @@ -0,0 +1,13 @@ +# System + +```{eval-rst} +.. automodule:: nautilus_trader.system +``` + +```{eval-rst} +.. automodule:: nautilus_trader.system.kernel + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/api_reference/trading.md b/nautilus-docs/docs/api_reference/trading.md new file mode 100644 index 0000000..190ab69 --- /dev/null +++ b/nautilus-docs/docs/api_reference/trading.md @@ -0,0 +1,37 @@ +# Trading + +```{eval-rst} +.. automodule:: nautilus_trader.trading +``` + +```{eval-rst} +.. automodule:: nautilus_trader.trading.controller + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.trading.filters + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.trading.strategy + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` + +```{eval-rst} +.. automodule:: nautilus_trader.trading.trader + :show-inheritance: + :inherited-members: + :members: + :member-order: bysource +``` diff --git a/nautilus-docs/docs/concepts/actors.md b/nautilus-docs/docs/concepts/actors.md new file mode 100644 index 0000000..6e89871 --- /dev/null +++ b/nautilus-docs/docs/concepts/actors.md @@ -0,0 +1,342 @@ +# Actors + +An `Actor` receives data, handles events, and manages state. The `Strategy` class extends Actor +with order management capabilities. + +**Key capabilities**: + +- Data subscription and requests (market data, custom data). +- Event handling and publishing. +- Timers and alerts. +- Cache and portfolio access. +- Logging. + +## Basic example + +Actors support configuration through a pattern similar to strategies. + +```python +from nautilus_trader.config import ActorConfig +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Bar, BarType +from nautilus_trader.common.actor import Actor + + +class MyActorConfig(ActorConfig): + instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" + bar_type: BarType # example value: "ETHUSDT-PERP.BINANCE-15-MINUTE[LAST]-INTERNAL" + lookback_period: int = 10 + + +class MyActor(Actor): + def __init__(self, config: MyActorConfig) -> None: + super().__init__(config) + + # Custom state variables + self.count_of_processed_bars: int = 0 + + def on_start(self) -> None: + # Subscribe to bars matching the configured bar type + self.subscribe_bars(self.config.bar_type) + + def on_bar(self, bar: Bar) -> None: + self.count_of_processed_bars += 1 +``` + +## Lifecycle + +Actors follow a defined state machine through their lifecycle: + +```mermaid +stateDiagram-v2 + [*] --> PRE_INITIALIZED + PRE_INITIALIZED --> READY : register() + READY --> STARTING : start() + STARTING --> RUNNING : on_start() + RUNNING --> STOPPING : stop() + STOPPING --> STOPPED : on_stop() + STOPPED --> RUNNING : resume() + RUNNING --> DEGRADING : degrade() + DEGRADING --> DEGRADED : on_degrade() + DEGRADED --> RUNNING : resume() + RUNNING --> FAULTING : fault() + FAULTING --> FAULTED : on_fault() + RUNNING --> DISPOSED : dispose() +``` + +Override these methods to hook into lifecycle events: + +| Method | When called | +|-----------------|---------------------------------------------------------------------| +| `on_start()` | Actor is starting (subscribe to data here). | +| `on_stop()` | Actor is stopping (cancel timers, clean up resources). | +| `on_resume()` | Actor is resuming from a stopped state. | +| `on_reset()` | Reset indicators and internal state (called between backtest runs). | +| `on_degrade()` | Actor is entering a degraded state (partial functionality). | +| `on_fault()` | Actor has encountered a fault. | +| `on_dispose()` | Actor is being disposed (final cleanup). | + +## Timers and alerts + +Actors have access to a clock for scheduling: + +```python +def on_start(self) -> None: + # Set a recurring timer (fires every 5 seconds) + self.clock.set_timer("my_timer", timedelta(seconds=5)) + + # Set a one-time alert + self.clock.set_alert("my_alert", self.clock.utc_now() + timedelta(minutes=1)) + +def on_stop(self) -> None: + # Cancel timers to prevent resource leaks across stop/resume cycles + self.clock.cancel_timer("my_timer") + +def on_timer(self, event: TimeEvent) -> None: + if event.name == "my_timer": + self.log.info("Timer fired!") + +def on_alert(self, event: TimeEvent) -> None: + if event.name == "my_alert": + self.log.info("Alert triggered!") +``` + +## System access + +Actors have access to core system components: + +| Property | Description | +|-------------------|----------------------------------------------------------| +| `self.cache` | Shared state for instruments, orders, positions, etc. | +| `self.portfolio` | Portfolio state and calculations. | +| `self.clock` | Current time and timer/alert scheduling. | +| `self.log` | Structured logging. | +| `self.msgbus` | Publish/subscribe to custom messages. | + +For custom messaging between components, see the [Message Bus](message_bus.md) guide. + +## Data handling and callbacks + +The system uses different callback handlers depending on whether data is historical or real-time. +Understanding the relationship between data *requests/subscriptions* and their handlers is key. + +### Historical vs real-time data + +The system distinguishes between two data flows: + +1. **Historical data** (from *requests*): + - Obtained through methods like `request_bars()`, `request_quote_ticks()`, etc. + - Processed through the `on_historical_data()` handler. + - Used for initial data loading and historical analysis. + +2. **Real-time data** (from *subscriptions*): + - Obtained through methods like `subscribe_bars()`, `subscribe_quote_ticks()`, etc. + - Processed through specific handlers like `on_bar()`, `on_quote_tick()`, etc. + - Used for live data processing. + +### Callback handlers + +Different data operations map to these handlers: + +| Operation | Category | Handler | Purpose | +|--------------------------------------|------------|--------------------------|---------------------------------------------------| +| `subscribe_data()` | Real-time | `on_data()` | Live data updates. | +| `subscribe_instrument()` | Real-time | `on_instrument()` | Live instrument definition updates. | +| `subscribe_instruments()` | Real-time | `on_instrument()` | Live instrument definition updates (for venue). | +| `subscribe_order_book_deltas()` | Real-time | `on_order_book_deltas()` | Live order book deltas. | +| `subscribe_order_book_depth()` | Real-time | `on_order_book_depth()` | Live order book depth snapshots. | +| `subscribe_order_book_at_interval()` | Real-time | `on_order_book()` | Live order book snapshots at intervals. | +| `subscribe_quote_ticks()` | Real-time | `on_quote_tick()` | Live quote updates. | +| `subscribe_trade_ticks()` | Real-time | `on_trade_tick()` | Live trade updates. | +| `subscribe_mark_prices()` | Real-time | `on_mark_price()` | Live mark price updates. | +| `subscribe_index_prices()` | Real-time | `on_index_price()` | Live index price updates. | +| `subscribe_bars()` | Real-time | `on_bar()` | Live bar updates. | +| `subscribe_funding_rates()` | Real-time | `on_funding_rate()` | Live funding rate updates. | +| `subscribe_instrument_status()` | Real-time | `on_instrument_status()` | Live instrument status updates. | +| `subscribe_instrument_close()` | Real-time | `on_instrument_close()` | Live instrument close updates. | +| `subscribe_option_greeks()` | Real-time | `on_option_greeks()` | Live option greeks updates. | +| `subscribe_option_chain()` | Real-time | `on_option_chain()` | Live option chain slice snapshots. | +| `subscribe_order_fills()` | Real-time | `on_order_filled()` | Live order fill events for an instrument. | +| `subscribe_order_cancels()` | Real-time | `on_order_canceled()` | Live order cancel events for an instrument. | +| `request_data()` | Historical | `on_historical_data()` | Historical data processing. | +| `request_order_book_deltas()` | Historical | `on_historical_data()` | Historical order book deltas. | +| `request_order_book_depth()` | Historical | `on_historical_data()` | Historical order book depth. | +| `request_order_book_snapshot()` | Historical | `on_historical_data()` | Historical order book snapshot. | +| `request_instrument()` | Historical | `on_instrument()` | Instrument definition. | +| `request_instruments()` | Historical | `on_instrument()` | Instrument definitions. | +| `request_quote_ticks()` | Historical | `on_historical_data()` | Historical quotes processing. | +| `request_trade_ticks()` | Historical | `on_historical_data()` | Historical trades processing. | +| `request_bars()` | Historical | `on_historical_data()` | Historical bars processing. | +| `request_aggregated_bars()` | Historical | `on_historical_data()` | Historical aggregated bars (on-the-fly). | +| `request_funding_rates()` | Historical | `on_historical_data()` | Historical funding rates processing. | + +### Example + +This example shows both historical and real-time data handling: + +```python +from nautilus_trader.common.actor import Actor +from nautilus_trader.config import ActorConfig +from nautilus_trader.core.data import Data +from nautilus_trader.model import Bar, BarType +from nautilus_trader.model import ClientId, InstrumentId + + +class MyActorConfig(ActorConfig): + instrument_id: InstrumentId # example value: "AAPL.XNAS" + bar_type: BarType # example value: "AAPL.XNAS-1-MINUTE-LAST-EXTERNAL" + + +class MyActor(Actor): + def __init__(self, config: MyActorConfig) -> None: + super().__init__(config) + self.bar_type = config.bar_type + + def on_start(self) -> None: + # Request historical data - will be processed by on_historical_data() handler + self.request_bars( + bar_type=self.bar_type, + # Many optional parameters + start=None, # pd.Timestamp | None + end=None, # pd.Timestamp | None + callback=None, # Callable[[UUID4], None] | None + update_catalog_mode=None, # UpdateCatalogMode | None + params=None, # dict[str, Any] | None + ) + + # Subscribe to real-time data - will be processed by on_bar() handler + self.subscribe_bars( + bar_type=self.bar_type, + # Many optional parameters + client_id=None, # ClientId, optional + params=None, # dict[str, Any], optional + ) + + def on_historical_data(self, data: Data) -> None: + # Handle historical data (from requests) + if isinstance(data, Bar): + self.log.info(f"Received historical bar: {data}") + + def on_bar(self, bar: Bar) -> None: + # Handle real-time bar updates (from subscriptions) + self.log.info(f"Received real-time bar: {bar}") +``` + +Separating historical and real-time handlers lets you apply different processing logic +based on context. For example: + +- Use historical data to initialize indicators or establish baseline metrics. +- Process real-time data differently for live trading decisions. +- Apply different validation or logging for historical vs real-time data. + +:::tip +When debugging data flow issues, check that you're looking at the correct handler for your data source. +If you're not seeing data in `on_bar()` but see log messages about receiving bars, check `on_historical_data()` +as the data might be coming from a request rather than a subscription. +::: + +## Order fill subscriptions + +Actors can subscribe to order fill events for specific instruments using `subscribe_order_fills()`. +This is useful for monitoring trading activity, fill analysis, or tracking execution quality. + +When subscribed, the handler `on_order_filled()` receives all fills for the specified instrument, +regardless of which strategy or component generated the original order. + +### Example + +```python +from nautilus_trader.common.actor import Actor +from nautilus_trader.config import ActorConfig +from nautilus_trader.model import InstrumentId +from nautilus_trader.model.events import OrderFilled + + +class MyActorConfig(ActorConfig): + instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" + + +class FillMonitorActor(Actor): + def __init__(self, config: MyActorConfig) -> None: + super().__init__(config) + self.fill_count = 0 + self.total_volume = 0.0 + + def on_start(self) -> None: + # Subscribe to all fills for the instrument + self.subscribe_order_fills(self.config.instrument_id) + + def on_order_filled(self, event: OrderFilled) -> None: + # Handle order fill events + self.fill_count += 1 + self.total_volume += float(event.last_qty) + + self.log.info( + f"Fill received: {event.order_side} {event.last_qty} @ {event.last_px}, " + f"Total fills: {self.fill_count}, Volume: {self.total_volume}" + ) + + def on_stop(self) -> None: + # Unsubscribe from fills + self.unsubscribe_order_fills(self.config.instrument_id) +``` + +:::note +Order fill subscriptions use the message bus only and do not involve the data engine. +The `on_order_filled()` handler receives events only while the actor is running. +::: + +## Order cancel subscriptions + +Actors can subscribe to order cancel events for specific instruments using `subscribe_order_cancels()`. +This is useful for monitoring cancellations or tracking order lifecycle events. + +When subscribed, the handler `on_order_canceled()` receives all cancels for the specified instrument, +regardless of which strategy or component generated the original order. + +### Example + +```python +from nautilus_trader.common.actor import Actor +from nautilus_trader.config import ActorConfig +from nautilus_trader.model import InstrumentId +from nautilus_trader.model.events import OrderCanceled + + +class MyActorConfig(ActorConfig): + instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" + + +class CancelMonitorActor(Actor): + def __init__(self, config: MyActorConfig) -> None: + super().__init__(config) + self.cancel_count = 0 + + def on_start(self) -> None: + # Subscribe to all cancels for the instrument + self.subscribe_order_cancels(self.config.instrument_id) + + def on_order_canceled(self, event: OrderCanceled) -> None: + # Handle order cancel events + self.cancel_count += 1 + + self.log.info( + f"Cancel received: {event.client_order_id}, " + f"Total cancels: {self.cancel_count}" + ) + + def on_stop(self) -> None: + # Unsubscribe from cancels + self.unsubscribe_order_cancels(self.config.instrument_id) +``` + +:::note +Order cancel subscriptions use the message bus only and do not involve the data engine. +The `on_order_canceled()` handler receives events only while the actor is running. +::: + +## Related guides + +- [Strategies](strategies.md) - Strategies extend actors with order management capabilities. +- [Data](data.md) - Data types and subscriptions available to actors. +- [Message Bus](message_bus.md) - The messaging system actors use for communication. diff --git a/nautilus-docs/docs/concepts/adapters.md b/nautilus-docs/docs/concepts/adapters.md new file mode 100644 index 0000000..859ee9d --- /dev/null +++ b/nautilus-docs/docs/concepts/adapters.md @@ -0,0 +1,194 @@ +# Adapters + +Adapters integrate data providers and trading venues into NautilusTrader. +They can be found in the top-level `adapters` subpackage. + +An adapter typically comprises these components: + +```mermaid +flowchart LR + subgraph Venue ["Trading Venue"] + API[REST API] + WS[WebSocket] + end + + subgraph Adapter ["Adapter"] + HTTP[HttpClient] + WSC[WebSocketClient] + IP[InstrumentProvider] + DC[DataClient] + EC[ExecutionClient] + end + + subgraph Core ["Nautilus Core"] + DE[DataEngine] + EE[ExecutionEngine] + end + + API <--> HTTP + WS <--> WSC + HTTP --> IP + HTTP --> DC + HTTP --> EC + WSC --> DC + WSC --> EC + DC <--> DE + EC <--> EE +``` + +| Component | Purpose | +|----------------------|------------------------------------------------------------| +| `HttpClient` | REST API communication. | +| `WebSocketClient` | Real-time streaming connection. | +| `InstrumentProvider` | Loads and parses instrument definitions from the venue. | +| `DataClient` | Handles market data subscriptions and requests. | +| `ExecutionClient` | Handles order submission, modification, and cancellation. | + +## Instrument providers + +Instrument providers parse venue API responses into Nautilus `Instrument` objects. + +An `InstrumentProvider` serves two use cases: + +- Standalone discovery of available instruments for research or backtesting +- Runtime loading in a `sandbox` or `live` [environment context](architecture.md#environment-contexts) + for actors and strategies + +### Research and backtesting + +Here is an example of discovering the current instruments for the Binance Futures testnet: + +```python +import asyncio +import os + +from nautilus_trader.adapters.binance.common.enums import BinanceAccountType +from nautilus_trader.adapters.binance import get_cached_binance_http_client +from nautilus_trader.adapters.binance.futures.providers import BinanceFuturesInstrumentProvider +from nautilus_trader.common.component import LiveClock + + +async def main(): + clock = LiveClock() + + client = get_cached_binance_http_client( + clock=clock, + account_type=BinanceAccountType.USDT_FUTURES, + api_key=os.getenv("BINANCE_FUTURES_TESTNET_API_KEY"), + api_secret=os.getenv("BINANCE_FUTURES_TESTNET_API_SECRET"), + is_testnet=True, + ) + + provider = BinanceFuturesInstrumentProvider( + client=client, + account_type=BinanceAccountType.USDT_FUTURES, + ) + + await provider.load_all_async() + + # Access loaded instruments + instruments = provider.list_all() + print(f"Loaded {len(instruments)} instruments") + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Live trading + +Each integration handles this differently. An `InstrumentProvider` within a `TradingNode` +generally offers two loading behaviors: + +- Load all instruments on start: + +```python +from nautilus_trader.config import InstrumentProviderConfig + +InstrumentProviderConfig(load_all=True) +``` + +- Load only the instruments specified in configuration: + +```python +InstrumentProviderConfig(load_ids=["BTCUSDT-PERP.BINANCE", "ETHUSDT-PERP.BINANCE"]) +``` + +## Data clients + +Data clients handle market data subscriptions and requests for a venue. They connect to venue APIs +and normalize incoming data into Nautilus types. + +### Requesting data + +Actors and strategies can request data using built-in methods. Data returns via callbacks: + +```python +from nautilus_trader.model import Instrument, InstrumentId +from nautilus_trader.trading.strategy import Strategy + + +class MyStrategy(Strategy): + def on_start(self) -> None: + # Request an instrument definition + self.request_instrument(InstrumentId.from_str("BTCUSDT-PERP.BINANCE")) + + # Request historical bars + self.request_bars(BarType.from_str("BTCUSDT-PERP.BINANCE-1-HOUR-LAST-EXTERNAL")) + + def on_instrument(self, instrument: Instrument) -> None: + self.log.info(f"Received instrument: {instrument.id}") + + def on_historical_data(self, data) -> None: + self.log.info(f"Received historical data: {data}") +``` + +### Subscribing to data + +For real-time data, use subscription methods: + +```python +def on_start(self) -> None: + # Subscribe to live trade updates + self.subscribe_trade_ticks(InstrumentId.from_str("BTCUSDT-PERP.BINANCE")) + + # Subscribe to live bars + self.subscribe_bars(BarType.from_str("BTCUSDT-PERP.BINANCE-1-MINUTE-LAST-EXTERNAL")) + +def on_trade_tick(self, tick: TradeTick) -> None: + self.log.info(f"Trade: {tick}") + +def on_bar(self, bar: Bar) -> None: + self.log.info(f"Bar: {bar}") +``` + +:::tip +See the [Actors](actors.md) documentation for a complete reference of available +request and subscription methods with their corresponding callbacks. +::: + +## Execution clients + +Execution clients handle order management for a venue. They translate Nautilus order commands +into venue-specific API calls and process execution reports back into Nautilus events. + +Key responsibilities: + +- Submit, modify, and cancel orders. +- Process fills and execution reports. +- Reconcile order state with the venue. +- Handle account and position updates. + +The `ExecutionEngine` routes commands to the appropriate +execution client based on the order's venue. See the [Execution](execution.md) guide for details +on order management from a strategy perspective. + +:::tip +For building a custom adapter, see the [Adapter Developer Guide](../developer_guide/adapters.md). +::: + +## Related guides + +- [Live Trading](live.md) - Configure and run live trading with adapters. +- [Execution](execution.md) - Order execution through adapters. +- [Data](data.md) - Market data provided by adapters. diff --git a/nautilus-docs/docs/concepts/architecture.md b/nautilus-docs/docs/concepts/architecture.md new file mode 100644 index 0000000..f0faeb2 --- /dev/null +++ b/nautilus-docs/docs/concepts/architecture.md @@ -0,0 +1,641 @@ +# Architecture + +This guide covers the architectural principles and structure of NautilusTrader: + +- Design philosophy and quality attributes. +- Core components and how they interact. +- Environment contexts (backtest, sandbox, live). +- Framework organization and code structure. + +:::note +Throughout the documentation, the term *"Nautilus system boundary"* refers to operations within +the runtime of a single Nautilus node (also known as a "trader instance"). +::: + +## Design philosophy + +The major architectural techniques and design patterns employed by NautilusTrader are: + +- [Domain driven design (DDD)](https://en.wikipedia.org/wiki/Domain-driven_design) +- [Event-driven architecture](https://en.wikipedia.org/wiki/Event-driven_programming) +- [Messaging patterns](https://en.wikipedia.org/wiki/Messaging_pattern) (Pub/Sub, Req/Rep, point-to-point) +- [Ports and adapters](https://en.wikipedia.org/wiki/Hexagonal_architecture_(software)) +- [Crash-only design](#crash-only-design) + +These techniques help achieve certain architectural quality attributes. + +### Quality attributes + +Architectural decisions are often a trade-off between competing priorities. +The following quality attributes guide design and architectural decisions, +roughly in order of weighting. + +- Reliability +- Performance +- Modularity +- Testability +- Maintainability +- Deployability + +### Assurance-driven engineering + +NautilusTrader is incrementally adopting a high-assurance mindset: critical code +paths should carry executable invariants that verify behaviour matches the +business requirements. Practically this means we: + +- Identify the components whose failure has the highest blast radius (core + domain types, risk and execution flows) and write down their invariants in + plain language. +- Codify those invariants as executable checks (unit tests, property tests, + fuzzers, static assertions) that run in CI, keeping the feedback loop light. +- Prefer zero-cost safety techniques built into Rust (ownership, `Result` + surfaces, `panic = abort`) and add targeted formal tools only where they pay + for themselves. +- Track “assurance debt” alongside feature work so new integrations extend the + safety net rather than bypass it. + +This approach preserves the platform’s delivery cadence while giving +high-stakes flows the additional scrutiny they need. + +Further reading: [High Assurance Rust](https://highassurance.rs/). + +### Crash-only design + +NautilusTrader draws inspiration from [crash-only design](https://en.wikipedia.org/wiki/Crash-only_software) +principles, particularly for handling unrecoverable faults. The core insight is that systems which +can recover cleanly from crashes are more robust than those with separate (and rarely tested) +graceful shutdown paths. + +Key principles: + +- **Unified recovery path** - Startup and crash recovery share the same code path, ensuring it is well-tested. +- **Externalized state** - Critical state is meant to be persisted externally when configured, reducing data-loss risk; durability depends on the backing store. +- **Fast restart** - The system is designed to restart quickly after a crash, minimizing downtime. +- **Idempotent operations** - Operations are designed to be safely retried after restart. +- **Fail-fast for unrecoverable errors** - Data corruption or invariant violations trigger immediate termination rather than attempting to continue in a compromised state. + +:::note +The system does provide graceful shutdown flows (`stop`, `dispose`) for normal operation. These +tear down clients, persist state, and flush writers. The crash-only philosophy applies specifically +to *unrecoverable faults* where attempting graceful cleanup could cause further damage. +::: + +This design complements the [fail-fast policy](#data-integrity-and-fail-fast-policy), where +unrecoverable errors result in immediate process termination. + +**References:** + +- [Crash-Only Software](https://www.usenix.org/conference/hotos-ix/crash-only-software) - Candea & Fox, HotOS 2003 (original research paper) +- [Microreboot—A Technique for Cheap Recovery](https://www.usenix.org/conference/osdi-04/microreboot—-technique-cheap-recovery) - Candea et al., OSDI 2004 +- [The properties of crash-only software](https://brooker.co.za/blog/2012/01/22/crash-only.html) - Marc Brooker's blog +- [Crash-only software: More than meets the eye](https://lwn.net/Articles/191059/) - LWN.net article +- [Recovery-Oriented Computing (ROC) Project](http://roc.cs.berkeley.edu/) - UC Berkeley/Stanford research + +### Data integrity and fail-fast policy + +NautilusTrader prioritizes data integrity over availability for trading operations. The system employs +a strict fail-fast policy for arithmetic operations and data handling to prevent silent data corruption +that could lead to incorrect trading decisions. + +#### Fail-fast principles + +The system will fail fast (panic or return an error) when encountering: + +- Arithmetic overflow or underflow in operations on timestamps, prices, or quantities that exceed valid ranges. +- Invalid data during deserialization including NaN, Infinity, or out-of-range values in market data or configuration. +- Type conversion failures such as negative values where only positive values are valid (timestamps, quantities). +- Malformed input parsing for prices, timestamps, or precision values. + +Rationale: + +In trading systems, corrupt data is worse than no data. A single incorrect price, timestamp, or quantity +can cascade through the system, resulting in: + +- Incorrect position sizing or risk calculations. +- Orders placed at wrong prices. +- Backtests producing misleading results. +- Silent financial losses. + +By crashing immediately on invalid data, NautilusTrader aims to provide: + +1. **No silent corruption** - The fail-fast policy is intended to prevent invalid data from propagating; this relies on checks covering the inputs. +2. **Immediate feedback** - Issues are discovered during development and testing, not in production. +3. **Audit trail** - Crash logs clearly identify the source of invalid data. +4. **Deterministic behavior** - With deterministic ordering and configuration, the same invalid input should trigger the same failure; nondeterministic sources can vary outcomes. + +#### When fail-fast applies + +Panics are used for: + +- Programmer errors (logic bugs, incorrect API usage). +- Data that violates fundamental invariants (negative timestamps, NaN prices). +- Arithmetic that would silently produce incorrect results. + +Results or Options are used for: + +- Expected runtime failures (network errors, file I/O). +- Business logic validation (order constraints, risk limits). +- User input validation. +- Library APIs exposed to downstream crates where callers need explicit error handling without relying on panics for control flow. + +#### Example scenarios + +```rust +// CORRECT: Panics on overflow - prevents data corruption +let total_ns = timestamp1 + timestamp2; // Panics if result > u64::MAX + +// CORRECT: Rejects NaN during deserialization +let price = serde_json::from_str("NaN"); // Error: "must be finite" + +// CORRECT: Explicit overflow handling when needed +let total_ns = timestamp1.checked_add(timestamp2)?; // Returns Option +``` + +This policy is implemented throughout the core types (`UnixNanos`, `Price`, `Quantity`, etc.) +and helps NautilusTrader maintain strong data correctness for production trading. + +In production deployments, the system is typically configured with `panic = abort` in release builds, +ensuring that any panic results in a clean process termination that can be handled by process supervisors +or orchestration systems. This aligns with the [crash-only design](#crash-only-design) principle, where unrecoverable errors +lead to immediate restart rather than attempting to continue in a potentially corrupted state. + +## System architecture + +The NautilusTrader codebase is actually both a framework for composing trading + systems, and a set of default system implementations which can operate in various +[environment contexts](#environment-contexts). + +![Architecture](https://github.com/nautechsystems/nautilus_trader/blob/develop/assets/architecture-overview.png?raw=true "architecture") + +### Core components + +Several core components work together to form the trading system: + +#### `NautilusKernel` + +The central orchestration component responsible for: + +- Initializing and managing all system components. +- Configuring the messaging infrastructure. +- Maintaining environment-specific behaviors. +- Coordinating shared resources and lifecycle management. +- Providing a unified entry point for system operations. + +#### `MessageBus` + +The backbone of inter-component communication, implementing: + +- **Publish/Subscribe patterns**: For broadcasting events and data to multiple consumers. +- **Request/Response communication**: For operations requiring acknowledgment. +- **Command/Event messaging**: For triggering actions and notifying state changes. +- **Optional state persistence**: Using Redis for durability and restart capabilities. + +#### `Cache` + +High-performance in-memory storage system that: + +- Stores instruments, accounts, orders, positions, and more. +- Provides performant fetching capabilities for trading components. +- Maintains consistent state across the system. +- Supports both read and write operations with optimized access patterns. + +#### `DataEngine` + +Processes and routes market data throughout the system: + +- Handles multiple data types (quotes, trades, bars, order books, custom data, and more). +- Routes data to appropriate consumers based on subscriptions. +- Manages data flow from external sources to internal components. + +#### `ExecutionEngine` + +Manages order lifecycle and execution: + +- Routes trading commands to the appropriate adapter clients. +- Tracks order and position states. +- Coordinates with risk management systems. +- Handles execution reports and fills from venues. +- Handles reconciliation of external execution state. + +#### `RiskEngine` + +Provides risk management: + +- Pre-trade risk checks and validation. +- Position and exposure monitoring. +- Real-time risk calculations. +- Configurable risk rules and limits. + +### Environment contexts + +An environment context in NautilusTrader defines the type of data and trading venue you work with. +Understanding these contexts matters for backtesting, development, and live trading. + +Here are the available environments you can work with: + +- `Backtest`: Historical data with simulated venues. +- `Sandbox`: Real-time data with simulated venues. +- `Live`: Real-time data with live venues (paper trading or real accounts). + +### Common core + +The platform has been designed to share as much common code between backtest, sandbox and live trading systems as possible. +This is formalized in the `system` subpackage, where you will find the `NautilusKernel` class, +providing a common core system 'kernel'. + +The *ports and adapters* architectural style enables modular components to be integrated into the +core system, providing various hooks for user-defined or custom component implementations. + +### Data and execution flow patterns + +Understanding how data and execution flow through the system helps when working with the platform: + +#### Data flow pattern + +1. **External data ingestion**: Market data enters via venue-specific `DataClient` adapters where it is normalized. +2. **Data processing**: The `DataEngine` handles data processing for internal components. +3. **Caching**: Processed data is stored in the `Cache` for fast access. +4. **Event publishing**: Data events are published to the `MessageBus`. +5. **Consumer delivery**: Subscribed components (Actors, Strategies) receive relevant data events. + +#### Execution flow pattern + +1. **Command generation**: Strategies create trading commands. +2. **Command publishing**: Commands are sent through the `MessageBus`. +3. **Risk validation**: The `RiskEngine` validates trading commands against configured risk rules. +4. **Execution routing**: The `ExecutionEngine` routes commands to appropriate venues. +5. **External submission**: The `ExecutionClient` submits orders to external trading venues. +6. **Event flow back**: Order events (fills, cancellations) flow back through the system. +7. **State updates**: Portfolio and position states update based on execution events. + +#### Component state management + +All components follow a finite state machine pattern. The `ComponentState` enum defines both stable states and transitional states: + +```mermaid +stateDiagram-v2 + [*] --> PRE_INITIALIZED + + PRE_INITIALIZED --> READY : register() + + READY --> STARTING : start() + STARTING --> RUNNING + + RUNNING --> STOPPING : stop() + STOPPING --> STOPPED + + STOPPED --> STARTING : start() + STOPPED --> RESETTING : reset() + RESETTING --> READY + + RUNNING --> RESUMING : resume() + RESUMING --> RUNNING + + RUNNING --> DEGRADING : degrade() + DEGRADING --> DEGRADED + + DEGRADED --> STOPPING : stop() + DEGRADED --> FAULTING : fault() + + RUNNING --> FAULTING : fault() + FAULTING --> FAULTED + + STOPPED --> DISPOSING : dispose() + FAULTED --> DISPOSING : dispose() + DISPOSING --> DISPOSED + + DISPOSED --> [*] +``` + +**Stable states:** + +- **PRE_INITIALIZED**: Component is instantiated but not yet ready to fulfill its specification. +- **READY**: Component is configured and able to be started. +- **RUNNING**: Component is operating normally and can fulfill its specification. +- **STOPPED**: Component has successfully stopped. +- **DEGRADED**: Component has degraded and may not meet its full specification. +- **FAULTED**: Component has shut down due to a detected fault. +- **DISPOSED**: Component has shut down and released all of its resources. + +**Transitional states:** + +- **STARTING**: Component is executing its actions on `start`. +- **STOPPING**: Component is executing its actions on `stop`. +- **RESUMING**: Component is being started again after its initial start. +- **RESETTING**: Component is executing its actions on `reset`. +- **DISPOSING**: Component is executing its actions on `dispose`. +- **DEGRADING**: Component is executing its actions on `degrade`. +- **FAULTING**: Component is executing its actions on `fault`. + +Transitional states are brief intermediate states that occur during state transitions. Components should not remain in transitional states for extended periods. + +#### Actor vs Component traits + +At the Rust implementation level, the system distinguishes between two complementary traits: + +```mermaid +classDiagram + class Actor { + <> + +id() Ustr + +handle(message) + } + + class Component { + <> + +component_id() ComponentId + +state() ComponentState + +register() + +start() + +stop() + +reset() + +dispose() + } + + class ActorRegistry { + +insert(actor) + +get(id) ActorRef + } + + class ComponentRegistry { + +insert(component) + +get(id) ComponentRef + } + + Actor <|.. Throttler : implements + Actor <|.. Strategy : implements + Component <|.. Strategy : implements + Component <|.. DataEngine : implements + Component <|.. ExecutionEngine : implements + + ActorRegistry --> Actor : manages + ComponentRegistry --> Component : manages + + class Throttler { + Actor only + } + + class Strategy { + Actor + Component + } + + class DataEngine { + Component only + } + + class ExecutionEngine { + Component only + } +``` + +**`Actor` trait** - Message dispatch: + +- Provides the `handle` method for receiving messages dispatched through the actor registry. +- Enables type-safe lookup and message dispatch by actor ID. +- Used by components that need to receive targeted messages (strategies, throttlers). + +**`Component` trait** - Lifecycle management: + +- Manages state transitions (`start`, `stop`, `reset`, `dispose`). +- Provides registration with the system kernel (`register`). +- Tracks component state via the finite state machine described above. +- Used by all system components that need lifecycle management. + +:::note +All components can publish and subscribe to messages via the `MessageBus` directly - this is independent of the `Actor` trait. The `Actor` trait specifically enables the registry-based message dispatch pattern where messages are routed to a specific actor by ID. +::: + +This separation allows: + +- **Actor-only**: Lightweight message handlers without lifecycle (e.g., `Throttler`). +- **Component-only**: System infrastructure with lifecycle but using direct MessageBus pub/sub (e.g., `DataEngine`, `ExecutionEngine`). +- **Both traits**: Trading strategies that need lifecycle management AND targeted message dispatch. + +The traits are managed by separate registries to support their different access patterns - lifecycle methods are called sequentially, while message handlers may be invoked re-entrantly during callbacks. + +### Messaging + +For modularity and loose coupling, an efficient `MessageBus` passes messages (data, commands, and events) between components. + +#### Threading model + +Within a node, the *kernel* consumes and dispatches messages on a single thread. The kernel encompasses: + +- The `MessageBus` and actor callback dispatch. +- Strategy logic and order management. +- Risk engine checks and execution coordination. +- Cache reads and writes. + +This single-threaded core provides deterministic event ordering and helps maintain backtest-live parity, +though live inputs and latency can still cause behavioral differences. Components consume messages +synchronously in a pattern *similar* to the [actor model](https://en.wikipedia.org/wiki/Actor_model). + +:::note +Of interest is the LMAX exchange architecture, which achieves award winning performance running on +a single thread. You can read about their *disruptor* pattern based architecture in [this interesting article](https://martinfowler.com/articles/lmax.html) by Martin Fowler. +::: + +Background services use separate threads or async runtimes: + +- **Network I/O** - WebSocket connections, REST clients, and async data feeds. +- **Persistence** - DataFusion queries and database operations via multi-threaded Tokio runtime. +- **Adapters** - Async adapter operations via thread pool executors. + +These services communicate results back to the kernel via the `MessageBus`. The bus itself is thread-local, +so each thread has its own instance, with cross-thread communication occurring through channels that +ultimately deliver events to the single-threaded core. + +## Framework organization + +The codebase organizes into layers of abstraction, grouped into logical subpackages +of cohesive concepts. You can navigate to the documentation for each subpackage +from the left nav menu. + +### Core / low-level + +- `core`: Constants, functions and low-level components used throughout the framework. +- `common`: Common parts for assembling the frameworks various components. +- `network`: Low-level base components for networking clients. +- `serialization`: Serialization base components and serializer implementations. +- `model`: Defines a rich trading domain model. + +### Components + +- `accounting`: Different account types and account management machinery. +- `adapters`: Integration adapters for the platform including brokers and exchanges. +- `analysis`: Components relating to trading performance statistics and analysis. +- `cache`: Provides common caching infrastructure. +- `data`: The data stack and data tooling for the platform. +- `execution`: The execution stack for the platform. +- `indicators`: A set of efficient indicators and analyzers. +- `persistence`: Data storage, cataloging and retrieval, mainly to support backtesting. +- `portfolio`: Portfolio management functionality. +- `risk`: Risk specific components and tooling. +- `trading`: Trading domain specific components and tooling. + +### System implementations + +- `backtest`: Backtesting componentry as well as a backtest engine and node implementations. +- `live`: Live engine and client implementations as well as a node for live trading. +- `system`: The core system kernel common between `backtest`, `sandbox`, `live` [environment contexts](#environment-contexts). + +## Code structure + +The foundation of the codebase is the `crates` directory, containing a collection of Rust crates including a C foreign function interface (FFI) generated by `cbindgen`. + +The bulk of the production code resides in the `nautilus_trader` directory, which contains a collection of Python/Cython subpackages and modules. + +Python bindings for the Rust core are provided by statically linking the Rust libraries to the C extension modules generated by Cython at compile time (effectively extending the CPython API). + +### Dependency flow + +```mermaid +flowchart TB + subgraph trader["nautilus_trader
Python / Cython"] + end + + subgraph core["crates
Rust"] + end + + trader -->|"C API"| core +``` + +### Rust crates + +The `crates/` directory contains the Rust implementation organized into focused crates with clear dependency boundaries. +Feature flags control optional functionality - for example, `streaming` enables persistence for catalog-based data streaming, +and `cloud` enables cloud storage backends (S3, Azure, GCP). + +Dependency flow (arrows point to dependencies): + +```mermaid +flowchart BT + subgraph Foundation + core + model + common + system + trading + end + + subgraph Infrastructure + serialization + network + cryptography + persistence + end + + subgraph Engines + data + execution + portfolio + risk + end + + subgraph Runtime + live + backtest + end + + adapters + pyo3 + + model --> core + common --> core + common --> model + system --> common + trading --> common + serialization --> model + network --> common + network --> cryptography + persistence --> serialization + data --> common + execution --> common + portfolio --> common + risk --> portfolio + live --> system + live --> trading + backtest --> system + backtest --> persistence + adapters --> live + adapters --> network + pyo3 --> adapters +``` + +**Crate categories:** + +| Category | Crates | Purpose | +|----------------|-----------------------------------------------------------|----------------------------------------------------------| +| Foundation | `core`, `model`, `common`, `system`, `trading` | Primitives, domain model, kernel, actor & strategy base. | +| Engines | `data`, `execution`, `portfolio`, `risk` | Core trading engine components. | +| Infrastructure | `serialization`, `network`, `cryptography`, `persistence` | Encoding, networking, signing, storage. | +| Runtime | `live`, `backtest` | Environment-specific node implementations. | +| External | `adapters/*` | Venue and data integrations. | +| Bindings | `pyo3` | Python bindings. | + +**Feature flags:** + +| Feature | Crates | Effect | +|-------------|----------------------------|------------------------------------------------------------| +| `streaming` | `data`, `system`, `live` | Enables `persistence` dependency for catalog streaming. | +| `cloud` | `persistence` | Enables cloud storage backends (S3, Azure, GCP, HTTP). | +| `python` | most crates | Enables PyO3 bindings (auto-enables `streaming`, `cloud`). | +| `defi` | `common`, `model`, `data` | Enables DeFi/blockchain data types. | + +:::note +Both Rust and Cython are build dependencies. The binary wheels produced from a build do not require +Rust or Cython to be installed at runtime. +::: + +### Type safety + +The platform design prioritizes software correctness and safety. + +The Rust codebase under `crates/` relies on the `rustc` compiler's guarantees for safe code. +Any `unsafe` blocks are explicit opt-outs where we must uphold the required invariants ourselves +(see the Rust section of the [Developer Guide](../developer_guide/rust.md)); overall memory and type safety +depend on those invariants holding. + +Cython provides type safety at the C level at both compile time, and runtime: + +:::info +If you pass an argument with an invalid type to a Cython implemented module with typed parameters, +then you will receive a `TypeError` at runtime. +::: + +If a function or method's parameter is not explicitly typed to accept `None`, passing `None` as an +argument will result in a `ValueError` at runtime. + +:::warning +The above exceptions are not explicitly documented to prevent excessive bloating of the docstrings. +::: + +### Errors and exceptions + +The documentation aims to cover all possible exceptions that NautilusTrader code +can raise, and the conditions that trigger them. + +:::warning +There may be other undocumented exceptions which can be raised by Python's standard +library, or from third party library dependencies. +::: + +### Processes and threads + +:::warning[One node per process] +Running multiple `TradingNode` or `BacktestNode` instances **concurrently** in the same process is not supported due to global singleton state: + +- **Backtest force-stop flag** - The `_FORCE_STOP` global flag is shared across all engines in the process. +- **Logger mode and timestamps** - The logging subsystem uses global state; backtests flip between static and real-time modes. +- **Runtime singletons** - Global Tokio runtime, callback registries, and other `OnceLock` instances are process-wide. + +**Sequential execution** of multiple nodes (one after another with proper disposal between runs) is fully supported and used in the test suite. + +For production deployments, add multiple strategies to a **single TradingNode** within a process. +For parallel execution or workload isolation, run each node in its own separate process. +::: + +## Related guides + +- [Overview](overview.md) - High-level introduction to NautilusTrader. +- [Message Bus](message_bus.md) - Core messaging infrastructure. diff --git a/nautilus-docs/docs/concepts/backtesting.md b/nautilus-docs/docs/concepts/backtesting.md new file mode 100644 index 0000000..bb48b23 --- /dev/null +++ b/nautilus-docs/docs/concepts/backtesting.md @@ -0,0 +1,1636 @@ +# Backtesting + +Backtesting simulates trading using a specific system implementation. The system comprises the +built-in engines, `Cache`, [MessageBus](message_bus.md), `Portfolio`, [Actors](actors.md), +[Strategies](strategies.md), [Execution Algorithms](execution.md), and user-defined modules. +A `BacktestEngine` processes a stream of historical data. When the stream is exhausted, the +engine produces results and performance metrics for analysis. + +NautilusTrader offers two API levels for backtesting: + +- **High-level API**: Uses a `BacktestNode` and configuration objects (`BacktestEngine`s are used internally). +- **Low-level API**: Uses a `BacktestEngine` directly with more "manual" setup. + +## Choosing an API level + +Consider using the **low-level** API when: + +- Your entire data stream can be processed within the available machine resources (e.g., RAM). +- You prefer not to store data in the Nautilus-specific Parquet format. +- You have a specific need or preference to retain raw data in its original format (e.g., CSV, binary, etc.). +- You require fine-grained control over the `BacktestEngine`, such as the ability to re-run backtests on identical datasets while swapping out components (e.g., actors or strategies) or adjusting parameter configurations. + +Consider using the **high-level** API when: + +- Your data stream exceeds available memory, requiring streaming data in batches. +- You want the performance and convenience of the `ParquetDataCatalog` for storing data in the Nautilus-specific Parquet format. +- You value the flexibility and functionality of passing configuration objects to define and manage multiple backtest runs across various engines simultaneously. + +## Low-level API + +The low-level API centers around a `BacktestEngine`, where inputs are initialized and added manually via a Python script. +An instantiated `BacktestEngine` can accept the following: + +- Lists of `Data` objects, which are automatically sorted into monotonic order based on `ts_init`. +- Multiple venues, manually initialized. +- Multiple actors, manually initialized and added. +- Multiple execution algorithms, manually initialized and added. + +This approach offers detailed control over the backtesting process, allowing you to manually configure each component. + +### Loading large datasets efficiently + +When working with large amounts of data across multiple instruments, the way you load data +can significantly impact performance. + +#### The performance consideration + +By default, `BacktestEngine.add_data()` sorts the entire data stream (existing data + newly +added data) on each call when `sort=True` (the default). This means: + +- First call with 1M bars: sorts 1M bars. +- Second call with 1M bars: sorts 2M bars. +- Third call with 1M bars: sorts 3M bars. +- And so on... + +This repeated sorting of increasingly large datasets can become a bottleneck when loading +data for multiple instruments. + +#### Optimization strategies + +**Strategy 1: Defer sorting until the end (recommended for multiple instruments)** + +```python +from nautilus_trader.backtest.engine import BacktestEngine + +engine = BacktestEngine() + +# Setup venue and instruments +engine.add_venue(...) +engine.add_instrument(instrument1) +engine.add_instrument(instrument2) +engine.add_instrument(instrument3) + +# Load all data WITHOUT sorting on each call +engine.add_data(instrument1_bars, sort=False) +engine.add_data(instrument2_bars, sort=False) +engine.add_data(instrument3_bars, sort=False) + +# Sort once at the end - much more efficient! +engine.sort_data() + +# Now run your backtest +engine.add_strategy(strategy) +engine.run() +``` + +**Strategy 2: Collect and add in a single batch** + +```python +# Collect all data first +all_bars = [] +all_bars.extend(instrument1_bars) +all_bars.extend(instrument2_bars) +all_bars.extend(instrument3_bars) + +# Add once with sorting +engine.add_data(all_bars, sort=True) +``` + +**Strategy 3: Use streaming API for very large datasets** + +For datasets that don't fit in memory, there are two streaming approaches: + +**Automatic chunking** - supply a generator that yields batches. The engine pulls chunks +lazily during a single `run()` call: + +```python +def data_generator(): + # Yield chunks of data (each chunk is a list of Data objects) + yield load_chunk_1() + yield load_chunk_2() + yield load_chunk_3() + +engine.add_data_iterator( + data_name="my_data_stream", + generator=data_generator(), +) + +engine.run() # Chunks are consumed on-demand +``` + +**Manual chunking** - load and run each batch yourself. This is the pattern +used internally by `BacktestNode` and gives full control over batch boundaries: + +```python +engine.add_strategy(strategy) + +for batch in data_batches: + engine.add_data(batch) + engine.run(streaming=True) + engine.clear_data() + +engine.end() # Finalize: flushes remaining timers, stops engines, produces results +``` + +:::note +In streaming mode, timer advancement stops when data exhausts for each batch. Timers scheduled +past the last data point (e.g. bar aggregation intervals) are deferred until more data arrives +or `end()` is called, which flushes up to the `end` boundary from the last `run()` call. +::: + +:::tip[Performance impact] +For a backtest with 10 instruments, each with 1M bars: + +- Sorting on each call: ~10 sorts of increasing size (1M, 2M, 3M, ... 10M bars). +- Sorting once at the end: 1 sort of 10M bars. + +The deferred sorting approach can be **significantly faster** for large datasets. +::: + +### Data loading contract + +The `BacktestEngine` enforces important invariants to ensure data integrity: + +**Requirements:** + +- All data must be sorted before calling `run()`. +- When using `sort=False`, you **must** call `sort_data()` before running. +- The engine validates this and raises `RuntimeError` if unsorted data is detected. +- Calling `sort_data()` multiple times is safe (idempotent). + +**Safety guarantees:** + +- Data lists are always copied internally to prevent external mutations from affecting engine state. +- You can safely clear or modify data lists after passing them to `add_data()`. +- Adding data with `sort=True` makes it immediately available for backtesting. + +This design ensures data integrity while enabling performance optimizations for large datasets. + +## High-level API + +The high-level API centers around a `BacktestNode`, which orchestrates the management of multiple `BacktestEngine` instances, +each defined by a `BacktestRunConfig`. Multiple configurations can be bundled into a list and processed by the node in one run. + +Each `BacktestRunConfig` object consists of the following: + +- A list of `BacktestDataConfig` objects. +- A list of `BacktestVenueConfig` objects. +- A list of `ImportableActorConfig` objects. +- A list of `ImportableStrategyConfig` objects. +- A list of `ImportableExecAlgorithmConfig` objects. +- An optional `ImportableControllerConfig` object. +- An optional `BacktestEngineConfig` object, with a default configuration if not specified. + +## Repeated runs + +When conducting multiple backtest runs, it's important to understand how components reset to avoid unexpected behavior. + +### BacktestEngine.reset() + +The `.reset()` method returns all stateful fields to their **initial value**, except for data and instruments which persist. + +**What gets reset:** + +- All trading state (orders, positions, account balances). +- Strategy instances are removed (you must re-add strategies before the next run). +- Engine counters and timestamps. + +**What persists:** + +- Data added via `.add_data()` (use `.clear_data()` to remove). +- Instruments (must match the persisted data). +- Venue configurations. + +**Instrument handling:** + +For `BacktestEngine`, instruments persist across resets by default (because data persists and instruments must match data). +This is configured via `CacheConfig.drop_instruments_on_reset=False` in the default `BacktestEngineConfig`. + +### Approaches for multiple backtest runs + +There are two main approaches for running multiple backtests: + +#### 1. Use BacktestNode (recommended for production) + +The high-level API is designed for multiple backtest runs with different configurations: + +```python +from nautilus_trader.backtest.node import BacktestNode +from nautilus_trader.config import BacktestRunConfig + +# Define multiple run configurations +configs = [ + BacktestRunConfig(...), # Run 1 + BacktestRunConfig(...), # Run 2 + BacktestRunConfig(...), # Run 3 +] + +# Execute all runs +node = BacktestNode(configs=configs) +results = node.run() +``` + +Each run gets a fresh engine with clean state - no reset() needed. + +#### 2. Use BacktestEngine.reset() + +For fine-grained control with the low-level API: + +```python +from nautilus_trader.backtest.engine import BacktestEngine + +engine = BacktestEngine() + +# Setup once +engine.add_venue(...) +engine.add_instrument(ETHUSDT) +engine.add_data(data) + +# Run 1 +engine.add_strategy(strategy1) +engine.run() + +# Reset and run 2 - instruments and data persist +engine.reset() +engine.add_strategy(strategy2) +engine.run() + +# Reset and run 3 +engine.reset() +engine.add_strategy(strategy3) +engine.run() +``` + +:::note +Instruments and data persist across resets by default for `BacktestEngine`, making parameter optimizations straightforward. +::: + +:::tip[Best practices] + +- **For production backtesting:** Use `BacktestNode` with configuration objects. +- **For parameter optimizations:** Use `BacktestEngine.reset()` to run multiple strategies against the same data. +- **For quick experiments:** Either approach works - choose based on individual use case. + +::: + +## Data + +Data provided for backtesting drives the execution flow. Since a variety of data types can be used, +it's crucial that your venue configurations align with the data being provided for backtesting. +Mismatches between data and configuration can lead to unexpected behavior during execution. + +NautilusTrader is primarily designed and optimized for order book data, which provides +a complete representation of every price level or order in the market, reflecting the real-time behavior of a trading venue. +This provides the greatest execution granularity and realism. However, if granular order book data is either not +available or necessary, then the platform has the capability of processing market data in the following descending order of detail: + +```mermaid +flowchart LR + L3["L3 Order Book
(market-by-order)"] + L2["L2 Order Book
(market-by-price)"] + L1["L1 Quotes
(top of book)"] + T["Trades"] + B["Bars"] + + L3 --> L2 --> L1 --> T --> B + + style L3 fill:#2d5a3d,color:#fff + style L2 fill:#3d6a4d,color:#fff + style L1 fill:#4d7a5d,color:#fff + style T fill:#5d8a6d,color:#fff + style B fill:#6d9a7d,color:#fff +``` + +1. **Order Book Data/Deltas (L3 market-by-order)**: + - Full market depth with visibility of all individual orders. + +2. **Order Book Data/Deltas (L2 market-by-price)**: + - Market depth visibility across all price levels. + +3. **Quote Ticks (L1 market-by-price)**: + - Top of book only - best bid and ask prices and sizes. + +4. **Trade Ticks**: + - Actual executed trades. + +5. **Bars**: + - Aggregated trading activity over fixed time intervals (e.g., 1-minute, 1-hour, 1-day). + +### Choosing data: cost vs. accuracy + +For many trading strategies, bar data (e.g., 1-minute) can be sufficient for backtesting and strategy development. This is +particularly important because bar data is typically much more accessible and cost-effective compared to tick or order book data. + +Given this practical reality, Nautilus is designed to support bar-based backtesting with advanced features +that maximize simulation accuracy, even when working with lower granularity data. + +:::tip +For some trading strategies, it can be practical to start development with bar data to validate core trading ideas. +If the strategy looks promising, but is more sensitive to precise execution timing (e.g., requires fills at specific prices +between OHLC levels, or uses tight take-profit/stop-loss levels), you can then invest in higher granularity data +for more accurate validation. +::: + +## Venues + +When initializing a venue for backtesting, you must specify its internal order `book_type` for execution processing from the following options: + +- `L1_MBP`: Level 1 market-by-price (default). Only the top level of the order book is maintained. +- `L2_MBP`: Level 2 market-by-price. Order book depth is maintained, with a single order aggregated per price level. +- `L3_MBO`: Level 3 market-by-order. Order book depth is maintained, with all individual orders tracked as provided by the data. + +The `book_type` determines which data types the matching engine uses to update book +state and drive execution. Data types not applicable for a given `book_type` are +ignored for book and price updates, though precision validation still applies and +the engine clock still advances. Strategies always receive all subscribed data via +the data engine regardless of `book_type`. + +| Data Type | L1_MBP | L2_MBP | L3_MBO | +| ------------------ | ----------------- | ----------------- | ----------------- | +| `QuoteTick` | Updates book | *Ignored* | *Ignored* | +| `TradeTick` | Triggers matching | Triggers matching | Triggers matching | +| `Bar` | Updates book | *Ignored* | *Ignored* | +| `OrderBookDelta` | *Ignored* | Updates book | Updates book | +| `OrderBookDeltas` | *Ignored* | Updates book | Updates book | +| `OrderBookDepth10` | Updates book | Updates book | Updates book | + +:::note +The granularity of the data must match the specified order `book_type`. Nautilus +cannot generate higher granularity data (L2 or L3) from lower-level data such as +quotes, trades, or bars. +::: + +:::warning +If you specify `L2_MBP` or `L3_MBO` as the venue’s `book_type`, quotes and bars +will not update the book. Ensure you provide order book delta data, otherwise +orders may appear as though they are never filled. +::: + +:::warning +When using `L1_MBP` (the default), order book deltas are ignored by the matching +engine. If you subscribe to order book deltas, set the venue `book_type` to +`L2_MBP` or `L3_MBO`. This also applies to sandbox execution, where the matching +engine uses the same `book_type` configuration. +::: + +## Execution + +### Data and message sequencing + +In the main backtesting loop, new market data is processed for order execution before being dispatched to actors/strategies via the data engine. + +#### Main loop flow + +For each data point the engine runs three phases: + +- **Exchange processes data.** The simulated exchange updates its order book from + the incoming market data and iterates the matching engine. This fills any existing + orders that now match against the new market state. +- **Strategy receives data.** The data engine dispatches the data point to actors + and strategies via their callbacks (e.g. `on_quote_tick`, `on_bar`). Strategies + may submit, cancel, or modify orders during these callbacks. +- **Settle venues.** The engine drains all queued venue commands and then iterates + matching engines to fill newly submitted orders. This loop repeats until no + pending commands remain, so cascading orders (e.g. a hedge submitted from + `on_order_filled`) settle within the same timestamp. + +```mermaid +sequenceDiagram + participant BL as Backtest Loop + participant Exch as SimulatedExchange + participant ME as MatchingEngine + participant DE as DataEngine + participant Stgy as Strategy + + BL->>BL: next data point (ts=T) + + rect rgb(240, 248, 255) + note right of BL: Phase 1 - Exchange processes data + BL->>Exch: process_quote_tick / process_bar + Exch->>ME: update book + iterate() + note right of ME: Matches existing orders
against new market state + end + + rect rgb(245, 255, 245) + note right of BL: Phase 2 - Strategy receives data + BL->>DE: process(data) + DE->>Stgy: on_quote_tick() / on_bar() + Stgy-->>Exch: submit_order (queued or immediate) + end + + rect rgb(255, 248, 240) + note right of BL: Phase 3 - Settle venues + BL->>BL: _process_and_settle_venues(T) + BL->>Exch: _drain_commands(T) + note right of Exch: Processes queued commands,
adds orders to matching core + BL->>ME: _core.iterate(T) + note right of ME: Matches newly added orders
against current market state + note right of ME: Fills may trigger strategy callbacks
that enqueue further commands,
repeats until no pending commands + BL->>Exch: run simulation modules + BL->>Exch: check instrument expirations + end +``` + +Timer events use the same settle mechanism but batch by timestamp: all callbacks at +timestamp T execute first, then venues are settled for T before advancing to T+1. + +#### Command settling + +When an order fill triggers a strategy callback that submits additional orders (e.g., a stop-loss submitted +in `on_order_filled`), those cascading commands are settled within the same timestamp/event cycle. The engine +repeatedly drains venue command queues and any newly generated commands until no commands remain pending +for the current timestamp. Simulation modules are run only once per cycle, after all commands have settled. + +When a `LatencyModel` is configured, commands are placed in the venue's inflight queue with a future +timestamp derived from the simulated latency. The settle loop considers inflight commands that are due +at the current timestamp as pending, so zero-latency or same-tick latency configurations still settle +correctly. Commands with future timestamps are deferred and processed when the engine reaches that time. + +### Fill modeling philosophy + +NautilusTrader treats historical order book and trade data as **immutable** during backtesting. What happened in the market is preserved exactly as recorded. Fills never modify the underlying book state. + +This addresses a gap in academic literature: most research focuses on live market dynamics where the book actually evolves. Historical backtesting with frozen snapshots is a distinct engineering problem: how do we simulate realistic fills against data that doesn't change in response to our orders? + +**Design choices:** + +- **Immutable historical data**: Order book and trade data are never modified. +- **Optional consumption tracking**: When `liquidity_consumption=True`, the engine tracks consumed liquidity per price level to prevent duplicate fills. See [Order book immutability](#order-book-immutability) for configuration. +- **Deterministic results**: The same backtest with the same data and configuration produces identical results when probabilistic fill models use a fixed `random_seed`. + +### Fill price determination + +The matching engine determines fill prices based on order type, book type, and market state. + +#### L2/L3 order book data + +With full order book depth, fills are determined by actual book simulation: + +| Order Type | Fill Price | +| ---------------------- | ----------------------------------------------------------- | +| `MARKET` | Walks the book, filling at each price level (taker). | +| `MARKET_TO_LIMIT` | Walks the book, filling at each price level (taker). | +| `LIMIT` | Order's limit price when matched (maker). | +| `STOP_MARKET` | Walks the book when triggered. | +| `STOP_LIMIT` | Order's limit price when triggered and matched. | +| `MARKET_IF_TOUCHED` | Walks the book when triggered. | +| `LIMIT_IF_TOUCHED` | Order's limit price when triggered. | +| `TRAILING_STOP_MARKET` | Walks the book when activated and triggered. | +| `TRAILING_STOP_LIMIT` | Order's limit price when activated, triggered, and matched. | + +With L2/L3 data, market-type orders may partially fill across multiple price levels if insufficient liquidity exists at the top of book. +Limit-type orders act as resting orders after triggering and may remain unfilled if the market doesn't reach the limit price. +`MARKET_TO_LIMIT` fills as a taker first, then rests any remaining quantity as a limit order at its first fill price. + +#### L1 order book data (quotes, trades, bars) + +With only top-of-book data, the same book simulation is used with a single-level book: + +| Order Type | BUY Fill Price | SELL Fill Price | +| ---------------------- | -------------- | --------------- | +| `MARKET` | Best ask | Best bid | +| `MARKET_TO_LIMIT` | Best ask | Best bid | +| `LIMIT` | Limit price | Limit price | +| `STOP_MARKET` | Best ask | Best bid | +| `STOP_LIMIT` | Limit price | Limit price | +| `MARKET_IF_TOUCHED` | Best ask | Best bid | +| `LIMIT_IF_TOUCHED` | Limit price | Limit price | +| `TRAILING_STOP_MARKET` | Best ask | Best bid | +| `TRAILING_STOP_LIMIT` | Limit price | Limit price | + +With L1 data, the simulated book has a single price level. Orders fill against the available size at that level. If an order has remaining quantity after exhausting top-of-book liquidity, market and marketable limit-style orders will slip one tick to fill the residual. + +For bar data specifically, `STOP_MARKET` and `TRAILING_STOP_MARKET` orders may fill at the trigger price rather than best ask/bid when the bar moves through the trigger during its high/low processing. See [Stop order fill behavior with bar data](#stop-order-fill-behavior-with-bar-data) for details. + +:::note +Fill models can alter these fill prices. See the [Fill models](#fill-models) section for details on configuring execution simulation. +::: + +#### Order type semantics + +- **Market execution**: Fill at current market price (bid/ask). This models real exchange behavior where these orders execute at the best available price after triggering. Exception: with bar data, `STOP_MARKET` and `TRAILING_STOP_MARKET` orders triggered during H/L processing fill at the trigger price (see below). +- **Limit execution**: Fill at the order's limit price when matched. Provides price guarantee but may not fill if the market doesn't reach the limit. + +#### Stop order fill behavior with bar data + +When backtesting with bar data only (no tick data), the matching engine distinguishes between two scenarios for `STOP_MARKET` and `TRAILING_STOP_MARKET` orders: + +**Gap scenario** (bar opens past trigger): +When a bar's open price gaps past the trigger price, the stop triggers immediately and fills at the market price (the open). This models real exchange behavior where stop-market orders provide no price guarantee during gaps. + +Example - SELL `STOP_MARKET` with trigger at 100: + +- Previous bar closes at 105. +- Next bar opens at 90 (overnight gap down). +- Stop triggers at open and fills at 90. + +**Move-through scenario** (bar moves through trigger): +When a bar opens normally and then its high or low moves through the trigger price, the stop fills at the trigger price. Since we only have OHLC data, we assume the market moved smoothly through the trigger and the order would have filled there. + +Example - SELL `STOP_MARKET` with trigger at 100: + +- Bar opens at 102 (no gap). +- Bar low reaches 98, moving through trigger at 100. +- Stop fills at 100 (the trigger price). + +This behavior caps potential slippage during orderly market moves while still modeling gap slippage accurately. For tick-level precision, use quote or trade tick data instead of bars. + +### Price protection + +Price protection defines an exchange-calculated price boundary that prevents marketable orders from +executing at excessively aggressive prices. This models exchanges like Binance and CME that implement +protection mechanisms for market and stop-market orders. + +**Configuration:** + +```python +from nautilus_trader.backtest.config import BacktestVenueConfig + +venue_config = BacktestVenueConfig( + name="BINANCE", + oms_type="NETTING", + account_type="MARGIN", + starting_balances=["100_000 USDT"], + price_protection_points=100, # 100 points = 1.00 offset for 2-decimal instruments +) +``` + +**How it works:** + +The matching engine calculates the protection boundary from the current best bid/ask at fill time: + +- **BUY orders**: `protection_price = ask + (points × price_increment)` +- **SELL orders**: `protection_price = bid - (points × price_increment)` + +The engine filters out fills beyond the protection boundary. For example, with `price_protection_points=100` +on an instrument with `price_increment=0.01`: + +- Best ask is 1001.00. +- Protection price = 1001.00 + (100 × 0.01) = 1002.00. +- A BUY market order fills only at prices ≤ 1002.00. +- Liquidity at 1003.00 or higher is filtered, leaving the order partially filled. + +**Trigger-time semantics:** + +The engine computes protection at fill time, not order submission time: + +- **Market orders**: Protection computed immediately when the order processes. +- **Stop-market orders**: Protection computed when the stop triggers, using the bid/ask at that moment. + +This design allows stop orders to be submitted even when the opposite side of the book is empty, +since the engine computes protection later when the stop triggers. + +**Order types affected:** + +- `MARKET` +- `STOP_MARKET` + +Limit orders are unaffected since they already define a price boundary. + +:::note +Set `price_protection_points=0` to disable price protection (default behavior). +::: + +### Slippage and spread handling + +When backtesting with different types of data, Nautilus implements specific handling for slippage and spread simulation: + +For L2 (market-by-price) or L3 (market-by-order) data, slippage is simulated with high accuracy by: + +- Filling orders against actual order book levels. +- Matching available size at each price level sequentially. +- Maintaining realistic order book depth impact (per order fill). + +For L1 data types (e.g., L1 order book, trades, quotes, bars), slippage is handled through the `FillModel`: + +**Per-fill slippage** (`prob_slippage`): + +- Applies to each fill when using an L1 book with a configured `FillModel`. +- Affects all order types (market, limit, stop, etc.). +- When triggered, moves the fill price one tick against the order direction. +- Example: With `prob_slippage=0.5`, a BUY order has 50% chance of filling one tick above the best ask. + +:::note +When backtesting with bar data, be aware that the reduced granularity of price information affects the slippage mechanism. +For the most realistic backtesting results, consider using higher granularity data sources such as L2 or L3 order book data when available. +::: + +#### How simulation varies by data type + +The behavior of the `FillModel` adapts based on the order book type being used: + +**L2/L3 order book data** + +With full order book depth, the `FillModel` focuses purely on simulating queue position for limit orders through `prob_fill_on_limit`. +The order book itself handles slippage naturally based on available liquidity at each price level. + +- `prob_fill_on_limit` is active - simulates queue position. +- `prob_slippage` is not used - real order book depth determines price impact. + +:::warning +The historical order book is immutable during backtesting. Book depth is **not** decremented after fills. +By default (`liquidity_consumption=False`), the same liquidity can be consumed repeatedly within an iteration. +Enable `liquidity_consumption=True` to track consumed liquidity per price level. Consumption resets when fresh +data arrives at that level. See [Order book immutability](#order-book-immutability) for details. +::: + +**L1 order book data** + +With only best bid/ask prices available, the `FillModel` provides additional simulation: + +- `prob_fill_on_limit` is active - simulates queue position. +- `prob_slippage` is active - simulates basic price impact since we lack real depth information. + +**Bar/Quote/Trade data** + +When using less granular data, the same behaviors apply as L1: + +- `prob_fill_on_limit` is active - simulates queue position. +- `prob_slippage` is active - simulates basic price impact. + +#### Important considerations + +- **Partial fills**: With L2/L3 data, fills are limited to available liquidity at each price level. With L1 data, the full order quantity fills at the single available level. +- **Consumption tracking**: See [Order book immutability](#order-book-immutability) for details on preventing duplicate fills. + +### Order book immutability + +Historical order book data is immutable during backtesting. When your order fills against book liquidity, +the book state remains unchanged. This preserves historical data integrity. + +The matching engine can optionally use **per-level consumption tracking** to prevent duplicate fills while +allowing fills when fresh liquidity arrives. This behavior is controlled by the `liquidity_consumption` +configuration option. + +**Configuration:** + +```python +from nautilus_trader.backtest.config import BacktestVenueConfig + +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="CASH", + starting_balances=["100_000 USD"], + liquidity_consumption=True, # Enable consumption tracking (default: False) +) +``` + +- `liquidity_consumption=False` (default): Each iteration fills against the full book liquidity independently. + Simpler behavior, assumes you're a small participant whose orders don't meaningfully impact available liquidity. +- `liquidity_consumption=True`: Tracks consumed liquidity per price level. Prevents the same + displayed liquidity from generating multiple fills. Resets when fresh data arrives at that level. + +**How consumption tracking works (when enabled):** + +For each price level, the engine maintains: + +- `original_size`: The book's quantity when tracking began. +- `consumed`: How much has been filled against this level. + +When processing a fill: + +1. Check if the book's current size at this level matches `original_size` +2. If different (fresh data arrived), reset the entry: `original_size = current_size`, `consumed = 0` +3. Calculate `available = original_size - consumed` +4. After filling, increment `consumed` by the fill quantity + +**Example:** + +1. Order book shows 100 units at ask 100.00. Engine tracks: `(original=100, consumed=0)`. +2. Your BUY order fills 30 units. Engine updates: `(original=100, consumed=30)`. Available = 70. +3. Another BUY order attempts 50 units. Available = 70, so it fills 50. `(original=100, consumed=80)`. +4. A delta updates ask 100.00 to 120 units. Engine resets: `(original=120, consumed=0)`. +5. New orders can now fill against the fresh 120 units. + +**Passive limit order fills on L1 data:** + +With L1 data (quotes, trades, bars), the book has only a single price level per side. When the market +moves through a passive (MAKER) limit order's price, the engine must decide how to handle remaining +order quantity after exhausting displayed liquidity. + +| `liquidity_consumption` | Behavior when market moves through passive limit | +| ----------------------- | ----------------------------------------------------------------------------------------------- | +| `False` (default) | Fill entire order at limit price. Assumes market movement implies sufficient liquidity existed. | +| `True` | Fill only against displayed liquidity. Order remains open for subsequent fills. | + +**Example scenario** (`liquidity_consumption=True`): + +1. Quote shows ask 100.10 with 50 units. +2. You place BUY LIMIT at 100.05 for 1000 units (passive, resting below ask). +3. Next quote shows ask 100.00 with 30 units (market moved through your limit). +4. Order fills 30 units against displayed liquidity. 970 units remain open. +5. Next quote shows ask 99.95 with 200 units. +6. Order fills another 200 units. 770 units remain open. +7. Fills continue as fresh liquidity arrives at crossed price levels. + +This behavior provides conservative fill simulation: your order only fills against liquidity +actually observed in the data, rather than inferring liquidity from price movements. + +**Trade tick liquidity:** + +Trade ticks provide evidence of executable liquidity at the trade price. When a trade occurs at a price level +not reflected in the current book, the engine can use the trade quantity as available liquidity, subject to +the same consumption tracking rules (when enabled). + +**Trade consumption seeding:** + +When using L2/L3 book data and a trade tick triggers order matching (e.g., triggering a resting stop order), +the trade itself consumed liquidity from the book. Before simulating fills for triggered orders, the engine +pre-seeds the consumption maps with the trade's consumed volume. This prevents triggered orders from filling +against liquidity that the triggering trade already consumed. This seeding is skipped for L1 books, where the +trade tick has already updated the single top-of-book level directly. + +For example, if the book has 10 units at the best ask and a BUY trade of size 8 triggers a stop market BUY +for 5 units, the stop order sees only 2 units remaining at best ask (10 - 8) and must fill the remaining +3 units at the next price level. Without this seeding, the stop would incorrectly fill all 5 units at the +best ask price. + +The engine uses a timestamp guard to avoid double-counting: if the book's most recent update (`ts_last`) +is newer than the trade's event time (`ts_event`), seeding is skipped. This handles exchanges like Binance +where depth deltas arrive before the corresponding trade tick, so the book already reflects the consumed +liquidity, so additional seeding would over-penalize fills. + +:::note +As the `FillModel` continues to evolve, future versions may introduce more sophisticated simulation of order execution dynamics, including: + +- Variable slippage based on order size. +- More complex queue position modeling. + +::: + +#### Known limitations + +**No queue position within a level**: Consumption tracking determines *how much* liquidity remains at a level, +but doesn't model *where* your order sits in the queue relative to other participants. Use `prob_fill_on_limit` +to simulate queue position probabilistically. + +**Trade-driven fills are opportunistic**: When trade ticks indicate liquidity at a price not in the book, +the engine uses this as fill evidence. However, this represents liquidity that existed momentarily and may +not reflect sustained availability. + +### Trade based execution + +Trade tick data triggers order fills by default (`trade_execution=True`). A trade tick indicates that liquidity +was accessed at the trade price, allowing resting limit orders to match. This mirrors the default behavior +for bar data (`bar_execution=True`). + +Advanced users who want to isolate execution to L1 book data only (quotes or order book updates) can disable +trade-based execution: + +```python +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="CASH", + starting_balances=["100_000 USD"], + trade_execution=False, # Disable trade-based fills +) +``` + +When `trade_execution=False` or `bar_execution=False`, the respective data types skip order matching +and maintenance operations (GTD order expiry, trailing stop activation, instrument expiration checks). +Quote ticks always trigger maintenance, so this is typically acceptable when using multiple data types. + +The matching engine uses a "transient override" mechanism: during the matching process, it temporarily adjusts +the matching core's Best Bid (for BUYER trades) or Best Ask (for SELLER trades) toward the trade price. This allows +resting orders on the passive side to cross the spread and fill. Note: the underlying order book data is never +modified (it remains immutable); only the matching core's internal price references are adjusted. + +**Fill determination:** + +When a trade tick triggers order matching, the engine determines fills as follows: + +1. **Book reflects trade price**: If the order book has liquidity at the trade price, fills use book depth (standard behavior). +2. **Book doesn't reflect trade price**: If the book's liquidity is at a different price, the engine uses a "trade-driven fill" at the order's limit price, capped to `min(order.leaves_qty, trade.size)`. + +This ensures that when a trade prints through the spread but the book hasn't updated, fills are bounded by what the trade tick actually evidences. When `liquidity_consumption=False` (default), the same trade size can fill multiple orders within an iteration. When `liquidity_consumption=True`, consumption tracking applies to trade-driven fills as well. Repeated fills at the same trade price will be bounded by consumed liquidity until fresh data arrives. + +**Restoration behavior:** + +After matching, the core's bid/ask are only restored to their original values if the trade price improved them +(moved them away from the spread): + +- **SELLER trade**: Ask is restored only if trade price was below the original ask. +- **BUYER trade**: Bid is restored only if trade price was above the original bid. + +If the trade price didn't improve the quote (e.g., a SELLER trade at or above the ask), the core retains +the trade price. This means repeated trades at or beyond the spread can progressively move the core's bid/ask. + +**Fill price:** + +- **SELLER trade at P**: The engine sets the core's Best Ask to P (if P < current ask). Resting BUY LIMIT orders at P or higher will fill at their limit price (if book doesn't have that level) or at book prices (if book does). +- **BUYER trade at P**: The engine sets the core's Best Bid to P (if P > current bid). Resting SELL LIMIT orders at P or lower will fill at their limit price (if book doesn't have that level) or at book prices (if book does). + +This conservative approach ensures fills occur at the order's limit price rather than potentially better trade prices. For example, a BUY LIMIT at 100.05 triggered by a SELLER trade at 100.00 will fill at 100.05, not 100.00. + +:::tip +Combine trade data with book or quote data for best results: book/quote data establishes the baseline spread, +while trade ticks trigger execution for orders that might be inside the spread or ahead of the quote updates. +::: + +#### Understanding trade tick aggressor sides + +A common source of confusion is the `aggressor_side` field on trade ticks: + +- **SELLER trade**: A seller aggressed, selling into the bid. This provides evidence of fill-able liquidity for **BUY** orders at the trade price. +- **BUYER trade**: A buyer aggressed, buying from the ask. This provides evidence of fill-able liquidity for **SELL** orders at the trade price. + +In other words, trade ticks trigger fills for orders on the **opposite** side of the aggressor. A SELLER trade at 100.00 can fill your resting BUY LIMIT at 100.00, but cannot fill your SELL LIMIT, since the trade already represents someone else selling. + +#### Combining L2 book data with trade ticks + +When using L2 order book data (e.g., 100ms throttled depth snapshots) combined with trade tick data: + +1. **Book updates establish the spread**: Each book delta/snapshot updates the matching engine's view of available liquidity at each price level. + +2. **Trade ticks provide execution evidence**: Trade ticks indicate that liquidity was accessed at a specific price, potentially between book snapshots. + +3. **Fill quantity determination**: When a trade triggers a fill: + - If the book already reflects liquidity at the trade price, fills use book depth + - If the trade price is inside the spread (not in the current book), fills are capped by `min(order.leaves_qty, trade.size)` + +4. **Timing considerations**: With throttled book data (e.g., 100ms), the book may lag behind trades. A trade at a price not yet reflected in the book will use trade-driven fill logic. + +**Common misconception**: Users sometimes expect every trade tick to trigger fills. Remember: + +- Only trades on the **opposite** side can fill your orders. +- SELLER trades → potential BUY fills. +- BUYER trades → potential SELL fills. +- Book UPDATE events move the market but only trigger fills if prices cross your order. + +#### Queue position tracking + +When `queue_position=True` is enabled alongside `trade_execution=True`, the matching engine simulates +queue position for limit orders. This provides more realistic fill behavior by tracking how many +orders are "ahead" of your order at a given price level. + +**How it works:** + +1. **Order placement**: When a LIMIT order is accepted, the engine snapshots the current same-side + book depth at the order's price level. This represents the orders ahead in the queue. + +2. **Trade ticks**: When trade ticks occur at the order's price level, the "quantity ahead" is + decremented by the trade size. Only trades on the correct side affect the queue (BUYER trades + decrement queue for SELL orders, SELLER trades decrement queue for BUY orders). Trades with + `NO_AGGRESSOR` (common in historical datasets lacking aggressor metadata) affect both sides. + This is pessimistic but prevents orders from stalling indefinitely. + +3. **Fill eligibility**: The order becomes eligible to fill only when the quantity ahead reaches zero. + On the tick that clears the queue, only the excess volume (trade size minus queue ahead) is + available for fill, preventing overfill. + +4. **Price level DELETE**: If the order book level is deleted (BookAction.DELETE), the queue clears + immediately, making the order fill-eligible. UPDATE actions are ignored (queue unchanged). + +5. **Order modification**: If the order is modified (price or quantity change), the queue position + resets. The order moves to the back of the queue at its new price level. + +**Configuration:** + +```python +from nautilus_trader.backtest.config import BacktestVenueConfig + +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="MARGIN", + starting_balances=["100_000 USD"], + trade_execution=True, # Required for queue_position + queue_position=True, # Enable queue position tracking +) +``` + +**Example scenario:** + +1. Order book shows 100 units at bid 100.00. +2. You place a BUY LIMIT at 100.00 for 50 units. Queue ahead = 100. +3. SELLER trade of 80 units at 100.00 → queue ahead = 20. No fill yet. +4. SELLER trade of 30 units at 100.00 → queue clears with 10 excess. Fill = 10 units. +5. Next SELLER trade of 50 units → fill remaining 40 units. + +**Limitations:** + +- Only applies to `LIMIT` orders. Stop-limit and limit-if-touched orders are not tracked in this implementation. +- Queue position is per-order, not shared across multiple orders at the same price. +- The queue snapshot is based on book state at order acceptance time. +- Trades with `NO_AGGRESSOR` decrement queue for both sides, which may cause orders to fill sooner than in reality (pessimistic for queue estimation, but prevents stalling). + +**L1 quote-based mode:** + +When using `BookType.L1_MBP` (top-of-book quotes only), queue position tracking uses +trade ticks to decrement the queue (the same mechanism as L2/L3), while quote ticks +handle price-move detection and deferred snapshot resolution. + +- **Trade ticks**: Trades at the order's price level decrement the queue ahead by the trade + size, identical to L2/L3 behavior. Only trades on the correct aggressor side affect the + queue (SELLER trades decrement queue for BUY orders, BUYER trades for SELL orders). +- **Price moves away**: If the bid drops below a BUY order's price (or ask rises above a + SELL order's price), the order's price level has been "crossed" and the queue clears to zero, + making the order fill-eligible on the next matching trade. +- **Price moves toward**: If the bid rises (or ask drops), the level at the order's price was + not consumed, so queue positions are preserved. +- **Price returns to a level**: When the price returns after moving away, the queue ahead is + capped at the new displayed size if it was previously larger. +- **Orders behind BBO (pending)**: When a limit order is placed behind the best bid/ask + (e.g., BUY below best bid), the queue snapshot is deferred because L1 data has no visible + depth at that level. Fills are blocked until the BBO reaches the order's price, at which + point the queue is snapshotted from the displayed size. Pending orders are also resolved + when trades cross through their price level. + +L1 mode uses the same configuration: set `queue_position=True` with `book_type=BookType.L1_MBP`. +This provides a lightweight alternative to full L2/L3 data when only top-of-book quotes are +available. + +:::note +Queue position tracking provides a heuristic simulation of queue dynamics. Real exchange queue +behavior depends on many factors (order priority rules, hidden orders, etc.) that cannot be +perfectly reconstructed from historical data. +::: + +### Bar based execution + +Bar data provides a summary of market activity with four key prices for each time period (assuming bars are aggregated by trades): + +- **Open**: opening price (first trade) +- **High**: highest price traded +- **Low**: lowest price traded +- **Close**: closing price (last trade) + +While this gives us an overview of price movement, we lose some important information that we'd have with more granular data: + +- We don't know in what order the market hit the high and low prices. +- We can't see exactly when prices changed within the time period. +- We don't know the actual sequence of trades that occurred. + +This is why Nautilus processes bar data through a system that attempts to maintain +the most realistic yet conservative market behavior possible, despite these limitations. +At its core, the platform always maintains an order book simulation - even when you provide less +granular data such as quotes, trades, or bars (although the simulation will only have a top level book). + +:::warning +When using bars for execution simulation (enabled by default with `bar_execution=True` in venue configurations), +Nautilus strictly expects the initialization timestamp (`ts_init`) of each bar to represent its **closing time**. +This ensures accurate chronological processing, prevents look-ahead bias, and aligns market updates (Open → High → Low → Close) with the moment the bar is complete. + +The event timestamp (`ts_event`) can represent either the open or close time of the bar: + +- If `ts_event` is at the **close**, ensure `ts_init_delta=0` when processing bars (default). +- If `ts_event` is at the **open**, set `ts_init_delta` equal to the bar's duration to shift `ts_init` to the close. + +::: + +#### Bar timestamp convention + +If your data source provides bars timestamped at the **opening time** (common in some providers), you need to ensure `ts_init` is set to the closing time for correct execution simulation. There are two approaches: + +**Approach 1: Adjust data timestamps (recommended)** + +- Use adapter-specific configurations like `bars_timestamp_on_close=True` (e.g., for Bybit or Databento adapters) to handle this automatically during data ingestion. +- For custom data, manually shift the timestamps by the bar duration before loading (e.g., add 1 minute for `1-MINUTE` bars). +- This approach is clearest because the data itself reflects the close time. + +**Approach 2: Use `ts_init_delta` parameter** + +- When calling `BarDataWrangler.process()`, set `ts_init_delta` to the bar's duration in nanoseconds (e.g., `60_000_000_000` for 1-minute bars). +- The wrangler computes `ts_init = ts_event + ts_init_delta`, shifting execution timing to the close. +- Use this when you cannot or prefer not to modify source data timestamps. + +Always verify your data's timestamp convention with a small sample to avoid simulation inaccuracies. Incorrect timestamp handling can lead to look-ahead bias and unrealistic backtest results. + +#### Processing bar data + +Even when you provide bar data, Nautilus maintains an internal order book for each instrument, as a real venue would. + +1. **Time processing**: + - Nautilus has a specific way of handling the timing of bar data *for execution* that's crucial for accurate simulation. + - The initialization timestamp (`ts_init`) is used for execution timing and must represent the close time of the bar. This approach is most logical because it represents the moment when the bar is fully formed and its aggregation is complete. + - The event timestamp (`ts_event`) represents when the data event occurred and may differ from `ts_init` depending on your data source: + - If your bars are timestamped at the **close** (the recommended default), use `ts_init_delta=0` in `BarDataWrangler` so that `ts_init = ts_event`. + - If your bars are timestamped at the **open**, set `ts_init_delta` to the bar's duration in nanoseconds (e.g., 60_000_000_000 for 1-minute bars) to shift `ts_init` to the close time. + - The platform ensures all events happen in the correct sequence based on `ts_init`, preventing any possibility of look-ahead bias in your backtests. + +:::note[Exceptions for bar execution] +Bars will **not** be processed for execution (and will not update the order book) in the following cases: + +- **Internally aggregated bars**: Bars with `AggregationSource.INTERNAL` are skipped to avoid processing bars that are derived from already-processed tick data. +- **Non-L1 book types**: When the venue's `book_type` is configured as `L2_MBP` or `L3_MBO`, bar data is ignored for execution processing, as bars are derived from top-of-book prices only. + +In these cases, bars will still be received by strategies for analytics and decision-making, but they won't trigger order matching or update the simulated order book. +::: + +2. **Price processing**: + - The platform converts each bar's OHLC prices into a sequence of market updates. + - By default, updates follow the order: Open → High → Low → Close (configurable via `bar_adaptive_high_low_ordering`). + - If you provide multiple timeframes (like both 1-minute and 5-minute bars), the platform uses the more granular data for highest accuracy. + +3. **Executions**: + - When you place orders, they interact with the simulated order book as they would on a real venue. + - For MARKET orders, execution happens at the current simulated market price plus any configured latency. + - For LIMIT orders working in the market, they'll execute if any of the bar's prices reach or cross your limit price (see below). + - The matching engine continuously processes orders as OHLC prices move, rather than waiting for complete bars. + +#### OHLC prices simulation + +During backtest execution, each bar is converted into a sequence of four price points: + +1. Opening price +2. High price *(Order between High/Low is configurable. See `bar_adaptive_high_low_ordering` below.)* +3. Low price +4. Closing price + +The trading volume for that bar is **split evenly** among these four points (25% each), with any +remainder added to the closing price trade to preserve total volume. In marginal cases, if the +bar's volume divided by 4 is less than the instrument's minimum `size_increment`, we use the +minimum `size_increment` per price point to ensure valid market activity (e.g., 1 contract for +CME group exchanges). + +How these price points are sequenced can be controlled via the `bar_adaptive_high_low_ordering` parameter when configuring a venue. + +Nautilus supports two modes of bar processing: + +1. **Fixed ordering** (`bar_adaptive_high_low_ordering=False`, default) + - Processes every bar in a fixed sequence: `Open → High → Low → Close`. + - Simple and deterministic approach. + +2. **Adaptive ordering** (`bar_adaptive_high_low_ordering=True`) + - Uses bar structure to estimate likely price path: + - If Open is closer to High: processes as `Open → High → Low → Close`. + - If Open is closer to Low: processes as `Open → Low → High → Close`. + - [Research](https://gist.github.com/stefansimik/d387e1d9ff784a8973feca0cde51e363) shows this approach achieves ~75-85% accuracy in predicting correct High/Low sequence (compared to statistical ~50% accuracy with fixed ordering). + - This is particularly important when both take-profit and stop-loss levels occur within the same bar - as the sequence determines which order fills first. + +Here's how to configure adaptive bar ordering for a venue, including account setup: + +```python +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.model.enums import OmsType, AccountType +from nautilus_trader.model import Money, Currency + +# Initialize the backtest engine +engine = BacktestEngine() + +# Add a venue with adaptive bar ordering and required account settings +engine.add_venue( + venue=venue, # Your Venue identifier, e.g., Venue("BINANCE") + oms_type=OmsType.NETTING, + account_type=AccountType.CASH, + starting_balances=[Money(10_000, Currency.from_str("USDT"))], + bar_adaptive_high_low_ordering=True, # Enable adaptive ordering of High/Low bar prices +) +``` + +### Internal bar aggregation timing + +When aggregating time bars internally from tick data, the data engine uses timers to close bars at +interval boundaries. A timing edge case occurs when data arrives at the exact bar close timestamp: the +timer may fire before processing boundary data. + +Configure `time_bars_build_delay` in `DataEngineConfig` to delay bar close timers: + +```python +from nautilus_trader.config import BacktestEngineConfig +from nautilus_trader.data.config import DataEngineConfig + +config = BacktestEngineConfig( + data_engine=DataEngineConfig( + time_bars_build_delay=1, # Microseconds + ), +) +``` + +:::tip +A small delay (1 microsecond) ensures boundary data is processed before the bar closes. +Useful when tick data clusters at round interval timestamps. +::: + +:::note +Only affects internally aggregated bars (`AggregationSource.INTERNAL`). +::: + +### Timer-only backtests + +The backtest engine supports running with timers but no market data. This is useful for scheduled +operations or testing timer-based logic. Timers fire in chronological order, and timer callbacks +can dynamically add data via `add_data_iterator()` which will be processed in sequence. + +:::warning +Data added by timer callbacks at the exact start time should have timestamps **after** the start time. +The engine reads the first data point before processing start-time timers, so dynamically added data +with timestamps at or before the start time may not be processed in the expected order. +::: + +### Fill models + +Fill models simulate order execution dynamics during backtesting. They address a fundamental challenge: +*even with perfect historical market data, we can't fully simulate how orders may have interacted +with other market participants in real-time*. + +The base `FillModel` provides probabilistic parameters for queue position and slippage simulation. +Subclasses can override `get_orderbook_for_fill_simulation()` to generate synthetic order books +for more sophisticated liquidity modeling. + +#### Available fill models + +| Model | Description | Use Case | +| ---------------------------- | ------------------------------------------------------- | -------------------------------------------- | +| `FillModel` | Base model with probabilistic fill/slippage parameters. | Simple queue position and slippage. | +| `BestPriceFillModel` | Fills at best price with unlimited liquidity. | Testing basic strategy logic optimistically. | +| `OneTickSlippageFillModel` | Forces exactly one tick of slippage on all orders. | Conservative slippage testing. | +| `TwoTierFillModel` | 10 contracts at best price, remainder one tick worse. | Basic market depth simulation. | +| `ThreeTierFillModel` | 50/30/20 contracts across three price levels. | More realistic depth simulation. | +| `ProbabilisticFillModel` | 50% chance best price, 50% chance one tick slippage. | Randomized execution quality. | +| `SizeAwareFillModel` | Different execution based on order size (≤10 vs >10). | Size-dependent market impact. | +| `LimitOrderPartialFillModel` | Max 5 contracts fill per price touch. | Queue position via partial fills. | +| `MarketHoursFillModel` | Wider spreads during low liquidity periods. | Session-aware execution. | +| `VolumeSensitiveFillModel` | Liquidity based on recent trading volume. | Volume-adaptive depth. | +| `CompetitionAwareFillModel` | Only percentage of visible liquidity available. | Multi-participant competition. | + +#### Configuring fill models + +**Using the base FillModel with probabilistic parameters:** + +```python +from nautilus_trader.backtest.config import BacktestVenueConfig +from nautilus_trader.backtest.config import ImportableFillModelConfig + +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="CASH", + starting_balances=["100_000 USD"], + fill_model=ImportableFillModelConfig( + fill_model_path="nautilus_trader.backtest.models:FillModel", + config_path="nautilus_trader.backtest.config:FillModelConfig", + config={ + "prob_fill_on_limit": 0.2, # Chance a limit order fills when price matches + "prob_slippage": 0.5, # Chance of 1-tick slippage (L1 data only) + "random_seed": 42, # Optional: Set for reproducible results + }, + ), +) +``` + +**Using an order book simulation model:** + +```python +from nautilus_trader.backtest.config import BacktestVenueConfig +from nautilus_trader.backtest.config import ImportableFillModelConfig + +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="CASH", + starting_balances=["100_000 USD"], + fill_model=ImportableFillModelConfig( + fill_model_path="nautilus_trader.backtest.models:ThreeTierFillModel", + ), +) +``` + +#### Probabilistic parameters (base FillModel) + +**prob_fill_on_limit** (default: `1.0`) + +Simulates queue position by controlling the probability of a limit order filling when its price level is touched (but not crossed). + +- `0.0`: Never fills at touch (back of queue). +- `0.5`: 50% chance of filling (middle of queue). +- `1.0`: Always fills at touch (front of queue). + +**prob_slippage** (default: `0.0`) + +Simulates price slippage on each fill. Only applies to L1 data types (quotes, trades, bars) where real depth is unavailable. Affects all order types when executing as takers. + +- `0.0`: No slippage (fills at best price). +- `0.5`: 50% chance of one tick slippage per fill. +- `1.0`: Always slips one tick. + +#### Order book simulation models + +These models override the `get_orderbook_for_fill_simulation()` method to generate synthetic order books +representing expected market liquidity. The matching engine fills orders against this simulated book. + +**How it works:** + +1. Before processing a fill, the matching engine calls `get_orderbook_for_fill_simulation()`. +2. If the model returns a synthetic order book, fills execute against that book's liquidity. +3. If the model returns `None`, standard fill logic applies. + +:::note +When a custom fill model provides a simulated order book, the `liquidity_consumption` tracking is **not** applied. +Custom fill models are expected to manage their own liquidity simulation within the returned order book. +Liquidity consumption tracking only affects the built-in fill logic (when `get_orderbook_for_fill_simulation()` returns `None`). +::: + +**Example: ThreeTierFillModel** + +This model creates a book with liquidity distributed across three price levels: + +- 50 contracts at best price +- 30 contracts one tick worse +- 20 contracts two ticks worse + +A 100-contract market order would fill partially at each level, experiencing realistic price impact. + +**Creating custom fill models:** + +```python +from nautilus_trader.backtest.models import FillModel +from nautilus_trader.model.book import OrderBook, BookOrder +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.core.rust.model import BookType + +class MyCustomFillModel(FillModel): + def get_orderbook_for_fill_simulation( + self, + instrument, + order, + best_bid, + best_ask, + ): + book = OrderBook( + instrument_id=instrument.id, + book_type=BookType.L2_MBP, + ) + + # Add custom liquidity based on your market model + # ... + + return book +``` + +### Precision requirements and invariants + +The matching engine enforces strict precision invariants to ensure data integrity throughout the fill pipeline. +All prices and quantities must match the instrument's configured precision (`price_precision` and `size_precision`). +Mismatches raise a `RuntimeError` immediately, preventing silent corruption of fill quantities. + +| Data/Operation | Field | Required Precision | Validation Location | +| -------------- | ------------------------------ | ---------------------------- | --------------------------- | +| `QuoteTick` | `bid_price`, `ask_price` | `instrument.price_precision` | `process_quote_tick` | +| `QuoteTick` | `bid_size`, `ask_size` | `instrument.size_precision` | `process_quote_tick` | +| `TradeTick` | `price` | `instrument.price_precision` | `process_trade_tick` | +| `TradeTick` | `size` | `instrument.size_precision` | `process_trade_tick` | +| `Bar` | `open`, `high`, `low`, `close` | `instrument.price_precision` | `process_bar` | +| `Bar` | `volume` (base units) | `instrument.size_precision` | `process_bar` | +| `Order` | `quantity` | `instrument.size_precision` | `process_order` | +| `Order` | `price` | `instrument.price_precision` | `process_order` | +| `Order` | `trigger_price` | `instrument.price_precision` | `process_order` | +| `Order` | `activation_price`\* | `instrument.price_precision` | `process_order` | +| Order update | `quantity` | `instrument.size_precision` | `update_order` | +| Order update | `price`, `trigger_price` | `instrument.price_precision` | `update_order` | +| Fill | `fill_qty` | `instrument.size_precision` | `apply_fills`, `fill_order` | +| Fill | `fill_px` | `instrument.price_precision` | `apply_fills` | + +\*`activation_price` is immutable after order submission. + +:::warning +`Bar.volume` must be in **base currency units**. Some data providers report quote-currency volume; +convert to base units before loading (divide by price or use provider-specific fields). +::: + +:::tip +If you encounter a precision mismatch error, align your data to the instrument: + +```python +# Align price/quantity to instrument precision +price = instrument.make_price(raw_price) +qty = instrument.make_qty(raw_qty) +``` + +Also verify that: + +1. The instrument definition matches your data source's precision. +2. Data was not inadvertently rounded or truncated during loading. +3. Custom data loaders preserve the original precision metadata. + +::: + +## Account types + +When you attach a venue to the engine, either for live trading or a backtest, you must pick one of three accounting modes by passing the `account_type` parameter: + +| Account type | Typical use-case | What the engine locks | +| ------------ | ------------------------------------------------ | ------------------------------------------------------------------------- | +| Cash | Spot trading (e.g., BTC/USDT, stocks). | Notional value for every position a pending order would open. | +| Margin | Derivatives or any product that allows leverage. | Initial margin for each order plus maintenance margin for open positions. | +| Betting | Sports betting, bookmaking. | Stake required by the venue; no leverage. | + +Example of adding a `CASH` account for a backtest venue: + +```python +from nautilus_trader.adapters.binance import BINANCE_VENUE +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.model.currencies import USDT +from nautilus_trader.model.enums import OmsType, AccountType +from nautilus_trader.model import Money, Currency + +# Initialize the backtest engine +engine = BacktestEngine() + +# Add a CASH account for the venue +engine.add_venue( + venue=BINANCE_VENUE, # Create or reference a Venue identifier + oms_type=OmsType.NETTING, + account_type=AccountType.CASH, + starting_balances=[Money(10_000, USDT)], +) +``` + +### Cash accounts + +Cash accounts settle trades in full; there is no leverage and therefore no concept of margin. + +### Margin accounts + +A *margin account* supports trading of instruments requiring margin, such as futures or leveraged products. +It tracks account balances, calculates required margins, and manages leverage to ensure sufficient collateral for positions and orders. + +**Key concepts**: + +- **Leverage**: Amplifies trading exposure relative to account equity. Higher leverage increases potential returns and risks. +- **Initial Margin**: Collateral required to submit an order to open a position. +- **Maintenance Margin**: Minimum collateral required to maintain an open position. +- **Locked Balance**: Funds reserved as collateral, unavailable for new orders or withdrawals. + +:::note +Reduce-only orders **do not** contribute to `balance_locked` in cash accounts, +nor do they add to initial margin in margin accounts, as they can only reduce existing exposure. +::: + +### Betting accounts + +Betting accounts are specialised for venues where you stake an amount to win or lose a fixed payout (some prediction markets, sports books, etc.). +The engine locks only the stake required by the venue; leverage and margin are not applicable. + +## Margin models + +NautilusTrader provides flexible margin calculation models to accommodate different venue types and trading scenarios. + +### Overview + +Different venues and brokers have varying approaches to calculating margin requirements: + +- **Traditional Brokers** (Interactive Brokers, TD Ameritrade): Fixed margin percentages regardless of leverage. +- **Crypto Exchanges** (Binance, some others): Leverage may reduce margin requirements. +- **Futures Exchanges** (CME, ICE): Fixed margin amounts per contract. + +### Available models + +#### StandardMarginModel + +Uses fixed percentages without leverage division, matching traditional broker behavior. + +**Formula:** + +```python +# Fixed percentages - leverage ignored +margin = notional * instrument.margin_init +``` + +- Initial Margin = `notional_value * instrument.margin_init` +- Maintenance Margin = `notional_value * instrument.margin_maint` + +**Use cases:** + +- Traditional brokers (Interactive Brokers, TD Ameritrade). +- Futures exchanges (CME, ICE). +- Forex brokers with fixed margin requirements. + +#### LeveragedMarginModel + +Divides margin requirements by leverage. + +**Formula:** + +```python +# Leverage reduces margin requirements +adjusted_notional = notional / leverage +margin = adjusted_notional * instrument.margin_init +``` + +- Initial Margin = `(notional_value / leverage) * instrument.margin_init` +- Maintenance Margin = `(notional_value / leverage) * instrument.margin_maint` + +**Use cases:** + +- Crypto exchanges that reduce margin with leverage. +- Venues where leverage affects margin requirements. + +### Usage + +#### Programmatic configuration + +```python +from nautilus_trader.backtest.models import LeveragedMarginModel +from nautilus_trader.backtest.models import StandardMarginModel +from nautilus_trader.test_kit.stubs.execution import TestExecStubs + +# Create account +account = TestExecStubs.margin_account() + +# Set standard model for traditional brokers +standard_model = StandardMarginModel() +account.set_margin_model(standard_model) + +# Or use leveraged model for crypto exchanges +leveraged_model = LeveragedMarginModel() +account.set_margin_model(leveraged_model) +``` + +#### Backtest configuration + +```python +from nautilus_trader.backtest.config import BacktestVenueConfig +from nautilus_trader.backtest.config import MarginModelConfig + +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="MARGIN", + starting_balances=["1_000_000 USD"], + margin_model=MarginModelConfig(model_type="standard"), # Options: 'standard', 'leveraged' +) +``` + +#### Available model types + +- `"leveraged"`: Margin reduced by leverage (default). +- `"standard"`: Fixed percentages (traditional brokers). +- Custom class path: `"my_package.my_module.MyMarginModel"`. + +#### Default behavior + +By default, `MarginAccount` uses `LeveragedMarginModel`. + +#### Real-world example + +**EUR/USD Trading Scenario:** + +- **Instrument**: EUR/USD +- **Quantity**: 100,000 EUR +- **Price**: 1.10000 +- **Notional Value**: $110,000 +- **Leverage**: 50x +- **Instrument Margin Init**: 3% + +**Margin calculations:** + +| Model | Calculation | Result | Percentage | +| --------- | ---------------------- | ------ | ---------- | +| Standard | $110,000 × 0.03 | $3,300 | 3.00% | +| Leveraged | ($110,000 ÷ 50) × 0.03 | $66 | 0.06% | + +**Account balance impact:** + +- **Account Balance**: $10,000. +- **Standard Model**: Cannot trade (requires $3,300 margin). +- **Leveraged Model**: Can trade (requires only $66 margin). + +### Real-world scenarios + +#### Interactive Brokers EUR/USD futures + +```python +# IB requires fixed margin regardless of leverage +account.set_margin_model(StandardMarginModel()) +margin = account.calculate_margin_init(instrument, quantity, price) +# Result: Fixed percentage of notional value +``` + +#### Binance crypto trading + +```python +# Binance may reduce margin with leverage +account.set_margin_model(LeveragedMarginModel()) +margin = account.calculate_margin_init(instrument, quantity, price) +# Result: Margin reduced by leverage factor +``` + +### Model selection + +#### Using the default model + +The default `LeveragedMarginModel` works out of the box: + +```python +account = TestExecStubs.margin_account() +margin = account.calculate_margin_init(instrument, quantity, price) +``` + +#### Using the standard model + +For traditional broker behavior: + +```python +account.set_margin_model(StandardMarginModel()) +margin = account.calculate_margin_init(instrument, quantity, price) +``` + +### Custom models + +You can create custom margin models by inheriting from `MarginModel`. Custom models receive configuration through the `MarginModelConfig`: + +```python +from nautilus_trader.backtest.models import MarginModel +from nautilus_trader.backtest.config import MarginModelConfig + +class RiskAdjustedMarginModel(MarginModel): + def __init__(self, config: MarginModelConfig): + """Initialize with configuration parameters.""" + self.risk_multiplier = Decimal(str(config.config.get("risk_multiplier", 1.0))) + self.use_leverage = config.config.get("use_leverage", False) + + def calculate_margin_init(self, instrument, quantity, price, leverage, use_quote_for_inverse=False): + notional = instrument.notional_value(quantity, price, use_quote_for_inverse) + if self.use_leverage: + adjusted_notional = notional.as_decimal() / leverage + else: + adjusted_notional = notional.as_decimal() + margin = adjusted_notional * instrument.margin_init * self.risk_multiplier + return Money(margin, instrument.quote_currency) + + def calculate_margin_maint(self, instrument, side, quantity, price, leverage, use_quote_for_inverse=False): + return self.calculate_margin_init(instrument, quantity, price, leverage, use_quote_for_inverse) +``` + +#### Using custom models + +**Programmatic:** + +```python +from nautilus_trader.backtest.config import MarginModelConfig +from nautilus_trader.backtest.config import MarginModelFactory + +config = MarginModelConfig( + model_type="my_package.my_module:RiskAdjustedMarginModel", + config={"risk_multiplier": 1.5, "use_leverage": False} +) + +custom_model = MarginModelFactory.create(config) +account.set_margin_model(custom_model) +``` + +### High-level backtest API configuration + +When using the high-level backtest API, you can specify margin models in your venue configuration using `MarginModelConfig`: + +```python +from nautilus_trader.backtest.config import MarginModelConfig +from nautilus_trader.backtest.config import BacktestVenueConfig +from nautilus_trader.config import BacktestRunConfig + +# Configure venue with specific margin model +venue_config = BacktestVenueConfig( + name="SIM", + oms_type="NETTING", + account_type="MARGIN", + starting_balances=["1_000_000 USD"], + margin_model=MarginModelConfig( + model_type="standard" # Use standard model for traditional broker simulation + ), +) + +# Use in backtest configuration +config = BacktestRunConfig( + venues=[venue_config], + # ... other config +) +``` + +#### Configuration examples + +**Standard model (traditional brokers):** + +```python +margin_model=MarginModelConfig(model_type="standard") +``` + +**Leveraged model (default):** + +```python +margin_model=MarginModelConfig(model_type="leveraged") # Default +``` + +**Custom model with configuration:** + +```python +margin_model=MarginModelConfig( + model_type="my_package.my_module:CustomMarginModel", + config={ + "risk_multiplier": 1.5, + "use_leverage": False, + "volatility_threshold": 0.02, + } +) +``` + +The margin model will be automatically applied to the simulated exchange during backtest execution. + +## Related guides + +- [Strategies](strategies.md) - Develop strategies to backtest. +- [Visualization](visualization.md) - Generate tearsheets from backtest results. +- [Reports](reports.md) - Analyze backtest performance data. diff --git a/nautilus-docs/docs/concepts/cache.md b/nautilus-docs/docs/concepts/cache.md new file mode 100644 index 0000000..438c9f3 --- /dev/null +++ b/nautilus-docs/docs/concepts/cache.md @@ -0,0 +1,539 @@ +# Cache + +The `Cache` is a central in-memory database that stores and manages all trading-related data, +from market data to order history to custom calculations. + +The Cache serves multiple purposes: + +1. **Stores market data**: + - Stores recent market history (e.g., order books, quotes, trades, bars). + - Gives you access to both current and historical market data for your strategy. + +2. **Tracks trading data**: + - Maintains complete `Order` history and current execution state. + - Tracks all `Position`s and `Account` information. + - Stores `Instrument` definitions and `Currency` information. + +3. **Stores custom data**: + - You can store any user-defined objects or data in the `Cache` for later use. + - Enables data sharing between different strategies. + +## How caching works + +**Built-in types**: + +- The system automatically adds data to the `Cache` as it flows through. +- In live contexts, the engine applies updates asynchronously, so you might see a brief delay between an event and its appearance in the `Cache`. +- All data flows through the `Cache` before reaching your strategy’s callbacks – see the diagram below: + +```mermaid +flowchart LR + data[Data] + engine[DataEngine] + cache[Cache] + callback["Strategy callback:
on_data(...)"] + + data --> engine --> cache --> callback +``` + +### Basic example + +Within a strategy, you can access the `Cache` through `self.cache`. Here’s a typical example: + +:::note +Within a `Strategy` class, `self` refers to the strategy instance. +::: + +```python +def on_bar(self, bar: Bar) -> None: + # Current bar is provided in the parameter 'bar' + + # Get historical bars from Cache + last_bar = self.cache.bar(self.bar_type, index=0) # Last bar (practically the same as the 'bar' parameter) + previous_bar = self.cache.bar(self.bar_type, index=1) # Previous bar + third_last_bar = self.cache.bar(self.bar_type, index=2) # Third last bar + + # Get current position information + if self.last_position_opened_id is not None: + position = self.cache.position(self.last_position_opened_id) + if position.is_open: + # Check position details + current_pnl = position.unrealized_pnl + + # Get all open orders for our instrument + open_orders = self.cache.orders_open(instrument_id=self.instrument_id) +``` + +## Configuration + +Use the `CacheConfig` class to configure the `Cache` behavior and capacity. +You can provide this configuration either to a `BacktestEngine` or a `TradingNode`, depending on your [environment context](architecture.md#environment-contexts). + +Here's a basic example of configuring the `Cache`: + +```python +from nautilus_trader.config import CacheConfig, BacktestEngineConfig, TradingNodeConfig + +# For backtesting +engine_config = BacktestEngineConfig( + cache=CacheConfig( + tick_capacity=10_000, # Store last 10,000 ticks per instrument + bar_capacity=5_000, # Store last 5,000 bars per bar type + ), +) + +# For live trading +node_config = TradingNodeConfig( + cache=CacheConfig( + tick_capacity=10_000, + bar_capacity=5_000, + ), +) +``` + +:::tip +By default, the `Cache` keeps the last 10,000 bars for each bar type and 10,000 trade ticks per instrument. +These limits provide a good balance between memory usage and data availability. Increase them if your strategy needs more historical data. +::: + +### Configuration options + +The `CacheConfig` class supports these parameters: + +```python +from nautilus_trader.config import CacheConfig + +cache_config = CacheConfig( + database: DatabaseConfig | None = None, # Database configuration for persistence + encoding: str = "msgpack", # Data encoding format ('msgpack' or 'json') + timestamps_as_iso8601: bool = False, # Store timestamps as ISO8601 strings + buffer_interval_ms: int | None = None, # Buffer interval for batch operations + bulk_read_batch_size: int | None = None, # Batch size for bulk reads (e.g., MGET) + use_trader_prefix: bool = True, # Use trader prefix in keys + use_instance_id: bool = False, # Include instance ID in keys + flush_on_start: bool = False, # Clear database on startup + drop_instruments_on_reset: bool = True, # Clear instruments on reset + tick_capacity: int = 10_000, # Maximum ticks stored per instrument + bar_capacity: int = 10_000, # Maximum bars stored per each bar-type +) +``` + +:::note +Each bar type maintains its own separate capacity. For example, if you're using both 1-minute and 5-minute bars, each stores up to `bar_capacity` bars. +When `bar_capacity` is reached, the `Cache` automatically removes the oldest data. +::: + +### Database configuration + +For persistence between system restarts, you can configure a database backend. + +When is it useful to use persistence? + +- **Long-running systems**: If you want your data to survive system restarts, upgrading, or unexpected failures, having a database configuration helps to pick up exactly where you left off. +- **Historical insights**: When you need to preserve past trading data for detailed post-analysis or audits. +- **Multi-node or distributed setups**: If multiple services or nodes need to access the same state, a persistent store helps ensure shared and consistent data. + +```python +from nautilus_trader.config import DatabaseConfig + +config = CacheConfig( + database=DatabaseConfig( + type="redis", # Database type + host="localhost", # Database host + port=6379, # Database port + timeout=2, # Connection timeout (seconds) + ), +) +``` + +## Using the cache + +### Accessing market data + +The `Cache` provides a full interface for accessing order books, quotes, trades, and bars. +All market data in the cache uses reverse indexing, so the most recent entry sits at index 0. + +#### Bar access + +```python +# Get a list of all cached bars for a bar type +bars = self.cache.bars(bar_type) # Returns list[Bar] or an empty list if no bars found + +# Get the most recent bar +latest_bar = self.cache.bar(bar_type) # Returns Bar or None if no such object exists + +# Get a specific historical bar by index (0 = most recent) +second_last_bar = self.cache.bar(bar_type, index=1) # Returns Bar or None if no such object exists + +# Check if bars exist and get count +bar_count = self.cache.bar_count(bar_type) # Returns number of bars in cache for the specified bar type +has_bars = self.cache.has_bars(bar_type) # Returns bool indicating if any bars exist for the specified bar type +``` + +#### Quote ticks + +```python +# Get quotes +quotes = self.cache.quote_ticks(instrument_id) # Returns list[QuoteTick] or an empty list if no quotes found +latest_quote = self.cache.quote_tick(instrument_id) # Returns QuoteTick or None if no such object exists +second_last_quote = self.cache.quote_tick(instrument_id, index=1) # Returns QuoteTick or None if no such object exists + +# Check quote availability +quote_count = self.cache.quote_tick_count(instrument_id) # Returns the number of quotes in cache for this instrument +has_quotes = self.cache.has_quote_ticks(instrument_id) # Returns bool indicating if any quotes exist for this instrument +``` + +#### Trade ticks + +```python +# Get trades +trades = self.cache.trade_ticks(instrument_id) # Returns list[TradeTick] or an empty list if no trades found +latest_trade = self.cache.trade_tick(instrument_id) # Returns TradeTick or None if no such object exists +second_last_trade = self.cache.trade_tick(instrument_id, index=1) # Returns TradeTick or None if no such object exists + +# Check trade availability +trade_count = self.cache.trade_tick_count(instrument_id) # Returns the number of trades in cache for this instrument +has_trades = self.cache.has_trade_ticks(instrument_id) # Returns bool indicating if any trades exist +``` + +#### Order book + +```python +# Get current order book +book = self.cache.order_book(instrument_id) # Returns OrderBook or None if no such object exists + +# Check if order book exists +has_book = self.cache.has_order_book(instrument_id) # Returns bool indicating if an order book exists + +# Get count of order book updates +update_count = self.cache.book_update_count(instrument_id) # Returns the number of updates received +``` + +#### Price access + +```python +from nautilus_trader.core.rust.model import PriceType + +# Get current price by type; Returns Price or None. +price = self.cache.price( + instrument_id=instrument_id, + price_type=PriceType.MID, # Options: BID, ASK, MID, LAST +) +``` + +#### Bar types + +```python +from nautilus_trader.core.rust.model import PriceType, AggregationSource + +# Get all available bar types for an instrument; Returns list[BarType]. +bar_types = self.cache.bar_types( + instrument_id=instrument_id, + price_type=PriceType.LAST, # Options: BID, ASK, MID, LAST + aggregation_source=AggregationSource.EXTERNAL, +) +``` + +#### Simple example + +```python +class MarketDataStrategy(Strategy): + def on_start(self): + # Subscribe to 1-minute bars + self.bar_type = BarType.from_str(f"{self.instrument_id}-1-MINUTE-LAST-EXTERNAL") # example of instrument_id = "EUR/USD.FXCM" + self.subscribe_bars(self.bar_type) + + def on_bar(self, bar: Bar) -> None: + bars = self.cache.bars(self.bar_type)[:3] + if len(bars) < 3: # Wait until we have at least 3 bars + return + + # Access last 3 bars for analysis + current_bar = bars[0] # Most recent bar + prev_bar = bars[1] # Second to last bar + prev_prev_bar = bars[2] # Third to last bar + + # Get latest quote and trade + latest_quote = self.cache.quote_tick(self.instrument_id) + latest_trade = self.cache.trade_tick(self.instrument_id) + + if latest_quote is not None: + current_spread = latest_quote.ask_price - latest_quote.bid_price + self.log.info(f"Current spread: {current_spread}") +``` + +### Trading objects + +The `Cache` provides access to all trading objects within the system, including: + +- Orders +- Positions +- Accounts +- Instruments + +#### Orders + +You can access and query orders through multiple methods, with flexible filtering options by venue, strategy, instrument, and order side. + +##### Basic order access + +```python +# Get a specific order by its client order ID +order = self.cache.order(ClientOrderId("O-123")) + +# Get all orders in the system +orders = self.cache.orders() + +# Get orders filtered by specific criteria +orders_for_venue = self.cache.orders(venue=venue) # All orders for a specific venue +orders_for_strategy = self.cache.orders(strategy_id=strategy_id) # All orders for a specific strategy +orders_for_instrument = self.cache.orders(instrument_id=instrument_id) # All orders for an instrument +``` + +##### Order state queries + +```python +# Get orders by their current state +open_orders = self.cache.orders_open() # Orders currently active at the venue +closed_orders = self.cache.orders_closed() # Orders that have completed their lifecycle +emulated_orders = self.cache.orders_emulated() # Orders being simulated locally by the system +inflight_orders = self.cache.orders_inflight() # Orders submitted (or modified) to venue, but not yet confirmed + +# Check specific order states +exists = self.cache.order_exists(client_order_id) # Checks if an order with the given ID exists in the cache +is_open = self.cache.is_order_open(client_order_id) # Checks if an order is currently open +is_closed = self.cache.is_order_closed(client_order_id) # Checks if an order is closed +is_emulated = self.cache.is_order_emulated(client_order_id) # Checks if an order is being simulated locally +is_inflight = self.cache.is_order_inflight(client_order_id) # Checks if an order is submitted or modified, but not yet confirmed +``` + +##### Order statistics + +```python +# Get counts of orders in different states +open_count = self.cache.orders_open_count() # Number of open orders +closed_count = self.cache.orders_closed_count() # Number of closed orders +emulated_count = self.cache.orders_emulated_count() # Number of emulated orders +inflight_count = self.cache.orders_inflight_count() # Number of inflight orders +total_count = self.cache.orders_total_count() # Total number of orders in the system + +# Get filtered order counts +buy_orders_count = self.cache.orders_open_count(side=OrderSide.BUY) # Number of currently open BUY orders +venue_orders_count = self.cache.orders_total_count(venue=venue) # Total number of orders for a given venue +``` + +#### Positions + +The `Cache` maintains a record of all positions and offers several ways to query them. + +##### Position access + +```python +# Get a specific position by its ID +position = self.cache.position(PositionId("P-123")) + +# Get positions by their state +all_positions = self.cache.positions() # All positions in the system +open_positions = self.cache.positions_open() # All currently open positions +closed_positions = self.cache.positions_closed() # All closed positions + +# Get positions filtered by various criteria +venue_positions = self.cache.positions(venue=venue) # Positions for a specific venue +instrument_positions = self.cache.positions(instrument_id=instrument_id) # Positions for a specific instrument +strategy_positions = self.cache.positions(strategy_id=strategy_id) # Positions for a specific strategy +long_positions = self.cache.positions(side=PositionSide.LONG) # All long positions +``` + +##### Position state queries + +```python +# Check position states +exists = self.cache.position_exists(position_id) # Checks if a position with the given ID exists +is_open = self.cache.is_position_open(position_id) # Checks if a position is open +is_closed = self.cache.is_position_closed(position_id) # Checks if a position is closed + +# Get position and order relationships +orders = self.cache.orders_for_position(position_id) # All orders related to a specific position +position = self.cache.position_for_order(client_order_id) # Find the position associated with a specific order +``` + +##### Position statistics + +```python +# Get position counts in different states +open_count = self.cache.positions_open_count() # Number of currently open positions +closed_count = self.cache.positions_closed_count() # Number of closed positions +total_count = self.cache.positions_total_count() # Total number of positions in the system + +# Get filtered position counts +long_positions_count = self.cache.positions_open_count(side=PositionSide.LONG) # Number of open long positions +instrument_positions_count = self.cache.positions_total_count(instrument_id=instrument_id) # Number of positions for a given instrument +``` + +#### Accounts + +```python +# Access account information +account = self.cache.account(account_id) # Retrieve account by ID +account = self.cache.account_for_venue(venue) # Retrieve account for a specific venue +account_id = self.cache.account_id(venue) # Retrieve account ID for a venue +accounts = self.cache.accounts() # Retrieve all accounts in the cache +``` + +#### Purging cached state + +The cache exposes explicit maintenance hooks that remove closed or stale objects while preserving safety checks: + +- `purge_closed_orders(ts_now, buffer_secs=0, purge_from_database=False)` drops closed orders that have been inactive for at least `buffer_secs`. Linked contingency orders remain until every dependent child is closed. +- `purge_closed_positions(ts_now, buffer_secs=0, purge_from_database=False)` removes positions that have stayed closed beyond the buffer window and deletes associated indices. +- `purge_account_events(ts_now, lookback_secs=0, purge_from_database=False)` trims account event history outside the lookback window and can cascade deletes to the backing database. + +Key safeguards: + +- Open orders and positions are never purged; the cache logs a warning and leaves the item intact. +- Linked orders keep parents in the cache until all children have closed, preventing premature removal of contingency chains. +- Indices and reverse lookups are cleaned alongside the primary object to avoid dangling references. +- Database deletions occur only when `purge_from_database=True` and a cache database is configured, ensuring in-memory purges do not silently erase persisted data. + +Use the trading clock (for example, `self.clock.timestamp_ns()`) when supplying `ts_now`. Set `purge_from_database=True` only when you intend to delete persisted records from Redis or PostgreSQL as well. In live trading these methods run automatically when the execution engine is configured with purge intervals; see [Memory management](live.md#memory-management) for the scheduler settings. + +#### Instruments and currencies + +##### Instruments + +```python +# Get instrument information +instrument = self.cache.instrument(instrument_id) # Retrieve a specific instrument by its ID +all_instruments = self.cache.instruments() # Retrieve all instruments in the cache + +# Get filtered instruments +venue_instruments = self.cache.instruments(venue=venue) # Instruments for a specific venue +instruments_by_underlying = self.cache.instruments(underlying="ES") # Instruments by underlying + +# Get instrument identifiers +instrument_ids = self.cache.instrument_ids() # Get all instrument IDs +venue_instrument_ids = self.cache.instrument_ids(venue=venue) # Get instrument IDs for a specific venue +``` + +##### Currencies + +```python +# Get currency information +currency = self.cache.load_currency("USD") # Loads currency data for USD +``` + +--- + +### Custom data + +The `Cache` can also store and retrieve custom data types in addition to built-in market data and trading objects. +Use it to share any user-defined data between system components, primarily actors and strategies. + +#### Basic storage and retrieval + +```python +# Call this code inside Strategy methods (`self` refers to Strategy) + +# Store data +self.cache.add(key="my_key", value=b"some binary data") + +# Retrieve data +stored_data = self.cache.get("my_key") # Returns bytes or None +``` + +For more complex use cases, the `Cache` can store custom data objects that inherit from the `nautilus_trader.core.Data` base class. + +:::warning +The `Cache` is not designed to be a full database replacement. For large datasets or complex querying needs, consider using a dedicated database system. +::: + +## Best practices and common questions + +### Cache vs. portfolio usage + +The `Cache` and `Portfolio` components serve different but complementary purposes in NautilusTrader: + +**Cache**: + +- Maintains the historical knowledge and current state of the trading system. +- Updates immediately when local state changes (for example, initializing an order before submission). +- Updates asynchronously as external events occur (for example, when an order fills). +- Provides a complete history of trading activity and market data. +- Keeps every event the strategy receives in the cache. + +**Portfolio**: + +- Aggregates position, exposure, and account information. +- Provides current state without history. + +**Example**: + +```python +class MyStrategy(Strategy): + def on_position_changed(self, event: PositionEvent) -> None: + # Use Cache when you need historical perspective + position_history = self.cache.position_snapshots(event.position_id) + + # Use Portfolio when you need current real-time state + current_exposure = self.portfolio.net_exposure(event.instrument_id) +``` + +### Cache vs. strategy variables + +Choosing between storing data in the `Cache` versus strategy variables depends on your specific needs: + +**Cache storage**: + +- Use for data that needs to be shared between strategies. +- Best for data that needs to persist between system restarts. +- Acts as a central database accessible to all components. +- Ideal for state that needs to survive strategy resets. + +**Strategy variables**: + +- Use for strategy-specific calculations. +- Better for temporary values and intermediate results. +- Provides faster access and better encapsulation. +- Best for data that only your strategy needs. + +**Example**: + +The following example shows how you might store data in the `Cache` so multiple strategies can access the same information. + +```python +import pickle + +class MyStrategy(Strategy): + def on_start(self): + # Prepare data you want to share with other strategies + shared_data = { + "last_reset": self.clock.timestamp_ns(), + "trading_enabled": True, + # Include any other fields that you want other strategies to read + } + + # Store it in the cache with a descriptive key + # This way, multiple strategies can call self.cache.get("shared_strategy_info") + # to retrieve the same data + self.cache.add("shared_strategy_info", pickle.dumps(shared_data)) +``` + +Another strategy can retrieve the cached data as follows: + +```python +import pickle + +class AnotherStrategy(Strategy): + def on_start(self): + # Load the shared data from the same key + data_bytes = self.cache.get("shared_strategy_info") + if data_bytes is not None: + shared_data = pickle.loads(data_bytes) + self.log.info(f"Shared data retrieved: {shared_data}") +``` + +## Related guides + +- [Data](data.md) - Data types stored in the cache. +- [Strategies](strategies.md) - Strategies access cache for market data and state. +- [Reports](reports.md) - Generate reports from cached data. diff --git a/nautilus-docs/docs/concepts/custom_data.md b/nautilus-docs/docs/concepts/custom_data.md new file mode 100644 index 0000000..f92c7c4 --- /dev/null +++ b/nautilus-docs/docs/concepts/custom_data.md @@ -0,0 +1,396 @@ +# Custom Data + +Nautilus Trader supports custom data authored in Python and Rust, and moves +that data through the same runtime, persistence, and query pipeline used +by the rest of the platform. + +This document explains how custom data is: + +- Registered at runtime. +- Wrapped across the Python/Rust boundary. +- Serialized to and from Arrow/Parquet. +- Routed through actors and strategies. + +## Goals + +The custom-data architecture satisfies the following requirements: + +- Let users define custom data in pure Python without writing Rust code. +- Let Rust-defined custom data use native Rust JSON and Arrow handlers. +- Preserve a single user-facing `CustomData` wrapper at the PyO3 boundary. +- Support persistence in `ParquetDataCatalogV2` using dynamic type registration + instead of hardcoded schemas. +- Make custom data routable through the normal data-engine, actor, and strategy + subscription flow. + +## High-level model + +There are two supported authoring modes: + +| Mode | Example | Registration path | Encode/decode path | Wrapper backend | +|-------------------|----------------------------------------------------|---------------------------------------------------------|---------------------------------|---------------------------| +| Pure Python | `@customdataclass_pyo3` class | `register_custom_data_class(...)` | Python callback + Arrow C FFI | `PythonCustomDataWrapper` | +| Same-binary Rust | `#[custom_data]` or `#[custom_data(pyo3)]` type | `ensure_custom_data_registered::()` and native extractor | Native Rust | Native Rust payload | + +Both modes converge on the same outer PyO3 `CustomData` wrapper and the same +`DataType` identity model. + +## End-to-end flow + +```mermaid +sequenceDiagram + participant U as User code + participant P as Python layer + participant R as Rust model/catalog + participant G as Global DataRegistry + participant S as Storage + + U->>P: define class/type + U->>P: register_custom_data_class(...) or module init + P->>R: install type registration + R->>G: store JSON/Arrow/extractor handlers + + U->>P: CustomData(data_type, data) + P->>R: write_custom_data([...]) + R->>G: lookup encoder by type_name + G-->>R: encoder + R->>S: write RecordBatch to Parquet + + U->>P: query(type_name, ...) + P->>R: query catalog + R->>S: read RecordBatch + metadata + R->>G: lookup decoder by type_name + G-->>R: decoder + R-->>P: CustomData wrappers + P-->>U: typed data via .data +``` + +## Core components + +### `DataRegistry` + +`crates/model/src/data/registry.rs` is the central runtime registry module for +custom data in the main process. Registration uses atomic `DashMap::entry()` so +that concurrent `register_*` and `ensure_*` calls do not race. + +The module contains several `OnceLock`-initialized `DashMap` singletons: + +- JSON deserializers keyed by `type_name`. +- Arrow schemas, encoders, and decoders keyed by `type_name`. +- Python extractors that convert a Python object into + `Arc`. +- Rust extractor factories that produce Python extractors for same-binary types. + +Instead of hardcoding every type into the main binary, Nautilus resolves +handlers at runtime using the `type_name` stored in `DataType` and Parquet +metadata. + +### `CustomData` + +The outer PyO3 `CustomData` wrapper is the common container that crosses the +FFI boundary. + +Constructor signature: `CustomData(data_type, data)` -- the `DataType` comes +first, then the inner payload. + +It contains: + +- A `DataType`. +- An inner custom payload implementing `CustomDataTrait` (wrapped in + `Arc`). + +Timestamps (`ts_event`, `ts_init`) are delegated to the inner +`CustomDataTrait` implementation and exposed as properties on the wrapper. + +On the Python side, `CustomData` exposes value semantics: `__eq__` and +`__repr__` are implemented (equality uses the Rust `PartialEq` logic). +Instances are intentionally unhashable so that equality remains consistent with +the inner payload comparison. + +This wrapper is shared across both custom-data modes. User code interacts with +one API even though the underlying payload may be: + +- A Python-backed wrapper. +- A same-binary Rust value. + +#### `CustomData` JSON envelope + +When serialized to JSON (e.g. for `to_json_bytes` / `from_json_bytes`, SQL +cache, or Redis), `CustomData` uses a single canonical envelope so that +deserialization does not depend on user payload field names: + +- `type`: The custom type name (from `CustomDataTrait::type_name`). +- `data_type`: An object with `type_name`, `metadata`, and optional + `identifier`. +- `payload`: The inner payload only (the result of `CustomDataTrait::to_json` + parsed as a value). Registered deserializers receive only this value in + `from_json`, so user structs can use any field names (including `value`) + without conflicting with wrapper metadata. + +This envelope is produced by Rust `CustomData` serialization and consumed by +`DataRegistry` when deserializing custom data from JSON. + +### `DataType` + +`DataType` identifies custom data for routing and persistence. + +Constructor: `DataType(type_name, metadata=None, identifier=None)`. + +It includes: + +- `type_name`. +- Optional `metadata`. +- Optional `identifier` (used only for catalog pathing, not for routing or + equality). + +Equality, hashing, and topic routing are derived from `type_name` and +`metadata` only. Two `DataType` values with the same type name and metadata but +different identifiers compare equal and publish to the same message bus topic. +The `identifier` affects only the storage path under +`data/custom//`. + +Custom-data storage and queries use `DataType`, not just the bare Rust/Python +class name. This allows the same logical type to be stored under different +metadata or identifiers while still decoding through the same registered +handler. + +## Registration architecture + +Registration bridges the gap between Python objects and Rust trait objects. + +```mermaid +flowchart TD + A[User-defined custom type] --> B{Mode} + B --> C[Pure Python] + B --> D[Same-binary Rust] + + C --> F[register_custom_data_class] + D --> G[ensure_custom_data_registered and native extractor] + + F --> I[Python callbacks registered] + G --> J[Native JSON and Arrow handlers registered] + + I --> L[Main-process DataRegistry] + J --> L +``` + +### Pure Python registration + +When Python code calls `register_custom_data_class(MyType)`: + +1. The type is registered in the Python serialization layer for JSON and Arrow + support. +2. Rust registers a Python extractor that wraps Python instances as + `PythonCustomDataWrapper`. +3. Rust registers Arrow schema/encode/decode callbacks in `DataRegistry`. + +This path is flexible and user-friendly, but Arrow encoding and reconstruction +rely on Python callbacks. + +### Same-binary Rust registration + +For Rust types defined inside Nautilus: + +1. `#[custom_data]` or `#[custom_data(pyo3)]` generates the necessary trait, + JSON, and Arrow implementations. +2. `ensure_custom_data_registered::()` inserts native schema/encoder/decoder + handlers into `DataRegistry`. +3. For PyO3-exposed types, a native extractor can convert Python instances back + into the concrete Rust type rather than a Python fallback wrapper. + +This path stays fully native in Rust for encode/decode. + +### Registration precedence + +`register_custom_data_class(...)` resolves types in the following order: + +1. Same-binary native Rust registration. +2. Pure Python fallback registration. + +That ordering preserves the fastest available path for types already known +natively by the main binary. + +## Wrapper backends + +Internally, the outer `CustomData` wrapper can hold different payload +implementations. + +### `PythonCustomDataWrapper` + +Used for pure Python custom data. + +Responsibilities: + +- Stores a reference to the Python object. +- Caches `ts_event`, `ts_init`, and `type_name`. +- Implements `CustomDataTrait`. +- Calls Python methods for JSON and Arrow-related operations under the GIL. + +This is the fallback path when the main process does not have a native Rust +representation for the type. + +### Native same-binary Rust payload + +For Rust types compiled into Nautilus, the inner payload is the concrete Rust +type itself and can be downcast directly from `Arc`. + +No Python callback path is needed for serialization or decode. + +## Persistence architecture + +### Why dynamic Arrow registration is needed + +Built-in Nautilus data types have schemas and encoders known statically to the +Rust binary. Custom data does not. The persistence layer therefore resolves +custom data dynamically using the registered `type_name`. + +### Catalog write flow + +`ParquetDataCatalogV2` expects custom writes to come in as `CustomData` values. + +The custom-data write path: + +1. Extracts `type_name`, `metadata`, and `identifier` from `DataType`. +2. Looks up the Arrow encoder in `DataRegistry`. +3. Encodes the values to a `RecordBatch`. +4. Appends a `data_type` column containing the persisted `DataType`. +5. Attaches `type_name` and metadata to the Arrow schema. +6. Writes the batch to Parquet under the custom-data path. + +The path layout is: + +- `data/custom//` + +Identifiers are normalized before becoming path segments. + +### Catalog read flow + +On query: + +1. The catalog reads matching Parquet files. +2. Extracts `type_name` from schema metadata. +3. Asks `DataRegistry` for the registered decoder. +4. Decodes the `RecordBatch` into `Vec`. +5. Reconstructs `CustomData` with the original `DataType`. + +This makes custom-data query resolution symmetric with write-time registration. +When converting a Feather stream to Parquet (e.g. after a backtest), the +custom-data branch decodes batches and writes them via +`write_custom_data_batch` so that custom data written through the Feather +writer is correctly converted to Parquet. + +## The Arrow C FFI bridge + +Pure Python custom data cannot provide native Rust Arrow encode logic directly. +For those types, Nautilus uses the Arrow C FFI interface to pass `RecordBatch` +data between Python and Rust without serialization overhead. + +```mermaid +sequenceDiagram + participant R as Rust encoder + participant P as Python custom class + participant F as Arrow C FFI structs + participant C as Parquet writer + + R->>P: encode_record_batch_py(items) + P->>P: build pyarrow.RecordBatch + P-->>F: _export_to_c (FFI_ArrowArray + FFI_ArrowSchema) + F-->>R: reconstruct native RecordBatch + R->>C: write Parquet +``` + +### Pure Python encode path + +For pure Python classes: + +1. Rust acquires the GIL. +2. Rust calls `encode_record_batch_py(...)` on the Python class. +3. Python converts objects to a `pyarrow.RecordBatch`. +4. Python exports the batch via `_export_to_c` into Arrow C FFI structs. +5. Rust reconstructs a native `RecordBatch` from the FFI structs and writes it. + +### Pure Python decode path + +For the reverse direction: + +1. Rust converts its `RecordBatch` into Arrow C FFI structs. +2. Python imports the batch via `RecordBatch._import_from_c`. +3. Python calls `decode_record_batch_py(metadata, batch)` on the class. +4. Rust wraps the returned Python objects in `PythonCustomDataWrapper`. + +### Native paths + +The Arrow C FFI bridge is not used for same-binary Rust custom data. Those +types use native Rust encode/decode handlers registered in the main process. + +## Reconstruction on query + +When custom data is loaded back from the catalog, reconstruction depends on the +backend: + +- Same-binary Rust types decode directly to native Rust values. +- Pure Python types reconstruct through the registered Python class using + `from_dict` or `from_json`. + +In all cases the caller receives the same outer `CustomData` wrapper at the +PyO3 API boundary. + +## Runtime integration + +Custom data is not only a persistence feature. It also participates in Nautilus +runtime routing. + +Relevant integrations include: + +- `crates/data/src/engine/mod.rs` publishes `CustomData` through the message + bus. +- `crates/common/src/msgbus/switchboard.rs` derives custom topics from + `DataType`. +- `crates/common/src/actor/*` routes custom data into actor subscriptions. +- `crates/trading/src/python/strategy.rs` exposes custom data to Python + strategy `on_data`. +- `crates/backtest/src/engine.rs` treats `Data::Custom` as + data-engine-delivered input rather than exchange-routed data. + +A registered custom type can be persisted, queried, subscribed to, and consumed +through the same runtime interfaces as other data families. + +## SQL cache and database integration + +The SQL cache/database layer also supports `CustomData`. + +Current behavior: + +- PostgreSQL stores custom data in the `custom` table. +- The stored record includes `data_type`, `metadata`, `identifier`, and full + JSON payload. +- Reads reconstruct `CustomData` using `CustomData::from_json_bytes(...)`. +- Python SQL bindings expose `add_custom_data` and `load_custom_data`. +- Redis cache stores custom data under keys + `custom::` with full `CustomData` JSON as value. +- Redis `add_custom_data` and `load_custom_data` filter by `DataType` + (type_name, metadata, identifier) and return results sorted by `ts_init`; + this is exposed via the PyO3 `RedisCacheDatabase` API. + +## Relationship to legacy Cython custom data + +Legacy Cython `@customdataclass` remains separate from this architecture. + +This document describes the PyO3 custom-data system: + +- PyO3 `CustomData`. +- Dynamic runtime registration. +- Arrow/Parquet persistence. +- Native Rust execution paths. + +Legacy Cython support is intentionally left unchanged. + +## Practical implications + +This architecture gives Nautilus two important properties: + +1. Python-first extensibility for users who only want to write Python. +2. Native Rust performance for built-in or compiled custom types. + +The result is one conceptual custom-data system with two backends, rather than +separate feature silos for Python-only and Rust-only data types. diff --git a/nautilus-docs/docs/concepts/data.md b/nautilus-docs/docs/concepts/data.md new file mode 100644 index 0000000..c286f27 --- /dev/null +++ b/nautilus-docs/docs/concepts/data.md @@ -0,0 +1,1892 @@ +# Data + +NautilusTrader provides built-in data types for the trading domain: + +- `OrderBookDelta` (L1/L2/L3): Represents the most granular order book updates. +- `OrderBookDeltas` (L1/L2/L3): Batches multiple order book deltas for more efficient processing. +- `OrderBookDepth10`: Aggregated order book snapshot (up to 10 levels per bid and ask side). +- `QuoteTick`: Represents the best bid and ask prices along with their sizes at the top-of-book. +- `TradeTick`: A single trade/match event between counterparties. +- `Bar`: OHLCV (Open, High, Low, Close, Volume) bar/candle, aggregated using a specified *aggregation method*. +- `MarkPriceUpdate`: The current mark price for an instrument (typically used in derivatives trading). +- `IndexPriceUpdate`: The index price for an instrument (underlying price used for mark price calculations). +- `FundingRateUpdate`: The funding rate for perpetual contracts (periodic payments between long and short positions). +- `InstrumentStatus`: An instrument-level status event. +- `InstrumentClose`: The closing price of an instrument. + +NautilusTrader operates primarily on granular order book data for the highest realism +in execution simulations. Backtests can also run on any supported market data type, +depending on the desired simulation fidelity. + +## Order books + +A high-performance order book implemented in Rust is available to maintain order book state based on provided data. + +`OrderBook` instances are maintained per instrument for both backtesting and live trading, with the following book types available: + +- `L3_MBO`: **Market by order (MBO)** or L3 data, uses every order book event at every price level, keyed by order ID. +- `L2_MBP`: **Market by price (MBP)** or L2 data, aggregates order book events by price level. +- `L1_MBP`: **Market by price (MBP)** or L1 data, also known as best bid and offer (BBO), captures only top-level updates. + +:::note +Top-of-book data, such as `QuoteTick`, `TradeTick` and `Bar`, can also be used for backtesting, with markets operating on `L1_MBP` book types. +::: + +### Delta flags and event boundaries + +Each `OrderBookDelta` carries a `flags` field using `RecordFlag` bitmask values +to signal event boundaries to the `DataEngine`: + +- `F_LAST`: Marks the final delta in a logical event group. When `buffer_deltas` + is enabled, the `DataEngine` accumulates deltas and only publishes to + subscribers when it encounters `F_LAST`. Every event group **must** end with + a delta that has `F_LAST` set. +- `F_SNAPSHOT`: Marks deltas that belong to a snapshot (as opposed to an + incremental update). Snapshot sequences begin with a `Clear` action followed + by `Add` deltas reconstructing the full book state. The last delta in a + snapshot has both `F_SNAPSHOT | F_LAST` set. + +:::warning +A missing `F_LAST` on the final delta in an event group causes buffered consumers +to accumulate deltas indefinitely without publishing. This applies to incremental +updates and snapshots alike, including empty book snapshots where only a `Clear` +delta is emitted. +::: + +## Instruments + +NautilusTrader supports a variety of instrument types across spot, derivatives, and specialty markets: + +```mermaid +flowchart TD + I[Instrument Types] + I --> Spot + I --> Derivatives + I --> Other + + Spot --> Equity + Spot --> CurrencyPair + Spot --> Commodity + Spot --> IndexInstrument + + Derivatives --> Futures + Derivatives --> Options + Derivatives --> Cfd + + Futures --> FuturesContract + Futures --> FuturesSpread + Futures --> CryptoFuture + Futures --> CryptoPerpetual + Futures --> PerpetualContract + + Options --> OptionContract + Options --> OptionSpread + Options --> CryptoOption + Options --> BinaryOption + + Other --> BettingInstrument + Other --> SyntheticInstrument +``` + +| Instrument | Description | +|----------------------|----------------------------------------------------------------------------------| +| `Equity` | Generic equity instrument. | +| `CurrencyPair` | Currency pair in a spot/cash market. | +| `Commodity` | Commodity in a spot/cash market. | +| `IndexInstrument` | Spot index (reference price, not directly tradable). | +| `FuturesContract` | Generic deliverable futures contract. | +| `FuturesSpread` | Deliverable futures spread. | +| `CryptoFuture` | Deliverable futures with crypto assets as underlying and settlement. | +| `CryptoPerpetual` | Crypto perpetual futures (perpetual swap). | +| `PerpetualContract` | Asset-class agnostic perpetual swap (any underlying). | +| `OptionContract` | Generic option contract. | +| `OptionSpread` | Generic option spread. | +| `CryptoOption` | Crypto option contract. | +| `BinaryOption` | Binary option instrument. | +| `Cfd` | Contract for Difference (CFD). | +| `BettingInstrument` | Instrument in a betting market. | +| `SyntheticInstrument`| Synthetic instrument with prices derived from component instruments via formula. | + +## Bars and aggregation + +### Introduction to bars + +A *bar* (also known as a candle, candlestick or kline) is a data structure that represents +price and volume information over a specific period, including: + +- Opening price +- Highest price +- Lowest price +- Closing price +- Traded volume (or ticks as a volume proxy) + +The system generates bars using an *aggregation method* that groups data by specific criteria. + +### Purpose of data aggregation + +Data aggregation in NautilusTrader transforms granular market data into structured bars or candles for several reasons: + +- To provide data for technical indicators and strategy development. +- Because time-aggregated data (like minute bars) are often sufficient for many strategies. +- To reduce costs compared to high-frequency L1/L2/L3 market data. + +### Aggregation methods + +The platform implements various aggregation methods: + +| Name | Description | Category | +|:-------------------|:---------------------------------------------------------------------------|:-------------| +| `TICK` | Aggregation of a number of ticks. | Threshold | +| `TICK_IMBALANCE` | Aggregation of the buy/sell imbalance of ticks. | Threshold | +| `TICK_RUNS` | Aggregation of sequential buy/sell runs of ticks. | Information | +| `VOLUME` | Aggregation of traded volume. | Threshold | +| `VOLUME_IMBALANCE` | Aggregation of the buy/sell imbalance of traded volume. | Threshold | +| `VOLUME_RUNS` | Aggregation of sequential runs of buy/sell traded volume. | Information | +| `VALUE` | Aggregation of the notional value of trades (also known as "Dollar bars"). | Threshold | +| `VALUE_IMBALANCE` | Aggregation of the buy/sell imbalance of trading by notional value. | Threshold | +| `VALUE_RUNS` | Aggregation of sequential buy/sell runs of trading by notional value. | Information | +| `RENKO` | Aggregation based on fixed price movements (brick size in ticks). | Threshold | +| `MILLISECOND` | Aggregation of time intervals with millisecond granularity. | Time | +| `SECOND` | Aggregation of time intervals with second granularity. | Time | +| `MINUTE` | Aggregation of time intervals with minute granularity. | Time | +| `HOUR` | Aggregation of time intervals with hour granularity. | Time | +| `DAY` | Aggregation of time intervals with day granularity. | Time | +| `WEEK` | Aggregation of time intervals with week granularity. | Time | +| `MONTH` | Aggregation of time intervals with month granularity. | Time | +| `YEAR` | Aggregation of time intervals with year granularity. | Time | + +### Information-driven bars + +Information-driven bars adapt their sampling frequency to market activity rather than using fixed +intervals. They are based on the concept of *aggressor side* (whether the trade initiator was a +buyer or seller) and come in two families: **imbalance** and **runs**. + +**Imbalance bars** close when the *net* buy/sell activity reaches a threshold. Each trade contributes +a signed value: positive for buyer-initiated trades and negative for seller-initiated. The bar closes +when the absolute imbalance reaches the configured step. This means that opposing trades cancel each +other out, so imbalance bars tend to form more slowly in balanced markets and faster during directional moves. + +**Runs bars** close when *consecutive* activity from the same aggressor side reaches a threshold. +Unlike imbalance bars, runs bars reset their counter when the aggressor side changes. +This makes them sensitive to sustained one-sided pressure rather than net imbalance. + +Both families have three variants based on what is measured: + +| Variant | Imbalance | Runs | What is measured | +|:--------|:-------------------|:--------------|:------------------------------------------| +| Tick | `TICK_IMBALANCE` | `TICK_RUNS` | Number of trades (each trade counts as 1) | +| Volume | `VOLUME_IMBALANCE` | `VOLUME_RUNS` | Traded volume (quantity) | +| Value | `VALUE_IMBALANCE` | `VALUE_RUNS` | Notional value (price x quantity) | + +:::note +Information-driven bars require `TradeTick` data because they need the `aggressor_side` field +to classify each trade. They cannot be aggregated from `QuoteTick` data alone. +::: + +### Types of aggregation + +NautilusTrader implements three distinct data aggregation methods: + +1. **Trade-to-bar aggregation**: Creates bars from `TradeTick` objects (executed trades) + - Use case: For strategies analyzing execution prices or when working directly with trade data. + - Always uses the `LAST` price type in the bar specification. + +2. **Quote-to-bar aggregation**: Creates bars from `QuoteTick` objects (bid/ask prices) + - Use case: For strategies focusing on bid/ask spreads or market depth analysis. + - Uses `BID`, `ASK`, or `MID` price types in the bar specification. + +3. **Bar-to-bar aggregation**: Creates larger-timeframe `Bar` objects from smaller-timeframe `Bar` objects + - Use case: For resampling existing smaller timeframe bars (1-minute) into larger timeframes (5-minute, hourly). + - Always requires the `@` symbol in the specification. + +### Bar types + +NautilusTrader defines a unique *bar type* (`BarType` class) based on the following components: + +- **Instrument ID** (`InstrumentId`): Specifies the particular instrument for the bar. +- **Bar Specification** (`BarSpecification`): + - `step`: Defines the interval or frequency of each bar. + - `aggregation`: Specifies the method used for data aggregation (see the above table). + - `price_type`: Indicates the price basis of the bar (e.g., bid, ask, mid, last). +- **Aggregation Source** (`AggregationSource`): Indicates whether the bar was aggregated internally (within Nautilus). +- or externally (by a trading venue or data provider). + +Bar types can also be classified as either *standard* or *composite*: + +- **Standard**: Generated from granular market data, such as quote-ticks or trade-ticks. +- **Composite**: Derived from a higher-granularity bar type through subsampling (like 5-MINUTE bars aggregate from 1-MINUTE bars). + +### Aggregation sources + +Bar data aggregation can be either *internal* or *external*: + +- `INTERNAL`: The bar is aggregated inside the local Nautilus system boundary. +- `EXTERNAL`: The bar is aggregated outside the local Nautilus system boundary (typically by a trading venue or data provider). + +For bar-to-bar aggregation, the target bar type is always `INTERNAL` (since you're doing the aggregation within NautilusTrader), +but the source bars can be either `INTERNAL` or `EXTERNAL`, i.e., you can aggregate externally provided bars or already +aggregated internal bars. + +### Defining bar types with *string syntax* + +#### Standard bars + +You can define standard bar types from strings using the following convention: + +`{instrument_id}-{step}-{aggregation}-{price_type}-{INTERNAL | EXTERNAL}` + +For example, to define a `BarType` for AAPL trades (last price) on Nasdaq (XNAS) using a 5-minute interval +aggregated from trades locally by Nautilus: + +```python +bar_type = BarType.from_str("AAPL.XNAS-5-MINUTE-LAST-INTERNAL") +``` + +#### Composite bars + +Composite bars are derived by aggregating higher-granularity bars into the desired bar type. To define a composite bar, +use this convention: + +`{instrument_id}-{step}-{aggregation}-{price_type}-INTERNAL@{step}-{aggregation}-{INTERNAL | EXTERNAL}` + +**Notes**: + +- The derived bar type must use an `INTERNAL` aggregation source (since this is how the bar is aggregated). +- The sampled bar type must have a higher granularity than the derived bar type. +- The sampled instrument ID is inferred to match that of the derived bar type. +- Composite bars can be aggregated *from* `INTERNAL` or `EXTERNAL` aggregation sources. + +For example, to define a `BarType` for AAPL trades (last price) on Nasdaq (XNAS) using a 5-minute interval +aggregated locally by Nautilus, from 1-minute interval bars aggregated externally: + +```python +bar_type = BarType.from_str("AAPL.XNAS-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL") +``` + +### Aggregation syntax examples + +The `BarType` string format encodes both the target bar type and, optionally, the source data type: + +``` +{instrument_id}-{step}-{aggregation}-{price_type}-{source}@{step}-{aggregation}-{source} +``` + +The part after the `@` symbol is optional and only used for bar-to-bar aggregation: + +- **Without `@`**: Aggregates from `TradeTick` objects (when price_type is `LAST`) or `QuoteTick` objects (when price_type is `BID`, `ASK`, or `MID`). +- **With `@`**: Aggregates from existing `Bar` objects (specifying the source bar type). + +#### Trade-to-bar example + +```python +def on_start(self) -> None: + # Define a bar type for aggregating from TradeTick objects + # Uses price_type=LAST which indicates TradeTick data as source + bar_type = BarType.from_str("6EH4.XCME-50-VOLUME-LAST-INTERNAL") + + # Request historical data (will receive bars in on_historical_data handler) + self.request_bars(bar_type) + + # Subscribe to live data (will receive bars in on_bar handler) + self.subscribe_bars(bar_type) +``` + +#### Quote-to-bar example + +```python +def on_start(self) -> None: + # Create 1-minute bars from ASK prices (in QuoteTick objects) + bar_type_ask = BarType.from_str("6EH4.XCME-1-MINUTE-ASK-INTERNAL") + + # Create 1-minute bars from BID prices (in QuoteTick objects) + bar_type_bid = BarType.from_str("6EH4.XCME-1-MINUTE-BID-INTERNAL") + + # Create 1-minute bars from MID prices (middle between ASK and BID prices in QuoteTick objects) + bar_type_mid = BarType.from_str("6EH4.XCME-1-MINUTE-MID-INTERNAL") + + # Request historical data and subscribe to live data + self.request_bars(bar_type_ask) # Historical bars processed in on_historical_data + self.subscribe_bars(bar_type_ask) # Live bars processed in on_bar +``` + +#### Bar-to-bar example + +```python +def on_start(self) -> None: + # Create 5-minute bars from 1-minute bars (Bar objects) + # Format: target_bar_type@source_bar_type + # Note: price type (LAST) is only needed on the left target side, not on the source side + bar_type = BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL") + + # Request historical data (processed in on_historical_data(...) handler) + self.request_bars(bar_type) + + # Subscribe to live updates (processed in on_bar(...) handler) + self.subscribe_bars(bar_type) +``` + +#### Advanced bar-to-bar example + +You can create complex aggregation chains where you aggregate from already aggregated bars: + +```python +# First create 1-minute bars from TradeTick objects (LAST indicates TradeTick source) +primary_bar_type = BarType.from_str("6EH4.XCME-1-MINUTE-LAST-INTERNAL") + +# Then create 5-minute bars from 1-minute bars +# Note the @1-MINUTE-INTERNAL part identifying the source bars +intermediate_bar_type = BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-INTERNAL") + +# Then create hourly bars from 5-minute bars +# Note the @5-MINUTE-INTERNAL part identifying the source bars +hourly_bar_type = BarType.from_str("6EH4.XCME-1-HOUR-LAST-INTERNAL@5-MINUTE-INTERNAL") +``` + +### Working with bars: request vs. subscribe + +NautilusTrader provides two distinct operations for working with bars: + +- **`request_bars()`**: Fetches historical data processed by the `on_historical_data()` handler. +- **`subscribe_bars()`**: Establishes a real-time data feed processed by the `on_bar()` handler. + +These methods work together in a typical workflow: + +1. First, `request_bars()` loads historical data to initialize indicators or state of strategy with past market behavior. +2. Then, `subscribe_bars()` ensures the strategy continues receiving new bars as they form in real-time. + +Example usage in `on_start()`: + +```python +def on_start(self) -> None: + # Define bar type + bar_type = BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL") + + # Request historical data to initialize indicators + # These bars will be delivered to the on_historical_data(...) handler in strategy + self.request_bars(bar_type) + + # Subscribe to real-time updates + # New bars will be delivered to the on_bar(...) handler in strategy + self.subscribe_bars(bar_type) + + # Register indicators to receive bar updates (they will be automatically updated) + self.register_indicator_for_bars(bar_type, self.my_indicator) +``` + +Required handlers in your strategy to receive the data: + +```python +def on_historical_data(self, data): + # Processes batches of historical bars from request_bars() + # Note: indicators registered with register_indicator_for_bars + # are updated automatically with historical data + pass + +def on_bar(self, bar): + # Processes individual bars in real-time from subscribe_bars() + # Indicators registered with this bar type will update automatically and they will be updated before this handler is called + pass +``` + +### Historical data requests with aggregation + +When requesting historical bars for backtesting or initializing indicators, you can use the `request_bars()` method, which supports both direct requests and aggregation: + +```python +# Request raw 1-minute bars (aggregated from TradeTick objects as indicated by LAST price type) +self.request_bars(BarType.from_str("6EH4.XCME-1-MINUTE-LAST-EXTERNAL")) + +# Request 5-minute bars aggregated from 1-minute bars +self.request_bars(BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL")) +``` + +If historical aggregated bars are needed, you can use specialized request `request_aggregated_bars()` method: + +```python +# Request bars that are aggregated from historical trade ticks +self.request_aggregated_bars([BarType.from_str("6EH4.XCME-100-VOLUME-LAST-INTERNAL")]) + +# Request bars that are aggregated from other bars +self.request_aggregated_bars([BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL")]) +``` + +### Common pitfalls + +**Register indicators before requesting data**: Ensure indicators are registered before requesting historical data so they get updated properly. + +```python +# Correct order +self.register_indicator_for_bars(bar_type, self.ema) +self.request_bars(bar_type) + +# Incorrect order +self.request_bars(bar_type) # Indicator won't receive historical data +self.register_indicator_for_bars(bar_type, self.ema) +``` + +### Performance considerations + +Bar aggregators track OHLC prices via the fixed-point `Price` type. Threshold comparisons for +tick and volume aggregators use integer arithmetic, while value-based and imbalance/runs aggregators +currently use `f64` for notional value and signed accumulation (these are being migrated to +fixed-point integer arithmetic). The choice of aggregation method has a modest impact on per-update +overhead: + +- **Time bars** are the most efficient for high-throughput data. The aggregator simply accumulates + OHLCV state per update; bar emission is driven by a timer rather than per-tick logic. +- **Threshold bars** (tick, volume, value) add a lightweight counter or accumulator check per update. + Volume and value bars may split a single large trade across multiple bars when it exceeds the + remaining threshold. +- **Information-driven bars** (imbalance, runs) require tracking aggressor side and signed + accumulation per update. The overhead is slightly higher than threshold bars but still minimal. +- **Renko bars** are price-driven and can emit multiple bars from a single large price move. + Otherwise the per-update cost is comparable to threshold bars. +- **Composite bars** (bar-to-bar) are the most efficient way to produce higher-timeframe bars + when lower-timeframe bars are already available, as each input bar represents an already + aggregated period rather than a single tick. + +### Time bar configuration + +Time bar behavior is controlled through `DataEngineConfig`. The following options +apply to all time-based aggregation (millisecond through year): + +| Option | Type | Default | Description | +|:------------------------------------|:-------|:--------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| `time_bars_interval_type` | `str` | `"left-open"` | `"left-open"`: start excluded, end included. `"right-open"`: start included, end excluded. | +| `time_bars_timestamp_on_close` | `bool` | `True` | When `True`, `ts_event` is the bar close time. When `False`, `ts_event` is the bar open time. | +| `time_bars_skip_first_non_full_bar` | `bool` | `False` | Skip emitting a bar when aggregation starts mid-interval, avoiding partial bars on startup. | +| `time_bars_build_with_no_updates` | `bool` | `True` | When `True`, bars are emitted even if no market updates arrived during the interval. | +| `time_bars_origin_offset` | `dict` | `None` | Maps `BarAggregation` types to `pd.Timedelta` offsets for shifting bar alignment (e.g., align to 09:30 market open). | +| `time_bars_build_delay` | `int` | `0` | Delay in microseconds before building a bar. Useful in backtests to ensure data at bar boundary timestamps is processed before the timer fires. | + +```python +from nautilus_trader.data.config import DataEngineConfig + +config = DataEngineConfig( + time_bars_timestamp_on_close=True, + time_bars_build_with_no_updates=False, + time_bars_skip_first_non_full_bar=True, +) +``` + +## Timestamps + +The platform uses two fundamental timestamp fields that appear across many objects, including market data, orders, and events. +These timestamps serve distinct purposes and help maintain precise timing information throughout the system: + +- `ts_event`: UNIX timestamp (nanoseconds) representing when an event actually occurred. +- `ts_init`: UNIX timestamp (nanoseconds) representing when Nautilus created the internal object representing that event. + +### Examples + +| **Event Type** | **`ts_event`** | **`ts_init`** | +| -----------------| ------------------------------------------------------| --------------| +| `TradeTick` | Time when trade occurred at the exchange. | Time when Nautilus received the trade data. | +| `QuoteTick` | Time when quote occurred at the exchange. | Time when Nautilus received the quote data. | +| `OrderBookDelta` | Time when order book update occurred at the exchange. | Time when Nautilus received the order book update. | +| `Bar` | Time of the bar's closing (exact minute/hour). | Time when Nautilus generated (for internal bars) or received the bar data (for external bars). | +| `OrderFilled` | Time when order was filled at the exchange. | Time when Nautilus received and processed the fill confirmation. | +| `OrderCanceled` | Time when cancellation was processed at the exchange. | Time when Nautilus received and processed the cancellation confirmation. | +| `NewsEvent` | Time when the news was published. | Time when the event object was created (if internal event) or received (if external event) in Nautilus. | +| Custom event | Time when event conditions actually occurred. | Time when the event object was created (if internal event) or received (if external event) in Nautilus. | + +:::note +The `ts_init` field represents a more general concept than "time of reception" for events. +It denotes the timestamp when an object, such as a data point or command, was initialized within Nautilus. +This distinction is important because `ts_init` is not exclusive to "received events". It applies to any internal +initialization process. + +For example, the `ts_init` field is also used for commands, where the concept of reception does not apply. +This broader definition ensures consistent handling of initialization timestamps across various object types in the system. +::: + +### Latency analysis + +The dual timestamp system enables latency analysis within the platform: + +- Latency can be calculated as `ts_init - ts_event`. +- This difference represents total system latency, including network transmission time, processing overhead, and any queueing delays. +- It's important to remember that the clocks producing these timestamps are likely not synchronized. + +### Environment-specific behavior + +#### Backtesting environment + +- Data is ordered by `ts_init` using a stable sort. +- This behavior ensures deterministic processing order and simulates realistic system behavior, including latencies. + +#### Live trading environment + +- The system processes data as it arrives to minimize latency and enable real-time decisions. + - `ts_init` field records the exact moment when data is received by Nautilus in real-time. + - `ts_event` reflects the time the event occurred externally, enabling accurate comparisons between external event timing and system reception. +- We can use the difference between `ts_init` and `ts_event` to detect network or processing delays. + +### Other notes and considerations + +- For data from external sources, `ts_init` is always the same as or later than `ts_event`. +- For data created within Nautilus, `ts_init` and `ts_event` can be the same because the object is initialized at the same time the event happens. +- Not every type with a `ts_init` field necessarily has a `ts_event` field. This reflects cases where: + - The initialization of an object happens at the same time as the event itself. + - The concept of an external event time does not apply. + +#### Persisted data + +The `ts_init` field indicates when the message was originally received. + +## Data flow + +The platform ensures consistency by flowing data through the same pathways across all system [environment contexts](architecture.md#environment-contexts) +(e.g., `backtest`, `sandbox`, `live`). Data is primarily transported via the `MessageBus` to the `DataEngine` +and then distributed to subscribed or registered handlers. + +For users who need more flexibility, the platform also supports the creation of custom data types. +For details on how to implement user-defined data types, see the [Custom Data](#custom-data) section below. + +## Loading data + +NautilusTrader supports data loading and conversion for three main use cases: + +- Providing data for a `BacktestEngine` to run backtests. +- Persisting the Nautilus-specific Parquet format for the data catalog via `ParquetDataCatalog.write_data(...)` to be later used with a `BacktestNode`. +- For research purposes (to ensure data is consistent between research and backtesting). + +Regardless of the destination, the process remains the same: converting diverse external data formats into Nautilus data structures. + +To achieve this, two main components are necessary: + +- A type of DataLoader (normally specific per raw source/format) which can read the data and return a `pd.DataFrame` with the correct schema for the desired Nautilus object. +- A type of DataWrangler (specific per data type) which takes this `pd.DataFrame` and returns a `list[Data]` of Nautilus objects. + +### Data loaders + +Data loader components are typically specific for the raw source/format and per integration. For instance, Binance order book data is stored in its raw CSV file form with +an entirely different format to [Databento Binary Encoding (DBN)](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn) files. + +### Data wranglers + +Data wranglers are implemented per specific Nautilus data type, and can be found in the `nautilus_trader.persistence.wranglers` module. +Currently there exists: + +- `OrderBookDeltaDataWrangler` +- `OrderBookDepth10DataWrangler` +- `QuoteTickDataWrangler` +- `TradeTickDataWrangler` +- `BarDataWrangler` + +:::warning +There are a number of **DataWrangler v2** components, which will take a `pd.DataFrame` typically +with a different fixed width Nautilus Arrow v2 schema, and output PyO3 Nautilus objects which are only compatible with the new version +of the Nautilus core, currently in development. + +**These PyO3 provided data objects are not compatible where the legacy Cython objects are currently used (e.g., adding directly to a `BacktestEngine`).** +::: + +### Fixed-point precision and raw values + +NautilusTrader uses fixed-point arithmetic for `Price` and `Quantity` types for precise financial calculations without floating-point errors. Understanding how raw values work is essential when creating data or working with catalogs. + +#### Raw value requirements + +When constructing `Price` or `Quantity` using `from_raw()`, the raw value **must** be a valid multiple of the scale factor for the given precision. Valid raw values should come from: + +- Accessing the `.raw` field of an existing value (e.g., `price.raw`). +- Using the Nautilus fixed-point conversion functions. +- Values from Nautilus-produced Arrow data. + +:::warning +Raw values that are not valid multiples will cause a panic. The raw value must be divisible by `10^(FIXED_PRECISION - precision)` where `FIXED_PRECISION` is 9 (standard mode) or 16 (high-precision mode). +::: + +#### Legacy catalog data and floating-point errors + +Data written to catalogs using V2 wranglers before 16th December 2025 may contain raw values with floating-point precision errors. This occurred because the wranglers used: + +```python +int(value * FIXED_SCALAR) # Introduces floating-point errors +``` + +For example, `int(0.67068 * 1e9)` produces `670680000000001` instead of the expected `670680000000000`. The correct approach is: + +```python +round(value * 10**precision) * scale # Correct precision-aware conversion +``` + +#### Automatic correction during catalog reads + +To maintain backward compatibility with existing catalog data, the Arrow decode path automatically corrects raw values by rounding them to the nearest valid multiple. This ensures that legacy catalogs continue to work without requiring data migration. + +:::note +This automatic correction adds a small amount of overhead during data decoding. In a future version, once catalogs have been repaired or migrated, this correction will become opt-in. A catalog repair/migration script may be provided to permanently fix legacy data. +::: + +### Transformation pipeline + +**Process flow**: + +1. Raw data (e.g., CSV) is input into the pipeline. +2. DataLoader processes the raw data and converts it into a `pd.DataFrame`. +3. DataWrangler further processes the `pd.DataFrame` to generate a list of Nautilus objects. +4. The Nautilus `list[Data]` is the output of the data loading process. + +The following diagram illustrates how raw data is transformed into Nautilus data structures: + +```mermaid +flowchart LR + raw["Raw data (CSV)"] + loader[DataLoader] + wrangler[DataWrangler] + output["Nautilus list[Data]"] + + raw --> loader + loader -->|"pd.DataFrame"| wrangler + wrangler --> output +``` + +Concretely, this would involve: + +- `BinanceOrderBookDeltaDataLoader.load(...)` which reads CSV files provided by Binance from disk, and returns a `pd.DataFrame`. +- `OrderBookDeltaDataWrangler.process(...)` which takes the `pd.DataFrame` and returns `list[OrderBookDelta]`. + +The following example shows how to accomplish the above in Python: + +```python +from nautilus_trader import TEST_DATA_DIR +from nautilus_trader.adapters.binance.loaders import BinanceOrderBookDeltaDataLoader +from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler +from nautilus_trader.test_kit.providers import TestInstrumentProvider + + +# Load raw data +data_path = TEST_DATA_DIR / "binance" / "btcusdt-depth-snap.csv" +df = BinanceOrderBookDeltaDataLoader.load(data_path) + +# Set up a wrangler +instrument = TestInstrumentProvider.btcusdt_binance() +wrangler = OrderBookDeltaDataWrangler(instrument) + +# Process to a list `OrderBookDelta` Nautilus objects +deltas = wrangler.process(df) +``` + +## Data catalog + +The data catalog is a central store for Nautilus data, persisted in the [Parquet](https://parquet.apache.org) file format. It is the primary data management system for both backtesting and live trading scenarios, providing efficient storage, retrieval, and streaming capabilities for market data. + +### Overview and architecture + +The NautilusTrader data catalog is built on a dual-backend architecture that combines the performance of Rust with the flexibility of Python: + +**Core components:** + +- **ParquetDataCatalog**: The main Python interface for data operations. +- **Rust backend**: High-performance query engine for core data types (OrderBookDelta, QuoteTick, TradeTick, Bar, MarkPriceUpdate). +- **PyArrow backend**: Flexible fallback for custom data types and advanced filtering. +- **fsspec integration**: Support for local and cloud storage (S3, GCS, Azure, etc.). + +**Key benefits**: + +- **Performance**: Rust backend provides optimized query performance for core market data types. +- **Flexibility**: PyArrow backend handles custom data types and complex filtering scenarios. +- **Scalability**: Efficient compression and columnar storage reduce storage costs and improve I/O performance. +- **Cloud native**: Built-in support for cloud storage providers through fsspec. +- **No dependencies**: Self-contained solution requiring no external databases or services. + +**Storage format advantages:** + +- Superior compression ratio and read performance compared to CSV/JSON/HDF5. +- Columnar storage enables efficient filtering and aggregation. +- Schema evolution support for data model changes. +- Cross-language compatibility (Python, Rust, Java, C++, etc.). + +The Arrow schemas used for the Parquet format are primarily single-sourced in the core `persistence` Rust crate, with some legacy schemas available from the `/serialization/arrow/schema.py` module. + +:::note +The current plan is to eventually phase out the Python schemas module, so that all schemas are single sourced in the Rust core for consistency and performance. +::: + +### Initializing + +The data catalog can be initialized from a `NAUTILUS_PATH` environment variable, or by explicitly passing in a path like object. + +:::note[NAUTILUS_PATH environment variable] +The `NAUTILUS_PATH` environment variable should point to the **root** directory containing your Nautilus data. The catalog will automatically append `/catalog` to this path. + +For example: + +- If `NAUTILUS_PATH=/home/user/trading_data`. +- Then the catalog will be located at `/home/user/trading_data/catalog`. + +This is a common pattern when using `ParquetDataCatalog.from_env()` - make sure your `NAUTILUS_PATH` points to the parent directory, not the catalog directory itself. +::: + +The following example shows how to initialize a data catalog where there is pre-existing data already written to disk at the given path. + +```python +from pathlib import Path +from nautilus_trader.persistence.catalog import ParquetDataCatalog + + +CATALOG_PATH = Path.cwd() / "catalog" + +# Create a new catalog instance +catalog = ParquetDataCatalog(CATALOG_PATH) + +# Alternative: Environment-based initialization +catalog = ParquetDataCatalog.from_env() # Uses NAUTILUS_PATH environment variable +``` + +### Filesystem protocols and storage options + +The catalog supports multiple filesystem protocols through fsspec integration, working across local and cloud storage systems. + +#### Supported filesystem protocols + +**Local filesystem (`file`):** + +```python +catalog = ParquetDataCatalog( + path="/path/to/catalog", + fs_protocol="file", # Default protocol +) +``` + +**Amazon S3 (`s3`):** + +```python +catalog = ParquetDataCatalog( + path="s3://my-bucket/nautilus-data/", + fs_protocol="s3", + fs_storage_options={ + "key": "your-access-key-id", + "secret": "your-secret-access-key", + "endpoint_url": "https://s3.amazonaws.com", # Optional custom endpoint + } +) +``` + +**Google Cloud Storage (`gcs`):** + +```python +catalog = ParquetDataCatalog( + path="gcs://my-bucket/nautilus-data/", + fs_protocol="gcs", + fs_storage_options={ + "project": "my-project-id", + "token": "/path/to/service-account.json", # Or "cloud" for default credentials + } +) +``` + +**Azure Blob Storage :** + +`abfs` protocol + +```python +catalog = ParquetDataCatalog( + path="abfs://container@account.dfs.core.windows.net/nautilus-data/", + fs_protocol="abfs", + fs_storage_options={ + "account_name": "your-storage-account", + "account_key": "your-account-key", + # Or use SAS token: "sas_token": "your-sas-token" + } +) +``` + +`az` protocol + +```python +catalog = ParquetDataCatalog( + path="az://container/nautilus-data/", + fs_protocol="az", + fs_storage_options={ + "account_name": "your-storage-account", + "account_key": "your-account-key", + # Or use SAS token: "sas_token": "your-sas-token" + } +) +``` + +#### URI-based initialization + +For convenience, you can use URI strings that automatically parse protocol and storage options: + +```python +# Local filesystem +catalog = ParquetDataCatalog.from_uri("/path/to/catalog") + +# S3 bucket +catalog = ParquetDataCatalog.from_uri("s3://my-bucket/nautilus-data/") + +# With storage options +catalog = ParquetDataCatalog.from_uri( + "s3://my-bucket/nautilus-data/", + fs_storage_options={ + "access_key_id": "your-key", + "secret_access_key": "your-secret" + } +) +``` + +### Writing data + +Store data in the catalog using the `write_data()` method. All Nautilus built-in `Data` objects are supported, and any data which inherits from `Data` can be written. + +```python +# Write a list of data objects +catalog.write_data(quote_ticks) + +# Write with custom timestamp range +catalog.write_data( + trade_ticks, + start=1704067200000000000, # Optional start timestamp override (UNIX nanoseconds) + end=1704153600000000000, # Optional end timestamp override (UNIX nanoseconds) +) + +# Skip disjoint check for overlapping data +catalog.write_data(bars, skip_disjoint_check=True) +``` + +### File naming and data organization + +The catalog automatically generates filenames based on the timestamp range of the data being written. Files are named using the pattern `{start_timestamp}_{end_timestamp}.parquet` where timestamps are in ISO format. + +Data is organized in directories by data type and instrument ID: + +``` +catalog/ +├── data/ +│ ├── quote_ticks/ +│ │ └── eurusd.sim/ +│ │ └── 20240101T000000000000000_20240101T235959999999999.parquet +│ └── trade_ticks/ +│ └── btcusd.binance/ +│ └── 20240101T000000000000000_20240101T235959999999999.parquet +``` + +**Rust backend data types (enhanced performance):** + +The following data types use optimized Rust implementations: + +- `OrderBookDelta`. +- `OrderBookDeltas`. +- `OrderBookDepth10`. +- `QuoteTick`. +- `TradeTick`. +- `Bar`. +- `MarkPriceUpdate`. + +:::warning +By default, data that overlaps with existing files will cause an assertion error to maintain data integrity. Use `skip_disjoint_check=True` in `write_data()` to bypass this check when needed. +::: + +### Reading data + +Use the `query()` method to read data back from the catalog: + +```python +from nautilus_trader.model import QuoteTick, TradeTick + +# Query quote ticks for a specific instrument and time range +quotes = catalog.query( + data_cls=QuoteTick, + identifiers=["EUR/USD.SIM"], + start="2024-01-01T00:00:00Z", + end="2024-01-02T00:00:00Z" +) + +# Query trade ticks with filtering +trades = catalog.query( + data_cls=TradeTick, + identifiers=["BTC/USD.BINANCE"], + start="2024-01-01", + end="2024-01-02", + where="price > 50000" +) +``` + +### `BacktestDataConfig` - data specification for backtests + +The `BacktestDataConfig` class is the primary mechanism for specifying data requirements before a backtest starts. It defines what data should be loaded from the catalog and how it should be filtered and processed during the backtest execution. + +#### Core parameters + +**Required parameters:** + +- `catalog_path`: Path to the data catalog directory. +- `data_cls`: The data type class (e.g., QuoteTick, TradeTick, OrderBookDelta, Bar). + +**Optional parameters:** + +- `catalog_fs_protocol`: Filesystem protocol ('file', 's3', 'gcs', etc.). +- `catalog_fs_storage_options`: Storage-specific options (credentials, region, etc.). +- `instrument_id`: Specific instrument to load data for. +- `instrument_ids`: List of instruments (alternative to single instrument_id). +- `start_time`: Start time for data filtering (ISO string or UNIX nanoseconds). +- `end_time`: End time for data filtering (ISO string or UNIX nanoseconds). +- `filter_expr`: Additional PyArrow filter expressions. +- `client_id`: Client ID for custom data types. +- `metadata`: Additional metadata for data queries. +- `bar_spec`: Bar specification for bar data (e.g., "1-MINUTE-LAST"). +- `bar_types`: List of bar types (alternative to bar_spec). + +#### Basic usage examples + +**Loading quote ticks:** + +```python +from nautilus_trader.config import BacktestDataConfig +from nautilus_trader.model import QuoteTick, InstrumentId + +data_config = BacktestDataConfig( + catalog_path="/path/to/catalog", + data_cls=QuoteTick, + instrument_id=InstrumentId.from_str("EUR/USD.SIM"), + start_time="2024-01-01T00:00:00Z", + end_time="2024-01-02T00:00:00Z", +) +``` + +**Loading multiple instruments:** + +```python +data_config = BacktestDataConfig( + catalog_path="/path/to/catalog", + data_cls=TradeTick, + instrument_ids=["BTC/USD.BINANCE", "ETH/USD.BINANCE"], + start_time="2024-01-01T00:00:00Z", + end_time="2024-01-02T00:00:00Z", +) +``` + +**Loading Bar Data:** + +```python +data_config = BacktestDataConfig( + catalog_path="/path/to/catalog", + data_cls=Bar, + instrument_id=InstrumentId.from_str("AAPL.NASDAQ"), + bar_spec="5-MINUTE-LAST", + start_time="2024-01-01", + end_time="2024-01-31", +) +``` + +#### Advanced configuration examples + +**Cloud Storage with Custom Filtering:** + +```python +data_config = BacktestDataConfig( + catalog_path="s3://my-bucket/nautilus-data/", + catalog_fs_protocol="s3", + catalog_fs_storage_options={ + "key": "your-access-key", + "secret": "your-secret-key", + "region": "us-east-1" + }, + data_cls=OrderBookDelta, + instrument_id=InstrumentId.from_str("BTC/USD.COINBASE"), + start_time="2024-01-01T09:30:00Z", + end_time="2024-01-01T16:00:00Z", +) +``` + +**Custom Data with Client ID:** + +```python +data_config = BacktestDataConfig( + catalog_path="/path/to/catalog", + data_cls="my_package.data.NewsEventData", + client_id="NewsClient", + metadata={"source": "reuters", "category": "earnings"}, + start_time="2024-01-01", + end_time="2024-01-31", +) +``` + +#### Integration with BacktestRunConfig + +The `BacktestDataConfig` objects are integrated into the backtesting framework through `BacktestRunConfig`: + +```python +from nautilus_trader.config import BacktestRunConfig, BacktestVenueConfig + +# Define multiple data configurations +data_configs = [ + BacktestDataConfig( + catalog_path="/path/to/catalog", + data_cls=QuoteTick, + instrument_id="EUR/USD.SIM", + start_time="2024-01-01", + end_time="2024-01-02", + ), + BacktestDataConfig( + catalog_path="/path/to/catalog", + data_cls=TradeTick, + instrument_id="EUR/USD.SIM", + start_time="2024-01-01", + end_time="2024-01-02", + ), +] + +# Create backtest run configuration +run_config = BacktestRunConfig( + venues=[BacktestVenueConfig(name="SIM", oms_type="HEDGING")], + data=data_configs, # List of data configurations + start="2024-01-01T00:00:00Z", + end="2024-01-02T00:00:00Z", +) +``` + +#### Data loading process + +When a backtest runs, the `BacktestNode` processes each `BacktestDataConfig`: + +1. **Catalog Loading**: Creates a `ParquetDataCatalog` instance from the config. +2. **Query Construction**: Builds query parameters from config attributes. +3. **Data Retrieval**: Executes catalog queries using the appropriate backend. +4. **Instrument Loading**: Loads instrument definitions if needed. +5. **Engine Integration**: Adds data to the backtest engine with proper sorting. + +The system automatically handles: + +- Instrument ID resolution and validation. +- Data type validation and conversion. +- Memory-efficient streaming for large datasets. +- Error handling and logging. + +### DataCatalogConfig - on-the-fly data loading + +The `DataCatalogConfig` class provides configuration for on-the-fly data loading scenarios, particularly useful for backtests where the number of possible instruments is vast, +Unlike `BacktestDataConfig` which pre-specifies data for backtests, `DataCatalogConfig` enables flexible catalog access during runtime. +Catalogs defined this way can also be used for requesting historical data. + +#### Core parameters + +**Required Parameters:** + +- `path`: Path to the data catalog directory. + +**Optional Parameters:** + +- `fs_protocol`: Filesystem protocol ('file', 's3', 'gcs', 'azure', etc.). +- `fs_storage_options`: Protocol-specific storage options. +- `name`: Optional name identifier for the catalog configuration. + +#### Basic usage examples + +**Local Catalog Configuration:** + +```python +from nautilus_trader.persistence.config import DataCatalogConfig + +catalog_config = DataCatalogConfig( + path="/path/to/catalog", + fs_protocol="file", + name="local_market_data" +) + +# Convert to catalog instance +catalog = catalog_config.as_catalog() +``` + +**Cloud storage configuration:** + +```python +catalog_config = DataCatalogConfig( + path="s3://my-bucket/market-data/", + fs_protocol="s3", + fs_storage_options={ + "key": "your-access-key", + "secret": "your-secret-key", + "region": "us-west-2", + "endpoint_url": "https://s3.us-west-2.amazonaws.com" + }, + name="cloud_market_data" +) +``` + +#### Integration with live trading + +`DataCatalogConfig` is commonly used in live trading configurations for historical data access: + +```python +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.persistence.config import DataCatalogConfig + +# Configure catalog for live system +catalog_config = DataCatalogConfig( + path="/data/nautilus/catalog", + fs_protocol="file", + name="historical_data" +) + +# Use in trading node configuration +node_config = TradingNodeConfig( + # ... other configurations + catalog=catalog_config, # Enable historical data access +) +``` + +#### Streaming configuration + +For streaming data to catalogs during live trading or backtesting, use `StreamingConfig`: + +```python +from nautilus_trader.persistence.config import StreamingConfig, RotationMode +import pandas as pd + +streaming_config = StreamingConfig( + catalog_path="/path/to/streaming/catalog", + fs_protocol="file", + flush_interval_ms=1000, # Flush every second + replace_existing=False, + rotation_mode=RotationMode.DAILY, + rotation_interval=pd.Timedelta(hours=1), + max_file_size=1024 * 1024 * 100, # 100MB max file size +) +``` + +#### Use cases + +**Historical Data Analysis:** + +- Load historical data during live trading for strategy calculations. +- Access reference data for instrument lookups. +- Retrieve past performance metrics. + +**Dynamic data loading:** + +- Load data based on runtime conditions. +- Implement custom data loading strategies. +- Support multiple catalog sources. + +**Research and development:** + +- Interactive data exploration in Jupyter notebooks. +- Ad-hoc analysis and backtesting. +- Data quality validation and monitoring. + +### Query system and dual backend architecture + +The catalog's query system uses a dual-backend architecture that selects the query engine based on data type and query parameters. + +#### Backend selection logic + +**Rust backend (high performance):** + +- **Supported Types**: OrderBookDelta, OrderBookDeltas, OrderBookDepth10, QuoteTick, TradeTick, Bar, MarkPriceUpdate. +- **Conditions**: Used when `files` parameter is None (automatic file discovery). +- **Benefits**: Optimized performance, memory efficiency, native Arrow integration. + +**PyArrow backend (flexible):** + +- **Supported Types**: All data types including custom data classes. +- **Conditions**: Used for custom data types or when `files` parameter is specified. +- **Benefits**: Advanced filtering, custom data support, complex query expressions. + +#### Query methods and parameters + +**Core query parameters:** + +```python +catalog.query( + data_cls=QuoteTick, # Data type to query + identifiers=["EUR/USD.SIM"], # Instrument identifiers + start="2024-01-01T00:00:00Z", # Start time (various formats supported) + end="2024-01-02T00:00:00Z", # End time + where="bid > 1.1000", # PyArrow filter expression + files=None, # Specific files (forces PyArrow backend) +) +``` + +**Time format support:** + +- ISO 8601 strings: `"2024-01-01T00:00:00Z"`. +- UNIX nanoseconds: `1704067200000000000` (or ISO format: `"2024-01-01T00:00:00Z"`). +- Pandas Timestamps: `pd.Timestamp("2024-01-01", tz="UTC")`. +- Python datetime objects (timezone-aware recommended). + +**Advanced filtering examples:** + +```python +# Complex PyArrow expressions +catalog.query( + data_cls=TradeTick, + identifiers=["BTC/USD.BINANCE"], + where="price > 50000 AND size > 1.0", + start="2024-01-01", + end="2024-01-02", +) + +# Multiple instruments with metadata filtering +catalog.query( + data_cls=Bar, + identifiers=["AAPL.NASDAQ", "MSFT.NASDAQ"], + where="volume > 1000000", + metadata={"bar_type": "1-MINUTE-LAST"}, +) +``` + +### Catalog operations + +The catalog provides several operation functions for maintaining and organizing data files. These operations help optimize storage, improve query performance, and maintain data integrity. + +#### Reset file names + +Reset parquet file names to match their actual content timestamps. This ensures filename-based filtering works correctly. + +**Reset all files in catalog:** + +```python +# Reset all parquet files in the catalog +catalog.reset_all_file_names() +``` + +**Reset specific data type:** + +```python +# Reset filenames for all quote tick files +catalog.reset_data_file_names(QuoteTick) + +# Reset filenames for specific instrument's trade files +catalog.reset_data_file_names(TradeTick, "BTC/USD.BINANCE") +``` + +#### Consolidate catalog + +Combine multiple small parquet files into larger files to improve query performance and reduce storage overhead. + +**Consolidate entire catalog:** + +```python +# Consolidate all files in the catalog +catalog.consolidate_catalog() + +# Consolidate files within a specific time range +catalog.consolidate_catalog( + start="2024-01-01T00:00:00Z", + end="2024-01-02T00:00:00Z", + ensure_contiguous_files=True +) +``` + +**Consolidate specific data type:** + +```python +# Consolidate all quote tick files +catalog.consolidate_data(QuoteTick) + +# Consolidate specific instrument's files +catalog.consolidate_data( + TradeTick, + identifier="BTC/USD.BINANCE", + start="2024-01-01", + end="2024-01-31" +) +``` + +#### Consolidate catalog by period + +Split data files into fixed time periods for standardized file organization. + +**Consolidate entire catalog by period:** + +```python +import pandas as pd + +# Consolidate all files by 1-day periods +catalog.consolidate_catalog_by_period( + period=pd.Timedelta(days=1) +) + +# Consolidate by 1-hour periods within time range +catalog.consolidate_catalog_by_period( + period=pd.Timedelta(hours=1), + start="2024-01-01T00:00:00Z", + end="2024-01-02T00:00:00Z" +) +``` + +**Consolidate specific data by period:** + +```python +# Consolidate quote data by 4-hour periods +catalog.consolidate_data_by_period( + data_cls=QuoteTick, + period=pd.Timedelta(hours=4) +) + +# Consolidate specific instrument by 30-minute periods +catalog.consolidate_data_by_period( + data_cls=TradeTick, + identifier="EUR/USD.SIM", + period=pd.Timedelta(minutes=30), + start="2024-01-01", + end="2024-01-31" +) +``` + +#### Delete data range + +Remove data within a specified time range for specific data types and instruments. This operation permanently deletes data and handles file intersections intelligently. + +**Delete entire catalog range:** + +```python +# Delete all data within a time range across the entire catalog +catalog.delete_catalog_range( + start="2024-01-01T00:00:00Z", + end="2024-01-02T00:00:00Z" +) + +# Delete all data from the beginning up to a specific time +catalog.delete_catalog_range(end="2024-01-01T00:00:00Z") +``` + +**Delete specific data type:** + +```python +# Delete all quote tick data for a specific instrument +catalog.delete_data_range( + data_cls=QuoteTick, + identifier="BTC/USD.BINANCE" +) + +# Delete trade data within a specific time range +catalog.delete_data_range( + data_cls=TradeTick, + identifier="EUR/USD.SIM", + start="2024-01-01T00:00:00Z", + end="2024-01-31T23:59:59Z" +) +``` + +:::warning +Delete operations permanently remove data and cannot be undone. Files that partially overlap the deletion range are split to preserve data outside the range. +::: + +### Feather streaming and conversion + +The catalog supports streaming data to temporary feather files during backtests, which can then be converted to permanent parquet format for efficient querying. + +**Example: option greeks streaming** + +```python +from option_trader.greeks import GreeksData +from nautilus_trader.persistence.config import StreamingConfig + +# 1. Configure streaming for custom data +streaming = StreamingConfig( + catalog_path=catalog.path, + include_types=[GreeksData], + flush_interval_ms=1000, +) + +# 2. Run backtest with streaming enabled +engine_config = BacktestEngineConfig(streaming=streaming) +results = node.run() + +# 3. Convert streamed data to permanent catalog +catalog.convert_stream_to_data( + results[0].instance_id, + GreeksData, +) + +# 4. Query converted data +greeks_data = catalog.query( + data_cls=GreeksData, + start="2024-01-01", + end="2024-01-31", + where="delta > 0.5", +) +``` + +### Catalog summary + +The NautilusTrader data catalog provides market data management: + +**Core features**: + +- **Dual Backend**: Rust performance + Python flexibility. +- **Multi-Protocol**: Local, S3, GCS, Azure storage. +- **Streaming**: Feather → Parquet conversion pipeline. +- **Operations**: Reset file names, consolidate data, period-based organization. + +**Key use cases**: + +- **Backtesting**: Pre-configured data loading via BacktestDataConfig. +- **Live Trading**: On-demand data access via DataCatalogConfig. +- **Maintenance**: File consolidation and organization operations. +- **Research**: Interactive querying and analysis. + +## Data migrations + +NautilusTrader defines an internal data format specified in the `nautilus_model` crate. +These models are serialized into Arrow record batches and written to Parquet files. +Nautilus backtesting is most efficient when using these Nautilus-format Parquet files. + +However, migrating the data model between [precision modes](../getting_started/installation.md#precision-mode) and schema changes can be challenging. +This guide explains how to handle data migrations using our utility tools. + +### Migration tools + +The `nautilus_persistence` crate provides two key utilities: + +#### `to_json` + +Converts Parquet files to JSON while preserving metadata: + +- Creates two files: + + - `.json`: Contains the deserialized data + - `.metadata.json`: Contains schema metadata and row group configuration + +- Automatically detects data type from filename: + + - `OrderBookDelta` (contains "deltas" or "order_book_delta") + - `QuoteTick` (contains "quotes" or "quote_tick") + - `TradeTick` (contains "trades" or "trade_tick") + - `Bar` (contains "bars") + +#### `to_parquet` + +Converts JSON back to Parquet format: + +- Reads both the data JSON and metadata JSON files. +- Preserves row group sizes from original metadata. +- Uses ZSTD compression. +- Creates `.parquet`. + +### Migration process + +The following migration examples both use trades data (you can also migrate the other data types in the same way). +All commands should be run from the root of the `persistence` crate directory. + +#### Migrating from standard-precision (64-bit) to high-precision (128-bit) + +This example describes a scenario where you want to migrate from standard-precision schema to high-precision schema. + +:::note +If you're migrating from a catalog that used the `Int64` and `UInt64` Arrow data types for prices and sizes, +be sure to check out commit [e284162](https://github.com/nautechsystems/nautilus_trader/commit/e284162cf27a3222115aeb5d10d599c8cf09cf50) +**before** compiling the code that writes the initial JSON. +::: + +**1. Convert from standard-precision Parquet to JSON**: + +```bash +cargo run --bin to_json trades.parquet +``` + +This will create `trades.json` and `trades.metadata.json` files. + +**2. Convert from JSON to high-precision Parquet**: + +Add the `--features high-precision` flag to write data as high-precision (128-bit) schema Parquet. + +```bash +cargo run --features high-precision --bin to_parquet trades.json +``` + +This will create a `trades.parquet` file with high-precision schema data. + +#### Migrating schema changes + +This example describes a scenario where you want to migrate from one schema version to another. + +**1. Convert from old schema Parquet to JSON**: + +Add the `--features high-precision` flag if the source data uses a high-precision (128-bit) schema. + +```bash +cargo run --bin to_json trades.parquet +``` + +This will create `trades.json` and `trades.metadata.json` files. + +**2. Switch to new schema version**: + +```bash +git checkout +``` + +**3. Convert from JSON back to new schema Parquet**: + +```bash +cargo run --features high-precision --bin to_parquet trades.json +``` + +This will create a `trades.parquet` file with the new schema. + +### Best practices + +- Always test migrations with a small dataset first. +- Maintain backups of original files. +- Verify data integrity after migration. +- Perform migrations in a staging environment before applying them to production data. + +## Custom data + +Due to the modular nature of the Nautilus design, it is possible to set up systems +with very flexible data streams, including custom user-defined data types. This +guide covers some possible use cases for this functionality. + +It's possible to create custom data types within the Nautilus system. First you +will need to define your data by subclassing from `Data`. + +:::info +As `Data` holds no state, it is not strictly necessary to call `super().__init__()`. +::: + +```python +from nautilus_trader.core import Data + + +class MyDataPoint(Data): + """ + This is an example of a user-defined data class, inheriting from the base class `Data`. + + The fields `label`, `x`, `y`, and `z` in this class are examples of arbitrary user data. + """ + + def __init__( + self, + label: str, + x: int, + y: int, + z: int, + ts_event: int, + ts_init: int, + ) -> None: + self.label = label + self.x = x + self.y = y + self.z = z + self._ts_event = ts_event + self._ts_init = ts_init + + @property + def ts_event(self) -> int: + """ + UNIX timestamp (nanoseconds) when the data event occurred. + + Returns + ------- + int + + """ + return self._ts_event + + @property + def ts_init(self) -> int: + """ + UNIX timestamp (nanoseconds) when the object was initialized. + + Returns + ------- + int + + """ + return self._ts_init +``` + +The `Data` abstract base class acts as a contract within the system and requires two properties +for all types of data: `ts_event` and `ts_init`. These represent the UNIX nanosecond timestamps +for when the event occurred and when the object was initialized, respectively. + +The recommended approach to satisfy the contract is to assign `ts_event` and `ts_init` +to backing fields, and then implement the `@property` for each as shown above +(for completeness, the docstrings are copied from the `Data` base class). + +:::info +These timestamps enable Nautilus to correctly order data streams for backtests +using monotonically increasing `ts_init` UNIX nanoseconds. +::: + +We can now work with this data type for backtesting and live trading. For instance, +we could now create an adapter which is able to parse and create objects of this +type - and send them back to the `DataEngine` for consumption by subscribers. + +You can publish a custom data type within your actor/strategy using the message bus +in the following way: + +```python +self.publish_data( + DataType(MyDataPoint, metadata={"some_optional_category": 1}), + MyDataPoint(...), +) +``` + +The `metadata` dictionary optionally adds more granular information that is used in the +topic name to publish data with the message bus. + +Extra metadata information can also be passed to a `BacktestDataConfig` configuration object in order to +enrich and describe custom data objects used in a backtesting context: + +```python +from nautilus_trader.config import BacktestDataConfig + +data_config = BacktestDataConfig( + catalog_path=str(catalog.path), + data_cls=MyDataPoint, + metadata={"some_optional_category": 1}, +) +``` + +You can subscribe to custom data types within your actor/strategy in the following way: + +```python +self.subscribe_data( + data_type=DataType(MyDataPoint, + metadata={"some_optional_category": 1}), + client_id=ClientId("MY_ADAPTER"), +) +``` + +The `client_id` provides an identifier to route the data subscription to a specific client. + +This will result in your actor/strategy passing these received `MyDataPoint` +objects to your `on_data` method. You will need to check the type, as this +method acts as a flexible handler for all custom data. + +```python +def on_data(self, data: Data) -> None: + # First check the type of data + if isinstance(data, MyDataPoint): + # Do something with the data +``` + +### Publishing and receiving signal data + +Here is an example of publishing and receiving signal data using the `MessageBus` from an actor or strategy. +A signal is an automatically generated custom data identified by a name containing only one value of a basic type +(str, float, int, bool or bytes). + +```python +self.publish_signal("signal_name", value, ts_event) +self.subscribe_signal("signal_name") + +def on_signal(self, signal): + print("Signal", data) +``` + +### Option greeks example + +This example demonstrates how to create a custom data type for option Greeks, specifically the delta. +By following these steps, you can create custom data types, subscribe to them, publish them, and store +them in the `Cache` or `ParquetDataCatalog` for efficient retrieval. + +```python +import msgspec +from nautilus_trader.core import Data +from nautilus_trader.core.datetime import unix_nanos_to_iso8601 +from nautilus_trader.model import DataType +from nautilus_trader.serialization.base import register_serializable_type +from nautilus_trader.serialization.arrow.serializer import register_arrow +import pyarrow as pa + +from nautilus_trader.model import InstrumentId +from nautilus_trader.core.datetime import dt_to_unix_nanos, unix_nanos_to_dt, format_iso8601 + + +class GreeksData(Data): + def __init__( + self, instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX"), + ts_event: int = 0, + ts_init: int = 0, + delta: float = 0.0, + ) -> None: + self.instrument_id = instrument_id + self._ts_event = ts_event + self._ts_init = ts_init + self.delta = delta + + def __repr__(self): + return (f"GreeksData(ts_init={unix_nanos_to_iso8601(self._ts_init)}, instrument_id={self.instrument_id}, delta={self.delta:.2f})") + + @property + def ts_event(self): + return self._ts_event + + @property + def ts_init(self): + return self._ts_init + + def to_dict(self): + return { + "instrument_id": self.instrument_id.value, + "ts_event": self._ts_event, + "ts_init": self._ts_init, + "delta": self.delta, + } + + @classmethod + def from_dict(cls, data: dict): + return GreeksData(InstrumentId.from_str(data["instrument_id"]), data["ts_event"], data["ts_init"], data["delta"]) + + def to_bytes(self): + return msgspec.msgpack.encode(self.to_dict()) + + @classmethod + def from_bytes(cls, data: bytes): + return cls.from_dict(msgspec.msgpack.decode(data)) + + def to_catalog(self): + return pa.RecordBatch.from_pylist([self.to_dict()], schema=GreeksData.schema()) + + @classmethod + def from_catalog(cls, table: pa.Table): + return [GreeksData.from_dict(d) for d in table.to_pylist()] + + @classmethod + def schema(cls): + return pa.schema( + { + "instrument_id": pa.string(), + "ts_event": pa.int64(), + "ts_init": pa.int64(), + "delta": pa.float64(), + } + ) +``` + +#### Publishing and receiving data + +Here is an example of publishing and receiving data using the `MessageBus` from an actor or strategy: + +```python +register_serializable_type(GreeksData, GreeksData.to_dict, GreeksData.from_dict) + +def publish_greeks(self, greeks_data: GreeksData): + self.publish_data(DataType(GreeksData), greeks_data) + +def subscribe_to_greeks(self): + self.subscribe_data(DataType(GreeksData)) + +def on_data(self, data): + if isinstance(data, GreeksData): + print("Data", data) +``` + +#### Writing and reading data using the cache + +Here is an example of writing and reading data using the `Cache` from an actor or strategy: + +```python +def greeks_key(instrument_id: InstrumentId): + return f"{instrument_id}_GREEKS" + +def cache_greeks(self, greeks_data: GreeksData): + self.cache.add(greeks_key(greeks_data.instrument_id), greeks_data.to_bytes()) + +def greeks_from_cache(self, instrument_id: InstrumentId): + return GreeksData.from_bytes(self.cache.get(greeks_key(instrument_id))) +``` + +#### Writing and reading data using a catalog + +For streaming custom data to feather files or writing it to parquet files in a catalog +(`register_arrow` needs to be used): + +```python +register_arrow(GreeksData, GreeksData.schema(), GreeksData.to_catalog, GreeksData.from_catalog) + +from nautilus_trader.persistence.catalog import ParquetDataCatalog +catalog = ParquetDataCatalog('.') + +catalog.write_data([GreeksData()]) +``` + +### Creating a custom data class automatically + +The `@customdataclass` decorator enables the creation of a custom data class with default +implementations for all the features described above. + +Each method can also be overridden if needed. Here is an example of its usage: + +```python +from nautilus_trader.model.custom import customdataclass + + +@customdataclass +class GreeksTestData(Data): + instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX") + delta: float = 0.0 + + +GreeksTestData( + instrument_id=InstrumentId.from_str("CL.GLBX"), + delta=1000.0, + ts_event=1, + ts_init=2, +) +``` + +#### Python-only custom data with the PyO3 catalog + +To use custom data with the Rust-backed catalog (`ParquetDataCatalogV2` from `nautilus_pyo3`), use the +`@customdataclass_pyo3()` decorator instead of `@customdataclass`. This adds the methods the Rust catalog +expects (JSON and Arrow IPC serialization). After defining your class, register it once. You can pass either +the **type** (recommended) or a **sample instance**: + +```python +from nautilus_trader.core.nautilus_pyo3 import ParquetDataCatalogV2 +from nautilus_trader.core.nautilus_pyo3.model import register_custom_data_class +from nautilus_trader.model.custom import customdataclass_pyo3 + + +@customdataclass_pyo3() +class MarketTickPython: + symbol: str = "" + price: float = 0.0 + volume: int = 0 + + +# Register by type (no instance needed; call once, e.g. at startup) +register_custom_data_class(MarketTickPython) + +catalog = ParquetDataCatalogV2("/path/to/catalog") +catalog.write_custom_data([MarketTickPython(1, 1, "AAPL", 150.5, 1000)]) +result = catalog.query("MarketTickPython", None, None, None, None, None, True) +``` + +See `nautilus_trader.model.custom.customdataclass_pyo3` for details. + +#### Custom data type stub + +For better IDE code suggestions, you can create a `.pyi` +stub file with the proper constructor signature for your custom data types as well as type hints for attributes. +This is particularly useful when the constructor is dynamically generated at runtime, as it allows the IDE to recognize +and provide suggestions for the class's methods and attributes. + +For instance, if you have a custom data class defined in `greeks.py`, you can create a corresponding `greeks.pyi` file +with the following constructor signature: + +```python +from nautilus_trader.core import Data +from nautilus_trader.model import InstrumentId + + +class GreeksData(Data): + instrument_id: InstrumentId + delta: float + + def __init__( + self, + ts_event: int = 0, + ts_init: int = 0, + instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX"), + delta: float = 0.0, + ) -> GreeksData: ... +``` + +## Related guides + +- [Instruments](instruments.md) - Financial instruments referenced by data. +- [Options](options.md) - Option instruments, chain subscriptions, and strike filtering. +- [Greeks](greeks.md) - Venue-provided and locally computed option Greeks. +- [Cache](cache.md) - Data storage and retrieval. +- [Adapters](adapters.md) - Data sources and connectivity. diff --git a/nautilus-docs/docs/concepts/execution.md b/nautilus-docs/docs/concepts/execution.md new file mode 100644 index 0000000..b743041 --- /dev/null +++ b/nautilus-docs/docs/concepts/execution.md @@ -0,0 +1,504 @@ +# Execution + +NautilusTrader can handle trade execution and order management for multiple strategies and venues +simultaneously (per instance). Several interacting components are involved in execution, making it +important to understand the possible flows of execution messages (commands and events). + +The main execution-related components include: + +- `Strategy` +- `ExecAlgorithm` (execution algorithms) +- `OrderEmulator` +- `RiskEngine` +- `ExecutionEngine` or `LiveExecutionEngine` +- `ExecutionClient` or `LiveExecutionClient` + +## Execution flow + +The `Strategy` base class inherits from `Actor` and contains all common data methods. +It also provides methods for managing orders and trade execution: + +- `submit_order(...)` +- `submit_order_list(...)` +- `modify_order(...)` +- `cancel_order(...)` +- `cancel_orders(...)` +- `cancel_all_orders(...)` +- `close_position(...)` +- `close_all_positions(...)` +- `query_account(...)` +- `query_order(...)` + +These methods create the necessary execution commands and send them on the message bus to the +relevant components (point-to-point). They also publish events such as `OrderInitialized`. + +The general execution flow looks like the following (each arrow indicates movement across the message bus): + +`Strategy` -> `OrderEmulator` -> `ExecAlgorithm` -> `RiskEngine` -> `ExecutionEngine` -> `ExecutionClient` + +The `OrderEmulator` and `ExecAlgorithm`(s) components are optional in the flow, depending on +individual order parameters (as explained below). + +This diagram illustrates message flow (commands and events) across the Nautilus execution components. + +```mermaid +flowchart LR + strategy[Strategy] + emulator[OrderEmulator] + algo[ExecAlgorithm] + risk[RiskEngine] + engine[ExecutionEngine] + client[ExecutionClient] + + strategy <--> emulator + strategy <--> algo + strategy <--> risk + emulator --> risk + algo --> risk + risk <--> engine + engine <--> client +``` + +## Order Management System (OMS) + +An order management system (OMS) type refers to the method used for assigning orders to positions and tracking those positions for an instrument. +OMS types apply to both strategies and venues (simulated and real). Even if a venue doesn't explicitly +state the method in use, an OMS type is always in effect. The OMS type for a component can be specified +using the `OmsType` enum. + +The `OmsType` enum has three variants: + +- `UNSPECIFIED`: The OMS type defaults based on where it is applied (details below) +- `NETTING`: Positions are combined into a single position per instrument ID +- `HEDGING`: Multiple positions per instrument ID are supported (both long and short) + +The table below describes different configuration combinations and their applicable scenarios. +When the strategy and venue OMS types differ, the `ExecutionEngine` handles this by overriding or assigning `position_id` values for received `OrderFilled` events. +A "virtual position" refers to a position ID that exists within the Nautilus system but not on the venue in +reality. + +| Strategy OMS | Venue OMS | Description | +|:-----------------------------|:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `NETTING` | `NETTING` | The strategy uses the venue's native OMS type, with a single position ID per instrument ID. | +| `HEDGING` | `HEDGING` | The strategy uses the venue's native OMS type, with multiple position IDs per instrument ID (both `LONG` and `SHORT`). | +| `NETTING` | `HEDGING` | The strategy **overrides** the venue's native OMS type. The venue tracks multiple positions per instrument ID, but Nautilus maintains a single position ID. | +| `HEDGING` | `NETTING` | The strategy **overrides** the venue's native OMS type. The venue tracks a single position per instrument ID, but Nautilus maintains multiple position IDs. | + +:::note +Configuring OMS types separately for strategies and venues increases platform complexity but allows +for a wide range of trading styles and preferences (see below). +::: + +OMS config examples: + +- Most cryptocurrency exchanges use a `NETTING` OMS type, representing a single position per market. It may be desirable for a trader to track multiple "virtual" positions for a strategy. +- Some FX ECNs or brokers use a `HEDGING` OMS type, tracking multiple positions both `LONG` and `SHORT`. The trader may only care about the NET position per currency pair. + +:::info +Nautilus does not yet support venue-side hedging modes such as Binance `BOTH` vs. `LONG/SHORT` where the venue nets per direction. +It is advised to keep Binance account configurations as `BOTH` so that a single position is netted. +::: + +### OMS configuration + +If a strategy OMS type is not explicitly set using the `oms_type` configuration option, +it will default to `UNSPECIFIED`. This means the `ExecutionEngine` will not override any venue `position_id`s, +and the OMS type will follow the venue's OMS type. + +:::tip +When configuring a backtest, you can specify the `oms_type` for the venue. For accuracy, match this with the OMS type used by the venue. +::: + +## Risk engine + +The `RiskEngine` is a component of every Nautilus system, including backtest, sandbox, and live environments. +Every order command and event passes through the `RiskEngine` unless specifically bypassed in the `RiskEngineConfig`. + +The `RiskEngine` performs these pre-trade risk checks: + +- Price precisions correct for the instrument. +- Prices are positive (unless an option type instrument) +- Quantity precisions correct for the instrument. +- Below maximum notional for the instrument. +- Within maximum or minimum quantity for the instrument. +- Only reducing position when a `reduce_only` execution instruction is specified for the order. + +If any risk check fails, the system generates an `OrderDenied` event with a human-readable reason, +closing the order. + +### Trading state + +Additionally, the current trading state of a Nautilus system affects order flow. + +The `TradingState` enum has three variants: + +- `ACTIVE`: Operates normally. +- `HALTED`: Does not process further order commands until state changes. +- `REDUCING`: Only processes cancels or commands that reduce open positions. + +See the [`RiskEngineConfig` API Reference](/docs/python-api-latest/config.html#nautilus_trader.risk.config.RiskEngineConfig) for further details. + +## Execution algorithms + +The platform supports custom execution algorithm components and provides built-in algorithms +such as TWAP (Time-Weighted Average Price). + +### TWAP (Time-Weighted Average Price) + +The TWAP algorithm spreads execution evenly over a specified time horizon. It receives a +primary order representing the total size and direction, then spawns smaller child orders +executed at regular intervals. + +This reduces the market impact of the full order size by spreading trade volume over time. + +The algorithm will immediately submit the first order, with the final order submitted being the +primary order at the end of the horizon period. + +Using the TWAP algorithm as an example (found in `/examples/algorithms/twap.py`), this example +demonstrates how to initialize and register a TWAP execution algorithm directly with a +`BacktestEngine` (assuming an engine is already initialized): + +```python +from nautilus_trader.examples.algorithms.twap import TWAPExecAlgorithm + +# `engine` is an initialized BacktestEngine instance +exec_algorithm = TWAPExecAlgorithm() +engine.add_exec_algorithm(exec_algorithm) +``` + +For this particular algorithm, two parameters must be specified: + +- `horizon_secs` +- `interval_secs` + +The `horizon_secs` parameter determines the time period over which the algorithm will execute, while +the `interval_secs` parameter sets the time between individual order executions. These parameters +determine how a primary order is split into a series of spawned orders. + +```python +from decimal import Decimal +from nautilus_trader.model.data import BarType +from nautilus_trader.test_kit.providers import TestInstrumentProvider +from nautilus_trader.examples.strategies.ema_cross_twap import EMACrossTWAP, EMACrossTWAPConfig + +# Configure your strategy +config = EMACrossTWAPConfig( + instrument_id=TestInstrumentProvider.ethusdt_binance().id, + bar_type=BarType.from_str("ETHUSDT.BINANCE-250-TICK-LAST-INTERNAL"), + trade_size=Decimal("0.05"), + fast_ema_period=10, + slow_ema_period=20, + twap_horizon_secs=10.0, # execution algorithm parameter (total horizon in seconds) + twap_interval_secs=2.5, # execution algorithm parameter (seconds between orders) +) + +# Instantiate your strategy +strategy = EMACrossTWAP(config=config) +``` + +Alternatively, you can specify these parameters dynamically per order, determining them based on +actual market conditions. In this case, the strategy configuration parameters could be provided to +an execution model which determines the horizon and interval. + +:::info +There is no limit to the number of execution algorithm parameters you can create. The parameters +must be a dictionary with string keys and primitive values (values that can be serialized +over the wire, such as ints, floats, and strings). +::: + +### Writing execution algorithms + +To build a custom execution algorithm, define a class that inherits from `ExecAlgorithm`. + +An execution algorithm is a type of `Actor`, so it's capable of the following: + +- Request and subscribe to data. +- Access the `Cache`. +- Set time alerts and/or timers using a `Clock`. + +Additionally it can: + +- Access the central `Portfolio`. +- Spawn secondary orders from a received primary (original) order. + +Once an execution algorithm is registered, and the system is running, it will receive orders off the +messages bus which are addressed to its `ExecAlgorithmId` via the `exec_algorithm_id` order parameter. +The order may also carry the `exec_algorithm_params` being a `dict[str, Any]`. + +:::warning +Because of the flexibility of the `exec_algorithm_params` dictionary, it's important to thoroughly +validate all of the key value pairs for correct operation of the algorithm (for starters that the +dictionary is not `None` and all necessary parameters actually exist). +::: + +Received orders will arrive via the following `on_order(...)` method. These received orders are +known as "primary" (original) orders when being handled by an execution algorithm. + +```python +from nautilus_trader.model.orders.base import Order + +def on_order(self, order: Order) -> None: + # Handle the order here +``` + +When the algorithm is ready to spawn a secondary order, it can use one of the following methods: + +- `spawn_market(...)` (spawns a `MARKET` order) +- `spawn_market_to_limit(...)` (spawns a `MARKET_TO_LIMIT` order) +- `spawn_limit(...)` (spawns a `LIMIT` order) + +:::note +Additional order types will be implemented in future versions, as the need arises. +::: + +Each of these methods takes the primary (original) `Order` as the first argument. The primary order +quantity will be reduced by the `quantity` passed in (becoming the spawned orders quantity). + +:::warning +The spawned quantity must not exceed the primary order's `leaves_qty` (remaining unfilled quantity). +::: + +:::note +If a spawned order is denied or rejected before acceptance, the deducted quantity is automatically +restored to the primary order. Once accepted by the venue, the reduction is considered committed. +::: + +Once the desired number of secondary orders have been spawned, and the execution routine is over, +the intention is that the algorithm will then finally send the primary (original) order. + +### Spawned orders + +All secondary orders spawned from an execution algorithm will carry a `exec_spawn_id` which is +the `ClientOrderId` of the primary (original) order, and whose `client_order_id` +derives from this original identifier with the following convention: + +- `exec_spawn_id` (primary order `client_order_id` value) +- `spawn_sequence` (the sequence number for the spawned order) + +``` +{exec_spawn_id}-E{spawn_sequence} +``` + +e.g. `O-20230404-001-000-E1` (for the first spawned order) + +:::note +The "primary" and "secondary" / "spawn" terminology was specifically chosen to avoid conflict +or confusion with the "parent" and "child" contingent orders terminology (an execution algorithm may also deal with contingent orders). +::: + +### Managing execution algorithm orders + +The `Cache` provides several methods to aid in managing (keeping track of) the activity of +an execution algorithm. Calling the below method will return all execution algorithm orders +for the given query filters. + +```python +def orders_for_exec_algorithm( + self, + exec_algorithm_id: ExecAlgorithmId, + venue: Venue | None = None, + instrument_id: InstrumentId | None = None, + strategy_id: StrategyId | None = None, + side: OrderSide = OrderSide.NO_ORDER_SIDE, +) -> list[Order]: +``` + +As well as more specifically querying the orders for a certain execution series/spawn. +Calling the below method will return all orders for the given `exec_spawn_id` (if found). + +```python +def orders_for_exec_spawn(self, exec_spawn_id: ClientOrderId) -> list[Order]: +``` + +:::note +This also includes the primary (original) order. +::: + +## Own order books + +Own order books are L3 order books that track only your own (user) orders organized by price level, maintained separately from the venue's public order books. + +### Purpose + +Own order books serve several purposes: + +- Monitor the state of your orders within the venue's public book in real-time. +- Validate order placement by checking available liquidity at price levels before submission. +- Help prevent self-trading by identifying price levels where your own orders already exist. +- Support advanced order management strategies that depend on queue position. +- Enable reconciliation between internal state and venue state during live trading. + +### Lifecycle + +Own order books are maintained per instrument and automatically updated as orders transition through their lifecycle. +Orders are added when submitted or accepted, updated when modified, and removed when filled, canceled, rejected, or expired. + +Only orders with prices can be represented in own order books. Market orders and other order types without explicit prices are excluded since they cannot be positioned at specific price levels. + +### Safe cancellation queries + +When querying own order books for orders to cancel, use a `status` filter that **excludes** `PENDING_CANCEL` to avoid processing orders already being cancelled. + +:::warning +Including `PENDING_CANCEL` in status filters can cause: + +- Duplicate cancel attempts on the same order. +- Inflated open order counts (orders in `PENDING_CANCEL` remain "open" until confirmed canceled). +- Order state explosion when multiple strategies attempt to cancel the same orders. + +::: + +The optional `accepted_buffer_ns` many methods expose is a time-based guard that only returns orders whose `ts_accepted` is at least that many nanoseconds in the past. Orders that have not yet been accepted by the venue still have `ts_accepted = 0`, so they are included once the buffer window elapses. To exclude those inflight orders you must pair the buffer with an explicit status filter (for example, restrict to `ACCEPTED` / `PARTIALLY_FILLED`). + +### Auditing + +During live trading, own order books can be periodically audited against the cache's order indexes to ensure consistency. +The audit mechanism verifies that closed orders are properly removed and that inflight orders (submitted but not yet accepted) remain tracked during venue latency windows. + +The audit interval can be configured using the `own_books_audit_interval_secs` parameter in live trading configurations. + +## Overfills + +An overfill occurs when the cumulative fill quantity for an order exceeds the original order quantity. +For example, an order for 100 units that receives fills totaling 110 units has an overfill of 10 units. + +### How overfills occur + +Overfills can result from two fundamentally different causes: + +- Duplicate fill events (a network/messaging issue). +- Genuine overfills at the matching engine (a real execution outcome). + +**Genuine overfills at the matching engine** + +In some cases, the matching engine actually executes more quantity than the order requested. +This is a real execution outcome, not a duplicate event: + +- **Matching engine race conditions**: In fast markets with high concurrency, an order may match + against multiple counterparties nearly simultaneously before being fully removed from the book. +- **Minimum lot size constraints**: If an order's remaining quantity falls below the venue's minimum + tradeable lot, some matching engines fill the minimum lot anyway rather than leaving an untradeable remainder. +- **DEX/AMM mechanics**: Decentralized exchanges using automated market makers may have execution + mechanics where actual fill quantities differ slightly from requested due to price impact calculations. +- **Multi-fill atomicity**: Some venues do not guarantee atomic fill quantities across partial + executions, allowing aggregate fills to exceed the original order quantity. + +**Duplicate fill events** + +Separate from genuine overfills, the same fill event may be delivered multiple times: + +- WebSocket reconnection replaying previously received events. +- The venue's internal retry or delivery guarantee mechanisms. +- API timing issues in the venue's execution reporting. + +The system handles duplicate events via `trade_id` deduplication (see below), but duplicates with +different `trade_id` values require overfill handling. + +**Race conditions with reconciliation** + +During live trading, the system maintains state through two parallel channels: + +- Real-time fill events arriving via WebSocket. +- Periodic reconciliation polling the venue for fill history. + +If the same fill arrives through both channels with different identifiers before deduplication +can occur, both may be applied to the order. This is particularly likely during: + +- System startup when reconciliation runs while WebSocket connections are establishing. +- Network instability causing reconnections mid-fill. +- High-frequency trading where fills arrive faster than reconciliation cycles. + +The likelihood of reconciliation race conditions increases when: + +- **Thresholds are reduced**: The `open_check_threshold_ms` and `inflight_check_threshold_ms` settings + (both default to 5,000 ms) define how long the engine waits before acting on discrepancies. + Reducing these below the round-trip latency to your venue increases the chance of processing + a fill via reconciliation before the real-time event arrives (or vice versa). +- **Reconciliation frequency is increased**: Setting `open_check_interval_secs` to aggressive values + (e.g., 1-2 seconds) increases how often the system polls the venue, creating more opportunities + for race conditions with real-time events. +- **Startup delay is reduced**: The `reconciliation_startup_delay_secs` setting (default 10 seconds) + provides time for WebSocket connections to stabilize before continuous reconciliation begins. + Reducing this increases the chance of duplicate fills during the startup window. + +See [Continuous reconciliation](live.md#continuous-reconciliation) for configuration details. + +### System behavior + +The `ExecutionEngine` checks for potential overfills before applying each fill event by comparing +the order's current `filled_qty` plus the incoming `last_qty` against the original `quantity`. + +The `allow_overfills` configuration option (default: `False`) controls how overfills are handled: + +| `allow_overfills` | Behavior | +|-------------------|----------------------------------------------------------------------------| +| `False` | Logs an error and rejects the fill, preserving the order's current state. | +| `True` | Logs a warning, applies the fill, and tracks the excess in `overfill_qty`. | + +When overfills are allowed, the order's `overfill_qty` field tracks the excess quantity. +The order transitions to `FILLED` status and `leaves_qty` is clamped to zero. + +### Duplicate fill detection + +The `Order` model enforces that each `trade_id` can only be applied once. Inside `Order.apply()`, +a hard check raises an error if the incoming fill's `trade_id` already exists on the order. +This is the invariant that prevents double-counting executions. + +**Core engine path (backtest and real-time event processing)** + +In the core `ExecutionEngine` (used for backtests and processing real-time fill events), before +calling `apply()`, the engine checks `Order.is_duplicate_fill()` which compares: + +- `trade_id` +- `order_side` +- `last_px` +- `last_qty` + +If all fields match an existing fill exactly, the event is skipped gracefully with a warning log. +This avoids raising an error for benign exact replays (e.g., from WebSocket reconnection). +If the `trade_id` matches but other fields differ ("noisy replay"), the 4-field check passes +but `Order.apply()` will raise an error due to the duplicate `trade_id`. The engine catches +this error, logs the exception with full context, and drops the fill - it does not crash. + +**Live reconciliation sanitizer** + +During live reconciliation, `LiveExecutionEngine` pre-filters on `trade_id` alone *before* +generating fill events. This check runs before the 4-field check described above. If a fill +report arrives with a `trade_id` that already exists on the order, it is skipped regardless +of whether the price or quantity differs. When the data does differ, a warning is logged to +alert operators to potential venue data quality issues. + +This pre-filtering ensures that "noisy duplicates" from venue replays or reconciliation races +are filtered out before they can trigger model integrity errors. If a venue legitimately needs +to correct fill data, it should use proper execution report semantics rather than resending +with the same `trade_id`. + +### Configuration + +For live trading, enable overfill tolerance in the `LiveExecEngineConfig`: + +```python +from nautilus_trader.live.config import LiveExecEngineConfig + +config = LiveExecEngineConfig( + allow_overfills=True, # Log warning instead of rejecting +) +``` + +:::tip +Enable `allow_overfills=True` when trading on venues known to emit duplicate fills or when +position reconciliation races with exchange fill events are expected. Monitor the logs for +overfill warnings to identify patterns that may require venue-specific handling. +::: + +:::warning +When `allow_overfills=False` (the default), rejected fills may cause position discrepancies +between the system and the venue. Use the [reconciliation](live.md#execution-reconciliation) +features to detect and resolve such discrepancies. +::: + +## Related guides + +- [Orders](orders.md) - Order types and management. +- [Positions](positions.md) - Position tracking from executions. +- [Strategies](strategies.md) - Order submission from strategies. diff --git a/nautilus-docs/docs/concepts/greeks.md b/nautilus-docs/docs/concepts/greeks.md new file mode 100644 index 0000000..707b4a8 --- /dev/null +++ b/nautilus-docs/docs/concepts/greeks.md @@ -0,0 +1,321 @@ +# Greeks + +Nautilus provides two paths for working with option Greeks +(sensitivities of option prices to changes in market variables): + +1. **Venue-provided Greeks (Rust/PyO3)** -- real-time Greeks streamed from venues + like Deribit and Bybit via the `OptionGreeks` data type and the option chain + aggregation system. +2. **Local Greeks calculator (Cython/Python)** -- the `GreeksCalculator` class that + computes Black-Scholes Greeks from cached market data, with support for portfolio + aggregation, shock scenarios, and beta weighting. + +Either path works independently or together. Venue-provided Greeks arrive +through the data subscription system and require no local computation. The local +calculator covers venues that do not stream Greeks, backtesting, and custom +adjustments (shocks, beta weighting, percent Greeks). + +## Venue-provided Greeks (Rust/PyO3) + +### OptionGreeks + +The `OptionGreeks` type represents venue-provided sensitivities for a single option +contract. It is a Rust-native type exposed to Python via PyO3. + +| Field | Type | Description | +|--------------------|------------------|-----------------------------------------------------| +| `instrument_id` | `InstrumentId` | The option contract these Greeks apply to. | +| `delta` | `float` | Rate of change of option price per unit underlying. | +| `gamma` | `float` | Rate of change of delta per unit underlying. | +| `vega` | `float` | Sensitivity to a 1% change in implied volatility. | +| `theta` | `float` | Daily time decay (dV/dt / 365.25). | +| `rho` | `float` | Sensitivity to a change in interest rate. | +| `mark_iv` | `float` or None | Mark implied volatility. | +| `bid_iv` | `float` or None | Bid implied volatility. | +| `ask_iv` | `float` or None | Ask implied volatility. | +| `underlying_price` | `float` or None | Underlying price at time of calculation. | +| `open_interest` | `float` or None | Open interest for the contract. | +| `ts_event` | `int` | UNIX timestamp (nanoseconds) of the event. | +| `ts_init` | `int` | UNIX timestamp (nanoseconds) when initialized. | + +Subscribe from an actor or strategy: + +```python +self.subscribe_option_greeks(instrument_id, client_id=ClientId("DERIBIT")) +``` + +Handle updates: + +```python +def on_option_greeks(self, greeks: OptionGreeks) -> None: + self.log.info(f"delta={greeks.delta:.4f} gamma={greeks.gamma:.6f}") +``` + +See the [Options](options.md) guide for the full subscription API including option +chain aggregation, strike range filtering, and snapshot modes. + +### Underlying Rust types + +The core Rust implementation lives in `crates/model/src/data/greeks.rs`: + +- `OptionGreekValues` -- a plain struct with `delta`, `gamma`, `vega`, `theta`, `rho` + fields. Implements `Add` and `Mul` for aggregation. +- `OptionGreeks` (in `crates/model/src/data/option_chain.rs`) -- wraps + `OptionGreekValues` with `instrument_id`, implied volatility fields, and timestamps. + Implements `Deref` so you can access Greeks fields directly. +- `HasGreeks` trait -- provides a `greeks()` method returning `OptionGreekValues`. + Implemented by both `OptionGreekValues` and `OptionGreeks`. + +### Black-Scholes functions (Rust/PyO3) + +Low-level pricing functions exposed to Python from `crates/model/src/data/greeks.rs`: + +```python +from nautilus_trader.core.nautilus_pyo3 import ( + black_scholes_greeks, + imply_vol, + imply_vol_and_greeks, + refine_vol_and_greeks, +) + +# Compute Greeks given known volatility +result = black_scholes_greeks(s=100.0, r=0.05, b=0.0, vol=0.20, is_call=True, k=100.0, t=0.25) +# result.delta, result.gamma, result.vega, result.theta, result.price, result.vol + +# Imply volatility from market price, then compute Greeks +result = imply_vol_and_greeks(s=100.0, r=0.05, b=0.0, is_call=True, k=100.0, t=0.25, price=5.0) + +# Refine volatility from a starting vol estimate (faster convergence) +result = refine_vol_and_greeks(s=100.0, r=0.05, b=0.0, is_call=True, k=100.0, t=0.25, + target_price=5.0, initial_vol=0.18) +``` + +The `BlackScholesGreeksResult` returned by these functions contains: `price`, `vol`, +`delta`, `gamma`, `vega`, `theta`, and `itm_prob`. + +**Conventions:** + +- Vega is scaled by 0.01 (sensitivity to a 1 percentage point vol change). +- Theta is scaled by 1/365.25 (daily decay). +- American-style options are priced as European for Greeks computation. + +## Local Greeks calculator (Cython/Python) + +### GreeksCalculator + +The `GreeksCalculator` class in `nautilus_trader/model/greeks.pyx` computes +Black-Scholes Greeks from cached market data. It is accessible from any actor or +strategy. + +```python +from nautilus_trader.model.greeks import GreeksCalculator + +# Typically created in on_start() +calculator = GreeksCalculator(cache=self.cache, clock=self.clock) +``` + +#### Instrument Greeks + +Compute Greeks for a single instrument (option or underlying) with quantity of 1: + +```python +greeks = calculator.instrument_greeks( + instrument_id=option_id, + flat_interest_rate=0.0425, # used if no yield curve in cache +) +# Returns GreeksData or None +``` + +The calculator: + +1. Looks up the instrument and its underlying in the cache. +2. Retrieves current prices (MID preferred, LAST as fallback). +3. Looks up yield curves from the cache (falls back to `flat_interest_rate`). +4. Implies volatility from the market price using `imply_vol_and_greeks`. +5. Returns a `GreeksData` object with all computed values. + +For non-option instruments (futures, equities), the calculator returns a `GreeksData` +with `delta=1` (or beta-weighted delta) and no gamma/vega/theta. + +**Shock scenarios** -- apply hypothetical changes to spot, volatility, or time: + +```python +greeks = calculator.instrument_greeks( + instrument_id=option_id, + spot_shock=10.0, # +10 points on underlying + vol_shock=0.02, # +2% absolute vol increase + time_to_expiry_shock=1/365, # roll forward one day +) +``` + +**Volatility update** -- refine implied vol from a cached starting point for faster +convergence: + +```python +greeks = calculator.instrument_greeks( + instrument_id=option_id, + update_vol=True, # use cached vol as starting point + cache_greeks=True, # store result for next iteration +) +``` + +**Beta-weighted Greeks** -- express delta and gamma in terms of an index: + +```python +greeks = calculator.instrument_greeks( + instrument_id=option_id, + index_instrument_id=InstrumentId.from_str("SPX.CBOE"), + beta_weights={underlying_id: 1.15}, + percent_greeks=True, +) +``` + +**Time-weighted vega** -- normalize vega across different expirations: + +```python +greeks = calculator.instrument_greeks( + instrument_id=option_id, + vega_time_weight_base=30, # normalize to 30-day vega +) +``` + +#### Portfolio Greeks + +Aggregate Greeks across all open positions matching filter criteria: + +```python +portfolio = calculator.portfolio_greeks( + underlyings=["AAPL", "MSFT"], + venue=Venue("CBOE"), + strategy_id=StrategyId("DELTA_HEDGE-001"), + flat_interest_rate=0.0425, + index_instrument_id=InstrumentId.from_str("SPX.CBOE"), + beta_weights=beta_dict, + percent_greeks=True, +) +# Returns PortfolioGreeks: pnl, price, delta, gamma, vega, theta +``` + +Filters: + +- `underlyings` -- list of symbol prefixes (e.g., `["AAPL"]` matches AAPL stock and + all AAPL options). +- `venue` -- restrict to a single venue. +- `instrument_id` -- restrict to a single instrument. +- `strategy_id` -- restrict to a single strategy. +- `side` -- filter by position side (LONG, SHORT). +- `greeks_filter` -- callable that accepts `PortfolioGreeks` per position; return + `True` to include. + +### GreeksData + +`GreeksData` is a Python custom data class (`@customdataclass`) that carries the full +context of a single instrument's Greeks computation. It extends `Data` and supports +Arrow serialization, cache storage, and catalog persistence. + +| Field | Type | Description | +|---------------------|-----------------|--------------------------------------------------------| +| `instrument_id` | `InstrumentId` | The instrument. | +| `is_call` | `bool` | True for call, False for put. | +| `strike` | `float` | Strike price. | +| `expiry` | `int` | Expiry date as YYYYMMDD integer. | +| `expiry_in_days` | `int` | Days to expiry. | +| `expiry_in_years` | `float` | Years to expiry (days / 365.25). | +| `multiplier` | `float` | Contract multiplier. | +| `quantity` | `float` | Position quantity (always 1 from `instrument_greeks`). | +| `underlying_price` | `float` | Underlying price used in calculation. | +| `interest_rate` | `float` | Interest rate used. | +| `cost_of_carry` | `float` | Cost of carry (r - dividend yield; 0 for futures). | +| `vol` | `float` | Implied volatility. | +| `pnl` | `float` | PnL relative to position entry (if position provided). | +| `price` | `float` | Model price. | +| `delta` | `float` | Delta. | +| `gamma` | `float` | Gamma. | +| `vega` | `float` | Vega (dV / 1% vol change). | +| `theta` | `float` | Theta (daily decay). | +| `itm_prob` | `float` | In-the-money probability. | + +`GreeksData` scales to portfolio level via its `to_portfolio_greeks()` method, which +multiplies all values by the contract `multiplier`. The `*` operator applies position +quantity: + +```python +position_greeks = signed_qty * instrument_greeks # returns PortfolioGreeks +``` + +### PortfolioGreeks + +`PortfolioGreeks` is the aggregated result from `portfolio_greeks()`. It supports +addition (`+`) for combining positions and scalar multiplication (`*`) for scaling: + +| Field | Type | Description | +|---------|---------|------------------------| +| `pnl` | `float` | Aggregate PnL. | +| `price` | `float` | Aggregate model value. | +| `delta` | `float` | Portfolio delta. | +| `gamma` | `float` | Portfolio gamma. | +| `vega` | `float` | Portfolio vega. | +| `theta` | `float` | Portfolio theta. | + +### YieldCurveData + +`YieldCurveData` stores an interest rate or dividend yield curve. The `GreeksCalculator` +looks up curves from the cache by currency code (for interest rates) or by underlying +instrument ID (for dividend yields). + +```python +from nautilus_trader.model.greeks_data import YieldCurveData +import numpy as np + +curve = YieldCurveData( + ts_event=0, + ts_init=0, + curve_name="USD", + tenors=np.array([0.25, 0.5, 1.0, 2.0]), + interest_rates=np.array([0.04, 0.042, 0.045, 0.048]), +) + +# Callable: interpolates rate for a given tenor +rate = curve(0.75) # quadratic interpolation +``` + +## Choosing between the two paths + +| Criterion | Venue-provided (`OptionGreeks`) | Local calculator (`GreeksCalculator`) | +|------------------------------|----------------------------------------|------------------------------------------| +| Computation | Done by the venue | Local Black-Scholes | +| Latency | Arrives with market data | Computed on demand | +| Venues | Deribit, Bybit (adapters with support) | Any venue with option instruments | +| Shock scenarios | Not supported | Spot, vol, and time shocks | +| Portfolio aggregation | Manual (iterate `OptionChainSlice`) | Built-in via `portfolio_greeks()` | +| Beta weighting | Not supported | Built-in | +| Backtest support | Via recorded `OptionGreeks` data | From cached prices at any point in time | +| Greeks available | delta, gamma, vega, theta, rho, IV, OI | delta, gamma, vega, theta, itm_prob, vol | +| Data type | `OptionGreeks` (Rust/PyO3) | `GreeksData` (Python `@customdataclass`) | + +## Greek definitions + +For reference, the Greeks that Nautilus computes: + +| Greek | Symbol | Definition | +|------------|--------|-------------------------------------------------------------------------------| +| Delta | `d` | First derivative of option price with respect to underlying price (dV/dS). | +| Gamma | `g` | Second derivative of option price with respect to underlying price (d2V/dS2). | +| Vega | `v` | Sensitivity to a 1 percentage point change in implied volatility (dV/dVol). | +| Theta | `t` | Daily time decay: change in option price per calendar day (dV/dt / 365.25). | +| Rho | `r` | Sensitivity to a change in the risk-free interest rate (dV/dr). | +| ITM prob | - | Probability that the option finishes in the money: P(ϕS_T > ϕK), where ϕ = 1 for calls and ϕ = -1 for puts. | + +## Examples + +Complete working examples are available in the repository: + +- `examples/live/bybit/bybit_option_greeks.py` -- subscribe to Bybit venue-provided Greeks. +- `examples/live/deribit/deribit_option_greeks.py` -- subscribe to Deribit venue-provided Greeks. + +## Related guides + +- [Options](options.md) - Option instruments, chain subscriptions, and strike filtering. +- [Data](data.md) - Built-in data types, custom data, and the subscription model. +- [Actors](actors.md) - Subscription and handler reference. +- [Strategies](strategies.md) - Strategy implementation and handler methods. diff --git a/nautilus-docs/docs/concepts/index.md b/nautilus-docs/docs/concepts/index.md new file mode 100644 index 0000000..28068fe --- /dev/null +++ b/nautilus-docs/docs/concepts/index.md @@ -0,0 +1,113 @@ +# Concepts + +These guides explain the core components, architecture, and design of NautilusTrader. + +## Overview + +Main features and intended use cases for the platform. + +## Architecture + +The principles, structures, and designs that underpin the platform. + +## Actors + +The `Actor` is the base component for interacting with the trading system. +Covers capabilities and implementation details. + +## Strategies + +How to implement trading strategies using the `Strategy` component. + +## Instruments + +Instrument definitions for tradable assets and contracts. + +## Value types + +The immutable numeric types (`Price`, `Quantity`, `Money`) used throughout the platform, +including their arithmetic behavior, precision handling, and type-specific constraints. + +## Data + +Built-in data types for the trading domain, and how to work with custom data. + +## Options + +Option instrument types, venue-provided Greeks streaming, option chain subscriptions +with strike range filtering, and snapshot aggregation. + +## Greeks + +Option Greeks (delta, gamma, vega, theta) from two paths: venue-provided real-time +Greeks via the Rust/PyO3 `OptionGreeks` type, and the local `GreeksCalculator` for +Black-Scholes computation with shock scenarios, beta weighting, and portfolio aggregation. + +## Custom data + +How the custom data system works across Python and Rust: registration, persistence, +Arrow encoding, and runtime routing through actors and strategies. + +## Order book + +The high-performance order book, own order tracking, filtered views for net liquidity, and binary market support. + +## Execution + +Trade execution and order management across multiple strategies and venues simultaneously (per instance), +including the components involved and the flow of execution messages (commands and events). + +## Orders + +Available order types, supported execution instructions, advanced order types, and emulated orders. + +## Positions + +Position lifecycle, aggregation from order fills, PnL calculations, and position snapshotting +for netting OMS configurations. + +## Cache + +The `Cache` is the central in-memory store for all trading-related data. +Covers capabilities and best practices. + +## Message Bus + +The `MessageBus` enables decoupled messaging between components, supporting point-to-point, +publish/subscribe, and request/response patterns. + +## Portfolio + +The `Portfolio` tracks all positions across strategies and instruments, providing a unified view +of holdings, risk exposure, and performance. + +## Reports + +Execution reports, portfolio analysis, PnL accounting, and backtest post-run analysis. + +## Logging + +High-performance logging for both backtesting and live trading, implemented in Rust. + +## Backtesting + +Running simulated trading on historical data using a specific system implementation. + +## Visualization + +Interactive tearsheets for analyzing backtest results, including charts, themes, +customization options, and custom visualizations via the extensible chart registry. + +## Live trading + +Deploying backtested strategies in real-time without code changes, and the key differences +between backtesting and live trading. + +## Adapters + +Requirements and best practices for developing integration adapters for data providers and trading venues. + +:::note +The Python API reference (linked in the sidebar) is the source of truth for the platform. +If there are discrepancies between these guides and the API reference, the API reference is correct. +::: diff --git a/nautilus-docs/docs/concepts/instruments.md b/nautilus-docs/docs/concepts/instruments.md new file mode 100644 index 0000000..179fc34 --- /dev/null +++ b/nautilus-docs/docs/concepts/instruments.md @@ -0,0 +1,541 @@ +# Instruments + +An instrument represents the specification for any tradable asset or contract. All +instrument types are implemented as Rust structs that implement the `Instrument` +trait. In Python, these are exposed as Cython extension types (via +`nautilus_trader.model.instruments`), with parallel PyO3 representations that are +converted to Cython types at the boundary. Pure Rust systems use the Rust types +directly. The platform supports a range of asset classes and instrument classes: + +- `Equity`: Listed shares or ETFs traded on cash markets. +- `CurrencyPair`: Spot FX or crypto pair in BASE/QUOTE format traded in cash markets. +- `Commodity`: Spot commodity instrument (e.g., gold or oil) traded in cash markets. +- `IndexInstrument`: Spot index calculated from constituents; used as a reference price and not directly tradable. +- `FuturesContract`: Deliverable futures contract with defined underlying, expiry, and multiplier. +- `FuturesSpread`: Exchange-defined multi-leg futures strategy (e.g., calendar or inter-commodity) quoted as one instrument. +- `CryptoFuture`: Dated, deliverable crypto futures contract with fixed expiry, underlying crypto, and settlement currency. +- `CryptoPerpetual`: Perpetual futures contract (perpetual swap) on crypto with no expiry; can be inverse or quanto-settled. +- `PerpetualContract`: Asset-class agnostic perpetual swap for any underlying (FX, equities, commodities, indexes, crypto). +- `OptionContract`: Exchange-traded option (put or call) on an underlying with strike and expiry. +- `OptionSpread`: Exchange-defined multi-leg options strategy (e.g., vertical, calendar, straddle) quoted as one instrument. +- `CryptoOption`: Option on a crypto underlying with crypto quote/settlement; supports inverse or quanto styles. +- `BinaryOption`: Fixed-payout option that settles to 0 or 1 based on a binary outcome. +- `Cfd`: Over-the-counter Contract for Difference that tracks an underlying and is cash-settled. +- `BettingInstrument`: Sports/gaming market selection (e.g., team or runner) tradable on betting venues. +- `SyntheticInstrument`: Synthetic instrument with prices derived from component instruments using a formula. + +## Symbology + +All instruments should have a unique `InstrumentId`, which is made up of both the native symbol, and venue ID, separated by a period. +For example, on the Binance Futures crypto exchange, the Ethereum Perpetual Futures Contract has the instrument ID `ETHUSDT-PERP.BINANCE`. + +All native symbols *should* be unique for a venue (this is not always the case e.g. Binance share native symbols between spot and futures markets), +and the `{symbol.venue}` combination *must* be unique for a Nautilus system. + +:::warning +The correct instrument must be matched to a market dataset such as ticks or order book data for logically sound operation. +An incorrectly specified instrument may truncate data or otherwise produce surprising results. +::: + +## Backtesting + +Generic test instruments can be instantiated through the `TestInstrumentProvider`: + +```python +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +audusd = TestInstrumentProvider.default_fx_ccy("AUD/USD") +``` + +```python +from nautilus_trader.adapters.binance.spot.providers import BinanceSpotInstrumentProvider +from nautilus_trader.model import InstrumentId + +provider = BinanceSpotInstrumentProvider(client=binance_http_client) +await provider.load_all_async() + +btcusdt = InstrumentId.from_str("BTCUSDT.BINANCE") +instrument = provider.find(btcusdt) +``` + +Or defined directly by constructing a specific instrument type: + +```python +from nautilus_trader.model.instruments import OptionContract + +instrument = OptionContract(...) # provide all necessary parameters +``` + +```rust +use nautilus_model::instruments::CurrencyPair; +use nautilus_model::identifiers::{InstrumentId, Symbol}; +use nautilus_model::types::{Currency, Price, Quantity}; + +let instrument = CurrencyPair::new( + InstrumentId::from("EUR/USD.SIM"), + Symbol::from("EUR/USD"), + Currency::from("EUR"), + Currency::from("USD"), + 5, // price_precision + 0, // size_precision + Price::from("0.00001"), // price_increment + Quantity::from("1"), // size_increment + // ... remaining parameters +); +``` + +See the full instrument [API Reference](/docs/python-api-latest/model/instruments.html). + +## Live trading + +Live integration adapters have `InstrumentProvider` implementations that automatically +cache the latest instrument definitions for the venue. Refer to a particular instrument +by passing the matching `InstrumentId` to data and execution methods that require one. + +## Finding instruments + +Since the same actor/strategy classes can be used for both backtest and live trading, you can +get instruments in exactly the same way through the central cache: + +```python +from nautilus_trader.model import InstrumentId + +instrument_id = InstrumentId.from_str("ETHUSDT-PERP.BINANCE") +instrument = self.cache.instrument(instrument_id) +``` + +```rust +use nautilus_model::identifiers::InstrumentId; + +let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); +let instrument = cache.instrument(&instrument_id); +``` + +It's also possible to subscribe to any changes to a particular instrument: + +```python +self.subscribe_instrument(instrument_id) +``` + +Or subscribe to all instrument changes for an entire venue: + +```python +from nautilus_trader.model import Venue + +binance = Venue("BINANCE") +self.subscribe_instruments(binance) +``` + +When an update to the instrument(s) is received by the `DataEngine`, the object(s) will +be passed to the `on_instrument()` handler. Override this method with actions +to take upon receiving an instrument update: + +```python +from nautilus_trader.model.instruments import Instrument + +def on_instrument(self, instrument: Instrument) -> None: + # Take some action on an instrument update + pass +``` + +## Precision + +Precision defines the number of decimal places allowed for prices and quantities on a +given instrument. Every instrument specifies a `price_precision` and `size_precision` +that determine the valid fractional resolution for that market. + +NautilusTrader enforces precision strictly by design. This section explains the rationale +and mechanics behind this approach. + +### Why precision is enforced + +**Realistic market simulation.** Real exchanges only accept prices and sizes at specific +precisions. A crypto spot market may support prices to 2 decimal places (e.g., `50000.01`) +while a different market supports 8 (e.g., `0.00012345`). Allowing arbitrary precision +in a backtest would produce fills at price levels that could never exist in production, +leading to misleading performance metrics. + +**Venue compatibility.** Most exchanges validate price and size precision on incoming +orders and reject those that exceed the instrument's specification. Enforcing precision +at the platform level catches a common class of these issues early. Note that venues may +also enforce tick-multiple or step-size constraints beyond what the `RiskEngine` currently +validates, so precision compliance alone does not guarantee venue acceptance. + +**Deterministic calculations.** Fixed-point arithmetic with explicit precision eliminates +floating-point drift and ensures calculations are reproducible across platforms and +environments. Two systems processing the same data will always produce identical results. + +**Data integrity.** The backtesting matching engine validates that all incoming market +data (quotes, trades, bars) matches the instrument's declared precision. This catches +mismatches between instrument definitions and data sources early, preventing silent +corruption of fill prices and quantities. + +### How precision works + +Each instrument defines two precision values: + +| Field | Constrains | Example | +|-------------------|--------------------------------------|------------------| +| `price_precision` | Order prices, trigger prices, fills. | `2` → `50000.01` | +| `size_precision` | Order quantities, fill quantities. | `5` → `1.00001` | + +These precisions are paired with minimum increments: + +| Field | Purpose | +|-------------------|------------------------------------------| +| `price_increment` | Smallest valid price change (tick size). | +| `size_increment` | Smallest valid quantity change. | + +The increment's own precision must exactly match the instrument's declared precision. +For example, an instrument with `price_precision=2` and `price_increment=Price(0.01, 2)` +is valid, but a mismatch between these values will raise an error at instrument creation. + +### Where precision is enforced + +Precision is validated at multiple levels throughout the platform: + +1. **Instrument creation**: The precision of `price_increment` and `size_increment` must + match `price_precision` and `size_precision` respectively. +2. **Risk engine**: Before an order reaches the venue, the `RiskEngine` checks that the + order's price and quantity precision do not exceed the instrument's limits. Orders that + fail this check are denied. +3. **Matching engine**: During backtesting, the matching engine validates that all incoming + market data matches the instrument's precision. Mismatches raise a `RuntimeError` + immediately. + +:::warning +The `RiskEngine` does not round values automatically. If you create a `Price` with +5 decimal places on an instrument that supports 2, the order will be denied. Use +`instrument.make_price()` and `instrument.make_qty()` to round explicitly. +::: + +### Working with instrument precision + +Use the instrument's factory methods to create values with correct precision: + +```python +instrument = self.cache.instrument(instrument_id) + +price = instrument.make_price(0.90500) +quantity = instrument.make_qty(150) +``` + +These methods round the input to the instrument's declared precision, ensuring the +result will pass precision checks. Other validation rules still apply (e.g., min/max +quantity limits), and `make_qty()` will raise if the rounded value is zero. + +:::tip +Always use `instrument.make_price()` and `instrument.make_qty()` when creating order +parameters. This avoids precision mismatch errors and ensures your values have the +correct number of decimal places for the instrument. +::: + +If you encounter precision mismatch errors during backtesting, verify that: + +1. The instrument definition matches your data source's precision. +2. Data was not inadvertently rounded or truncated during loading. +3. Custom data loaders preserve the original precision metadata. + +## Limits + +Certain value limits are optional for instruments and can be `None`, these are exchange +dependent and can include: + +- `max_quantity` (maximum quantity for a single order). +- `min_quantity` (minimum quantity for a single order). +- `max_notional` (maximum value of a single order). +- `min_notional` (minimum value of a single order). +- `max_price` (maximum valid quote or order price). +- `min_price` (minimum valid quote or order price). + +:::note +Most of these limits are checked by the Nautilus `RiskEngine`, otherwise exceeding +published limits *can* result in the exchange rejecting orders. +::: + +## Margins and fees + +Margin calculations are handled by the `MarginAccount` class. This section explains how margins work and introduces key concepts you need to know. + +### When do margins apply? + +Each exchange (e.g., CME or Binance) operates with a specific account type that determines whether margin calculations are applicable. +When setting up an exchange venue, you'll specify one of these account types: + +- `AccountType.MARGIN`: Accounts that use margin calculations, which are explained below. +- `AccountType.CASH`: Simple accounts where margin calculations do not apply. +- `AccountType.BETTING`: Accounts designed for betting, which also do not involve margin calculations. + +### Vocabulary + +To understand trading on margin, let’s start with some key terms: + +**Notional Value**: The total contract value in the quote currency. It represents the full market value of your position. For example, with EUR/USD futures on CME (symbol 6E). + +- Each contract represents 125,000 EUR (EUR is base currency, USD is quote currency). +- If the current market price is 1.1000, the notional value equals 125,000 EUR × 1.1000 (price of EUR/USD) = 137,500 USD. + +**Leverage** (`leverage`): The ratio that determines how much market exposure you can control relative to your account deposit. For example, with 10× leverage, you can control 10,000 USD worth of positions with 1,000 USD in your account. + +**Initial Margin** (`margin_init`): The margin rate required to open a position. It represents the minimum amount of funds that must be available in your account to open new positions. This is only a pre-check; no funds are actually locked. + +**Maintenance Margin** (`margin_maint`): The margin rate required to keep a position open. This amount is locked in your account to maintain the position. It is always lower than the initial margin. You can view the total blocked funds (sum of maintenance margins for open positions) using the following in your strategy: + +```python +self.portfolio.balances_locked(venue) +``` + +**Maker/Taker Fees**: The fees charged by exchanges based on your order's interaction with the market: + +- Maker Fee (`maker_fee`): A fee (typically lower) charged when you "make" liquidity by placing an order that remains on the order book. For example, a limit buy order below the current price adds liquidity, and the *maker* fee applies when it fills. +- Taker Fee (`taker_fee`): A fee (typically higher) charged when you "take" liquidity by placing an order that executes immediately. For instance, a market buy order or a limit buy above the current price removes liquidity, and the *taker* fee applies. + +**Fee rate sign convention**: Nautilus uses a consistent sign convention for fee rates across all adapters and the backtesting engine: + +- **Positive fee rate** = commission (fee charged, reducing account balance). +- **Negative fee rate** = rebate (fee earned, increasing account balance). + +For example, a maker fee of `-0.00025` means you receive a 0.025% rebate for providing liquidity, while a taker fee of `0.00075` means you pay a 0.075% commission for taking liquidity. + +:::note +Different exchanges use different sign conventions in their APIs. Nautilus adapters normalize these to the convention above. If you're manually specifying fee rates for backtesting, ensure you follow this convention. +::: + +:::tip +Not all exchanges or instruments implement maker/taker fees. If absent, set both `maker_fee` and `taker_fee` to 0 for the `Instrument` (e.g., `FuturesContract`, `Equity`, `CurrencyPair`, `Commodity`, `Cfd`, `BinaryOption`, `BettingInstrument`). +::: + +### Margin calculation formula + +The `MarginAccount` class calculates margins using the following formulas: + +```python +# Initial margin calculation +margin_init = (notional_value / leverage * margin_init) + (notional_value / leverage * taker_fee) + +# Maintenance margin calculation +margin_maint = (notional_value / leverage * margin_maint) + (notional_value / leverage * taker_fee) +``` + +**Key Points**: + +- Both formulas follow the same structure but use their respective margin rates (`margin_init` and `margin_maint`). +- Each formula consists of two parts: + - **Primary margin calculation**: Based on notional value, leverage, and margin rate. + - **Fee Adjustment**: Accounts for the maker/taker fee. + +### Implementation details + +For those interested in exploring the technical implementation: + +- [nautilus_trader/accounting/accounts/margin.pyx](https://github.com/nautechsystems/nautilus_trader/blob/develop/nautilus_trader/accounting/accounts/margin.pyx) +- Key methods: `calculate_margin_init(self, ...)` and `calculate_margin_maint(self, ...)` + +## Commissions + +Trading commissions represent the fees charged by exchanges or brokers for executing trades. +While maker/taker fees are common in cryptocurrency markets, traditional exchanges like CME often +employ other fee structures, such as per-contract commissions. +NautilusTrader supports multiple commission models to accommodate diverse fee structures across different markets. + +### Built-in fee models + +The framework provides two built-in fee model implementations: + +1. `MakerTakerFeeModel`: Implements the maker/taker fee structure common in cryptocurrency exchanges, where fees are + calculated as a percentage of the trade value. +2. `FixedFeeModel`: Applies a fixed commission per trade, regardless of the trade size. + +### Creating custom fee models + +While the built-in fee models cover common scenarios, you might encounter situations requiring specific commission structures. +NautilusTrader's flexible architecture allows you to implement custom fee models by inheriting from `FeeModel`. + +For example, if you're trading futures on exchanges that charge per-contract commissions (like CME), you can implement +a custom fee model. When creating custom fee models, we inherit from the `FeeModel` base class, which is implemented +in Cython for performance reasons. This Cython implementation is reflected in the parameter naming convention, +where type information is incorporated into parameter names using underscores (like `Order_order` or `Quantity_fill_qty`). + +While these parameter names might look unusual to Python developers, they're a result of Cython's type system and help +maintain consistency with the framework's core components. Here's how you could create a per-contract commission model: + +```python +class PerContractFeeModel(FeeModel): + def __init__(self, commission: Money): + super().__init__() + self.commission = commission + + def get_commission(self, Order_order, Quantity_fill_qty, Price_fill_px, Instrument_instrument): + total_commission = Money(self.commission * Quantity_fill_qty, self.commission.currency) + return total_commission +``` + +This custom implementation calculates the total commission by multiplying a `fixed per-contract fee` by the `number +of contracts` traded. The `get_commission(...)` method receives information about the order, fill quantity, fill price +and instrument, allowing for flexible commission calculations based on these parameters. + +Our new class `PerContractFeeModel` inherits class `FeeModel`, which is implemented in Cython, +so notice the Cython-style parameter names in the method signature: + +- `Order_order`: The order object, with type prefix `Order_`. +- `Quantity_fill_qty`: The fill quantity, with type prefix `Quantity_`. +- `Price_fill_px`: The fill price, with type prefix `Price_`. +- `Instrument_instrument`: The instrument object, with type prefix `Instrument_`. + +These parameter names follow NautilusTrader's Cython naming conventions, where the prefix indicates the expected type. +While this might seem verbose compared to typical Python naming conventions, it ensures type safety and consistency +with the framework's Cython codebase. + +### Using fee models in practice + +To use any fee model in your trading system, whether built-in or custom, you specify it when setting up the venue. +Here's an example using the custom per-contract fee model: + +```python +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.objects import Money, Currency + +engine.add_venue( + venue=venue, + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, + base_currency=USD, + fee_model=PerContractFeeModel(Money(2.50, USD)), # 2.50 USD per contract + starting_balances=[Money(1_000_000, USD)], # Starting with 1,000,000 USD balance +) +``` + +:::tip +When implementing custom fee models, ensure they accurately reflect the fee structure of your target exchange. +Even small discrepancies in commission calculations can significantly impact strategy performance metrics during backtesting. +::: + +### Additional info + +The raw instrument definition as provided by the exchange (typically from JSON serialized data) is also +included as a generic Python dictionary. This is to retain all information +which is not necessarily part of the unified Nautilus API, and is available to the user +at runtime by calling the `.info` property. + +## Synthetic instruments + +The platform supports creating customized synthetic instruments, which can generate synthetic quote +and trades. These are useful for: + +- Enabling `Actor` and `Strategy` components to subscribe to quote or trade feeds. +- Triggering emulated orders. +- Constructing bars from synthetic quotes or trades. + +Synthetic instruments cannot be traded directly, as they are constructs that only exist locally +within the platform. They serve as analytical tools, providing useful metrics based on their component +instruments. + +In the future, we plan to support order management for synthetic instruments, enabling trading of +their component instruments based on the synthetic instrument's behavior. + +:::info +The venue for a synthetic instrument is always designated as `'SYNTH'`. +::: + +### Formula + +A synthetic instrument is composed of a combination of two or more component instruments (which +can include instruments from multiple venues), as well as a "derivation formula". +Utilizing the dynamic expression engine powered by the [evalexpr](https://github.com/ISibboI/evalexpr) +Rust crate, the platform can evaluate the formula to calculate the latest synthetic price tick +from the incoming component instrument prices. + +See the `evalexpr` documentation for a full description of available features, operators and precedence. + +:::tip +Before defining a new synthetic instrument, ensure that all component instruments are already defined and exist in the cache. +::: + +### Subscribing + +The following example demonstrates the creation of a new synthetic instrument with an actor/strategy. +This synthetic instrument will represent a simple spread between Bitcoin and +Ethereum spot prices on Binance. For this example, it is assumed that spot instruments for +`BTCUSDT.BINANCE` and `ETHUSDT.BINANCE` are already present in the cache. + +```python +from nautilus_trader.model.instruments import SyntheticInstrument + +btcusdt_binance_id = InstrumentId.from_str("BTCUSDT.BINANCE") +ethusdt_binance_id = InstrumentId.from_str("ETHUSDT.BINANCE") + +# Define the synthetic instrument +synthetic = SyntheticInstrument( + symbol=Symbol("BTC-ETH:BINANCE"), + price_precision=8, + components=[ + btcusdt_binance_id, + ethusdt_binance_id, + ], + formula=f"{btcusdt_binance_id} - {ethusdt_binance_id}", + ts_event=self.clock.timestamp_ns(), + ts_init=self.clock.timestamp_ns(), +) + +# Recommended to store the synthetic instruments ID somewhere +self._synthetic_id = synthetic.id + +# Add the synthetic instrument for use by other components +self.add_synthetic(synthetic) + +# Subscribe to quotes for the synthetic instrument +self.subscribe_quote_ticks(self._synthetic_id) +``` + +:::note +The `instrument_id` for the synthetic instrument in the above example will be structured as `{symbol}.{SYNTH}`, resulting in `'BTC-ETH:BINANCE.SYNTH'`. +::: + +### Updating formulas + +It's also possible to update a synthetic instrument formulas at any time. The following example +shows how to achieve this with an actor/strategy. + +```python +# Recover the synthetic instrument from the cache (assuming `synthetic_id` was assigned) +synthetic = self.cache.synthetic(self._synthetic_id) + +# Update the formula to take the average +new_formula = "(BTCUSDT.BINANCE + ETHUSDT.BINANCE) / 2" +synthetic.change_formula(new_formula) + +# Now update the synthetic instrument +self.update_synthetic(synthetic) +``` + +### Trigger instrument IDs + +The platform allows for emulated orders to be triggered based on synthetic instrument prices. In +the following example, we build upon the previous one to submit a new emulated order. +This order will be retained in the emulator until a trigger from synthetic quotes releases it. +It will then be submitted to Binance as a MARKET order: + +```python +order = self.strategy.order_factory.limit( + instrument_id=ETHUSDT_BINANCE.id, + order_side=OrderSide.BUY, + quantity=Quantity.from_str("1.5"), + price=Price.from_str("30000.00000000"), # <-- Synthetic instrument price + emulation_trigger=TriggerType.DEFAULT, + trigger_instrument_id=self._synthetic_id, # <-- Synthetic instrument identifier +) + +self.strategy.submit_order(order) +``` + +### Error handling + +The platform validates inputs including the derivation formula for synthetic instruments. +Invalid or erroneous inputs may still lead to undefined behavior. + +See the [`SyntheticInstrument` API Reference](/docs/python-api-latest/model/instruments.html#nautilus_trader.model.instruments.synthetic.SyntheticInstrument) for input requirements and potential exceptions. + +## Related guides + +- [Data](data.md) - Market data types for instruments. +- [Orders](orders.md) - Orders reference instruments. diff --git a/nautilus-docs/docs/concepts/live.md b/nautilus-docs/docs/concepts/live.md new file mode 100644 index 0000000..25caf48 --- /dev/null +++ b/nautilus-docs/docs/concepts/live.md @@ -0,0 +1,597 @@ +# Live Trading + +NautilusTrader deploys backtested strategies to live markets with no code changes. + +**Live trading involves real financial risk. Before deploying to production, understand +system configuration, node operations, execution reconciliation, and the differences +between backtesting and live trading.** + +:::danger[Jupyter notebooks not recommended for live trading] +Do not run live trading nodes in Jupyter notebooks. Event loop conflicts and +operational risks make them unsuitable: + +- Jupyter runs its own asyncio event loop, which conflicts with `TradingNode`'s event loop. +- Workarounds like `nest_asyncio` are not production-grade. +- Cells can run out of order, kernels can crash, and state can disappear. +- Notebooks lack the logging, monitoring, and graceful shutdown needed for production trading. + +Use Jupyter for backtesting, analysis, and experimentation. For live trading, run nodes +as standalone Python scripts or services. +::: + +:::warning[One TradingNode per process] +Running multiple `TradingNode` instances concurrently in the same process is not supported due to global singleton state. +Add multiple strategies to a single node, or run additional nodes in separate processes for parallel execution. + +See [Processes and threads](architecture.md#processes-and-threads) for details. +::: + +:::warning[Do not block the event loop] +User code on the event loop thread (strategy callbacks, actor handlers, `on_event` methods) +must return quickly. This applies to both Python and Rust. Blocking operations like model +inference, heavy calculations, or synchronous I/O cause missed fills, stale data, and +delayed order submissions. Offload long-running work to an executor or a separate thread/process. +::: + +:::info[Platform differences] +Windows signal handling differs from Unix-like systems. If you are running on Windows, please read +the note on [Windows signal handling](#windows-signal-handling) for guidance on graceful shutdown +behavior and Ctrl+C (SIGINT) support. +::: + +## Configuration + +### `TradingNodeConfig` + +`TradingNodeConfig` inherits from `NautilusKernelConfig` and adds live-specific options: + +```python +from nautilus_trader.config import TradingNodeConfig + +config = TradingNodeConfig( + trader_id="MyTrader-001", + + # Component configurations + cache=CacheConfig(), + message_bus=MessageBusConfig(), + data_engine=LiveDataEngineConfig(), + risk_engine=LiveRiskEngineConfig(), + exec_engine=LiveExecEngineConfig(), + portfolio=PortfolioConfig(), + + # Client configurations + data_clients={ + "BINANCE": BinanceDataClientConfig(), + }, + exec_clients={ + "BINANCE": BinanceExecClientConfig(), + }, +) +``` + +#### Core configuration parameters + +| Setting | Default | Description | +|--------------------------|--------------|---------------------------------------------| +| `trader_id` | "TRADER-001" | Unique trader identifier (name-tag format). | +| `instance_id` | `None` | Optional unique instance identifier. | +| `timeout_connection` | 30.0 | Connection timeout in seconds. | +| `timeout_reconciliation` | 10.0 | Reconciliation timeout in seconds. | +| `timeout_portfolio` | 10.0 | Portfolio initialization timeout. | +| `timeout_disconnection` | 10.0 | Disconnection timeout. | +| `timeout_post_stop` | 5.0 | Post-stop cleanup timeout. | + +#### Cache database configuration + +```python +from nautilus_trader.config import CacheConfig +from nautilus_trader.config import DatabaseConfig + +cache_config = CacheConfig( + database=DatabaseConfig( + host="localhost", + port=6379, + username="nautilus", + password="pass", + timeout=2.0, + ), + encoding="msgpack", # or "json" + timestamps_as_iso8601=True, + buffer_interval_ms=100, + flush_on_start=False, +) +``` + +#### MessageBus configuration + +```python +from nautilus_trader.config import MessageBusConfig +from nautilus_trader.config import DatabaseConfig + +message_bus_config = MessageBusConfig( + database=DatabaseConfig(timeout=2), + timestamps_as_iso8601=True, + use_instance_id=False, + types_filter=[QuoteTick, TradeTick], # Filter specific message types + stream_per_topic=False, + autotrim_mins=30, # Automatic message trimming + heartbeat_interval_secs=1, +) +``` + +### Multi-venue configuration + +A node can connect to multiple venues. This example configures both +spot and futures markets for Binance: + +```python +config = TradingNodeConfig( + trader_id="MultiVenue-001", + + # Multiple data clients for different market types + data_clients={ + "BINANCE_SPOT": BinanceDataClientConfig( + account_type=BinanceAccountType.SPOT, + testnet=False, + ), + "BINANCE_FUTURES": BinanceDataClientConfig( + account_type=BinanceAccountType.USDT_FUTURES, + testnet=False, + ), + }, + + # Corresponding execution clients + exec_clients={ + "BINANCE_SPOT": BinanceExecClientConfig( + account_type=BinanceAccountType.SPOT, + testnet=False, + ), + "BINANCE_FUTURES": BinanceExecClientConfig( + account_type=BinanceAccountType.USDT_FUTURES, + testnet=False, + ), + }, +) +``` + +### ExecutionEngine configuration + +`LiveExecEngineConfig` controls order processing, execution events, and +venue reconciliation. For full details see the +[API Reference](/docs/python-api-latest/config.html#nautilus_trader.live.config.LiveExecEngineConfig). + +#### Reconciliation + +Recovers missed order and position events to keep system state consistent with the venue. + +| Setting | Default | Description | +|---------------------------------|---------|---------------------------------------------------------------------------------| +| `reconciliation` | True | Activate reconciliation at startup to align internal state with the venue. | +| `reconciliation_lookback_mins` | None | How far back (minutes) to request past events for reconciling uncached state. | +| `reconciliation_instrument_ids` | None | Include list of instrument IDs to reconcile. | +| `filtered_client_order_ids` | None | Client order IDs to skip during reconciliation (for venue-side duplicates). | + +See [Execution reconciliation](#execution-reconciliation) for details. + +#### Order filtering + +Controls which order events and reports the system processes, preventing conflicts +across trading nodes. + +| Setting | Default | Description | +|------------------------------------|---------|-------------------------------------------------------------------------------| +| `filter_unclaimed_external_orders` | False | Drop unclaimed external orders so they do not affect the strategy. | +| `filter_position_reports` | False | Drop position status reports. Useful when multiple nodes trade one account. | + +:::note[Order tagging behavior] +Reconciliation tags orders by origin: + +- **`VENUE` tag**: external orders discovered at the venue (placed outside this system). +- **`RECONCILIATION` tag**: synthetic orders generated to align position discrepancies. + +When `filter_unclaimed_external_orders` is enabled, only `VENUE`-tagged orders are filtered. +`RECONCILIATION`-tagged orders are never filtered, so position alignment always succeeds. +::: + +#### Continuous reconciliation + +A background loop starts after startup reconciliation completes. It: + +- Monitors in-flight orders for delays exceeding a configured threshold. +- Reconciles open orders with the venue at configurable intervals. +- Audits internal *own* order books against the venue's public books. + +The loop waits for startup reconciliation to finish before starting periodic checks. +The `reconciliation_startup_delay_secs` parameter adds a further delay *after* startup +reconciliation completes, giving the system time to stabilize. + +When retries are exhausted, the engine resolves the order as follows: + +**In-flight order timeout resolution** (venue does not respond after max retries): + +| Current status | Resolved to | Rationale | +|------------------|-------------|--------------------------------------------| +| `SUBMITTED` | `REJECTED` | No confirmation received from venue. | +| `PENDING_UPDATE` | `CANCELED` | Modification remains unacknowledged. | +| `PENDING_CANCEL` | `CANCELED` | Venue never confirmed the cancellation. | + +**Order consistency checks** (when cache state differs from venue state): + +| Cache status | Venue status | Resolution | Rationale | +|--------------------|--------------|-------------|---------------------------------------------------------------------| +| `SUBMITTED` | Not found | `REJECTED` | Order never confirmed by venue (e.g., lost during network error). | +| `ACCEPTED` | Not found | `REJECTED` | Order doesn't exist at venue, likely was never successfully placed. | +| `ACCEPTED` | `CANCELED` | `CANCELED` | Venue canceled the order (user action or venue-initiated). | +| `ACCEPTED` | `EXPIRED` | `EXPIRED` | Order reached GTD expiration at venue. | +| `ACCEPTED` | `REJECTED` | `REJECTED` | Venue rejected after initial acceptance (rare but possible). | +| `PARTIALLY_FILLED` | `CANCELED` | `CANCELED` | Order canceled at venue with fills preserved. | +| `PARTIALLY_FILLED` | Not found | `CANCELED` | Order doesn't exist but had fills (reconciles fill history). | + +:::note +**Reconciliation caveats:** + +- **"Not found" resolutions** only apply in full-history mode (`open_check_open_only=False`). + Open-only mode (the default) skips these checks because venue "open orders" endpoints + exclude closed orders by design, making it impossible to distinguish missing orders from + recently closed ones. +- **Recent order protection**: the engine skips reconciliation for orders whose last event + falls within the `open_check_threshold_ms` window (default 5s). This prevents false + positives from race conditions where the venue is still processing. +- **Targeted query safeguard**: before marking an order `REJECTED` or `CANCELED` when + "not found", the engine issues a single-order query to the venue. + This catches false negatives from bulk query limitations or timing delays. +- **`FILLED` orders** that are "not found" at the venue are silently ignored. Venues + commonly drop completed orders from their query results. + +::: + +#### Retry coordination and lookback behavior + +The inflight loop and open-order loop share a single retry counter +(`_recon_check_retries`), bounded by `inflight_check_retries` and +`open_check_missing_retries` respectively. The stricter limit wins, +and avoids duplicate venue queries for the same order state. + +When the open-order loop exhausts retries, the engine issues one targeted +`GenerateOrderStatusReport` probe before applying a terminal state. If the +venue returns the order, reconciliation proceeds and the retry counter resets. + +**Single-order query protection**: the engine caps single-order queries per +cycle via `max_single_order_queries_per_cycle` (default: 10). Remaining +orders are deferred to the next cycle. A configurable delay +(`single_order_query_delay_ms`, default: 100ms) spaces out consecutive +queries to avoid rate limits. This handles bulk query failures across hundreds of orders +without overwhelming the venue API. + +Orders older than `open_check_lookback_mins` rely on this targeted probe. +Keep the lookback generous for venues with short history windows. Increase +`open_check_threshold_ms` if venue timestamps lag the local clock, so +recently updated orders are not marked missing prematurely. + +| Setting | Default | Description | +|--------------------------------------|----------------|--------------------------------------------------------------------------------------------------| +| `inflight_check_interval_ms` | 2,000 ms | How often to check in-flight order status. Set to 0 to disable. | +| `inflight_check_threshold_ms` | 5,000 ms | Time before an in-flight order triggers a venue status check. Lower if colocated. | +| `inflight_check_retries` | 5 retries | Retry attempts to verify an in-flight order with the venue. | +| `open_check_interval_secs` | None | How often (seconds) to check open orders at the venue. None or 0.0 disables. Recommended: 5-10s.| +| `open_check_open_only` | True | When true, query only open orders; when false, fetch full history (resource-intensive). | +| `open_check_lookback_mins` | 60 min | Lookback window (minutes) for order status polling. Only orders modified within this window. | +| `open_check_threshold_ms` | 5,000 ms | Minimum time since last cached event before acting on venue discrepancies. | +| `open_check_missing_retries` | 5 retries | Max retries before resolving an order open in cache but not found at venue. | +| `max_single_order_queries_per_cycle` | 10 | Cap on single-order queries per cycle. Prevents rate-limit exhaustion. | +| `single_order_query_delay_ms` | 100 ms | Delay (ms) between single-order queries to avoid rate limits. | +| `reconciliation_startup_delay_secs` | 10.0 s | Delay (seconds) *after* startup reconciliation before continuous checks begin. | +| `own_books_audit_interval_secs` | None | Interval (seconds) between auditing own order books against public books. | +| `position_check_interval_secs` | None | Interval (seconds) between position consistency checks. On discrepancy, queries for missing fills. None disables. Recommended: 30-60s. | +| `position_check_lookback_mins` | 60 min | Lookback window (minutes) for querying fill reports on position discrepancy. | +| `position_check_threshold_ms` | 5,000 ms | Minimum time since last local activity before acting on position discrepancies. | +| `position_check_retries` | 3 retries | Max attempts per instrument before the engine stops retrying that discrepancy. Once exceeded, an error is logged and the discrepancy is no longer actively reconciled until it clears. | + +:::warning + +- **`open_check_lookback_mins`**: do not reduce below 60 minutes. A short window + triggers false "missing order" resolutions because orders fall outside the query range. +- **`reconciliation_startup_delay_secs`**: do not reduce below 10 seconds in production. + The delay lets the system stabilize after startup reconciliation before continuous + checks begin. + +::: + +#### Additional options + +| Setting | Default | Description | +|------------------------------------|---------|-------------------------------------------------------------------------------------------------| +| `allow_overfills` | False | Allow fills exceeding order quantity (logs warning). Useful when reconciliation races fills. | +| `generate_missing_orders` | True | Generate LIMIT orders during reconciliation to align position discrepancies (strategy `EXTERNAL`, tag `RECONCILIATION`). | +| `snapshot_orders` | False | Take order snapshots on order events. | +| `snapshot_positions` | False | Take position snapshots on position events. | +| `snapshot_positions_interval_secs` | None | Interval (seconds) between position snapshots. | +| `debug` | False | Enable debug logging for execution. | + +#### Memory management + +Periodically purges closed orders, closed positions, and account events from the +in-memory cache, keeping memory bounded during long-running or HFT sessions. + +| Setting | Default | Description | +|----------------------------------------|---------|------------------------------------------------------------------------------------| +| `purge_closed_orders_interval_mins` | None | How often (minutes) to purge closed orders from memory. Recommended: 10-15 min. | +| `purge_closed_orders_buffer_mins` | None | How long (minutes) an order must be closed before purging. Recommended: 60 min. | +| `purge_closed_positions_interval_mins` | None | How often (minutes) to purge closed positions from memory. Recommended: 10-15 min. | +| `purge_closed_positions_buffer_mins` | None | How long (minutes) a position must be closed before purging. Recommended: 60 min. | +| `purge_account_events_interval_mins` | None | How often (minutes) to purge account events from memory. Recommended: 10-15 min. | +| `purge_account_events_lookback_mins` | None | How old (minutes) an account event must be before purging. Recommended: 60 min. | +| `purge_from_database` | False | Also delete from the backing database (Redis/PostgreSQL). **Use with caution**. | + +Setting an interval enables the purge loop; leaving it unset disables scheduling and +deletion. Database records are unaffected unless `purge_from_database` is true. Each +loop delegates to the cache APIs described in +[Purging cached state](cache.md#purging-cached-state). + +#### Queue management + +| Setting | Default | Description | +|----------------------------------|---------|---------------------------------------------------------------------------------| +| `qsize` | 100,000 | Size of internal queue buffers. | +| `graceful_shutdown_on_exception` | False | Gracefully shut down on unexpected queue processing exceptions (not user code). | + +### Strategy configuration + +For a complete parameter list see the `StrategyConfig` +[API Reference](/docs/python-api-latest/config.html#nautilus_trader.trading.config.StrategyConfig). + +#### Identification + +| Setting | Default | Description | +|----------------|---------|---------------------------------------------------------------| +| `strategy_id` | None | Unique strategy identifier. | +| `order_id_tag` | None | Unique tag appended to this strategy's order IDs. | + +#### Order management + +| Setting | Default | Description | +|-----------------------------|---------|--------------------------------------------------------------------------------------------| +| `oms_type` | None | [OMS type](../concepts/execution#oms-configuration) for position ID and order processing. | +| `use_uuid_client_order_ids` | False | Use UUID4 values for client order IDs. | +| `external_order_claims` | None | Instrument IDs whose external orders this strategy claims. | +| `manage_contingent_orders` | False | Automatically manage OTO, OCO, and OUO contingent orders. | +| `manage_gtd_expiry` | False | Manage GTD expirations for orders. | + +### Windows signal handling + +:::warning +Windows: asyncio event loops do not implement `loop.add_signal_handler`. As a result, the legacy +`TradingNode` does not receive OS signals via asyncio on Windows. Use Ctrl+C (SIGINT) handling or +programmatic shutdown; SIGTERM parity is not expected on Windows. +::: + +On Windows, asyncio event loops do not implement `loop.add_signal_handler`, so Unix-style +signal integration is unavailable. `TradingNode` does not receive OS signals via asyncio +on Windows and will not stop gracefully unless you intervene. + +Recommended approaches: + +- Wrap `run` with `try/except KeyboardInterrupt` and call `node.stop()` then `node.dispose()`. + Ctrl+C raises `KeyboardInterrupt` in the main thread, giving you a clean teardown path. +- Publish a `ShutdownSystem` command programmatically (or call `shutdown_system(...)` from + an actor/component) to trigger the same shutdown path. + +The “inflight check loop task still pending” message appears because the normal graceful +shutdown path is not triggered. This is tracked as +[#2785](https://github.com/nautechsystems/nautilus_trader/issues/2785). + +The v2 `LiveNode` already handles Ctrl+C via `tokio::signal::ctrl_c()` and a Python SIGINT +bridge, so runner and tasks shut down cleanly. + +Example pattern for Windows: + +```python +try: + node.run() +except KeyboardInterrupt: + pass +finally: + try: + node.stop() + finally: + node.dispose() +``` + +## Execution reconciliation + +Execution reconciliation aligns the venue's actual order and position state with the +system's internal state built from events. Only the `LiveExecutionEngine` performs +reconciliation, since backtesting controls both sides. + +:::note[Terminology] +An **in-flight order** is one awaiting venue acknowledgement: + +- `SUBMITTED` - initial submission, awaiting accept/reject. +- `PENDING_UPDATE` - modification requested, awaiting confirmation. +- `PENDING_CANCEL` - cancellation requested, awaiting confirmation. + +These orders are monitored by the continuous reconciliation loop to detect stale or lost messages. +::: + +Two scenarios: + +- **Cached state exists**: report data generates missing events to align the state. +- **No cached state**: all orders and positions at the venue are generated from scratch. + +:::tip +Persist all execution events to the cache database. This reduces reliance on venue history +and allows full recovery even with short lookback windows. +::: + +### Reconciliation configuration + +Unless `reconciliation` is set to false, the execution engine reconciles state for each +venue at startup. The `reconciliation_lookback_mins` parameter controls how far back the +engine requests history. + +:::tip +Leave `reconciliation_lookback_mins` unset. This lets the engine request the maximum +execution history the venue provides. +::: + +:::warning +Executions before the lookback window still generate alignment events, but with some +information loss that a longer window would avoid. Some venues also filter or drop +older execution data. Persisting all events to the cache database prevents both issues. +::: + +Each strategy can claim external orders for an instrument ID generated during reconciliation +via the `external_order_claims` config parameter. This lets a strategy resume managing open +orders when no cached state exists. + +Orders generated with strategy ID `EXTERNAL` and tag `RECONCILIATION` during position +reconciliation are internal to the engine. They cannot be claimed via `external_order_claims` +and should not be managed by user strategies. + +:::tip +To detect external orders in your strategy, check `order.strategy_id.value == "EXTERNAL"`. These orders participate in portfolio calculations and position tracking like any other order. +::: + +For all live trading options, see the `LiveExecEngineConfig` [API Reference](/docs/python-api-latest/config.html#nautilus_trader.live.config.LiveExecEngineConfig). + +### Reconciliation procedure + +All adapter execution clients follow the same reconciliation procedure, calling three methods +to produce an execution mass status: + +- `generate_order_status_reports` +- `generate_fill_reports` +- `generate_position_status_reports` + +```mermaid +flowchart TD + Start[Startup Reconciliation] --> Fetch[Fetch venue reports
orders, fills, positions] + Fetch --> Dedup[Deduplicate reports
log warnings for duplicates] + Dedup --> Orders[Order Reconciliation
align order states, generate missing events] + Orders --> Fills[Fill Reconciliation
verify fills, generate missing OrderFilled events] + Fills --> Pos[Position Reconciliation
compare net positions per instrument] + Pos --> Match{Positions
match venue?} + Match -->|Yes| Done[Reconciliation complete
system ready for trading] + Match -->|No| Gen[Generate missing orders
strategy: EXTERNAL, tag: RECONCILIATION] + Gen --> Done +``` + +The system reconciles its state against these reports, which represent external reality: + +- **Duplicate check**: + - Deduplicates order reports within the batch and logs warnings. + - Logs duplicate trade IDs as warnings for investigation. +- **Order reconciliation**: + - Generates and applies events to move orders from cached state to current state. + - Infers `OrderFilled` events for missing trade reports. + - Generates external order events for unrecognized client order IDs or reports missing a client order ID. + - Verifies fill report data consistency with tolerance-based price and commission comparisons. +- **Position reconciliation**: + - Matches the net position per instrument against venue position reports using instrument precision. + - Generates external order events when order reconciliation leaves a position that differs from the venue. + - When `generate_missing_orders` is enabled (default: True), generates orders with strategy ID `EXTERNAL` and tag `RECONCILIATION` to align discrepancies. + - Falls through a price hierarchy when generating reconciliation orders: + 1. **Calculated reconciliation price** (preferred): targets the correct average position. + 2. **Market mid-price**: uses the current bid-ask midpoint. + 3. **Current position average**: uses the existing position's average price. + 4. **MARKET order** (last resort): used only when no price data exists (no positions, no market data). + - Uses LIMIT orders when a price can be determined (cases 1-3) to preserve PnL accuracy. + - Skips zero quantity differences after precision rounding. +- **Partial window adjustment**: + - When `reconciliation_lookback_mins` is set, the window may miss opening fills. + - The system adjusts fills using lifecycle analysis to reconstruct positions accurately: + - Detects zero-crossings (position qty crosses through FLAT) to identify separate lifecycles. + - Adds synthetic opening fills when the earliest lifecycle is incomplete. + - Filters out closed lifecycles when the current lifecycle matches the venue position. + - Replaces a mismatched current lifecycle with a synthetic fill reflecting the venue position. + - Synthetic fills use calculated reconciliation prices to target correct average positions. + - See [Partial window adjustment scenarios](#partial-window-adjustment-scenarios) for details. +- **Exception handling**: + - Individual adapter failures do not abort the entire reconciliation process. + - Fill reports arriving before order status reports are deferred until order state is available. + +If reconciliation fails, the system logs an error and does not start. + +### Common reconciliation scenarios + +The tables below cover startup reconciliation (mass status) and runtime checks (in-flight order checks, open-order polls, own-books audits). + +#### Startup reconciliation + +| Scenario | Description | System behavior | +|----------------------------------------|------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------| +| **Order state discrepancy** | Local state differs from venue (e.g., local `SUBMITTED`, venue `REJECTED`). | Updates local order to match venue state, emits missing events. | +| **Missed fills** | Venue filled an order but the engine missed the event. | Generates missing `OrderFilled` events. | +| **Multiple fills** | Order has partial fills, some missed by the engine. | Reconstructs complete fill history from venue reports. | +| **External orders** | Orders exist on venue but not in local cache. | Creates orders with strategy ID `EXTERNAL` and tag `VENUE`. | +| **Partially filled then canceled** | Order partially filled then canceled by venue. | Updates state to `CANCELED`, preserves fill history. | +| **Different fill data** | Venue reports different fill price/commission than cached. | Preserves cached data, logs discrepancies. | +| **Filtered orders** | Orders marked for filtering via config. | Skips based on `filtered_client_order_ids` or instrument filters. | +| **Duplicate order reports** | Multiple orders share the same identifier. | Deduplicates with warning logged. | +| **Position quantity mismatch (long)** | Internal long position differs from venue (e.g., 100 vs 150). | Generates BUY LIMIT with calculated price when `generate_missing_orders=True`. | +| **Position quantity mismatch (short)** | Internal short position differs from venue (e.g., -100 vs -150). | Generates SELL LIMIT with calculated price when `generate_missing_orders=True`. | +| **Position reduction** | Venue position smaller than internal (e.g., internal 150 long, venue 100 long). | Generates opposite-side LIMIT order with calculated price. | +| **Position side flip** | Internal position opposite of venue (e.g., internal 100 long, venue 50 short). | Generates LIMIT order to close internal and open external position. | +| **Internal reconciliation orders** | Orders with strategy ID `EXTERNAL` and tag `RECONCILIATION`. | Never filtered, regardless of `filter_unclaimed_external_orders`. | + +#### Runtime checks + +| Scenario | Description | System behavior | +|-----------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| +| **In-flight order timeout** | Order remains unconfirmed beyond threshold. | After `inflight_check_retries`, resolves to `REJECTED`. | +| **Open orders check discrepancy** | Periodic poll detects a venue state change. | Confirms status at `open_check_interval_secs` and applies transitions. | +| **Own books audit mismatch** | Own order books diverge from venue public books. | Audits at `own_books_audit_interval_secs`, logs inconsistencies. | + +### Common reconciliation issues + +- **Missing trade reports**: Some venues filter out older trades. Increase `reconciliation_lookback_mins` or cache all events locally. +- **Position mismatches**: External orders that predate the lookback window cause position drift. Flatten the account before restarting to reset state. +- **Duplicate order IDs**: Deduplicated with warnings logged. Frequent duplicates may indicate venue data integrity issues. +- **Precision differences**: Small decimal differences are handled using instrument precision. Large discrepancies may indicate missing orders. +- **Out-of-order reports**: Fill reports arriving before order status reports are deferred until order state is available. + +:::tip +For persistent issues, drop cached state or flatten accounts before restarting. +::: + +### Reconciliation invariants + +The reconciliation system maintains three invariants: + +1. **Position quantity**: the final quantity matches the venue within instrument precision. +2. **Average entry price**: the position's average entry price matches the venue's reported price within tolerance (default 0.01%). +3. **PnL integrity**: all generated fills, including synthetic fills, use calculated prices that preserve correct unrealized PnL. + +These hold even when: + +- The reconciliation window misses complete fill history. +- Fills are missing from venue reports. +- Position lifecycles span beyond the lookback window. +- Multiple zero-crossings have occurred. + +### Partial window adjustment scenarios + +When `reconciliation_lookback_mins` limits the window, the system analyzes position lifecycles +from fills and adjusts to reconstruct positions accurately. + +| Scenario | Description | System behavior | +|--------------------------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------------------| +| **Complete lifecycle** | All fills from opening to current state are captured. | No adjustment. | +| **Incomplete single lifecycle** | Window misses opening fills, no zero-crossings. | Adds synthetic opening fill with calculated price. | +| **Multiple lifecycles, current matches** | Zero-crossings detected, current lifecycle matches venue. | Filters out old lifecycles, returns current only. | +| **Multiple lifecycles, current mismatch** | Zero-crossings detected, current lifecycle differs from venue. | Replaces current lifecycle with a single synthetic fill. | +| **Flat position** | Venue reports FLAT regardless of fill history. | No adjustment. | +| **No fills** | Window contains no fill reports. | No adjustment, empty result. | + +**Key concepts:** + +- **Zero-crossing**: position quantity crosses through zero (FLAT), marking a lifecycle boundary. +- **Lifecycle**: a sequence of fills between zero-crossings representing one open-close cycle. +- **Synthetic fill**: a calculated fill report representing missing activity, priced to achieve the correct average position. +- **Tolerance**: position matching uses configurable price tolerance (default 0.0001 = 0.01%) to absorb minor calculation differences. + +## Related guides + +- [Adapters](adapters.md) - Venue connectivity. +- [Execution](execution.md) - Order execution in live environments. +- [Backtesting](backtesting.md) - Testing strategies before deployment. diff --git a/nautilus-docs/docs/concepts/logging.md b/nautilus-docs/docs/concepts/logging.md new file mode 100644 index 0000000..91e9b2d --- /dev/null +++ b/nautilus-docs/docs/concepts/logging.md @@ -0,0 +1,470 @@ +# Logging + +The platform provides logging for both backtesting and live trading using a high-performance logging subsystem implemented in Rust +with a standardized facade from the `log` crate. + +The core logger operates in a separate thread and uses a multi-producer single-consumer (MPSC) channel to receive log messages. +This design ensures that the main thread remains performant, avoiding potential bottlenecks caused by log string formatting or file I/O operations. + +Logging output is configurable and supports: + +- **stdout/stderr writer** for console output +- **file writer** for persistent storage of logs + +:::info +Infrastructure such as [Vector](https://github.com/vectordotdev/vector) can be integrated to collect and aggregate events within your system. +::: + +## Architecture + +The logging subsystem captures events from multiple sources and routes them through an MPSC channel to a dedicated logging thread: + +```mermaid +flowchart TB + subgraph Sources["Log Sources"] + PY["Python Logger"] + NAUT["Nautilus Rust Components"] + LOG["External Rust Libraries
(using log crate)
rustls, etc."] + end + + subgraph Filtering["Filtering"] + LF["log_level / log_level_file
(LoggingConfig)"] + end + + subgraph Logger["Nautilus Logger"] + NL["Logger
(implements log::Log)"] + end + + subgraph Channel["MPSC Channel"] + TX["Sender (tx)"] + RX["Receiver (rx)"] + end + + subgraph Thread["Logging Thread"] + LT["Log Writer"] + end + + subgraph Output["Output"] + STDOUT["stdout/stderr"] + FILE["Log Files"] + end + + PY --> NL + NAUT --> NL + LOG --> LF --> NL + + NL --> TX --> RX --> LT + LT --> STDOUT + LT --> FILE + + subgraph Tracing["Tracing Subscriber (optional)"] + TRACE["External Rust Libraries
(using tracing crate)
hyper_util, h2, tokio, etc."] + EF["RUST_LOG
(EnvFilter)"] + FMT["fmt::Layer"] + end + + TRACE --> EF --> FMT --> STDOUT +``` + +- **Python and Nautilus components**: Log directly through the Nautilus Logger. +- **External `log` crate users**: Filtered by `log_level`/`log_level_file` in `LoggingConfig`. +- **External `tracing` crate users**: When enabled, output goes directly to stdout (separate from Nautilus logging), filtered by the `RUST_LOG` environment variable. +- **Logging thread**: All Nautilus log events are sent through an MPSC channel to a dedicated thread, ensuring the main thread isn't blocked by I/O operations. + +## Configuration + +Logging can be configured by importing the `LoggingConfig` object. +By default, log events with an 'INFO' `LogLevel` and higher are written to stdout/stderr. + +Log level (`LogLevel`) values include the following (matching standard log level conventions). + +The following log levels are supported: + +- `OFF` - Disable logging. +- `TRACE` - Most verbose; only emitted by Rust components (cannot be generated from Python). +- `DEBUG` - Detailed diagnostic information. +- `INFO` - General operational messages. +- `WARNING` - Potential issues that don't prevent operation. +- `ERROR` - Errors that may affect functionality. + +:::tip +You can set `TRACE` as a filter level to capture trace logs from Rust components, even though Python code cannot emit them directly. +::: + +See the `LoggingConfig` [API Reference](/docs/python-api-latest/config.html#nautilus_trader.common.config.LoggingConfig) for further details. + +Logging can be configured in the following ways: + +- Minimum `LogLevel` for stdout/stderr. +- Minimum `LogLevel` for log files. +- Maximum size before rotating a log file. +- Maximum number of backup log files to maintain when rotating. +- Automatic log file naming with date or timestamp components, or custom log file name. +- Directory for writing log files. +- Plain text or JSON log file formatting. +- Filtering of individual components by log level. +- ANSI colors in log lines. +- Bypass logging entirely. +- Print Rust config to stdout at initialization. +- Optionally initialize logging via the PyO3 bridge (`use_pyo3`) to capture log events emitted by Rust components. +- Truncate existing log file on startup if it already exists (`clear_log_file`) + +### Standard output logging + +Log messages are written to the console via stdout/stderr writers. The minimum log level can be configured using the `log_level` parameter. + +### File logging + +Log files are written to the current working directory by default. The naming convention and rotation behavior are configurable and follow specific patterns based on your settings. + +You can specify a custom log directory using `log_directory` and/or a custom file basename using `log_file_name`. + +**Log file formats:** + +- `None` (default) - Plain text format with `.log` extension. +- `"json"` - JSON format with `.json` extension, useful for log aggregation tools. + +For detailed information about log file naming conventions and rotation behavior, see the [Log file rotation](#log-file-rotation) and [Log file naming convention](#log-file-naming-convention) sections below. + +#### Log file rotation + +Rotation behavior depends on both the presence of a size limit and whether a custom file name is provided: + +- **Size-based rotation**: + - Enabled by specifying the `log_file_max_size` parameter (e.g., `100_000_000` for 100 MB). + - When writing a log entry would make the current file exceed this size, the file is closed and a new one is created. +- **Date-based rotation (default naming only)**: + - Applies when no `log_file_max_size` is specified and no custom `log_file_name` is provided. + - At each UTC date change (midnight), the current log file is closed and a new one is started, creating one file per UTC day. +- **No rotation**: + - When a custom `log_file_name` is provided without a `log_file_max_size`, logs continue to append to the same file. + - Note: Size-based rotation takes precedence - if both a custom name and size limit are provided, rotation still occurs. +- **Backup file management**: + - Controlled by the `log_file_max_backup_count` parameter (default: 5), limiting the total number of rotated files kept. + - When this limit is exceeded, the oldest backup files are automatically removed. + +#### Log file naming convention + +The default naming convention ensures log files are uniquely identifiable and timestamped. +The format depends on whether file rotation is enabled: + +**With file rotation enabled**: + +- **Format**: `{trader_id}_{%Y-%m-%d_%H%M%S:%3f}_{instance_id}.{log|json}` +- **Example**: `TESTER-001_2025-04-09_210721:521_d7dc12c8-7008-4042-8ac4-017c3db0fc38.log` +- **Components**: + - `{trader_id}`: The trader identifier (e.g., `TESTER-001`). + - `{%Y-%m-%d_%H%M%S:%3f}`: Full ISO 8601-compliant datetime with millisecond resolution. + - `{instance_id}`: A unique instance identifier. + - `{log|json}`: File suffix based on format setting. + +**Without size-based rotation (default naming)**: + +- **Format**: `{trader_id}_{%Y-%m-%d}_{instance_id}.{log|json}` +- **Example**: `TESTER-001_2025-04-09_d7dc12c8-7008-4042-8ac4-017c3db0fc38.log` +- **Components**: + - `{trader_id}`: The trader identifier. + - `{%Y-%m-%d}`: Date only (YYYY-MM-DD). + - `{instance_id}`: A unique instance identifier. + - `{log|json}`: File suffix based on format setting. +- **Note**: With default naming and no size limit, logs rotate daily at UTC midnight. + +**Custom naming**: + +If `log_file_name` is set (e.g., `my_custom_log`): + +- With rotation disabled: The file will be named exactly as provided (e.g., `my_custom_log.log`). +- With rotation enabled: The file will include the custom name and timestamp (e.g., `my_custom_log_2025-04-09_210721:521.log`). + +### Component log filtering + +The `log_component_levels` parameter can be used to set log levels for each component individually. +The input value should be a dictionary of component ID strings to log level strings: `dict[str, str]`. + +Below is an example of a trading node logging configuration that includes some of the options mentioned above: + +```python +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import TradingNodeConfig + +config_node = TradingNodeConfig( + trader_id="TESTER-001", + logging=LoggingConfig( + log_level="INFO", + log_level_file="DEBUG", + log_file_format="json", + log_component_levels={ "Portfolio": "INFO" }, + ), + ... # Omitted +) +``` + +For backtesting, the `BacktestEngineConfig` class can be used instead of `TradingNodeConfig`, as the same options are available. + +### Environment variable configuration + +The `NAUTILUS_LOG` environment variable provides an alternative way to configure logging using a semicolon-separated spec string. This is useful for Rust-only binaries or when you want to override logging settings without modifying code. + +```bash +export NAUTILUS_LOG="stdout=Info;fileout=Debug;RiskEngine=Error;is_colored" +``` + +**Supported keys:** + +| Key | Type | Description | +|-----------------------|-----------|--------------------------------------------------| +| `stdout` | Log level | Maximum level for stdout output. | +| `fileout` | Log level | Maximum level for file output. | +| `is_colored` | Flag | Enable ANSI colors (default: true). | +| `print_config` | Flag | Print config to stdout at startup. | +| `log_components_only` | Flag | Only log components with explicit filters. | +| `` | Log level | Component-specific level (exact match). | +| `` | Log level | Module-specific level (prefix match, Rust only). | + +Flags are enabled by their presence in the spec string (no value needed). Log levels are case-insensitive: `Off`, `Trace`, `Debug`, `Info`, `Warning` (or `Warn`), `Error`. + +:::note +For Rust-only binaries, setting `NAUTILUS_LOG` enables lazy initialization of the logging subsystem on first use, without requiring explicit `init_logging()` calls. +::: + +### Components-only logging + +When focusing on a subset of noisy systems, enable `log_components_only` to log messages only from components explicitly listed in `log_component_levels`. All other components are suppressed regardless of the global `log_level` or file level. + +Example (Python configuration): + +```python +logging = LoggingConfig( + log_level="INFO", + log_component_levels={ + "RiskEngine": "DEBUG", + "Portfolio": "INFO", + }, + log_components_only=True, +) +``` + +If configuring via the environment using the Rust spec string, include `log_components_only` alongside component filters, for example: + +```bash +export NAUTILUS_LOG="stdout=Info;log_components_only;RiskEngine=Debug;Portfolio=Info" +``` + +### Module path filtering (Rust only) + +When using the `NAUTILUS_LOG` environment variable, you can filter by Rust module paths in addition to component names. Keys containing `::` are treated as module path filters with prefix matching, while keys without `::` are component filters with exact matching. + +```bash +# Filter all adapters to Warn, but allow Debug for OKX specifically +export NAUTILUS_LOG="stdout=Info;nautilus_okx=Warn;nautilus_okx::websocket=Debug" +``` + +The longest matching prefix takes precedence. In the example above, `nautilus_okx::websocket::handler` would use the `Debug` level (longer prefix), while `nautilus_okx::data` would use `Warn`. + +:::tip +Rust log macros automatically capture the module path when no explicit component is provided. This enables module-level filtering to work with standard logging calls. +::: + +:::note +Module path filtering is only available via the `NAUTILUS_LOG` environment variable. The Python `log_component_levels` configuration uses component name matching only. +::: + +:::warning +If `log_components_only=True` (or `log_components_only` is present in the spec string) and `log_component_levels` is empty, no log messages will be emitted to stdout/stderr or files. Add at least one component filter or disable components-only logging. +::: + +### Log colors + +ANSI color codes improve log readability in terminals. +In environments that do not support ANSI color rendering (such as some cloud environments or text editors), +these color codes may not be appropriate as they can appear as raw text. + +To accommodate for such scenarios, the `LoggingConfig.log_colors` option can be set to `false`. +Disabling `log_colors` will prevent the addition of ANSI color codes to the log messages, +which avoids raw escape codes in environments without color support. + +## Using a logger directly + +It's possible to use `Logger` objects directly, and these can be initialized anywhere (very similar to the Python built-in `logging` API). + +If you ***aren't*** using an object which already initializes a `NautilusKernel` (and logging) such as `BacktestEngine` or `TradingNode`, +then you can activate logging in the following way: + +```python +from nautilus_trader.common.component import init_logging +from nautilus_trader.common.component import Logger + +log_guard = init_logging() +logger = Logger("MyLogger") +``` + +See the [`init_logging` API Reference](/docs/python-api-latest/common.html) for further details. + +:::warning +Only one logging subsystem can be initialized per process with an `init_logging` call. Multiple `LogGuard` instances (up to 255) can exist concurrently, and the logging thread will remain active until all guards are dropped. +::: + +## LogGuard: managing log lifecycle + +The `LogGuard` ensures that the logging subsystem remains active and operational throughout the lifecycle of a process. +It prevents premature shutdown of the logging subsystem when running multiple engines in the same process. + +### Reference counting implementation + +The logging system uses reference counting to track active `LogGuard` instances: + +- **Counter increments**: When a new `LogGuard` is created, an atomic counter is incremented. +- **Counter decrements**: When a `LogGuard` is dropped, the counter is decremented. +- **Logging thread termination**: When the counter reaches zero (last `LogGuard` dropped), the logging thread is properly joined to ensure all pending log messages are written before the process terminates. +- **Maximum guards**: The system supports up to 255 concurrent `LogGuard` instances. Attempting to create more raises a `RuntimeError`. + +This mechanism ensures that: + +1. `LogGuard` keeps the logging thread alive and flushes on drop; abrupt termination (crashes, kill signals) can still lose buffered logs. +2. The logging thread remains active as long as any `LogGuard` exists. +3. On graceful shutdown, all buffered logs are properly flushed to their destinations. + +### Why use LogGuard? + +Without a `LogGuard`, any attempt to run sequential engines in the same process may result in errors such as: + +``` +Error sending log event: [INFO] ... +``` + +This occurs because the logging subsystem's underlying channel and Rust `Logger` are closed when the first engine is disposed. +As a result, subsequent engines lose access to the logging subsystem, leading to these errors. + +By using a `LogGuard`, you can ensure consistent logging behavior across multiple backtests or engine runs in the same process. +The `LogGuard` retains the resources of the logging subsystem and ensures that logs continue to function correctly, +even as engines are disposed and initialized. + +:::note +Using `LogGuard` is required to maintain consistent logging behavior throughout a process with multiple engines. +::: + +## Running multiple engines + +The following example demonstrates how to use a `LogGuard` when running multiple engines sequentially in the same process: + +```python +log_guard = None # Initialize LogGuard reference + +for i in range(number_of_backtests): + engine = setup_engine(...) + + # Assign reference to LogGuard + if log_guard is None: + log_guard = engine.get_log_guard() + + # Add actors and execute the engine + actors = setup_actors(...) + engine.add_actors(actors) + engine.run() + engine.dispose() # Dispose safely +``` + +### Steps + +- **Initialize LogGuard once**: The `LogGuard` is obtained from the first engine (`engine.get_log_guard()`) and is retained throughout the process. This ensures that the logging subsystem remains active. +- **Dispose engines safely**: Each engine is safely disposed of after its backtest completes. The `LogGuard` remains valid after `engine.dispose()` - only the engine is cleaned up, not the logging subsystem. +- **Reuse LogGuard**: The same `LogGuard` instance is reused for subsequent engines, preventing the logging subsystem from shutting down prematurely. + +### Considerations + +- **Multiple LogGuards per process**: The system supports up to 255 concurrent `LogGuard` instances per process. Each guard increments a reference counter when created and decrements it when dropped. +- **Thread safety**: The logging subsystem, including `LogGuard`, is thread-safe, ensuring consistent behavior even in multi-threaded environments. +- **Automatic cleanup**: When the last `LogGuard` is dropped (reference count reaches zero), the logging thread is properly joined to ensure all pending logs are written before the process terminates. + +## Tracing subscriber for external Rust libraries + +External Rust crates that use the `tracing` crate can have their log output displayed by enabling +the tracing subscriber. This is useful for debugging external dependencies or when integrating +custom Rust components (such as feature extractors or adapters) compiled as separate PyO3 extensions. + +### Enabling the subscriber + +Enable the tracing subscriber by setting `use_tracing=True` in `LoggingConfig`: + +```python +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import TradingNodeConfig + +config_node = TradingNodeConfig( + trader_id="TESTER-001", + logging=LoggingConfig( + log_level="INFO", + use_tracing=True, + ), + ... # Omitted +) +``` + +Alternatively, call `init_tracing()` directly: + +```python +from nautilus_trader.core import nautilus_pyo3 + +nautilus_pyo3.init_tracing() +``` + +### Filtering with RUST_LOG + +The `RUST_LOG` environment variable controls which tracing events are displayed: + +```bash +# Show debug logs from your crate, warn and above from hyper +RUST_LOG=my_feature_extractor=debug,hyper=warn python my_script.py +``` + +If `RUST_LOG` is not set, the default filter level is `warn`. + +### How it works + +The tracing subscriber uses a `tracing-subscriber` fmt layer with a custom formatter to output +directly to stdout. This is separate from the Nautilus logging infrastructure - tracing output +uses a Nautilus-aligned format with nanosecond timestamps. + +Example tracing output: + +``` +2026-01-24T05:51:42.809619000Z [DEBUG] hyper_util::client::legacy::connect::http: connecting to 104.18.5.240:443 +2026-01-24T05:51:42.810543000Z [DEBUG] hyper_util::client::legacy::pool: pooling idle connection for ("https", api.example.com) +``` + +**Differences from Nautilus logging:** + +- Tracing output goes directly to stdout, not through the Nautilus logging thread. +- Tracing events are not written to Nautilus log files. +- Filtering is controlled exclusively by `RUST_LOG`, independent of `LoggingConfig`. + +For external libraries that use the `log` crate (such as `rustls`), their events go through +the Nautilus logger and are filtered by `log_level`/`log_level_file` in `LoggingConfig`. + +:::tip +`RUST_LOG` only affects crates using `tracing`. For crates using `log`, configure verbosity +via `LoggingConfig` or the `NAUTILUS_LOG` environment variable (e.g., `NAUTILUS_LOG=stdout=Debug`). +::: + +:::note +The tracing subscriber can only be initialized once per process. When using `use_tracing=True` in +`LoggingConfig`, subsequent kernel creations safely skip re-initialization. Direct calls to +`init_tracing()` when already initialized will raise an error. +::: + +## Platform-specific considerations + +### Windows shutdown behavior + +On Windows, non-deterministic garbage collection during interpreter shutdown can occasionally +prevent the logging thread from joining properly. When the last `LogGuard` is dropped, the +logging subsystem signals the background thread to close and joins it to ensure all pending +messages are written. If Python's garbage collector delays dropping the guard until after +interpreter shutdown has begun, this join may not complete, resulting in truncated logs. + +This issue is tracked in GitHub [issue #3027](https://github.com/nautechsystems/nautilus_trader/issues/3027). +A more deterministic shutdown mechanism is under consideration. + +## Related guides + +- [Architecture](architecture.md) - System architecture including logging infrastructure. diff --git a/nautilus-docs/docs/concepts/message_bus.md b/nautilus-docs/docs/concepts/message_bus.md new file mode 100644 index 0000000..e94f182 --- /dev/null +++ b/nautilus-docs/docs/concepts/message_bus.md @@ -0,0 +1,473 @@ +# Message Bus + +The `MessageBus` enables communication between system components through message passing. +This design creates a loosely coupled architecture where components interact without +direct dependencies. + +The *messaging patterns* include: + +- Point-to-Point +- Publish/Subscribe +- Request/Response + +Messages exchanged via the `MessageBus` fall into three categories: + +- Data +- Events +- Commands + +## Data and signal publishing + +While the `MessageBus` is a lower-level component that users typically interact with indirectly, +`Actor` and `Strategy` classes provide convenient methods built on top of it: + +```python +def publish_data(self, data_type: DataType, data: Data) -> None: +def publish_signal(self, name: str, value, ts_event: int = 0) -> None: +``` + +These methods allow you to publish custom data and signals efficiently without needing to work directly with the `MessageBus` interface. + +## Direct access + +For advanced users or specialized use cases, direct access to the message bus is available within `Actor` and `Strategy` +classes through the `self.msgbus` reference, which provides the full message bus interface. + +To publish a custom message directly, you can specify a topic as a `str` and any Python `object` as the message payload, for example: + +```python +self.msgbus.publish("MyTopic", "MyMessage") +``` + +## Messaging styles + +NautilusTrader is an **event-driven** framework where components communicate by sending and receiving messages. +Understanding the different messaging styles helps when building trading systems. + +This guide explains the three primary messaging patterns available in NautilusTrader: + +| **Messaging Style** | **Purpose** | **Best For** | +|:---------------------------------------------|:--------------------------------------------|:------------------------------------------------------| +| **MessageBus - Publish/Subscribe to topics** | Low-level, direct access to the message bus | Custom events, system-level communication | +| **Actor-Based - Publish/Subscribe Data** | Structured trading data exchange | Trading metrics, indicators, data needing persistence | +| **Actor-Based - Publish/Subscribe Signal** | Lightweight notifications | Simple alerts, flags, status updates | + +Each approach serves different purposes. This section helps you decide which pattern to use. + +### MessageBus publish/subscribe to topics + +#### Concept + +The `MessageBus` is the central hub for all messages in NautilusTrader. It enables a **publish/subscribe** pattern +where components can publish events to **named topics**, and other components can subscribe to receive those messages. +This decouples components, allowing them to interact indirectly via the message bus. + +#### Key benefits and use cases + +The message bus approach is ideal when you need: + +- **Cross-component communication** within the system. +- **Flexibility** to define any topic and send any type of payload (any Python object). +- **Decoupling** between publishers and subscribers who don't need to know about each other. +- **Global Reach** where messages can be received by multiple subscribers. +- Working with events that don't fit within the predefined `Actor` model. +- Advanced scenarios requiring full control over messaging. + +#### Considerations + +- You must track topic names manually (typos could result in missed messages). +- You must define handlers manually. + +#### Quick overview code + +```python +from nautilus_trader.core.message import Event + +# Define a custom event +class Each10thBarEvent(Event): + TOPIC = "each_10th_bar" # Topic name + def __init__(self, bar): + self.bar = bar + +# Subscribe in a component (in Strategy) +self.msgbus.subscribe(Each10thBarEvent.TOPIC, self.on_each_10th_bar) + +# Publish an event (in Strategy) +event = Each10thBarEvent(bar) +self.msgbus.publish(Each10thBarEvent.TOPIC, event) + +# Handler (in Strategy) +def on_each_10th_bar(self, event: Each10thBarEvent): + self.log.info(f"Received 10th bar: {event.bar}") +``` + +#### Full example + +[MessageBus Example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/example_09_messaging_with_msgbus) + +### Actor-based publish/subscribe data + +#### Concept + +This approach provides a way to exchange trading specific data between `Actor`s in the system. +(note: each `Strategy` inherits from `Actor`). It inherits from `Data`, which ensures proper timestamping +and ordering of events - crucial for correct backtest processing. + +#### Key benefits and use cases + +The Data publish/subscribe approach works well when you need: + +- **Exchange of structured trading data** like market data, indicators, custom metrics, or option greeks. +- **Proper event ordering** via built-in timestamps (`ts_event`, `ts_init`) crucial for backtest accuracy. +- **Data persistence and serialization** through the `@customdataclass` decorator, integrating with NautilusTrader's data catalog system. +- **Standardized trading data exchange** between system components. + +#### Considerations + +- Requires defining a class that inherits from `Data` or uses `@customdataclass`. + +#### Inheriting from `Data` vs. using `@customdataclass` + +**Inheriting from `Data` class:** + +- Defines abstract properties `ts_event` and `ts_init` that must be implemented by the subclass. These ensure proper data ordering in backtests based on timestamps. + +**The `@customdataclass` decorator:** + +- Adds `ts_event` and `ts_init` attributes if they are not already present. +- Provides serialization functions: `to_dict()`, `from_dict()`, `to_bytes()`, `to_arrow()`, etc. +- Enables data persistence and external communication. + +#### Quick overview code + +```python +from nautilus_trader.core.data import Data +from nautilus_trader.model.custom import customdataclass + +@customdataclass +class GreeksData(Data): + delta: float + gamma: float + +# Publish data (in Actor / Strategy) +data = GreeksData(delta=0.75, gamma=0.1, ts_event=1_630_000_000_000_000_000, ts_init=1_630_000_000_000_000_000) +self.publish_data(GreeksData, data) + +# Subscribe to receiving data (in Actor / Strategy) +self.subscribe_data(GreeksData) + +# Handler (this is static callback function with fixed name) +def on_data(self, data: Data): + if isinstance(data, GreeksData): + self.log.info(f"Delta: {data.delta}, Gamma: {data.gamma}") +``` + +#### Full example + +[Actor-Based Data Example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/example_10_messaging_with_actor_data) + +### Actor-based publish/subscribe signal + +#### Concept + +**Signals** are a lightweight way to publish and subscribe to simple notifications within the actor framework. +This is the simplest messaging approach, requiring no custom class definitions. + +#### Key benefits and use cases + +The Signal messaging approach works well when you need: + +- **Simple, lightweight notifications/alerts** like "RiskThresholdExceeded" or "TrendUp". +- **Quick, on-the-fly messaging** without defining custom classes. +- **Broadcasting alerts or flags** as primitive data (`int`, `float`, or `str`). +- **Easy API integration** with straightforward methods (`publish_signal`, `subscribe_signal`). +- **Multiple subscriber communication** where all subscribers receive signals when published. +- **Minimal setup overhead** with no class definitions required. + +#### Considerations + +- Each signal can contain only **single value** of type: `int`, `float`, and `str`. That means no support for complex data structures or other Python types. +- In the `on_signal` handler, you can only differentiate between signals using `signal.value`, as the signal name is not accessible in the handler. + +#### Quick overview code + +```python +# Define signal constants for better organization (optional but recommended) +import types +from nautilus_trader.core.datetime import unix_nanos_to_dt +from nautilus_trader.common.enums import LogColor + +signals = types.SimpleNamespace() +signals.NEW_HIGHEST_PRICE = "NewHighestPriceReached" +signals.NEW_LOWEST_PRICE = "NewLowestPriceReached" + +# Subscribe to signals (in Actor/Strategy) +self.subscribe_signal(signals.NEW_HIGHEST_PRICE) +self.subscribe_signal(signals.NEW_LOWEST_PRICE) + +# Publish a signal (in Actor/Strategy) +self.publish_signal( + name=signals.NEW_HIGHEST_PRICE, + value=signals.NEW_HIGHEST_PRICE, # value can be the same as name for simplicity + ts_event=bar.ts_event, # timestamp from triggering event +) + +# Handler (this is static callback function with fixed name) +def on_signal(self, signal): + # IMPORTANT: We match against signal.value, not signal.name + match signal.value: + case signals.NEW_HIGHEST_PRICE: + self.log.info( + f"New highest price was reached. | " + f"Signal value: {signal.value} | " + f"Signal time: {unix_nanos_to_dt(signal.ts_event)}", + color=LogColor.GREEN + ) + case signals.NEW_LOWEST_PRICE: + self.log.info( + f"New lowest price was reached. | " + f"Signal value: {signal.value} | " + f"Signal time: {unix_nanos_to_dt(signal.ts_event)}", + color=LogColor.RED + ) +``` + +#### Full example + +[Actor-Based Signal Example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/example_11_messaging_with_actor_signals) + +### Summary and decision guide + +Here's a quick reference to help you decide which messaging style to use: + +#### Decision guide: Which style to choose? + +| **Use Case** | **Recommended Approach** | **Setup required** | +|:--------------------------------------------|:--------------------------------------------------------------------------------|:-------------------| +| Custom events or system-level communication | `MessageBus` + Pub/Sub to topic | Topic + Handler management | +| Structured trading data | `Actor` + Pub/Sub Data + optional `@customdataclass` if serialization is needed | New class definition inheriting from `Data` (handler `on_data` is predefined) | +| Simple alerts/notifications | `Actor` + Pub/Sub Signal | Signal name only | + +## External publishing + +The `MessageBus` can be *backed* with any database or message broker technology which has an +integration written for it, this then enables external publishing of messages. + +:::info +Redis is currently supported for all serializable messages which are published externally. +The minimum supported Redis version is 6.2 (required for [streams](https://redis.io/docs/latest/develop/data-types/streams/) functionality). +::: + +Under the hood, when a backing database (or any other compatible technology) is configured, +all outgoing messages are first serialized, then transmitted via a Multiple-Producer Single-Consumer (MPSC) channel to a separate thread (implemented in Rust). +In this separate thread, the message is written to its final destination, which is presently Redis streams. + +Offloading I/O to a separate thread keeps the main thread unblocked. + +### Serialization + +Nautilus supports serialization for: + +- All Nautilus built-in types (serialized as dictionaries `dict[str, Any]` containing serializable primitives). +- Python primitive types (`str`, `int`, `float`, `bool`, `bytes`). + +You can add serialization support for custom types by registering them through the `serialization` subpackage. + +```python +def register_serializable_type( + cls, + to_dict: Callable[[Any], dict[str, Any]], + from_dict: Callable[[dict[str, Any]], Any], +): + ... +``` + +- `cls`: The type to register. +- `to_dict`: The delegate to instantiate a dict of primitive types from the object. +- `from_dict`: The delegate to instantiate the object from a dict of primitive types. + +## Configuration + +The message bus external backing technology can be configured by importing the `MessageBusConfig` object and passing this to +your `TradingNodeConfig`. Each of these config options will be described below. + +```python +... # Other config omitted +message_bus=MessageBusConfig( + database=DatabaseConfig(), + encoding="json", + timestamps_as_iso8601=True, + buffer_interval_ms=100, + autotrim_mins=30, + use_trader_prefix=True, + use_trader_id=True, + use_instance_id=False, + streams_prefix="streams", + types_filter=[QuoteTick, TradeTick], +) +... +``` + +### Database config + +A `DatabaseConfig` must be provided, for a default Redis setup on the local +loopback you can pass a `DatabaseConfig()`, which will use defaults to match. + +### Encoding + +Two encodings are currently supported by the built-in `Serializer` used by the `MessageBus`: + +- JSON (`json`) +- MessagePack (`msgpack`) + +Use the `encoding` config option to control the message writing encoding. + +:::tip +The `msgpack` encoding is used by default as it offers the most optimal serialization and memory performance. +We recommend using `json` encoding for human readability when performance is not a primary concern. +::: + +### Timestamp formatting + +By default timestamps are formatted as UNIX epoch nanosecond integers. Alternatively you can +configure ISO 8601 string formatting by setting the `timestamps_as_iso8601` to `True`. + +### Message stream keys + +Message stream keys are essential for identifying individual trader nodes and organizing messages within streams. +They can be tailored to meet your specific requirements and use cases. In the context of message bus streams, a trader key is typically structured as follows: + +``` +trader:{trader_id}:{instance_id}:{streams_prefix} +``` + +The following options are available for configuring message stream keys: + +#### Trader prefix + +If the key should begin with the `trader` string. + +#### Trader ID + +If the key should include the trader ID for the node. + +#### Instance ID + +Each trader node is assigned a unique 'instance ID,' which is a UUIDv4. This instance ID helps distinguish individual traders when messages +are distributed across multiple streams. You can include the instance ID in the trader key by setting the `use_instance_id` configuration option to `True`. +This is particularly useful when you need to track and identify traders across various streams in a multi-node trading system. + +#### Streams prefix + +The `streams_prefix` string enables you to group all streams for a single trader instance or organize +messages for multiple instances. Configure this by passing a string to the `streams_prefix` configuration +option, ensuring other prefixes are set to false. + +#### Stream per topic + +Indicates whether the producer will write a separate stream for each topic. This is particularly +useful for Redis backings, which do not support wildcard topics when listening to streams. +If set to False, all messages will be written to the same stream. + +:::info +Redis does not support wildcard stream topics. For better compatibility with Redis, it is recommended to set this option to False. +::: + +### Types filtering + +When messages are published on the message bus, they are serialized and written to a stream if a backing +for the message bus is configured and enabled. To prevent flooding the stream with data like high-frequency +quotes, you may filter out certain types of messages from external publication. + +To enable this filtering mechanism, pass a list of `type` objects to the `types_filter` parameter in the message bus configuration, +specifying which types of messages should be excluded from external publication. + +```python +from nautilus_trader.config import MessageBusConfig +from nautilus_trader.model.data import QuoteTick +from nautilus_trader.model.data import TradeTick + +# Create a MessageBusConfig instance with types filtering +message_bus = MessageBusConfig( + types_filter=[QuoteTick, TradeTick] +) +``` + +### Stream auto-trimming + +The `autotrim_mins` configuration parameter allows you to specify the lookback window in minutes for automatic stream trimming in your message streams. +Automatic stream trimming helps manage the size of your message streams by removing older messages, ensuring that the streams remain manageable in terms of storage and performance. + +:::info +The current Redis implementation will maintain the `autotrim_mins` as a maximum width (plus roughly a minute, as streams are trimmed no more than once per minute). +Rather than a maximum lookback window based on the current wall clock time. +::: + +## External streams + +The message bus within a `TradingNode` (node) is referred to as the "internal message bus". +A producer node is one which publishes messages onto an external stream (see [external publishing](#external-publishing)). +The consumer node listens to external streams to receive and publish deserialized message payloads on its internal message bus. + +```mermaid +flowchart TB + producer[Producer Node] + stream[Stream] + consumer1[Consumer Node 1] + consumer2[Consumer Node 2] + + producer --> stream + stream --> consumer1 + stream --> consumer2 +``` + +:::tip +Set the `LiveDataEngineConfig.external_clients` with the list of `client_id`s intended to represent the external streaming clients. +The `DataEngine` will filter out subscription commands for these clients, ensuring that the external streaming provides the necessary data for any subscriptions to these clients. +::: + +### Example configuration + +The following example details a streaming setup where a producer node publishes Binance data externally, +and a downstream consumer node publishes these data messages onto its internal message bus. + +#### Producer node + +We configure the `MessageBus` of the producer node to publish to a `"binance"` stream. +The settings `use_trader_id`, `use_trader_prefix`, and `use_instance_id` are all set to `False` +to ensure a simple and predictable stream key that the consumer nodes can register for. + +```python +message_bus=MessageBusConfig( + database=DatabaseConfig(timeout=2), + use_trader_id=False, + use_trader_prefix=False, + use_instance_id=False, + streams_prefix="binance", # <--- + stream_per_topic=False, + autotrim_mins=30, +), +``` + +#### Consumer node + +We configure the `MessageBus` of the consumer node to receive messages from the same `"binance"` stream. +The node will listen to the external stream keys to publish these messages onto its internal message bus. +Additionally, we declare the client ID `"BINANCE_EXT"` as an external client. This ensures that the +`DataEngine` does not attempt to send data commands to this client ID, as we expect these messages to be +published onto the internal message bus from the external stream, to which the node has subscribed to the relevant topics. + +```python +data_engine=LiveDataEngineConfig( + external_clients=[ClientId("BINANCE_EXT")], +), +message_bus=MessageBusConfig( + database=DatabaseConfig(timeout=2), + external_streams=["binance"], # <--- +), +``` + +## Related guides + +- [Actors](actors.md) - Actors use the message bus for event handling. +- [Architecture](architecture.md) - Message bus role in system architecture. diff --git a/nautilus-docs/docs/concepts/options.md b/nautilus-docs/docs/concepts/options.md new file mode 100644 index 0000000..514bc9b --- /dev/null +++ b/nautilus-docs/docs/concepts/options.md @@ -0,0 +1,291 @@ +# Options + +Nautilus provides first-class support for options trading across traditional +and crypto markets. This includes option-specific instrument types, venue-provided +Greeks streaming, option chain aggregation, and a local Black-Scholes Greeks calculator +for risk management. + +## Option instrument types + +The platform defines several option instrument types: + +| Instrument | Description | +|------------------|----------------------------------------------------------------------------------------| +| `OptionContract` | Exchange-traded option (put or call) on an underlying with strike and expiry. | +| `OptionSpread` | Exchange-defined multi-leg options strategy (vertical, calendar, straddle) as one line. | +| `CryptoOption` | Option on a crypto underlying with crypto quote/settlement; inverse or quanto styles. | +| `BinaryOption` | Fixed-payout option that settles to 0 or 1 based on a binary outcome. | + +Greeks-relevant metadata varies by instrument type: + +- `OptionContract`, `CryptoOption` -- full Greeks inputs: `strike_price`, + `option_kind` (CALL/PUT), `expiration_utc`, `underlying`, `multiplier`. +- `OptionSpread` -- a combination of up to 4 option legs, each weighted by a + ratio. Has `underlying`, `expiration_utc`, and `strategy_type` (vertical, + calendar, straddle, etc.). Per-leg `strike_price` and `option_kind` live on + each leg's `OptionContract`, not on the spread itself. Greeks are computed + per leg and aggregated. Spreads are commonly used for orders (the exchange + executes as a single order), while the individual legs appear as positions. +- `BinaryOption` -- has `expiration_utc` and `outcome`/`description`, but no + `strike_price`, `option_kind`, or `underlying`. + +## Subscribing to Greeks + +Venues like Deribit and Bybit publish real-time Greeks alongside their options markets. +Nautilus provides two subscription levels: + +- **Per-instrument Greeks** -- subscribe to individual option contracts. +- **Option chain slices** -- subscribe to an aggregated view of an entire option series. + +### Per-instrument Greeks + +Subscribe to venue-provided Greeks for a single option contract from an actor or strategy: + +```python +from nautilus_trader.model.identifiers import ClientId + +client_id = ClientId("DERIBIT") +self.subscribe_option_greeks(instrument_id, client_id=client_id) +``` + +Handle incoming updates by implementing the `on_option_greeks` handler: + +```python +def on_option_greeks(self, greeks) -> None: + self.log.info( + f"{greeks.instrument_id}: " + f"delta={greeks.delta:.4f} gamma={greeks.gamma:.6f} " + f"vega={greeks.vega:.4f} theta={greeks.theta:.4f} " + f"mark_iv={greeks.mark_iv} underlying={greeks.underlying_price}" + ) +``` + +To stop receiving updates: + +```python +self.unsubscribe_option_greeks(instrument_id, client_id=client_id) +``` + +### Option chain subscriptions + +An option chain subscription aggregates quotes and Greeks across all strikes in an +option series into periodic `OptionChainSlice` snapshots. The `DataEngine` creates +one `OptionChainManager` per series and owns the full lifecycle: routing incoming +data through the manager, publishing snapshots, and managing wire subscriptions. + +```python +from nautilus_trader.core import nautilus_pyo3 + +series_id = nautilus_pyo3.OptionSeriesId(...) # identifies the series (venue, underlying, expiry) + +# Subscribe to 5 strikes above and below ATM, snapshot every 1000ms +strike_range = nautilus_pyo3.StrikeRange.atm_relative(strikes_above=5, strikes_below=5) +self.subscribe_option_chain( + series_id, + strike_range=strike_range, + snapshot_interval_ms=1000, +) +``` + +Handle snapshots by implementing the `on_option_chain` handler: + +```python +def on_option_chain(self, chain) -> None: + for strike in chain.strikes(): + call = chain.get_call(strike) + put = chain.get_put(strike) + if call and call.greeks: + self.log.info(f"Call {strike}: delta={call.greeks.delta:.4f}") +``` + +### Strike range filtering + +`StrikeRange` controls which strikes are active in a chain subscription: + +| Variant | Description | Example | +|----------------|------------------------------------------------------|-----------------------------------------------| +| `Fixed` | Subscribe to an explicit set of strikes. | `nautilus_pyo3.StrikeRange.fixed([...])` | +| `AtmRelative` | N strikes above and N below the current ATM strike. | `nautilus_pyo3.StrikeRange.atm_relative(5, 5)` | +| `AtmPercent` | All strikes within a percentage band around ATM. | `nautilus_pyo3.StrikeRange.atm_percent(0.10)` | + +For ATM-based variants, subscriptions are deferred until the ATM price is determined. +ATM is derived from the forward price embedded in venue-provided `OptionGreeks` updates +(the `underlying_price` field). It can also be seeded from an initial forward price +fetched via HTTP, allowing instant bootstrap before live WebSocket ticks arrive. As ATM +shifts, the active strike set rebalances automatically. + +### Snapshot vs. raw mode + +The `snapshot_interval_ms` parameter controls publishing behavior: + +- **Snapshot mode** (`snapshot_interval_ms=1000`): Quotes and Greeks accumulate in a + buffer and publish as an `OptionChainSlice` on a timer. Suitable for periodic + portfolio rebalancing or UI display. +- **Raw mode** (`snapshot_interval_ms=None`): Each quote or Greeks update publishes + a slice immediately. Suitable for latency-sensitive strategies that react to + individual updates. + +## Option chain architecture + +The option chain system is event-driven and built around per-series isolation. The +`DataEngine` creates one `OptionChainManager` (a PyO3 wrapper around the Rust +`OptionChainAggregator` and `AtmTracker`) per subscribed option series. The engine +owns the lifecycle: subscription routing, timer management, and message bus publishing. +The manager handles only aggregation state and ATM tracking. + +```mermaid +flowchart TD + subgraph DataEngine + DE[DataEngine] + TMR[SnapshotTimer] + end + + subgraph "OptionChainManager (per series)" + MGR[Manager / PyO3] + AGG[OptionChainAggregator] + ATM[AtmTracker] + end + + DC[DataClient] -- QuoteTick --> DE + DC -- OptionGreeks --> DE + DE -- "handle_quote()" --> MGR + DE -- "handle_greeks()" --> MGR + MGR --> AGG + MGR --> ATM + ATM -- "forward price" --> AGG + TMR -- "timer tick" --> DE + DE -- "snapshot()" --> MGR + MGR -- "OptionChainSlice" --> DE + DE -- publish --> MB((MessageBus)) + MB -- "on_option_chain" --> S[Actor / Strategy] + DE -- "sub/unsub" --> DC +``` + +### Component responsibilities + +#### DataEngine + +Holds one `OptionChainManager` per active `OptionSeriesId`. On +`SubscribeOptionChain`, it resolves instruments from the cache, creates the +manager, subscribes active instruments to the data client, and sets up the +snapshot timer. On each timer tick, it calls `manager.check_rebalance()` and +`manager.snapshot()`, forwarding any subscription changes directly to the data +client. On `UnsubscribeOptionChain` or when all instruments expire, it tears +down the manager, cancels the timer, and unsubscribes wire-level feeds. + +#### OptionChainManager (PyO3) + +A thin PyO3 wrapper around `OptionChainAggregator` and `AtmTracker`. It does +not interact with the message bus, clock, or data clients. The `DataEngine` +feeds it market data through `handle_quote()` and `handle_greeks()`, and +retrieves snapshots via `snapshot()`. Both `handle_*` methods return a boolean +indicating whether ATM bootstrap occurred (first ATM price arrived), which the +engine uses to trigger subscription of the real active instrument set. + +#### OptionChainAggregator + +Accumulates quotes and Greeks into call/put buffers using keep-latest semantics. +Instruments that did not update since the last snapshot are still included. Greeks +that arrive before any quote for an instrument are held in a `pending_greeks` +buffer and attached when the first quote arrives. On each `snapshot()` call, the +aggregator produces an immutable `OptionChainSlice`. + +#### AtmTracker + +Derives the ATM price reactively from the `underlying_price` field in incoming +`OptionGreeks` events (the venue-provided forward price for that expiry). It can +be pre-seeded from an HTTP forward price response for instant bootstrap without +waiting for WebSocket ticks. + +### Bootstrap and rebalancing + +For ATM-based strike ranges (`AtmRelative`, `AtmPercent`), the active instrument +set cannot be determined until the ATM price is known. There are two bootstrap +paths: + +**Instant bootstrap (forward price available):** + +1. `DataEngine` receives `SubscribeOptionChain`, resolves all instruments for the + series from the cache, and requests forward prices from the data client. +2. When the forward price response arrives, the engine creates the manager with + the ATM price pre-seeded. The manager computes the active strike set during + construction. +3. The engine subscribes the active instruments immediately. + +**Deferred bootstrap (no forward price):** + +1. Same as above, but no matching forward price is found in the response. +2. The engine creates the manager with no initial ATM price. The active set is + empty and no wire subscriptions are made for the chain. +3. Bootstrap depends on relevant Greeks data already flowing from other + subscriptions (e.g., per-instrument `subscribe_option_greeks` calls). When + the engine feeds an `OptionGreeks` event with `underlying_price` through + `handle_greeks()`, the manager bootstraps and returns `True`. The engine + then subscribes the now-active instrument set. + +Once bootstrapped, the aggregator monitors ATM drift. On each snapshot timer tick, +the engine calls `check_rebalance()` which returns any instruments to add or +remove. A hysteresis threshold and cooldown period prevent thrashing near strike +boundaries. + +## OptionGreeks data type + +`OptionGreeks` carries venue-provided sensitivities and implied volatility for a +single option contract: + +| Field | Type | Description | +|--------------------|------------------|-------------------------------------------------------| +| `instrument_id` | `InstrumentId` | The option contract these Greeks apply to. | +| `delta` | `float` | Rate of change of option price per unit underlying. | +| `gamma` | `float` | Rate of change of delta per unit underlying. | +| `vega` | `float` | Sensitivity to a 1% change in implied volatility. | +| `theta` | `float` | Daily time decay (dV/dt / 365.25). | +| `rho` | `float` | Sensitivity to a change in interest rate. | +| `mark_iv` | `float` or None | Mark implied volatility. | +| `bid_iv` | `float` or None | Bid implied volatility. | +| `ask_iv` | `float` or None | Ask implied volatility. | +| `underlying_price` | `float` or None | Underlying price at time of calculation. | +| `open_interest` | `float` or None | Open interest for the contract. | +| `ts_event` | `int` | UNIX timestamp (nanoseconds) of the event. | +| `ts_init` | `int` | UNIX timestamp (nanoseconds) when initialized. | + +## OptionChainSlice data type + +`OptionChainSlice` is a point-in-time snapshot of an entire option series. + +Properties: + +| Property | Type | Description | +|--------------|----------------------|------------------------------------------| +| `series_id` | `OptionSeriesId` | The option series identifier. | +| `atm_strike` | `Price` or None | Current ATM strike (if determined). | +| `ts_event` | `int` | UNIX timestamp (nanoseconds). | +| `ts_init` | `int` | UNIX timestamp (nanoseconds). | + +Call and put data are accessed through methods, not as direct properties. +Each `OptionStrikeData` returned by these methods contains a `quote` (`QuoteTick`) +and an optional `greeks` (`OptionGreeks`) for that strike. + +Methods: + +- `strikes()` -- all unique strike prices in the chain. +- `strike_count()`, `call_count()`, `put_count()` -- counts. +- `get_call(strike)`, `get_put(strike)` -- full `OptionStrikeData`. +- `get_call_greeks(strike)`, `get_put_greeks(strike)` -- Greeks only. +- `get_call_quote(strike)`, `get_put_quote(strike)` -- quote only. +- `is_empty()` -- true if the chain has no data. + +## Adapter support + +The following adapters currently support option Greeks subscriptions: + +| Adapter | Per-instrument Greeks | Option chains | +|---------|:---------------------:|:-------------:| +| Deribit | ✓ | ✓ | +| Bybit | ✓ | ✓ | + +## See also + +- [Greeks](greeks.md) -- local Greeks calculation and portfolio risk management. +- [Data](data.md) -- built-in data types and the subscription model. +- [Actors](actors.md) -- subscription and handler reference table. diff --git a/nautilus-docs/docs/concepts/order_book.md b/nautilus-docs/docs/concepts/order_book.md new file mode 100644 index 0000000..8240996 --- /dev/null +++ b/nautilus-docs/docs/concepts/order_book.md @@ -0,0 +1,246 @@ +# Order Book + +NautilusTrader provides a high-performance order book implemented in Rust, capable of +maintaining full book state from L1 through L3 data. The `OrderBook` is the primary +component for tracking public market depth, while the `OwnOrderBook` tracks your own +orders separately, enabling filtered views that show true available liquidity. + +:::note +This guide documents the Rust API. These types are also available from Python via +PyO3 bindings (`nautilus_pyo3.OrderBook`, `nautilus_pyo3.OwnOrderBook`). The legacy +Cython `OrderBook` (`nautilus_trader.model.book.OrderBook`) returned by +`cache.order_book()` has a similar but not identical interface. Refer to the +API reference for differences. +::: + +## Book types + +`OrderBook` instances are maintained per instrument for both backtesting and live trading: + +- `L3_MBO`: **Market by order** data. Tracks every order at every price level, keyed by order ID. +- `L2_MBP`: **Market by price** data. Aggregates orders by price level (one entry per price). +- `L1_MBP`: **Top-of-book** data, also known as best bid and offer (BBO). Captures only the + best prices. + +:::note +Top-of-book data such as `QuoteTick`, `TradeTick` and `Bar` can also maintain `L1_MBP` books. +::: + +## Subscribing to book data + +Strategies and actors subscribe to order book updates through the following methods. +Subscriptions and handlers are part of the Python strategy/actor layer: + +```python +# L3/L2 incremental deltas +self.subscribe_order_book_deltas(instrument_id) + +# Aggregated depth snapshots (up to 10 levels) +self.subscribe_order_book_depth(instrument_id) + +# Full book snapshots at a timed interval +self.subscribe_order_book_at_interval(instrument_id, interval_ms=1000) +``` + +Each subscription type delivers data to the corresponding handler: + +```python +def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: + ... + +def on_order_book_depth(self, depth: OrderBookDepth10) -> None: + ... + +def on_order_book(self, order_book: OrderBook) -> None: + ... +``` + +## Accessing the book + +The `OrderBook` exposes top-of-book accessors: + +```rust +let best_bid: Option = book.best_bid_price(); +let best_ask: Option = book.best_ask_price(); +let spread: Option = book.spread(); +let midpoint: Option = book.midpoint(); +``` + +## Analysis methods + +The `OrderBook` supports market depth analysis and execution simulation: + +```rust +// Average fill price for a given quantity +let avg_px = book.get_avg_px_for_quantity(quantity, OrderSide::Buy); + +// Average price and quantity for a target exposure (notional) +let (price, qty, exposure) = + book.get_avg_px_qty_for_exposure(target_exposure, OrderSide::Buy); + +// Cumulative quantity available at or better than a price +let qty = book.get_quantity_for_price(price, OrderSide::Buy); + +// Quantity at a specific price level only +let qty = book.get_quantity_at_level(price, OrderSide::Buy, 2); + +// Simulate fills against the book +let fills: Vec<(Price, Quantity)> = book.simulate_fills(&order); + +// All crossed levels regardless of order quantity +let levels = book.get_all_crossed_levels(OrderSide::Buy, price, 2); +``` + +## Integrity checks + +The `book_check_integrity` function validates that the book state is consistent +with its type: + +- **L1_MBP**: No more than one level per side. +- **L2_MBP**: No more than one order per price level. +- **L3_MBO**: No structural constraints (any number of orders at any level). +- **All types**: Best bid must not exceed best ask (crossed book). Locked markets + (bid == ask) are considered valid. + +These checks run internally during delta application. The instrument ID of incoming +deltas is also validated against the book's instrument ID, returning +`BookIntegrityError::InstrumentMismatch` on mismatch. + +## Pretty printing + +Both `OrderBook` and `OwnOrderBook` provide a `pprint` method that renders the book +as a human-readable table: + +```rust +book.pprint(5, None); +book.pprint(5, Some(Decimal::new(1, 2))); // group_size = 0.01 +``` + +The `group_size` parameter buckets price levels into coarser groups for instruments +with fine tick sizes. The output is a formatted table with bids on the left, prices +in the center, and asks on the right. + +## Own order book + +The `OwnOrderBook` tracks your own working orders separately from the public book. +Market making and other strategies use it to find the true available liquidity +at each price level (public size minus your own orders). + +The cache maintains own order books automatically as orders are submitted, accepted, +and filled. + +### Order lifecycle + +The `OwnOrderBook` tracks orders through their lifecycle. Orders are added when +submitted and updated as events arrive (accepted, partially filled, etc.). +Each `OwnBookOrder` carries: + +- `status`: Current order status (SUBMITTED, ACCEPTED, PARTIALLY_FILLED, etc.). +- `ts_accepted`: Timestamp when the order was accepted by the venue. +- `ts_submitted`: Timestamp when the order was submitted. + +These fields are used by the filtering logic to selectively include or exclude +orders from filtered views (see [Status and time filtering](#status-and-time-filtering)). + +### Auditing + +The `audit_open_orders` method reconciles the book against a set of known open +order IDs. Any orders in the book not in the provided set are removed and logged +as audit errors. The cache calls this periodically to keep the own book in sync +with the execution system. + +### Querying + +```rust +// Check if a specific order is tracked +let in_book = own_book.is_order_in_book(&client_order_id); + +// Get all tracked order IDs per side +let bid_ids = own_book.bid_client_order_ids(); +let ask_ids = own_book.ask_client_order_ids(); + +// Aggregated quantities per price level +let bid_qty = own_book.bid_quantity(None, None, None, None, None); +let ask_qty = own_book.ask_quantity(None, None, None, None, None); + +// Pretty print +own_book.pprint(5, None); +``` + +### Filtered views + +Subtract your own orders from the public book to see net available liquidity: + +```rust +// Filtered maps of price -> quantity (own orders subtracted) +let net_bids = book.bids_filtered_as_map(Some(10), Some(&own_book), None, None, None); +let net_asks = book.asks_filtered_as_map(Some(10), Some(&own_book), None, None, None); + +// Full filtered OrderBook with all analysis methods available +let filtered = book.filtered_view(Some(&own_book), Some(10), None, None, None); +let avg_px = filtered.get_avg_px_for_quantity(quantity, OrderSide::Buy); +``` + +The `filtered_view` method returns a new `OrderBook` with your own sizes subtracted, +giving access to the full set of analysis methods (`spread`, `midpoint`, +`get_avg_px_for_quantity`, etc.) on the net book. + +### Status and time filtering + +Filtered views support optional status and time-based filtering for own orders: + +```rust +let status = Some(AHashSet::from([OrderStatus::Accepted])); + +// Only subtract ACCEPTED orders (ignore SUBMITTED, PENDING_CANCEL, etc.) +let filtered = book.filtered_view(Some(&own_book), None, status, None, None); +``` + +The `accepted_buffer_ns` parameter provides a grace period: when set, only orders +where `ts_accepted + buffer <= now` are included. This excludes recently accepted +orders that may not yet appear in the public book feed. The buffer applies to the +`ts_accepted` field regardless of order status. Combine with a status filter to +also exclude non-accepted orders. + +```rust +// Only subtract orders accepted at least 500ms ago +let filtered = book.filtered_view( + Some(&own_book), + None, + None, + Some(500_000_000), + Some(clock.timestamp_ns()), +); +``` + +## Binary markets + +For binary/prediction markets (e.g., Polymarket), instruments have two complementary +sides (YES and NO) where prices sum to 1.0. A bid on the NO side at 0.40 is +economically equivalent to an ask on the YES side at 0.60. + +The `OwnOrderBook::combined_with_opposite` method handles this transformation, +merging your orders from both sides into a single view: + +```rust +let yes_own = own_yes_book + .cloned() + .unwrap_or_else(|| OwnOrderBook::new(yes_instrument_id)); + +let no_own = own_no_book + .cloned() + .unwrap_or_else(|| OwnOrderBook::new(no_instrument_id)); + +// Merge NO-side orders with parity price transform (1 - price) +let combined = yes_own.combined_with_opposite(&no_own).unwrap(); + +// Filter the public YES book using the combined own book +let filtered = book.filtered_view(Some(&combined), None, None, None, None); +``` + +The transformation works as follows: + +- NO asks at price P become bids at price 1 - P in the combined book. +- NO bids at price P become asks at price 1 - P in the combined book. + +This gives a complete picture of your own liquidity across both sides of the market. diff --git a/nautilus-docs/docs/concepts/orders.md b/nautilus-docs/docs/concepts/orders.md new file mode 100644 index 0000000..aecdc1e --- /dev/null +++ b/nautilus-docs/docs/concepts/orders.md @@ -0,0 +1,838 @@ +# Orders + +NautilusTrader supports a broad set of order types and execution instructions, exposing as much +of a trading venue's functionality as possible. Traders can define instructions and contingencies +for order execution and management across any trading strategy. + +## Overview + +All order types are derived from two fundamentals: *Market* and *Limit* orders. In terms of liquidity, they are opposites. +*Market* orders consume liquidity by executing immediately at the best available price, whereas *Limit* +orders provide liquidity by resting in the order book at a specified price until matched. + +The order types available for the platform are (using the `OrderType` enum values): + +- `MARKET` +- `LIMIT` +- `STOP_MARKET` +- `STOP_LIMIT` +- `MARKET_TO_LIMIT` +- `MARKET_IF_TOUCHED` +- `LIMIT_IF_TOUCHED` +- `TRAILING_STOP_MARKET` +- `TRAILING_STOP_LIMIT` + +:::info +NautilusTrader provides a unified API for many order types and execution instructions, but not all venues support every option. +If an order includes an instruction or option the target venue does not support, the system does not submit it. +Instead, it logs a clear, explanatory error. +::: + +### Terminology + +- An order is **aggressive** if its type is `MARKET` or if it executes as a *marketable* order (i.e., takes liquidity). +- An order is **passive** if it is not marketable (i.e., provides liquidity). +- An order is **active local** if it remains within the local system boundary in one of the following three non-terminal statuses: + - `INITIALIZED` + - `EMULATED` + - `RELEASED` +- An order is **in-flight** when at one of the following statuses: + - `SUBMITTED` + - `PENDING_UPDATE` + - `PENDING_CANCEL` +- An order is **open** when at one of the following (non-terminal) statuses: + - `ACCEPTED` + - `TRIGGERED` + - `PENDING_UPDATE` + - `PENDING_CANCEL` + - `PARTIALLY_FILLED` +- An order is **closed** when at one of the following (terminal) statuses: + - `DENIED` + - `REJECTED` + - `CANCELED` + - `EXPIRED` + - `FILLED` + +### Order state flow + +The following diagram illustrates the order lifecycle and primary state transitions: + +```mermaid +flowchart TB + subgraph local ["Active Local"] + Initialized + Emulated + Released + end + + subgraph flight ["In-Flight"] + Submitted + PendingUpdate + PendingCancel + end + + subgraph open ["Open (on venue)"] + Accepted + Triggered + PartiallyFilled + end + + subgraph closed ["Closed (terminal)"] + Denied + Rejected + Canceled + Expired + Filled + end + + Initialized -->|"Emulation trigger"| Emulated + Initialized -->|"Submit"| Submitted + Initialized -->|"System denied"| Denied + Emulated -->|"Triggered locally"| Released + Released --> Submitted + + Submitted -->|"Venue ACK"| Accepted + Submitted --> Rejected + + Accepted -->|"Stop hit"| Triggered + Accepted --> PartiallyFilled + Triggered --> PartiallyFilled + PartiallyFilled -->|"More fills"| PartiallyFilled + + Accepted --> PendingUpdate + Accepted --> PendingCancel + PartiallyFilled --> PendingUpdate + PartiallyFilled --> PendingCancel + PendingUpdate --> Accepted + PendingCancel --> Canceled + + Accepted --> Filled + Triggered --> Filled + PartiallyFilled --> Filled + PartiallyFilled --> Canceled + Accepted --> Expired +``` + +### Order status definitions + +| Status | Description | +|--------------------|-------------------------------------------------------------------------------------------| +| `INITIALIZED` | Order is instantiated within the Nautilus system. | +| `DENIED` | Order was denied by Nautilus for being invalid, unprocessable, or exceeding a risk limit. | +| `EMULATED` | Order is being emulated by the `OrderEmulator` component. | +| `RELEASED` | Order was released from the `OrderEmulator` component. | +| `SUBMITTED` | Order was submitted to the venue (awaiting acknowledgement). | +| `ACCEPTED` | Order was acknowledged by the venue as received and valid (may now be working). | +| `REJECTED` | Order was rejected by the trading venue. | +| `CANCELED` | Order was canceled (terminal). | +| `EXPIRED` | Order reached its GTD expiration (terminal). | +| `TRIGGERED` | Order's STOP price was triggered on the venue. | +| `PENDING_UPDATE` | Order is pending a modification request on the venue. | +| `PENDING_CANCEL` | Order is pending a cancellation request on the venue. | +| `PARTIALLY_FILLED` | Order has been partially filled on the venue. | +| `FILLED` | Order has been completely filled (terminal). | + +## Execution instructions + +Certain venues allow a trader to specify conditions and restrictions on +how an order will be processed and executed. The following is a brief +summary of the different execution instructions available. + +### Time in force + +The order's time in force specifies how long the order will remain open or active before any +remaining quantity is canceled. + +- `GTC` **(Good Till Cancel)**: The order remains active until canceled by the trader or the venue. +- `IOC` **(Immediate or Cancel / Fill and Kill)**: The order executes immediately, with any unfilled portion canceled. +- `FOK` **(Fill or Kill)**: The order executes immediately in full or not at all. +- `GTD` **(Good Till Date)**: The order remains active until a specified expiration date and time. +- `DAY` **(Good for session/day)**: The order remains active until the end of the current trading session. +- `AT_THE_OPEN` **(OPG)**: The order is only active at the open of the trading session. +- `AT_THE_CLOSE`: The order is only active at the close of the trading session. + +### Expire time + +This instruction is to be used in conjunction with the `GTD` time in force to specify the time +at which the order will expire and be removed from the venue's order book (or order management system). + +### Post-only + +An order which is marked as `post_only` will only ever participate in providing liquidity to the +limit order book, and never initiating a trade which takes liquidity as an aggressor. This option is +important for market makers, or traders seeking to restrict the order to a liquidity *maker* fee tier. + +### Reduce-only + +An order which is set as `reduce_only` will only ever reduce an existing position on an instrument and +never open a new position (if already flat). The exact behavior of this instruction can vary between venues. + +However, the behavior in the Nautilus `SimulatedExchange` is typical of a real venue. + +- Order will be canceled if the associated position is closed (becomes flat). +- Order quantity will be reduced as the associated position's size decreases. + +### Display quantity + +The `display_qty` specifies the portion of a *Limit* order which is displayed on the limit order book. +These are also known as iceberg orders as there is a visible portion to be displayed, with more quantity which is hidden. +Specifying a display quantity of zero is also equivalent to setting an order as `hidden`. + +### Trigger type + +Also known as [trigger method](https://www.interactivebrokers.com/en/software/tws/usersguidebook/configuretws/Modify%20the%20Stop%20Trigger%20Method.htm) +which is applicable to conditional trigger orders, specifying the method of triggering the stop price. + +- `DEFAULT`: The default trigger type for the venue (typically `LAST_PRICE` or `BID_ASK`). +- `LAST_PRICE`: The trigger price will be based on the last traded price. +- `BID_ASK`: The trigger price will be based on the bid for buy orders and ask for sell orders. +- `DOUBLE_LAST`: The trigger price will be based on the last two consecutive last prices. +- `DOUBLE_BID_ASK`: The trigger price will be based on the last two consecutive bid or ask prices as applicable. +- `LAST_OR_BID_ASK`: The trigger price will be based on either the last price or bid/ask. +- `MID_POINT`: The trigger price will be based on the mid-point between the bid and ask. +- `MARK_PRICE`: The trigger price will be based on the venue's mark price for the instrument. +- `INDEX_PRICE`: The trigger price will be based on the venue's index price for the instrument. + +### Trigger offset type + +Applicable to conditional trailing-stop trigger orders, specifies the method of triggering modification +of the stop price based on the offset from the *market* (bid, ask or last price as applicable). + +- `DEFAULT`: The default offset type for the venue (typically `PRICE`). +- `PRICE`: The offset is based on a price difference. +- `BASIS_POINTS`: The offset is based on a price percentage difference expressed in basis points (100bp = 1%). +- `TICKS`: The offset is based on a number of ticks. +- `PRICE_TIER`: The offset is based on a venue-specific price tier. + +### Contingent orders + +More advanced relationships can be specified between orders. +For example, child orders can be assigned to trigger only when the parent is activated or filled, or orders can be +linked so that one cancels or reduces the quantity of another. See the [Advanced Orders](#advanced-orders) section for more details. + +## Order factory + +The easiest way to create new orders is by using the built-in `OrderFactory`, which is +automatically attached to every `Strategy` class. This factory will take care +of lower level details - such as ensuring the correct trader ID and strategy ID are assigned, generation +of a necessary initialization ID and timestamp, and abstracts away parameters which don't necessarily +apply to the order type being created, or are only needed to specify more advanced execution instructions. + +This leaves the factory with simpler order creation methods to work with, all the +examples use an `OrderFactory` from within a `Strategy` context. + +See the [`OrderFactory` API Reference](/docs/python-api-latest/common.html#nautilus_trader.common.factories.OrderFactory) for further details. + +## Order types + +The following describes the order types which are available for the platform with a code example. +Any optional parameters will be clearly marked with a comment which includes the default value. + +### Market + +A *Market* order is an instruction by the trader to immediately trade +the given quantity at the best price available. You can also specify several +time in force options, and indicate whether this order is only intended to reduce +a position. + +In the following example we create a *Market* order on the Interactive Brokers [IdealPro](https://ibkr.info/node/1708) Forex ECN +to BUY 100,000 AUD using USD: + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import MarketOrder + +order: MarketOrder = self.order_factory.market( + instrument_id=InstrumentId.from_str("AUD/USD.IDEALPRO"), + order_side=OrderSide.BUY, + quantity=Quantity.from_int(100_000), + time_in_force=TimeInForce.IOC, # <-- optional (default GTC) + reduce_only=False, # <-- optional (default False) + tags=["ENTRY"], # <-- optional (default None) +) +``` + +See the [`MarketOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.market.MarketOrder) for further details. + +### Limit + +A *Limit* order is placed on the limit order book at a specific price, and will only +execute at that price (or better). + +In the following example we create a *Limit* order on the Binance Futures Crypto exchange to SELL 20 ETHUSDT-PERP Perpetual Futures +contracts at a limit price of 5000 USDT, as a market maker. + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import LimitOrder + +order: LimitOrder = self.order_factory.limit( + instrument_id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), + order_side=OrderSide.SELL, + quantity=Quantity.from_int(20), + price=Price.from_str("5_000.00"), + time_in_force=TimeInForce.GTC, # <-- optional (default GTC) + expire_time=None, # <-- optional (default None) + post_only=True, # <-- optional (default False) + reduce_only=False, # <-- optional (default False) + display_qty=None, # <-- optional (default None which indicates full display) + tags=None, # <-- optional (default None) +) +``` + +See the [`LimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.limit.LimitOrder) for further details. + +### Stop-Market + +A *Stop-Market* order is a conditional order which once triggered, will immediately +place a *Market* order. This order type is often used as a stop-loss to limit losses, either +as a SELL order against LONG positions, or as a BUY order against SHORT positions. + +In the following example we create a *Stop-Market* order on the Binance Spot/Margin exchange +to SELL 1 BTC at a trigger price of 100,000 USDT, active until further notice: + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import StopMarketOrder + +order: StopMarketOrder = self.order_factory.stop_market( + instrument_id=InstrumentId.from_str("BTCUSDT.BINANCE"), + order_side=OrderSide.SELL, + quantity=Quantity.from_int(1), + trigger_price=Price.from_int(100_000), + trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) + time_in_force=TimeInForce.GTC, # <-- optional (default GTC) + expire_time=None, # <-- optional (default None) + reduce_only=False, # <-- optional (default False) + tags=None, # <-- optional (default None) +) +``` + +See the [`StopMarketOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.stop_market.StopMarketOrder) for further details. + +### Stop-Limit + +A *Stop-Limit* order is a conditional order which once triggered will immediately place +a *Limit* order at the specified price. + +In the following example we create a *Stop-Limit* order on the Currenex FX ECN to BUY 50,000 GBP at a limit price of 1.3000 USD +once the market hits the trigger price of 1.30010 USD, active until midday 6th June, 2022 (UTC): + +```python +import pandas as pd +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import StopLimitOrder + +order: StopLimitOrder = self.order_factory.stop_limit( + instrument_id=InstrumentId.from_str("GBP/USD.CURRENEX"), + order_side=OrderSide.BUY, + quantity=Quantity.from_int(50_000), + price=Price.from_str("1.30000"), + trigger_price=Price.from_str("1.30010"), + trigger_type=TriggerType.BID_ASK, # <-- optional (default DEFAULT) + time_in_force=TimeInForce.GTD, # <-- optional (default GTC) + expire_time=pd.Timestamp("2022-06-06T12:00"), + post_only=True, # <-- optional (default False) + reduce_only=False, # <-- optional (default False) + tags=None, # <-- optional (default None) +) +``` + +See the [`StopLimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.stop_limit.StopLimitOrder) for further details. + +### Market-To-Limit + +A *Market-To-Limit* order submits as a market order at the current best price. +If the order partially fills, the system cancels the remainder and resubmits it as a *Limit* order at the executed price. + +In the following example we create a *Market-To-Limit* order on the Interactive Brokers [IdealPro](https://ibkr.info/node/1708) Forex ECN +to BUY 200,000 USD using JPY: + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import MarketToLimitOrder + +order: MarketToLimitOrder = self.order_factory.market_to_limit( + instrument_id=InstrumentId.from_str("USD/JPY.IDEALPRO"), + order_side=OrderSide.BUY, + quantity=Quantity.from_int(200_000), + time_in_force=TimeInForce.GTC, # <-- optional (default GTC) + reduce_only=False, # <-- optional (default False) + display_qty=None, # <-- optional (default None which indicates full display) + tags=None, # <-- optional (default None) +) +``` + +See the [`MarketToLimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.market_to_limit.MarketToLimitOrder) for further details. + +### Market-If-Touched + +A *Market-If-Touched* order is a conditional order which once triggered will immediately +place a *Market* order. This order type is often used to enter a new position on a stop price, +or to take profits for an existing position, either as a SELL order against LONG positions, +or as a BUY order against SHORT positions. + +In the following example we create a *Market-If-Touched* order on the Binance Futures exchange +to SELL 10 ETHUSDT-PERP Perpetual Futures contracts at a trigger price of 10,000 USDT, active until further notice: + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import MarketIfTouchedOrder + +order: MarketIfTouchedOrder = self.order_factory.market_if_touched( + instrument_id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), + order_side=OrderSide.SELL, + quantity=Quantity.from_int(10), + trigger_price=Price.from_str("10_000.00"), + trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) + time_in_force=TimeInForce.GTC, # <-- optional (default GTC) + expire_time=None, # <-- optional (default None) + reduce_only=False, # <-- optional (default False) + tags=["ENTRY"], # <-- optional (default None) +) +``` + +See the [`MarketIfTouchedOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.market_if_touched.MarketIfTouchedOrder) for further details. + +### Limit-If-Touched + +A *Limit-If-Touched* order is a conditional order which once triggered will immediately place +a *Limit* order at the specified price. + +In the following example we create a *Limit-If-Touched* order to BUY 5 BTCUSDT-PERP Perpetual Futures contracts on the +Binance Futures exchange at a limit price of 30,100 USDT (once the market hits the trigger price of 30,150 USDT), +active until midday 6th June, 2022 (UTC): + +```python +import pandas as pd +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import LimitIfTouchedOrder + +order: LimitIfTouchedOrder = self.order_factory.limit_if_touched( + instrument_id=InstrumentId.from_str("BTCUSDT-PERP.BINANCE"), + order_side=OrderSide.BUY, + quantity=Quantity.from_int(5), + price=Price.from_str("30_100"), + trigger_price=Price.from_str("30_150"), + trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) + time_in_force=TimeInForce.GTD, # <-- optional (default GTC) + expire_time=pd.Timestamp("2022-06-06T12:00"), + post_only=True, # <-- optional (default False) + reduce_only=False, # <-- optional (default False) + tags=["TAKE_PROFIT"], # <-- optional (default None) +) +``` + +See the [`LimitIfTouchedOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.limit_if_touched.LimitIfTouchedOrder) for further details. + +### Trailing-Stop-Market + +A *Trailing-Stop-Market* order is a conditional order which trails a stop trigger price +a fixed offset away from the defined market price. Once triggered a *Market* order will +immediately be placed. + +In the following example we create a *Trailing-Stop-Market* order on the Binance Futures exchange to SELL 10 ETHUSD-PERP COIN_M margined +Perpetual Futures Contracts activating at a price of 5,000 USD, then trailing at an offset of 1% (in basis points) away from the current last traded price: + +```python +import pandas as pd +from decimal import Decimal +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model.enums import TrailingOffsetType +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import TrailingStopMarketOrder + +order: TrailingStopMarketOrder = self.order_factory.trailing_stop_market( + instrument_id=InstrumentId.from_str("ETHUSD-PERP.BINANCE"), + order_side=OrderSide.SELL, + quantity=Quantity.from_int(10), + activation_price=Price.from_str("5_000"), + trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) + trailing_offset=Decimal(100), + trailing_offset_type=TrailingOffsetType.BASIS_POINTS, + time_in_force=TimeInForce.GTC, # <-- optional (default GTC) + expire_time=None, # <-- optional (default None) + reduce_only=True, # <-- optional (default False) + tags=["TRAILING_STOP-1"], # <-- optional (default None) +) +``` + +See the [`TrailingStopMarketOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.trailing_stop_market.TrailingStopMarketOrder) for further details. + +### Trailing-Stop-Limit + +A *Trailing-Stop-Limit* order is a conditional order which trails a stop trigger price +a fixed offset away from the defined market price. Once triggered a *Limit* order will +immediately be placed at the defined price (which is also updated as the market moves until triggered). + +In the following example we create a *Trailing-Stop-Limit* order on the Currenex FX ECN to BUY 1,250,000 AUD using USD +at a limit price of 0.71000 USD, activating at 0.72000 USD then trailing at a stop offset of 0.00100 USD +away from the current ask price, active until further notice: + +```python +import pandas as pd +from decimal import Decimal +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model.enums import TrailingOffsetType +from nautilus_trader.model import InstrumentId +from nautilus_trader.model import Price +from nautilus_trader.model import Quantity +from nautilus_trader.model.orders import TrailingStopLimitOrder + +order: TrailingStopLimitOrder = self.order_factory.trailing_stop_limit( + instrument_id=InstrumentId.from_str("AUD/USD.CURRENEX"), + order_side=OrderSide.BUY, + quantity=Quantity.from_int(1_250_000), + price=Price.from_str("0.71000"), + activation_price=Price.from_str("0.72000"), + trigger_type=TriggerType.BID_ASK, # <-- optional (default DEFAULT) + limit_offset=Decimal("0.00050"), + trailing_offset=Decimal("0.00100"), + trailing_offset_type=TrailingOffsetType.PRICE, + time_in_force=TimeInForce.GTC, # <-- optional (default GTC) + expire_time=None, # <-- optional (default None) + reduce_only=True, # <-- optional (default False) + tags=["TRAILING_STOP"], # <-- optional (default None) +) +``` + +See the [`TrailingStopLimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.trailing_stop_limit.TrailingStopLimitOrder) for further details. + +## Advanced orders + +The following guide should be read in conjunction with the specific documentation from the broker or venue +involving these order types, lists/groups and execution instructions (such as for Interactive Brokers). + +### Order lists + +Combinations of contingent orders, or larger order bulks can be grouped together into a list with a common +`order_list_id`. The orders contained in this list may or may not have a contingent relationship with +each other, as this is specific to how the orders themselves are constructed, and the +specific venue they are being routed to. + +### Contingency types + +- **OTO (One-Triggers-Other)** – a parent order that, once executed, automatically places one or more child orders. + - *Full-trigger model*: child order(s) are released **only after the parent is completely filled**. Common at most retail equity/option brokers (e.g. Schwab, Fidelity, TD Ameritrade) and many spot-crypto venues (Binance, Coinbase). + - *Partial-trigger model*: child order(s) are released **pro-rata to each partial fill**. Used by professional-grade platforms such as Interactive Brokers, most futures/FX OMSs, and Kraken Pro. + +- **OCO (One-Cancels-Other)** – two (or more) linked live orders where executing one cancels the remainder. + +- **OUO (One-Updates-Other)** – two (or more) linked live orders where executing one reduces the open quantity of the remainder. + +:::info +These contingency types relate to ContingencyType FIX tag <1385> . +::: + +#### One-Triggers-Other (OTO) + +An OTO order involves two parts: + +1. **Parent order** – submitted to the matching engine immediately. +2. **Child order(s)** – held *off-book* until the trigger condition is met. + +##### Trigger models + +| Trigger model | When are child orders released? | +|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| **Full trigger** | When the parent order’s cumulative quantity equals its original quantity (i.e., it is *fully* filled). | +| **Partial trigger** | Immediately upon each partial execution of the parent; the child’s quantity matches the executed amount and is increased as further fills occur. | + +:::info +The default backtest venue for NautilusTrader uses a *partial-trigger model* for OTO orders. +To opt-in to a *full-trigger mode*, set `oto_trigger_mode="FULL"` for the venue (e.g. via `BacktestVenueConfig`). +::: + +**Working with partial-trigger in production:** + +If your strategy requires full-trigger semantics but the venue or backtest engine uses partial-trigger: + +1. Submit the parent order without contingent children. +2. Subscribe to `OrderFilled` events for the parent order. +3. Only submit child orders (stop-loss, take-profit) after confirming the parent is fully filled. +4. Use `order.is_closed` and `order.filled_qty == order.quantity` to verify complete fill. + +> **Why the distinction matters** +> *Full trigger* leaves a risk window: any partially filled position is live without its protective exit until the remaining quantity fills. +> *Partial trigger* mitigates that risk by ensuring every executed lot instantly has its linked stop/limit, at the cost of creating more order traffic and updates. + +An OTO order can use any supported asset type on the venue (e.g. stock entry with option hedge, futures entry with OCO bracket, crypto spot entry with TP/SL). + +| Venue / Adapter ID | Asset classes | Trigger rule for child | Practical notes | +|----------------------------------------------|---------------------------|---------------------------------------------|-------------------------------------------------------------------| +| Binance / Binance Futures (`BINANCE`) | Spot, perpetual futures | **Partial or full** – fires on first fill. | OTOCO/TP-SL children appear instantly; monitor margin usage. | +| Bybit Spot (`BYBIT`) | Spot | **Full** – child placed after completion. | TP-SL preset activates only once the limit order is fully filled. | +| Bybit Perps (`BYBIT`) | Perpetual futures | **Partial and full** – configurable. | “Partial-position” mode sizes TP-SL as fills arrive. | +| Kraken Futures (`KRAKEN`) | Futures & perps | **Partial and full** – automatic. | Child quantity matches every partial execution. | +| OKX (`OKX`) | Spot, futures, options | **Full** – attached stop waits for fill. | Position-level TP-SL can be added separately. | +| Interactive Brokers (`INTERACTIVE_BROKERS`) | Stocks, options, FX, fut | **Configurable** – OCA can pro-rate. | `OcaType 2/3` reduces remaining child quantities. | +| dYdX v4 (`DYDX`) | Perpetual futures (DEX) | On-chain condition (size exact). | TP-SL triggers by oracle price; partial fill not applicable. | +| Polymarket (`POLYMARKET`) | Prediction market (DEX) | N/A. | Advanced contingency handled entirely at the strategy layer. | +| Betfair (`BETFAIR`) | Sports betting | N/A. | Advanced contingency handled entirely at the strategy layer. | + +#### One-Cancels-Other (OCO) + +An OCO order is a set of linked orders where the execution of **any** order (full *or partial*) triggers a best-efforts cancellation of the others. +Both orders are live simultaneously; once one starts filling, the venue attempts to cancel the unexecuted portion of the remainder. + +#### One-Updates-Other (OUO) + +An OUO order is a set of linked orders where execution of one order causes an immediate *reduction* of open quantity in the other order(s). +Both orders are live concurrently, and each partial execution proportionally updates the remaining quantity of its peer order on a best-effort basis. + +### Contingent order validation + +When working with contingent orders (OTO, OCO, OUO), be aware of the following validation rules and error scenarios: + +**Order list requirements:** + +- All orders in a contingent group must share the same `order_list_id`. +- Parent orders must be submitted before or simultaneously with their children. +- Child orders reference their parent via `parent_order_id`. + +**Modification rules:** + +- Parent orders can typically be modified while pending, but modifications may cascade to children. +- Child orders can be modified independently on most venues, but check venue-specific behavior. +- Canceling a parent order will cancel all associated child orders. + +**Common error scenarios:** + +| Scenario | System behavior | +|----------|-----------------| +| Child references non-existent parent | Order denied with `INVALID_ORDER` error | +| Parent canceled before children trigger | Children automatically canceled | +| OCO sibling filled before cancel propagates | Partial fill honored, remaining quantity canceled | +| Insufficient margin for bracket | Entry may execute, children rejected separately | + +:::warning +Always handle `OrderDenied` and `OrderRejected` events in your strategy, especially for contingent orders where +partial failures can leave positions unprotected. +::: + +### Bracket orders + +Bracket orders are an advanced order type that allows traders to set both take-profit and stop-loss +levels for a position simultaneously. This involves placing a parent order (entry order) and two child +orders: a take-profit `LIMIT` order and a stop-loss `STOP_MARKET` order. When the parent order executes, +the system places the child orders. The take-profit closes the position if the market moves favorably, and the stop-loss limits losses if it moves unfavorably. + +Bracket orders can be easily created using the [OrderFactory](/docs/python-api-latest/common.html#nautilus_trader.common.factories.OrderFactory), +which supports various order types, parameters, and instructions. + +:::warning +You should be aware of the margin requirements of positions, as bracketing a position will consume +more order margin. +::: + +## Emulated orders + +### Introduction + +Emulation lets you use order types even when your trading venue does not natively support them. + +Nautilus locally mimics the behavior of these order types (such as `STOP_LIMIT` or `TRAILING_STOP` orders) +while using only `MARKET` and `LIMIT` orders for actual execution on the venue. + +When you create an emulated order, Nautilus continuously tracks a specific type of market price (specified by the +`emulation_trigger` parameter) and based on the order type and conditions you've set, will automatically submit +the appropriate fundamental order (`MARKET` / `LIMIT`) when the triggering condition is met. + +For example, if you create an emulated `STOP_LIMIT` order, Nautilus will monitor the market price until your `stop` +price is reached, and then automatically submits a `LIMIT` order to the venue. + +To perform emulation, Nautilus needs to know which **type of market price** it should monitor. +By default, it uses bid and ask prices (quotes), which is why you'll often see `emulation_trigger=TriggerType.DEFAULT` +in examples (this is equivalent to using `TriggerType.BID_ASK`). However, Nautilus supports various other price types, +that can guide the emulation process. + +### Submitting an order for emulation + +The only requirement to emulate an order is to pass a `TriggerType` to the `emulation_trigger` +parameter of an `Order` constructor, or `OrderFactory` creation method. The following +emulation trigger types are currently supported: + +- `NO_TRIGGER`: disables local emulation completely and order is fully submitted to the venue. +- `DEFAULT`: which is the same as `BID_ASK`. +- `BID_ASK`: emulated using quotes to trigger. +- `LAST_PRICE`: emulated using trades to trigger. + +The choice of trigger type determines how the order emulation will behave: + +- For `STOP` orders, the trigger price will be compared against the specified trigger type. +- For `TRAILING_STOP` orders, the trailing offset will be updated based on the specified trigger type. +- For `LIMIT` orders being emulated, the limit price will be compared against the specified trigger type to determine when to release the order as a `MARKET` order. + +Here are all the available values you can set into `emulation_trigger` parameter and their purposes: + +| Trigger Type | Description | Common use cases | +|:------------------|:-----------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------| +| `NO_TRIGGER` | Disables emulation completely. The order is sent directly to the venue without any local processing. | When you want to use the venue's native order handling, or for simple order types that don't need emulation. | +| `DEFAULT` | Same as `BID_ASK`. This is the standard choice for most emulated orders. | General-purpose emulation when you want to work with the "default" type of market prices. | +| `BID_ASK` | Uses the best bid and ask prices (quotes) to guide emulation. | Stop orders, trailing stops, and other orders that should react to the current market spread. | +| `LAST_PRICE` | Uses the price of the most recent trade to guide emulation. | Orders that should trigger based on actual executed trades rather than quotes. | +| `DOUBLE_LAST` | Uses two consecutive last trade prices to confirm the trigger condition. | When you want additional confirmation of price movement before triggering. | +| `DOUBLE_BID_ASK` | Uses two consecutive bid/ask price updates to confirm the trigger condition. | When you want extra confirmation of quote movements before triggering. | +| `LAST_OR_BID_ASK` | Triggers on either last trade price or bid/ask prices. | When you want to be more responsive to any type of price movement. | +| `MID_POINT` | Uses the middle point between the best bid and ask prices. | Orders that should trigger based on the theoretical fair price. | +| `MARK_PRICE` | Uses the mark price (common in derivatives markets) for triggering. | Particularly useful for futures and perpetual contracts. | +| `INDEX_PRICE` | Uses an underlying index price for triggering. | When trading derivatives that track an index. | + +### Technical details + +The platform makes it possible to emulate most order types locally, regardless +of whether the type is supported on a trading venue. The logic and code paths for +order emulation are exactly the same for all [environment contexts](architecture.md#environment-contexts) +and use a common `OrderEmulator` component. + +:::note +There is no limitation on the number of emulated orders you can have per running instance. +::: + +### Lifecycle + +An emulated order will progress through the following stages: + +1. Submitted by a `Strategy` through the `submit_order` method. +2. Sent to the `RiskEngine` for pre-trade risk checks (it may be denied at this point). +3. Sent to the `OrderEmulator` where it is *held* / emulated. +4. Once triggered, emulated order is transformed into a `MARKET` or `LIMIT` order and released (submitted to the venue). +5. Released order undergoes final risk checks before venue submission. + +:::note +Emulated orders are subject to the same risk controls as *regular* orders, and can be +modified and canceled by a trading strategy in the normal way. They will also be included +when canceling all orders. +::: + +:::info +An emulated order will retain its original client order ID throughout its entire life cycle, making it easy to query +through the cache. +::: + +#### Held emulated orders + +The following will occur for an emulated order now *held* by the `OrderEmulator` component: + +- The original `SubmitOrder` command will be cached. +- The emulated order will be processed inside a local `MatchingCore` component. +- The `OrderEmulator` will subscribe to any needed market data (if not already) to update the matching core. +- The emulated order can be modified (by the trader) and updated (by the market) until *released* or canceled. + +#### Released emulated orders + +Once data arrival triggers / matches an emulated order locally, the following +*release* actions will occur: + +- The order will be transformed to either a `MARKET` or `LIMIT` order (see below table) through an additional `OrderInitialized` event. +- The orders `emulation_trigger` will be set to `NONE` (it will no longer be treated as an emulated order by any component). +- The order attached to the original `SubmitOrder` command will be sent back to the `RiskEngine` for additional checks since any modification/updates. +- If not denied, then the command will continue to the `ExecutionEngine` and on to the trading venue via an `ExecutionClient` as normal. + +### Order types which can be emulated + +The following table lists which order types are possible to emulate, and +which order type they transform to when being released for submission to the +trading venue. + +| Order type for emulation | Can emulate | Released type | +|:-------------------------|:------------|:--------------| +| `MARKET` | | n/a | +| `MARKET_TO_LIMIT` | | n/a | +| `LIMIT` | ✓ | `MARKET` | +| `STOP_MARKET` | ✓ | `MARKET` | +| `STOP_LIMIT` | ✓ | `LIMIT` | +| `MARKET_IF_TOUCHED` | ✓ | `MARKET` | +| `LIMIT_IF_TOUCHED` | ✓ | `LIMIT` | +| `TRAILING_STOP_MARKET` | ✓ | `MARKET` | +| `TRAILING_STOP_LIMIT` | ✓ | `LIMIT` | + +### Querying + +When writing trading strategies, it may be necessary to know the state of emulated orders in the system. +There are several ways to query emulation status: + +#### Through the Cache + +The following `Cache` methods are available: + +- `self.cache.orders_emulated(...)`: Returns all currently emulated orders. +- `self.cache.is_order_emulated(...)`: Checks if a specific order is emulated. +- `self.cache.orders_emulated_count(...)`: Returns the count of emulated orders. + +See the full [API reference](/docs/python-api-latest/cache.html) for additional details. + +#### Direct order queries + +You can query order objects directly using: + +- `order.is_emulated` + +If either of these return `False`, then the order has been *released* from the +`OrderEmulator`, and so is no longer considered an emulated order (or was never an emulated order). + +:::warning +Do not hold a local reference to an emulated order. The order object transforms +when the emulated order is *released*. Use the `Cache` instead. +::: + +### Persistence and recovery + +If a running system either crashes or shuts down with active emulated orders, then +they will be reloaded inside the `OrderEmulator` from any configured cache database. +This preserves order state across system restarts and recoveries. + +### Best practices + +When working with emulated orders, consider the following best practices: + +1. Always use the `Cache` for querying or tracking emulated orders rather than storing local references +2. Be aware that emulated orders transform to different types when released +3. Remember that emulated orders undergo risk checks both at submission and release + +:::note +Order emulation allows you to use advanced order types even on venues that don't natively support them, +making your trading strategies more portable across different venues. +::: + +## Related guides + +- [Execution](execution.md) - Order execution and fill handling. +- [Positions](positions.md) - Positions created from order fills. +- [Strategies](strategies.md) - Order management from strategies. diff --git a/nautilus-docs/docs/concepts/overview.md b/nautilus-docs/docs/concepts/overview.md new file mode 100644 index 0000000..d60e92f --- /dev/null +++ b/nautilus-docs/docs/concepts/overview.md @@ -0,0 +1,255 @@ +# Overview + +## Introduction + +NautilusTrader is an open-source, production-grade, Rust-native engine for multi-asset, +multi-venue trading systems. + +The system spans research, deterministic simulation, and live execution within a single +event-driven architecture, with Python serving as the control plane for strategy logic, +configuration, and orchestration. + +This separation provides the performance and safety of a compiled trading engine with +the flexibility of Python for system composition and strategy development. +Trading systems can also be written entirely in Rust for mission-critical workloads. + +The same execution semantics and deterministic time model operate in both research and +live systems. Strategies deploy from research to production with no code changes, +providing research-to-live parity and reducing the divergence that typically introduces +deployment risk. + +NautilusTrader is asset-class-agnostic. Any venue with a REST API or WebSocket feed can be +integrated through modular adapters. Current integrations span crypto exchanges (CEX and +DEX), traditional markets (FX, equities, futures, options), and betting exchanges. + +## Features + +- **Fast**: Rust core with asynchronous networking using [tokio](https://crates.io/crates/tokio). +- **Reliable**: Type- and thread-safety backed by Rust, with optional Redis-backed state persistence. +- **Portable**: Runs on Linux, macOS, and Windows. Deploy using Docker. +- **Flexible**: Modular adapters integrate any REST API or WebSocket feed. +- **Advanced**: Time in force `IOC`, `FOK`, `GTC`, `GTD`, `DAY`, `AT_THE_OPEN`, `AT_THE_CLOSE`, advanced order types and conditional triggers. Execution instructions `post-only`, `reduce-only`, and icebergs. Contingency orders including `OCO`, `OUO`, `OTO`. +- **Customizable**: User-defined components, or assemble entire systems from scratch using the [cache](cache.md) and [message bus](message_bus.md). +- **Backtesting**: Multiple venues, instruments, and strategies simultaneously using historical quote tick, trade tick, bar, order book, and custom data with nanosecond resolution. +- **Live**: Identical strategy implementations between research and live deployment. +- **Multi-venue**: Run market-making and cross-venue strategies across multiple venues simultaneously. +- **AI training**: Engine fast enough to train AI trading agents (RL/ES). + +## Why NautilusTrader? + +Trading strategy research typically happens in Python using vectorized approaches, while +production trading systems are built separately using event-driven architectures in +compiled languages. + +NautilusTrader removes this separation. + +A Rust-native core provides a deterministic event-driven runtime for both research and live +execution, while Python serves as the control plane. The same architecture, execution +semantics, and time model operate across both environments, allowing strategies to move +from research to production without reimplementation. + +Python bindings are provided via [PyO3](https://pyo3.rs), with an ongoing migration from +Cython. No Rust toolchain is required at install time. + +## Use cases + +There are three main use cases for this software package: + +- Backtest trading systems on historical data (`backtest`). +- Simulate trading systems with real-time data and virtual execution (`sandbox`). +- Deploy trading systems live on real or paper accounts (`live`). + +The codebase provides a framework for building the software layer of systems that achieve the above. +The default `backtest` and `live` system implementations live in their respectively named subpackages. +A `sandbox` environment can be built using the sandbox adapter. + +:::note + +- All examples will use these default system implementations. +- We consider trading strategies to be subcomponents of end-to-end trading systems, these systems +include the application and infrastructure layers. + +::: + +## Distributed + +The platform integrates into larger distributed systems. +Nearly all configuration and domain objects serialize using JSON, MessagePack, or Apache Arrow +(Feather) for communication over the network. + +## Common core + +The common system core is used by all node [environment contexts](architecture.md#environment-contexts) (`backtest`, `sandbox`, and `live`). +User-defined `Actor`, `Strategy` and `ExecAlgorithm` components are managed consistently across these environment contexts. + +## Backtesting + +Feed data to a `BacktestEngine` either directly or through a higher-level `BacktestNode` and +`ParquetDataCatalog`, then run the data through the system with nanosecond resolution. + +## Live trading + +A `TradingNode` ingests data and events from multiple data and execution clients, supporting both +demo/paper trading accounts and real accounts. Running asynchronously on a single +[event loop](https://docs.python.org/3/library/asyncio-eventloop.html) provides high performance, +with the option to use the [uvloop](https://github.com/MagicStack/uvloop) implementation +(available for Linux and macOS) for additional throughput. + +## Domain model + +The platform features a trading domain model that includes various value types such as +`Price` and `Quantity`, as well as more complex entities such as `Order` and `Position` objects, +which are used to aggregate multiple events to determine state. + +## Timestamps + +All timestamps use nanosecond precision in UTC. + +Timestamp strings follow ISO 8601 (RFC 3339) format with either 9 digits (nanoseconds) or 3 digits (milliseconds) of decimal precision, +(but mostly nanoseconds) always maintaining all digits including trailing zeros. +These can be seen in log messages, and debug/display outputs for objects. + +A timestamp string consists of: + +- Full date component always present: `YYYY-MM-DD`. +- `T` separator between date and time components. +- Always nanosecond precision (9 decimal places) or millisecond precision (3 decimal places) for certain cases such as GTD expiry times. +- Always UTC timezone designated by `Z` suffix. + +Example: `2024-01-05T15:30:45.123456789Z` + +For the complete specification, refer to [RFC 3339: Date and Time on the Internet](https://datatracker.ietf.org/doc/html/rfc3339). + +## UUIDs + +The platform uses Universally Unique Identifiers (UUID) version 4 (RFC 4122) for unique identifiers. +Our high-performance implementation uses the `uuid` crate for correctness validation when parsing from strings, +ensuring input UUIDs comply with the specification. + +A valid UUID v4 consists of: + +- 32 hexadecimal digits displayed in 5 groups. +- Groups separated by hyphens: `8-4-4-4-12` format. +- Version 4 designation (indicated by the third group starting with "4"). +- RFC 4122 variant designation (indicated by the fourth group starting with "8", "9", "a", or "b"). + +Example: `2d89666b-1a1e-4a75-b193-4eb3b454c757` + +For the complete specification, refer to [RFC 4122: A Universally Unique Identifier (UUID) URN Namespace](https://datatracker.ietf.org/doc/html/rfc4122). + +## Data types + +The following market data types can be requested historically, and also subscribed to as live streams when available from a venue / data provider, and implemented in an integrations adapter. + +- `OrderBookDelta` (L1/L2/L3) +- `OrderBookDeltas` (container type) +- `OrderBookDepth10` (fixed depth of 10 levels per side) +- `QuoteTick` +- `TradeTick` +- `Bar` +- `Instrument` +- `InstrumentStatus` +- `InstrumentClose` + +The following `PriceType` options can be used for bar aggregations: + +- `BID` +- `ASK` +- `MID` +- `LAST` + +## Bar aggregations + +The following `BarAggregation` methods are available: + +- `MILLISECOND` +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `YEAR` +- `TICK` +- `VOLUME` +- `VALUE` (a.k.a Dollar bars) +- `RENKO` (price-based bricks) +- `TICK_IMBALANCE` +- `TICK_RUNS` +- `VOLUME_IMBALANCE` +- `VOLUME_RUNS` +- `VALUE_IMBALANCE` +- `VALUE_RUNS` + +Currently implemented aggregations: + +- `MILLISECOND` +- `SECOND` +- `MINUTE` +- `HOUR` +- `DAY` +- `WEEK` +- `MONTH` +- `YEAR` +- `TICK` +- `VOLUME` +- `VALUE` +- `RENKO` + +Aggregations listed above that are not repeated in the implemented list are planned but not yet available. + +The price types and bar aggregations can be combined with step sizes >= 1 in any way through a `BarSpecification`. +This allows alternative bars to be aggregated for live trading. + +## Account types + +The following account types are available for both live and backtest environments: + +- `Cash` single-currency (base currency) +- `Cash` multi-currency +- `Margin` single-currency (base currency) +- `Margin` multi-currency +- `Betting` single-currency + +## Order types + +The following order types are available (when possible on a venue): + +- `MARKET` +- `LIMIT` +- `STOP_MARKET` +- `STOP_LIMIT` +- `MARKET_TO_LIMIT` +- `MARKET_IF_TOUCHED` +- `LIMIT_IF_TOUCHED` +- `TRAILING_STOP_MARKET` +- `TRAILING_STOP_LIMIT` + +## Value types + +The following value types are backed by either 128-bit or 64-bit raw integer values, depending on the +[precision mode](../getting_started/installation.md#precision-mode) used during compilation. + +- `Price` +- `Quantity` +- `Money` + +### High-precision mode (128-bit) + +When the `high-precision` feature flag is **enabled** (default), values use the specification: + +| Type | Raw backing | Max precision | Min value | Max value | +|:-------------|:------------|:--------------|:--------------------|:-------------------| +| `Price` | `i128` | 16 | -17,014,118,346,046 | 17,014,118,346,046 | +| `Money` | `i128` | 16 | -17,014,118,346,046 | 17,014,118,346,046 | +| `Quantity` | `u128` | 16 | 0 | 34,028,236,692,093 | + +### Standard-precision mode (64-bit) + +When the `high-precision` feature flag is **disabled**, values use the specification: + +| Type | Raw backing | Max precision | Min value | Max value | +|:-------------|:------------|:--------------|:--------------------|:-------------------| +| `Price` | `i64` | 9 | -9,223,372,036 | 9,223,372,036 | +| `Money` | `i64` | 9 | -9,223,372,036 | 9,223,372,036 | +| `Quantity` | `u64` | 9 | 0 | 18,446,744,073 | diff --git a/nautilus-docs/docs/concepts/portfolio.md b/nautilus-docs/docs/concepts/portfolio.md new file mode 100644 index 0000000..16177ff --- /dev/null +++ b/nautilus-docs/docs/concepts/portfolio.md @@ -0,0 +1,169 @@ +# Portfolio + +The Portfolio is the central hub for managing and tracking all positions across active strategies for the trading node or backtest. +It consolidates position data from multiple instruments, providing a unified view of your holdings, risk exposure, and overall performance. + +## Currency conversion + +The Portfolio supports automatic currency conversion for PnL and exposure calculations, +allowing you to view results in your preferred currency. This is particularly useful when +trading across multiple instruments with different settlement currencies or managing multiple +accounts with different base currencies. + +### Supported conversions + +Currency conversion is available for the following portfolio queries: + +- `realized_pnl()` / `realized_pnls()` - Convert realized PnL to target currency. +- `unrealized_pnl()` / `unrealized_pnls()` - Convert unrealized PnL to target currency. +- `total_pnl()` / `total_pnls()` - Convert total PnL to target currency. +- `net_exposure()` / `net_exposures()` - Convert net exposure to target currency. + +All methods accept an optional `target_currency` parameter to specify the desired output +currency. + +### Single account behavior + +When querying a single account without specifying `target_currency`, the Portfolio +automatically converts values to that account's base currency: + +```python +# Returns exposure in the account's base currency (e.g., USD) +exposure = portfolio.net_exposures(venue=BINANCE, account_id=account_id) +``` + +### Multi-account behavior + +When querying multiple accounts simultaneously, behavior depends on whether you query +all instruments (`net_exposures()`) or a single instrument (`net_exposure()`): + +**For `net_exposures()` (all instruments):** + +- **Same base currency**: Automatically converts to the common base currency. +- **Different base currencies**: Returns a dict with multiple currencies, each converted + to its account's base currency. Provide `target_currency` for single-currency results. + +**For `net_exposure()` (single instrument across accounts):** + +- **Different base currencies**: Returns `None` unless you provide `target_currency`. + +```python +# Scenario 1: Multiple accounts, all with USD base currency +exposures = portfolio.net_exposures(venue=BINANCE) +# Returns {USD: Money(...)} + +# Scenario 2: Multiple accounts with different base currencies (USD and EUR) +exposures = portfolio.net_exposures(venue=BINANCE) +# Returns {USD: Money(...), EUR: Money(...)} + +# Force single currency across accounts +exposures = portfolio.net_exposures(venue=BINANCE, target_currency=USD) +# Returns {USD: Money(...)} +``` + +### Conversion failures + +When `target_currency` is provided and currency conversion fails, behavior depends on +the method type: + +- **Single-value methods** (`realized_pnl`, `unrealized_pnl`, `total_pnl`, `net_exposure`): + Return `None` and log an error to prevent incorrect values. +- **Dict-returning methods** (`realized_pnls`, `unrealized_pnls`, `total_pnls`, `net_exposures`): + Omit instruments that fail conversion but return results for successful conversions. + +:::warning +Exchange rate data must be available when using `target_currency` for cross-currency +aggregation. +::: + +### Conversion price types + +When converting exposures to a target currency, the Portfolio uses different price types +depending on the position composition: + +- **All long positions**: Uses `BID` prices (conservative for long exposure). +- **All short positions**: Uses `ASK` prices (conservative for short exposure). +- **Mixed positions**: Uses `MID` prices (neutral when both long and short exist). + +This ensures conversions reflect realistic market conditions where you would liquidate +long positions at bid and cover short positions at ask. For mixed positions, mid-pricing +provides a neutral valuation. + +If `use_mark_xrates` is enabled in the portfolio configuration, `MARK` prices replace +`MID` prices for mixed positions and general conversions. + +## Portfolio statistics + +There are a variety of [built-in portfolio statistics](https://github.com/nautechsystems/nautilus_trader/tree/develop/crates/analysis/src/statistics) +which analyse a trading portfolio's performance for both backtests and live trading. + +The statistics are generally categorized as follows. + +- PnLs based statistics (per currency) +- Returns based statistics +- Positions based statistics +- Orders based statistics + +You can also call a trader's `PortfolioAnalyzer` and calculate statistics at any arbitrary +time, including *during* a backtest, or live trading session. + +## Custom statistics + +Custom portfolio statistics can be defined by inheriting from the `PortfolioStatistic` +base class, and implementing any of the `calculate_` methods. + +For example, the following is the implementation for the built-in `WinRate` statistic: + +```python +import pandas as pd +from typing import Any +from nautilus_trader.analysis.statistic import PortfolioStatistic + + +class WinRate(PortfolioStatistic): + """ + Calculates the win rate from a realized PnLs series. + """ + + def calculate_from_realized_pnls(self, realized_pnls: pd.Series) -> Any | None: + # Preconditions + if realized_pnls is None or realized_pnls.empty: + return 0.0 + + # Calculate statistic + winners = [x for x in realized_pnls if x > 0.0] + losers = [x for x in realized_pnls if x <= 0.0] + + return len(winners) / float(max(1, (len(winners) + len(losers)))) +``` + +These statistics can then be registered with a traders `PortfolioAnalyzer`. + +```python +stat = WinRate() + +# Register with the portfolio analyzer +engine.portfolio.analyzer.register_statistic(stat) +``` + +See the [`PortfolioAnalyzer` API Reference](/docs/python-api-latest/analysis.html#nautilus_trader.analysis.analyzer.PortfolioAnalyzer) for all available methods. + +:::tip +Your statistic should handle degenerate inputs such as `None`, empty series, or insufficient data. +Return `None` for unknown/incalculable values, or a reasonable default like `0.0` when semantically appropriate (e.g., win rate with no trades). +::: + +## Backtest analysis + +Following a backtest run, the engine passes realized PnLs, returns, positions, and orders data to each registered +statistic. Any output is then displayed in the tear sheet under the `Portfolio Performance` heading, grouped as: + +- Realized PnL statistics (per currency) +- Returns statistics (for the entire portfolio) +- General statistics derived from position and order data (for the entire portfolio) + +## Related guides + +- [Positions](positions.md) - Position tracking within portfolios. +- [Reports](reports.md) - Generate portfolio analysis reports. +- [Visualization](visualization.md) - Visualize portfolio performance. diff --git a/nautilus-docs/docs/concepts/positions.md b/nautilus-docs/docs/concepts/positions.md new file mode 100644 index 0000000..61b92dd --- /dev/null +++ b/nautilus-docs/docs/concepts/positions.md @@ -0,0 +1,460 @@ +# Positions + +This guide explains how positions work in NautilusTrader, including their lifecycle, aggregation +from order fills, profit and loss calculations, and the important concept of position snapshotting +for netting OMS configurations. + +## Overview + +A position represents an open exposure to a particular instrument in the market. Positions are +fundamental to tracking trading performance and risk, as they aggregate all fills for a particular +instrument and continuously calculate metrics like unrealized PnL, average entry price, and total +exposure. + +The system automatically creates positions when orders fill and tracks them +from open to close. The platform supports both netting +and hedging position management styles through its OMS (Order Management System) configuration. + +## Position lifecycle + +### Creation + +The system opens a position on the first fill: + +- **NETTING OMS**: Opens on first fill for an instrument (one position per instrument). +- **HEDGING OMS**: Opens on first fill for a new `position_id` (multiple positions per instrument). + +A position tracks: + +- Opening order and fill details. +- Entry side (`LONG` or `SHORT`). +- Initial quantity and average price. +- Timestamps for initialization and opening. + +:::tip +You can access positions through the Cache using `self.cache.position(position_id)` or +`self.cache.positions(instrument_id=instrument_id)` from within your actors/strategies. +::: + +### Updates + +As additional fills occur, the position: + +- Aggregates quantities from buy and sell fills. +- Recalculates average entry and exit prices. +- Updates peak quantity (maximum exposure reached). +- Tracks all associated order IDs and trade IDs. +- Accumulates commissions by currency. + +### Closure + +A position closes when the net quantity becomes zero (`FLAT`). At closure: + +- The closing order ID is recorded. +- Duration is calculated from open to close. +- Final realized PnL is computed. +- In `NETTING` OMS, when the position later reopens, the engine snapshots the closed state to preserve historical PnL (see [Position snapshotting](#position-snapshotting)). + +## Order fill aggregation + +Positions aggregate order fills to maintain an accurate view of market exposure. The aggregation +process handles both sides of trading activity: + +### Buy fills + +When a BUY order fills: + +- Increases long exposure or reduces short exposure. +- Updates average entry price for opening trades. +- Updates average exit price for closing trades. +- Calculates realized PnL for any closed portion. + +### Sell fills + +When a SELL order fills: + +- Increases short exposure or reduces long exposure. +- Updates average entry price for opening trades. +- Updates average exit price for closing trades. +- Calculates realized PnL for any closed portion. + +### Net position calculation + +The position maintains a `signed_qty` field representing the net exposure: + +- Positive values indicate `LONG` positions. +- Negative values indicate `SHORT` positions. +- Zero indicates a `FLAT` (closed) position. + +```python +# Example: Position aggregation +# Initial BUY 100 units at $50 +signed_qty = +100 # LONG position + +# Subsequent SELL 150 units at $55 +signed_qty = -50 # Now SHORT position + +# Final BUY 50 units at $52 +signed_qty = 0 # Position FLAT (closed) +``` + +## Position adjustments + +Position adjustments track quantity or PnL changes that occur outside of normal order fills, +ensuring the position quantity accurately reflects the true net asset position. The system +generates `PositionAdjusted` events for these scenarios. + +### Base currency commissions + +When trading spot currency pairs (e.g., BTC/USDT) or FX spot, commissions paid in the base +currency directly affect the net quantity received or delivered: + +- **Opening fills**: Commission is deducted from the traded quantity. A buy of 1.0 BTC with + 0.001 BTC commission results in a net long position of 0.999 BTC. +- **Closing fills**: Commission is applied to `signed_qty` because it affects actual inventory. + Selling a 0.999 BTC LONG position with 0.000999 BTC commission leaves you SHORT 0.000999 BTC, + not FLAT, because you gave up 0.999999 BTC total. +- **Flips**: Commission affects the final position size on both sides of the flip. + +:::note +Base currency commissions only apply to spot currency pairs and FX spot instruments where the +commission currency matches `instrument.base_currency`. For other instruments, commissions are +tracked separately and do not affect position quantity. +::: + +### Funding payments + +Funding adjustments track periodic payments for perpetual futures without affecting position +quantity. These are logged with `quantity_change = None` and can include PnL impacts. + +### Adjustment tracking + +All adjustments are preserved in the position event history: + +- `position.adjustments` returns the list of all `PositionAdjusted` events. +- Each adjustment includes type (`COMMISSION` or `FUNDING`), quantity change, and timestamps. +- The adjustment history is cleared when positions close and reopen. When events are purged, + commission adjustments tied to the removed fills are regenerated while non-commission adjustments + (for example funding) are preserved. + +## OMS types and position management + +NautilusTrader supports two primary OMS types that fundamentally affect how positions are tracked +and managed. An `OmsType.UNSPECIFIED` option also exists, which defaults to the component's +context. For full details, see the [Execution guide](execution.md#order-management-system-oms). + +### `NETTING` + +In `NETTING` mode, all fills for an instrument are aggregated into a single position: + +- One position per instrument ID. +- All fills contribute to the same position. +- Position flips from `LONG` to `SHORT` (or vice versa) as net quantity changes. +- Historical snapshots preserve closed position states. + +### `HEDGING` + +In `HEDGING` mode, multiple positions can exist for the same instrument: + +- Multiple simultaneous `LONG` and `SHORT` positions. +- Each position has a unique position ID. +- Positions are tracked independently. +- No automatic netting across positions. +- Closed positions remain in cache history but do not reopen; new fills create new positions. + +:::warning +When using `HEDGING` mode, be aware of increased margin requirements as each position +consumes margin independently. Some venues may not support true hedging mode and will +net positions automatically. +::: + +### Strategy vs venue OMS + +The platform allows different OMS configurations for strategies and venues: + +| Strategy OMS | Venue OMS | Behavior | +|--------------|-----------|-------------------------------------------------------------| +| `NETTING` | `NETTING` | Single position per instrument at both strategy and venue. | +| `HEDGING` | `HEDGING` | Multiple positions supported at both levels. | +| `NETTING` | `HEDGING` | Venue tracks multiple, Nautilus maintains single position. | +| `HEDGING` | `NETTING` | Venue tracks single, Nautilus maintains virtual positions. | + +:::tip +For most trading scenarios, keeping strategy and venue OMS types aligned simplifies +position management. Override configurations are primarily useful for prop trading +desks or when interfacing with legacy systems. See the [Live guide](live.md) +for venue-specific OMS configuration. +::: + +## Position snapshotting + +Position snapshotting is an important feature for `NETTING` OMS configurations that preserves +the state of closed positions for accurate PnL tracking and reporting. + +### Why snapshotting matters + +In a `NETTING` system, when a position closes (becomes `FLAT`) and then reopens with a new trade, +the position object is reset to track the new exposure. Without snapshotting, the historical +realized PnL from the previous position cycle would be lost. + +### How it works + +When a `NETTING` position closes and then receives a new fill for the same instrument, the execution +engine snapshots the closed position state before resetting it, preserving: + +- Final quantities and prices. +- Realized PnL. +- All fill events. +- Commission totals. + +This snapshot is stored in the cache indexed by position ID. The position then resets for the new +cycle while previous snapshots remain accessible. The Portfolio aggregates PnL across all snapshots +for accurate totals. + +:::note +This historical snapshot mechanism differs from optional position state snapshots (`snapshot_positions`), +which periodically record open-position state for telemetry. See the [Live guide](live.md) for +`snapshot_positions` and `snapshot_positions_interval_secs` settings. +::: + +### Example scenario + +```python +# NETTING OMS Example +# Cycle 1: Open LONG position +BUY 100 units at $50 # Position opens +SELL 100 units at $55 # Position closes, PnL = $500 +# Snapshot taken preserving $500 realized PnL + +# Cycle 2: Open SHORT position +SELL 50 units at $54 # Position reopens (SHORT) +BUY 50 units at $52 # Position closes, PnL = $100 +# Snapshot taken preserving $100 realized PnL + +# Total realized PnL = $500 + $100 = $600 (from snapshots) +``` + +Without snapshotting, only the most recent cycle's PnL would be available, leading to +incorrect reporting and analysis. + +## PnL calculations + +NautilusTrader provides PnL calculations that account for instrument +specifications and market conventions. + +### Realized PnL + +Calculated when positions are partially or fully closed: + +```python +# For standard instruments +realized_pnl = (exit_price - entry_price) * closed_quantity * multiplier + +# For inverse instruments (side-aware) +# LONG: realized_pnl = closed_quantity * multiplier * (1/entry_price - 1/exit_price) +# SHORT: realized_pnl = closed_quantity * multiplier * (1/exit_price - 1/entry_price) +``` + +The engine automatically applies the correct formula based on position side. + +### Unrealized PnL + +Calculated using current market prices for open positions. The `price` parameter accepts any +reference price (bid, ask, mid, last, or mark): + +```python +position.unrealized_pnl(last_price) # Using last traded price +position.unrealized_pnl(bid_price) # Conservative for LONG positions +position.unrealized_pnl(ask_price) # Conservative for SHORT positions +``` + +Returns `Money(0, settlement_currency)` for `FLAT` positions regardless of the price provided. + +### Total PnL + +Combines realized and unrealized components: + +```python +total_pnl = position.total_pnl(current_price) +# Returns realized_pnl + unrealized_pnl +``` + +### Currency considerations + +- PnL is calculated in the instrument's settlement currency. +- For Forex, this is typically the quote currency. +- For inverse contracts, PnL may be in the base currency. +- Portfolio aggregates realized PnL per instrument in settlement currency. +- Multi-currency totals require conversion outside the Position class. + +## Commissions and costs + +Positions track all trading costs: + +- Commissions are accumulated by currency. +- Each fill's commission is added to the running total. +- Multiple commission currencies are supported. +- Realized PnL includes commissions only when denominated in the settlement currency. +- Other commissions are tracked separately and may require conversion. + +```python +commissions = position.commissions() +# Returns list[Money] with aggregated commission totals per currency + +notional = position.notional_value(current_price) +# Returns Money in quote currency (standard) or base currency (inverse) +``` + +**Limitations:** + +- Panics if inverse instrument has no `base_currency` set. +- Does not handle quanto contracts (returns quote currency instead of settlement currency). +- For quanto instruments, use `instrument.calculate_notional_value()` instead. + +## Position properties and state + +### Identifiers + +- `id`: Unique position identifier. +- `instrument_id`: The traded instrument. +- `account_id`: Account where position is held. +- `trader_id`: The trader who owns the position. +- `strategy_id`: The strategy managing the position. +- `opening_order_id`: Client order ID that opened the position. +- `closing_order_id`: Client order ID that closed the position. + +### Position state + +- `side`: Current position side (`LONG`, `SHORT`, or `FLAT`). +- `entry`: Direction of the currently open position (`Buy` for `LONG`, `Sell` for `SHORT`). Updates when position flips direction. +- `quantity`: Current absolute position size. +- `signed_qty`: Signed position size (positive for `LONG`, negative for `SHORT`). +- `peak_qty`: Maximum quantity reached during position lifetime. +- `is_open`: Whether position is currently open. +- `is_closed`: Whether position is closed (`FLAT`). +- `is_long`: Whether position side is `LONG`. +- `is_short`: Whether position side is `SHORT`. + +### Pricing and valuation + +- `avg_px_open`: Average entry price. +- `avg_px_close`: Average exit price when closing. +- `realized_pnl`: Realized profit/loss. +- `realized_return`: Realized return as decimal (e.g., 0.05 for 5%). +- `quote_currency`: Quote currency of the instrument. +- `base_currency`: Base currency if applicable. +- `settlement_currency`: Currency for PnL settlement. + +### Instrument specifications + +- `multiplier`: Contract multiplier. +- `price_precision`: Decimal precision for prices. +- `size_precision`: Decimal precision for quantities. +- `is_inverse`: Whether instrument is inverse. + +### Timestamps + +- `ts_init`: When position was initialized. +- `ts_opened`: When position was opened. +- `ts_last`: Last update timestamp. +- `ts_closed`: When position was closed. +- `duration_ns`: Duration from open to close in nanoseconds. + +### Associated data + +- `symbol`: The instrument's ticker symbol. +- `venue`: The trading venue. +- `client_order_ids`: All client order IDs associated with position. +- `venue_order_ids`: All venue order IDs associated with position. +- `trade_ids`: All trade/fill IDs from venue. +- `events`: All order fill events applied to position. +- `event_count`: Total number of fill events applied. +- `last_event`: Most recent fill event. +- `last_trade_id`: Most recent trade ID. + +:::info +For complete type information and detailed property documentation, see the Position +[API Reference](/docs/python-api-latest/model/position.html#nautilus_trader.model.position.Position). +::: + +## Events and tracking + +Positions maintain a complete history of events: + +- All order fill events are stored chronologically. +- Associated client order IDs are tracked. +- Trade IDs from the venue are preserved. +- Event count indicates total fills applied. + +This historical data enables: + +- Detailed position analysis. +- Trade reconciliation. +- Performance attribution. +- Audit trails. + +:::tip +Use `position.events` to access the full history of fills for reconciliation. +The `position.trade_ids` property helps match against broker statements. +See the [Execution guide](execution.md) for reconciliation best practices. +::: + +## Numerical precision + +Position calculations use 64-bit floating-point (`f64`) arithmetic for PnL and average price computations. +While fixed-point types (`Price`, `Quantity`, `Money`) preserve exact precision at configured decimal places, +internal calculations convert to `f64` for performance and overflow safety. + +### Design rationale + +The platform uses `f64` for position calculations to balance performance and accuracy: + +- Floating-point operations are significantly faster than arbitrary-precision arithmetic. +- Raw integer multiplication can overflow even with 128-bit integers. +- Each calculation starts from precise fixed-point values, avoiding cumulative error. +- IEEE-754 double precision provides ~15 decimal digits of accuracy. + +### Validated precision characteristics + +Testing confirms `f64` arithmetic maintains accuracy for typical trading scenarios: + +- Standard amounts: No precision loss for amounts ≥ 0.01 in standard currencies. +- High-precision instruments: 9-decimal crypto prices preserved within 1e-6 tolerance. +- Sequential fills: 100 fills show no drift (commission accuracy to 1e-10). +- Extreme prices: Handles range from 0.00001 to 99,999.99999 without overflow. +- Round-trip trades: Opening and closing at same price produces exact PnL (commissions only). + +For implementation details, see `test_position_pnl_precision_*` tests in `crates/model/src/position.rs`. + +:::note +For regulatory compliance or audit trails requiring exact decimal arithmetic, consider using `Decimal` +types from external libraries. Very small amounts below `f64` epsilon (~1e-15) may round to zero. +This does not affect realistic trading scenarios with standard currency precisions (typically 2-9 decimals). +::: + +## Integration with other components + +Positions interact with several key components: + +- **Portfolio**: Aggregates positions across instruments and strategies. +- **ExecutionEngine**: Creates and updates positions from fills. +- **Cache**: Stores position state and snapshots. +- **RiskEngine**: Monitors position limits and exposure. + +:::note +Positions are not created for spread instruments. While contingent orders can still trigger for spreads, +they operate without position linkage. The engine handles spread instruments separately from regular positions. +::: + +## Summary + +Positions are central to tracking trading activity and performance. Understanding how positions +aggregate fills, calculate PnL, and handle different OMS configurations matters when building +trading strategies. Position snapshotting provides accurate historical tracking in `NETTING` +mode, and the event history supports detailed analysis and reconciliation. + +## Related guides + +- [Orders](orders.md) - Orders that create and modify positions. +- [Execution](execution.md) - Fill handling that updates positions. +- [Portfolio](portfolio.md) - Portfolio-level position aggregation. diff --git a/nautilus-docs/docs/concepts/reports.md b/nautilus-docs/docs/concepts/reports.md new file mode 100644 index 0000000..1f599ff --- /dev/null +++ b/nautilus-docs/docs/concepts/reports.md @@ -0,0 +1,435 @@ +# Reports + +This guide explains the portfolio analysis and reporting capabilities provided by the `ReportProvider` +class, and how these reports are used for PnL accounting and backtest post-run analysis. + +## Overview + +The `ReportProvider` class in NautilusTrader generates structured analytical reports from +trading data, transforming raw orders, fills, positions, and account states into pandas DataFrames +for analysis and visualization. These reports help you evaluate strategy performance, +analyze execution quality, and verify PnL accounting. + +Reports can be generated using two approaches: + +- **Trader helper methods** (recommended): Convenient methods like `trader.generate_orders_report()`. +- **ReportProvider directly**: For more control over data selection and filtering. + +Reports provide consistent analytics across both backtesting and live trading environments, +enabling reliable performance evaluation and strategy comparison. + +## Available reports + +The `ReportProvider` class offers several static methods to generate reports from trading data. +Each report returns a pandas DataFrame with specific columns and indexing for easy analysis. + +### Orders report + +Generates a full view of all orders: + +```python +# Using Trader helper method (recommended) +orders_report = trader.generate_orders_report() + +# Or using ReportProvider directly +from nautilus_trader.analysis import ReportProvider + +orders = cache.orders() +orders_report = ReportProvider.generate_orders_report(orders) +``` + +**Returns `pd.DataFrame`. Key columns include:** + +| Column | Description | +|--------------------|---------------------------------------------------------| +| `client_order_id` | Index - unique order identifier. | +| `instrument_id` | Trading instrument. | +| `strategy_id` | Strategy that created the order. | +| `trader_id` | Trader identifier. | +| `account_id` | Account identifier (if assigned). | +| `venue_order_id` | Venue-assigned order ID (if accepted). | +| `side` | BUY or SELL. | +| `type` | MARKET, LIMIT, etc. | +| `status` | Current order status. | +| `quantity` | Original order quantity (string). | +| `filled_qty` | Amount filled (string). | +| `price` | Limit price (order-type dependent). | +| `avg_px` | Average fill price (if filled). | +| `time_in_force` | Time-in-force instruction. | +| `ts_init` | Order initialization timestamp (Unix nanoseconds). | +| `ts_last` | Last update timestamp (Unix nanoseconds). | + +Additional columns vary by order type (e.g., `trigger_price` for stop orders, `expire_time` for +GTD orders). See `Order.to_dict()` for the complete field list. + +### Order fills report + +Provides a summary of filled orders (one row per order): + +```python +# Using Trader helper method (recommended) +fills_report = trader.generate_order_fills_report() + +# Or using ReportProvider directly +orders = cache.orders() +fills_report = ReportProvider.generate_order_fills_report(orders) +``` + +This report includes only orders with `filled_qty > 0` and contains the same columns as the +orders report, but filtered to executed orders only. Note that `ts_init` and `ts_last` are +converted to datetime objects in this report for easier analysis. + +### Fills report + +Details individual fill events (one row per fill): + +```python +# Using Trader helper method (recommended) +fills_report = trader.generate_fills_report() + +# Or using ReportProvider directly +orders = cache.orders() +fills_report = ReportProvider.generate_fills_report(orders) +``` + +**Returns `pd.DataFrame`. Key columns include:** + +| Column | Description | +|--------------------|------------------------------------------| +| `client_order_id` | Index - order identifier. | +| `trade_id` | Unique trade/fill identifier. | +| `venue_order_id` | Venue-assigned order ID. | +| `instrument_id` | Trading instrument. | +| `strategy_id` | Strategy that created the order. | +| `account_id` | Account identifier. | +| `position_id` | Associated position ID (if applicable). | +| `order_side` | BUY or SELL. | +| `order_type` | Order type (MARKET, LIMIT, etc.). | +| `last_px` | Fill execution price (string). | +| `last_qty` | Fill execution quantity (string). | +| `currency` | Currency of the fill. | +| `liquidity_side` | MAKER or TAKER. | +| `commission` | Commission amount and currency. | +| `ts_event` | Fill timestamp (datetime). | +| `ts_init` | Initialization timestamp (datetime). | + +See `OrderFilled.to_dict()` for the complete field list. + +### Positions report + +Position analysis including snapshots: + +```python +# Using Trader helper method (recommended) +# Automatically includes snapshots for NETTING OMS +positions_report = trader.generate_positions_report() + +# Or using ReportProvider directly +positions = cache.positions() +snapshots = cache.position_snapshots() # For NETTING OMS +positions_report = ReportProvider.generate_positions_report( + positions=positions, + snapshots=snapshots +) +``` + +**Returns `pd.DataFrame`. Key columns include:** + +| Column | Description | +|--------------------|------------------------------------------| +| `position_id` | Index - unique position identifier. | +| `instrument_id` | Trading instrument. | +| `strategy_id` | Strategy that managed the position. | +| `trader_id` | Trader identifier. | +| `account_id` | Account identifier. | +| `opening_order_id` | Order ID that opened the position. | +| `closing_order_id` | Order ID that closed the position. | +| `entry` | Entry side (BUY or SELL). | +| `side` | Position side (LONG, SHORT, or FLAT). | +| `quantity` | Current position size. | +| `peak_qty` | Maximum size reached. | +| `avg_px_open` | Average entry price. | +| `avg_px_close` | Average exit price (if closed). | +| `commissions` | List of commissions paid. | +| `realized_pnl` | Realized profit/loss. | +| `realized_return` | Return percentage. | +| `ts_init` | Position initialization timestamp. | +| `ts_opened` | Opening timestamp (datetime). | +| `ts_last` | Last update timestamp. | +| `ts_closed` | Closing timestamp (datetime or NA). | +| `duration_ns` | Position duration in nanoseconds. | +| `is_snapshot` | Whether this is a historical snapshot. | + +### Account report + +Tracks account balance and margin changes over time: + +```python +# Using Trader helper method (recommended) +# Requires venue parameter +from nautilus_trader.model.identifiers import Venue +venue = Venue("BINANCE") +account_report = trader.generate_account_report(venue) + +# Or using ReportProvider directly +account = cache.account(account_id) +account_report = ReportProvider.generate_account_report(account) +``` + +**Returns `pd.DataFrame`. Columns include:** + +| Column | Description | +|-----------------|--------------------------------------------| +| `ts_event` | Index - timestamp of account state change. | +| `account_id` | Account identifier. | +| `account_type` | Type of account (e.g., SPOT, MARGIN). | +| `base_currency` | Base currency for the account. | +| `total` | Total balance amount (string). | +| `free` | Available balance (string). | +| `locked` | Balance locked in orders (string). | +| `currency` | Currency of the balance. | +| `reported` | Whether balance was reported by venue. | +| `margins` | Margin information (list, if applicable). | +| `info` | Additional venue-specific information. | + +Each row represents a balance entry; accounts with multiple currencies produce multiple rows +per account state event. + +## PnL accounting considerations + +Accurate PnL accounting requires careful consideration of several factors: + +### Position-based PnL + +- **Realized PnL**: Calculated when positions are partially or fully closed. +- **Unrealized PnL**: Marked-to-market using current prices. +- **Commission impact**: Only included when in settlement currency. + +:::warning +PnL calculations depend on the OMS type. In `NETTING` OMS, position snapshots +preserve historical PnL when positions reopen. Always include snapshots in +reports for accurate total PnL calculation. In `HEDGING` OMS, snapshots are +not used since each position has a unique ID and is never reopened. +::: + +### Multi-currency accounting + +When dealing with multiple currencies: + +- Each position tracks PnL in its settlement currency. +- Portfolio aggregation requires currency conversion. +- Commission currencies may differ from settlement currency. + +```python +# Accessing PnL across positions +for position in positions: + realized = position.realized_pnl # In settlement currency + unrealized = position.unrealized_pnl(last_price) + + # Handle multi-currency aggregation (illustrative) + # Note: Currency conversion requires user-provided exchange rates + if position.settlement_currency != base_currency: + # Apply conversion rate from your data source + # rate = get_exchange_rate(position.settlement_currency, base_currency) + # realized_converted = realized.as_double() * rate + pass +``` + +### Snapshot considerations + +For `NETTING` OMS: + +```python +from nautilus_trader.model.objects import Money + +# Include snapshots for complete PnL (per currency) +pnl_by_currency = {} + +# Add PnL from current positions +for position in cache.positions(instrument_id=instrument_id): + if position.realized_pnl: + currency = position.realized_pnl.currency + if currency not in pnl_by_currency: + pnl_by_currency[currency] = 0.0 + pnl_by_currency[currency] += position.realized_pnl.as_double() + +# Add PnL from historical snapshots +for snapshot in cache.position_snapshots(instrument_id=instrument_id): + if snapshot.realized_pnl: + currency = snapshot.realized_pnl.currency + if currency not in pnl_by_currency: + pnl_by_currency[currency] = 0.0 + pnl_by_currency[currency] += snapshot.realized_pnl.as_double() + +# Create Money objects for each currency +total_pnls = [Money(amount, currency) for currency, amount in pnl_by_currency.items()] +``` + +## Backtest post-run analysis + +After a backtest completes, analysis is available through various reports +and the portfolio analyzer. + +### Accessing backtest results + +```python +# After backtest run +engine.run(start=start_time, end=end_time) + +# Generate reports using Trader helper methods +orders_report = engine.trader.generate_orders_report() +positions_report = engine.trader.generate_positions_report() +fills_report = engine.trader.generate_fills_report() + +# Or access data directly for custom analysis +orders = engine.cache.orders() +positions = engine.cache.positions() +snapshots = engine.cache.position_snapshots() +``` + +### Portfolio statistics + +The portfolio analyzer provides performance metrics: + +```python +# Access portfolio analyzer +portfolio = engine.portfolio + +# Get different categories of statistics +stats_pnls = portfolio.analyzer.get_performance_stats_pnls() +stats_returns = portfolio.analyzer.get_performance_stats_returns() +stats_general = portfolio.analyzer.get_performance_stats_general() +``` + +:::info +For detailed information about available statistics and creating custom metrics, +see the [Portfolio guide](portfolio.md#portfolio-statistics). The Portfolio guide covers: + +- Built-in statistics categories (PnLs, returns, positions, orders based). +- Creating custom statistics with `PortfolioStatistic`. +- Registering and using custom metrics. + +::: + +### Visualization + +NautilusTrader provides interactive tearsheets and plots via Plotly: + +```python +from nautilus_trader.analysis import create_tearsheet + +# After backtest run +engine.run() + +# Generate interactive HTML tearsheet +create_tearsheet(engine, output_path="tearsheet.html") +``` + +This creates an interactive HTML report with: + +- Equity curve +- Drawdown analysis +- Monthly returns heatmap +- Performance statistics table +- Returns distribution + +For more control, generate individual plots: + +```python +from nautilus_trader.analysis import create_equity_curve + +returns = engine.portfolio.analyzer.returns() +fig = create_equity_curve(returns, title="My Strategy Equity") +fig.show() # Display in browser +fig.write_image("equity.png") # Export to PNG (requires kaleido) +``` + +Install visualization dependencies: + +```bash +uv pip install "nautilus_trader[visualization]" +``` + +## Report generation patterns + +### Live trading + +During live trading, generate reports periodically: + +```python +import pandas as pd + +class ReportingActor(Actor): + def on_start(self): + # Schedule periodic reporting + self.clock.set_timer( + name="generate_reports", + interval=pd.Timedelta(minutes=30), + callback=self.generate_reports + ) + + def generate_reports(self, event): + # Generate and log reports + positions_report = self.trader.generate_positions_report() + + # Save or transmit report + positions_report.to_csv(f"positions_{event.ts_event}.csv") +``` + +### Performance analysis + +For backtest analysis: + +```python +import pandas as pd + +# Run the backtest +engine.run(start=start_time, end=end_time) + +# Collect results +positions_closed = engine.cache.positions_closed() +stats_pnls = engine.portfolio.analyzer.get_performance_stats_pnls() +stats_returns = engine.portfolio.analyzer.get_performance_stats_returns() +stats_general = engine.portfolio.analyzer.get_performance_stats_general() + +# Create summary dictionary +results = { + "total_positions": len(positions_closed), + "pnl_total": stats_pnls.get("PnL (total)"), + "sharpe_ratio": stats_returns.get("Sharpe Ratio (252 days)"), + "profit_factor": stats_general.get("Profit Factor"), + "win_rate": stats_general.get("Win Rate"), +} + +# Display results +results_df = pd.DataFrame([results]) +print(results_df.T) # Transpose for vertical display +``` + +:::info +Reports are generated from in-memory data structures. For large-scale analysis +or long-running systems, consider persisting reports to a database for efficient +querying. See the [Cache guide](cache.md) for persistence options. +::: + +## Integration with other components + +The `ReportProvider` works with several system components: + +- **Cache**: Source of all trading data (orders, positions, accounts) for reports. +- **Portfolio**: Uses reports for performance analysis and metrics calculation. +- **BacktestEngine**: Uses reports for post-run analysis and visualization. +- **Position snapshots**: Required for accurate PnL reporting in `NETTING` OMS. + +## Summary + +The `ReportProvider` generates reports from orders, fills, positions, and account +states as structured DataFrames for analysis and visualization. For accurate total +PnL in `NETTING` OMS, include position snapshots when generating reports. + +## Related guides + +- [Visualization](visualization.md) - Interactive tearsheets and charts from backtest results. +- [Portfolio](portfolio.md) - Portfolio statistics and performance metrics. +- [Backtesting](backtesting.md) - Running backtests that generate reports. +- [Cache](cache.md) - Cache system that stores data for reports. diff --git a/nautilus-docs/docs/concepts/strategies.md b/nautilus-docs/docs/concepts/strategies.md new file mode 100644 index 0000000..b5520c6 --- /dev/null +++ b/nautilus-docs/docs/concepts/strategies.md @@ -0,0 +1,692 @@ +# Strategies + +A strategy inherits the `Strategy` class and implements +the methods its logic requires. + +**Capabilities**: + +- All `Actor` capabilities. +- Order management. + +**Relationship with actors**: +The `Strategy` class inherits from `Actor`, which means strategies have access to all actor functionality +plus order management capabilities. + +:::tip +We recommend reviewing the [Actors](actors.md) guide before diving into strategy development. +::: + +Strategies can be added to Nautilus systems in any [environment contexts](architecture.md#environment-contexts) and will start sending commands and receiving +events based on their logic as soon as the system starts. + +With these building blocks of data ingest, event handling, and order management (discussed below), +you can build any type of strategy including directional, momentum, re-balancing, +pairs, market making, etc. + +See the [`Strategy` API Reference](/docs/python-api-latest/trading.html) for all available methods. + +There are two main parts of a Nautilus trading strategy: + +- The strategy implementation itself, defined by inheriting the `Strategy` class. +- The *optional* strategy configuration, defined by inheriting the `StrategyConfig` class. + +:::tip +Once a strategy is defined, the same source code can be used for backtesting and live trading. +::: + +The main capabilities of a strategy include: + +- Historical data requests. +- Live data feed subscriptions. +- Setting time alerts or timers. +- Cache access. +- Portfolio access. +- Creating and managing orders and positions. + +## Strategy implementation + +A trading strategy inherits from `Strategy`, so you must define a constructor. +At minimum, initialize the base class: + +```python +from nautilus_trader.trading.strategy import Strategy + +class MyStrategy(Strategy): + def __init__(self) -> None: + super().__init__() # <-- the superclass must be called to initialize the strategy +``` + +From here, you can implement handlers as necessary to perform actions based on state transitions +and events. + +:::warning +Do not call components such as `clock` and `logger` in the `__init__` constructor (which is prior to registration). +This is because the systems clock and logging subsystem have not yet been initialized. +::: + +### Handlers + +Handlers are methods on the `Strategy` class that perform actions based on events or state changes. +These methods use the `on_*` prefix. Implement any or all of them as your strategy requires. + +Multiple handlers exist for similar event types to give you control over granularity. +Respond to a specific event with a dedicated handler, or use a generic handler for a range +of related events (using typical switch statement logic). +The system calls handlers in sequence from most specific to most general. + +#### Stateful actions + +Lifecycle state changes trigger these handlers. Recommendations: + +- Use the `on_start` method to initialize your strategy (e.g., fetch instruments, subscribe to data). +- Use the `on_stop` method for cleanup tasks (e.g., cancel open orders, close open positions, unsubscribe from data). + +```python +def on_start(self) -> None: +def on_stop(self) -> None: +def on_resume(self) -> None: +def on_reset(self) -> None: +def on_dispose(self) -> None: +def on_degrade(self) -> None: +def on_fault(self) -> None: +def on_save(self) -> dict[str, bytes]: # Returns user-defined dictionary of state to be saved +def on_load(self, state: dict[str, bytes]) -> None: +``` + +#### Data handling + +These handlers receive data updates, including built-in market data and custom user-defined data. + +```python +from nautilus_trader.core import Data +from nautilus_trader.model import OrderBook +from nautilus_trader.model import Bar +from nautilus_trader.model import QuoteTick +from nautilus_trader.model import TradeTick +from nautilus_trader.model import OrderBookDeltas +from nautilus_trader.model import InstrumentClose +from nautilus_trader.model import InstrumentStatus +from nautilus_trader.model import OptionChainSlice +from nautilus_trader.model import OptionGreeks +from nautilus_trader.model.instruments import Instrument + +def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: +def on_order_book(self, order_book: OrderBook) -> None: +def on_quote_tick(self, tick: QuoteTick) -> None: +def on_trade_tick(self, tick: TradeTick) -> None: +def on_bar(self, bar: Bar) -> None: +def on_instrument(self, instrument: Instrument) -> None: +def on_instrument_status(self, data: InstrumentStatus) -> None: +def on_instrument_close(self, data: InstrumentClose) -> None: +def on_option_greeks(self, greeks: OptionGreeks) -> None: +def on_option_chain(self, chain: OptionChainSlice) -> None: +def on_historical_data(self, data: Data) -> None: +def on_data(self, data: Data) -> None: # Custom data passed to this handler +def on_signal(self, signal: Data) -> None: # Custom signals passed to this handler +``` + +#### Order management + +These handlers receive events related to orders. +`OrderEvent` type messages are passed to handlers in the following sequence: + +1. Specific handler (e.g., `on_order_accepted`, `on_order_rejected`, etc.) +2. `on_order_event(...)` +3. `on_event(...)` + +```python +from nautilus_trader.model.events import OrderAccepted +from nautilus_trader.model.events import OrderCanceled +from nautilus_trader.model.events import OrderCancelRejected +from nautilus_trader.model.events import OrderDenied +from nautilus_trader.model.events import OrderEmulated +from nautilus_trader.model.events import OrderEvent +from nautilus_trader.model.events import OrderExpired +from nautilus_trader.model.events import OrderFilled +from nautilus_trader.model.events import OrderInitialized +from nautilus_trader.model.events import OrderModifyRejected +from nautilus_trader.model.events import OrderPendingCancel +from nautilus_trader.model.events import OrderPendingUpdate +from nautilus_trader.model.events import OrderRejected +from nautilus_trader.model.events import OrderReleased +from nautilus_trader.model.events import OrderSubmitted +from nautilus_trader.model.events import OrderTriggered +from nautilus_trader.model.events import OrderUpdated + +def on_order_initialized(self, event: OrderInitialized) -> None: +def on_order_denied(self, event: OrderDenied) -> None: +def on_order_emulated(self, event: OrderEmulated) -> None: +def on_order_released(self, event: OrderReleased) -> None: +def on_order_submitted(self, event: OrderSubmitted) -> None: +def on_order_rejected(self, event: OrderRejected) -> None: +def on_order_accepted(self, event: OrderAccepted) -> None: +def on_order_canceled(self, event: OrderCanceled) -> None: +def on_order_expired(self, event: OrderExpired) -> None: +def on_order_triggered(self, event: OrderTriggered) -> None: +def on_order_pending_update(self, event: OrderPendingUpdate) -> None: +def on_order_pending_cancel(self, event: OrderPendingCancel) -> None: +def on_order_modify_rejected(self, event: OrderModifyRejected) -> None: +def on_order_cancel_rejected(self, event: OrderCancelRejected) -> None: +def on_order_updated(self, event: OrderUpdated) -> None: +def on_order_filled(self, event: OrderFilled) -> None: +def on_order_event(self, event: OrderEvent) -> None: # All order event messages are eventually passed to this handler +``` + +#### Position management + +These handlers receive events related to positions. +`PositionEvent` type messages are passed to handlers in the following sequence: + +1. Specific handler (e.g., `on_position_opened`, `on_position_changed`, etc.) +2. `on_position_event(...)` +3. `on_event(...)` + +```python +from nautilus_trader.model.events import PositionChanged +from nautilus_trader.model.events import PositionClosed +from nautilus_trader.model.events import PositionEvent +from nautilus_trader.model.events import PositionOpened + +def on_position_opened(self, event: PositionOpened) -> None: +def on_position_changed(self, event: PositionChanged) -> None: +def on_position_closed(self, event: PositionClosed) -> None: +def on_position_event(self, event: PositionEvent) -> None: # All position event messages are eventually passed to this handler +``` + +#### Generic event handling + +This handler will eventually receive all event messages which arrive at the strategy, including those for +which no other specific handler exists. + +```python +from nautilus_trader.core.message import Event + +def on_event(self, event: Event) -> None: +``` + +#### Handler example + +The following example shows a typical `on_start` handler method implementation (taken from the example EMA cross strategy). +Here we can see the following: + +- Indicators being registered to receive bar updates. +- Historical data being requested (to hydrate the indicators). +- Live data being subscribed to. + +```python +def on_start(self) -> None: + """ + Actions to be performed on strategy start. + """ + self.instrument = self.cache.instrument(self.instrument_id) + if self.instrument is None: + self.log.error(f"Could not find instrument for {self.instrument_id}") + self.stop() # Transitions strategy to STOPPED state + return + + # Register the indicators for updating + self.register_indicator_for_bars(self.bar_type, self.fast_ema) + self.register_indicator_for_bars(self.bar_type, self.slow_ema) + + # Get historical data + self.request_bars(self.bar_type) + + # Subscribe to live data + self.subscribe_bars(self.bar_type) + self.subscribe_quote_ticks(self.instrument_id) +``` + +### Clock and timers + +Strategies have access to a `Clock` which provides a number of methods for creating +different timestamps, as well as setting time alerts or timers to trigger `TimeEvent`s. + +See the [`Clock` API Reference](/docs/python-api-latest/common.html) for all available methods. + +#### Current timestamps + +While there are multiple ways to obtain current timestamps, here are two commonly used methods as examples: + +To get the current UTC timestamp as a tz-aware `pd.Timestamp`: + +```python +import pandas as pd + + +now: pd.Timestamp = self.clock.utc_now() +``` + +To get the current UTC timestamp as nanoseconds since the UNIX epoch: + +```python +unix_nanos: int = self.clock.timestamp_ns() +``` + +#### Time alerts + +Time alerts can be set which will result in a `TimeEvent` being dispatched to the `on_event` handler at the +specified alert time. In a live context, this might be slightly delayed by a few microseconds. + +This example sets a time alert to trigger one minute from the current time: + +```python +import pandas as pd + +# Fire a TimeEvent one minute from now +self.clock.set_time_alert( + name="MyTimeAlert1", + alert_time=self.clock.utc_now() + pd.Timedelta(minutes=1), +) +``` + +#### Timers + +Continuous timers can be set up which will generate a `TimeEvent` at regular intervals until the timer expires +or is canceled. + +This example sets a timer to fire once per minute, starting immediately: + +```python +import pandas as pd + +# Fire a TimeEvent every minute +self.clock.set_timer( + name="MyTimer1", + interval=pd.Timedelta(minutes=1), +) +``` + +### Cache access + +The trader's central `Cache` stores data and execution objects (orders, positions, etc). +Many methods are available with filtering. Here are some basic use cases. + +#### Fetching data + +The following example fetches data from the cache (assuming some instrument ID attribute is assigned). +These methods return `None` if the requested data is not available. + +```python +last_quote = self.cache.quote_tick(self.instrument_id) +last_trade = self.cache.trade_tick(self.instrument_id) +last_bar = self.cache.bar(bar_type) +``` + +#### Fetching execution objects + +The following example shows how individual order and position objects can be fetched from the cache: + +```python +order = self.cache.order(client_order_id) +position = self.cache.position(position_id) +``` + +See the [`Cache` API Reference](/docs/python-api-latest/cache.html) for all available methods. + +### Portfolio access + +The trader's central `Portfolio` provides account and positional information. +The following shows a general outline of available methods. + +#### Account and positional information + +```python +import decimal + +from nautilus_trader.accounting.accounts.base import Account +from nautilus_trader.model import Venue +from nautilus_trader.model import Currency +from nautilus_trader.model import Money +from nautilus_trader.model import InstrumentId + +def account(self, venue: Venue) -> Account + +def balances_locked(self, venue: Venue) -> dict[Currency, Money] +def margins_init(self, venue: Venue) -> dict[Currency, Money] +def margins_maint(self, venue: Venue) -> dict[Currency, Money] +def unrealized_pnls(self, venue: Venue) -> dict[Currency, Money] +def realized_pnls(self, venue: Venue) -> dict[Currency, Money] +def net_exposures(self, venue: Venue) -> dict[Currency, Money] + +def unrealized_pnl(self, instrument_id: InstrumentId) -> Money +def realized_pnl(self, instrument_id: InstrumentId) -> Money +def net_exposure(self, instrument_id: InstrumentId) -> Money +def net_position(self, instrument_id: InstrumentId) -> decimal.Decimal + +def is_net_long(self, instrument_id: InstrumentId) -> bool +def is_net_short(self, instrument_id: InstrumentId) -> bool +def is_flat(self, instrument_id: InstrumentId) -> bool +def is_completely_flat(self) -> bool +``` + +See the [`Portfolio` API Reference](/docs/python-api-latest/portfolio.html) for all available methods. + +#### Reports and analysis + +The `Portfolio` also exposes a `PortfolioAnalyzer`, which accepts a flexible amount of data +(to accommodate different lookback windows). The analyzer tracks and generates performance +metrics and statistics. + +See the [`PortfolioAnalyzer` API Reference](/docs/python-api-latest/analysis.html) and [Portfolio statistics](portfolio.md#portfolio-statistics) guide. + +### Trading commands + +The following trading commands are available for order management. +See also the [Execution](../concepts/execution.md) guide for the full flow through the system. + +#### Submitting orders + +An `OrderFactory` is provided on the base class for every `Strategy` as a convenience, reducing +the amount of boilerplate required to create different `Order` objects (although these objects +can still be initialized directly with the `Order.__init__(...)` constructor if the trader prefers). + +The component a `SubmitOrder` or `SubmitOrderList` command will flow to for execution depends on the following: + +- If an `emulation_trigger` is specified, the command will *firstly* be sent to the `OrderEmulator`. +- If an `exec_algorithm_id` is specified (with no `emulation_trigger`), the command will *firstly* be sent to the relevant `ExecAlgorithm`. +- Otherwise, the command will *firstly* be sent to the `RiskEngine`. + +This example submits a `LIMIT` BUY order for emulation (see [Emulated Orders](orders.md#emulated-orders)): + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model.orders import LimitOrder + + +def buy(self) -> None: + """ + Users simple buy method (example). + """ + order: LimitOrder = self.order_factory.limit( + instrument_id=self.instrument_id, + order_side=OrderSide.BUY, + quantity=self.instrument.make_qty(self.trade_size), + price=self.instrument.make_price(5000.00), + emulation_trigger=TriggerType.LAST_PRICE, + ) + + self.submit_order(order) +``` + +:::info +You can specify both order emulation and an execution algorithm. In this case, the order is +first sent to the `OrderEmulator`, and upon release is then routed to the `ExecAlgorithm`. +::: + +This example submits a `MARKET` BUY order to a TWAP execution algorithm: + +```python +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model import ExecAlgorithmId + + +def buy(self) -> None: + """ + Users simple buy method (example). + """ + order: MarketOrder = self.order_factory.market( + instrument_id=self.instrument_id, + order_side=OrderSide.BUY, + quantity=self.instrument.make_qty(self.trade_size), + time_in_force=TimeInForce.FOK, + exec_algorithm_id=ExecAlgorithmId("TWAP"), + exec_algorithm_params={"horizon_secs": 20, "interval_secs": 2.5}, + ) + + self.submit_order(order) +``` + +#### Canceling orders + +Orders can be canceled individually, as a batch, or all orders for an instrument (with an optional side filter). + +If the order is already *closed* or already pending cancel, then a warning will be logged. + +If the order is currently *open* then the status will become `PENDING_CANCEL`. + +The component a `CancelOrder`, `CancelAllOrders` or `BatchCancelOrders` command will flow to for execution depends on the following: + +- If the order is currently emulated, the command will *firstly* be sent to the `OrderEmulator`. +- If an `exec_algorithm_id` is specified (with no `emulation_trigger`), and the order is still active within the local system, the command will *firstly* be sent to the relevant `ExecAlgorithm`. +- Otherwise, the order will *firstly* be sent to the `ExecutionEngine`. + +:::info +Any managed GTD timer will also be canceled after the command has left the strategy. +::: + +The following shows how to cancel an individual order: + +```python +self.cancel_order(order) +``` + +The following shows how to cancel a batch of orders: + +```python +from nautilus_trader.model.orders import Order + + +my_order_list: list[Order] = [order1, order2, order3] +self.cancel_orders(my_order_list) +``` + +The following shows how to cancel all orders: + +```python +self.cancel_all_orders() +``` + +#### Modifying orders + +Orders can be modified individually when emulated, or *open* on a venue (if supported). + +If the order is already *closed* or already pending cancel, then a warning will be logged. +If the order is currently *open* then the status will become `PENDING_UPDATE`. + +:::warning +At least one value must differ from the original order for the command to be valid. +::: + +The component a `ModifyOrder` command will flow to for execution depends on the following: + +- If the order is currently emulated, the command will *firstly* be sent to the `OrderEmulator`. +- Otherwise, the order will *firstly* be sent to the `RiskEngine`. + +:::info +Once an order is under the control of an execution algorithm, it cannot be directly modified by a strategy (only canceled). +::: + +The following shows how to modify the size of `LIMIT` BUY order currently *open* on a venue: + +```python +from nautilus_trader.model import Quantity + + +new_quantity: Quantity = Quantity.from_int(5) +self.modify_order(order, new_quantity) +``` + +:::info +The price and trigger price can also be modified (when emulated or supported by a venue). +::: + +#### Market exit + +The `market_exit()` method provides a graceful way to exit all positions and cancel all orders +for a strategy. The strategy remains running after the exit completes, allowing you to re-enter +positions later if desired. + +```python +self.market_exit() +``` + +The market exit process: + +1. Cancels all open and in-flight orders for the strategy. +2. Closes all open positions with market orders. +3. Periodically checks (at `market_exit_interval_ms`) until all orders resolve and positions close. +4. Calls `post_market_exit()` once flat, or after `market_exit_max_attempts` is reached. + +Two hooks are available for custom logic: + +- `on_market_exit()` - Called when the exit process begins. +- `post_market_exit()` - Called when the exit process completes. + +```python +class MyStrategy(Strategy): + def on_market_exit(self) -> None: + self.log.info("Beginning market exit...") + + def post_market_exit(self) -> None: + self.log.info("Market exit complete") +``` + +During a market exit, non-reduce-only orders are automatically denied. For order lists, +if any order in the list is non-reduce-only, the entire list is denied to preserve list +semantics (e.g., bracket orders with interdependencies). + +To check if an exit is in progress (e.g., to skip order submission logic), use `is_exiting()`: + +```python +def on_quote_tick(self, tick: QuoteTick) -> None: + if self.is_exiting(): + return # Skip order logic during exit + # ... normal order logic +``` + +To automatically perform a market exit when the strategy is stopped, set `manage_stop=True`: + +```python +config = StrategyConfig(manage_stop=True) +``` + +With this option, calling `stop()` will first perform a market exit, then stop the strategy +once flat. + +Configuration options in `StrategyConfig`: + +- `manage_stop` (default: False) - If True, `stop()` performs a market exit before stopping. +- `market_exit_interval_ms` (default: 100) - Interval between exit completion checks. +- `market_exit_max_attempts` (default: 100) - Maximum checks before completing the exit. +- `market_exit_time_in_force` (default: None/GTC) - Time in force for closing market orders. +- `market_exit_reduce_only` (default: True) - If closing market orders should be reduce only. + +## Strategy configuration + +A separate configuration class gives full flexibility over where and how a strategy +is instantiated. Configurations serialize over the wire, enabling distributed backtesting +and remote live trading. + +This is opt-in. You can skip configuration and pass parameters directly to your +strategy constructor. If you want distributed backtests or remote live trading, +define a configuration. + +Here is an example configuration: + +```python +from decimal import Decimal +from nautilus_trader.config import StrategyConfig +from nautilus_trader.model import Bar, BarType +from nautilus_trader.model import InstrumentId +from nautilus_trader.trading.strategy import Strategy + + +# Configuration definition +class MyStrategyConfig(StrategyConfig): + instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" + bar_type: BarType # example value: "ETHUSDT-PERP.BINANCE-15-MINUTE[LAST]-EXTERNAL" + fast_ema_period: int = 10 + slow_ema_period: int = 20 + trade_size: Decimal + order_id_tag: str + + +# Strategy definition +class MyStrategy(Strategy): + def __init__(self, config: MyStrategyConfig) -> None: + # Always initialize the parent Strategy class + # After this, configuration is stored and available via `self.config` + super().__init__(config) + + # Custom state variables + self.time_started = None + self.count_of_processed_bars: int = 0 + + def on_start(self) -> None: + self.time_started = self.clock.utc_now() # Remember time, when strategy started + self.subscribe_bars(self.config.bar_type) # See how configuration data are exposed via `self.config` + + def on_bar(self, bar: Bar): + self.count_of_processed_bars += 1 # Update count of processed bars + + +# Instantiate configuration with specific values. By setting: +# - InstrumentId - we parameterize the instrument the strategy will trade. +# - BarType - we parameterize bar-data, that strategy will trade. +config = MyStrategyConfig( + instrument_id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), + bar_type=BarType.from_str("ETHUSDT-PERP.BINANCE-15-MINUTE[LAST]-EXTERNAL"), + trade_size=Decimal(1), + order_id_tag="001", +) + +# Pass configuration to our trading strategy. +strategy = MyStrategy(config=config) +``` + +Access configuration values through `self.config`. +This provides clear separation between: + +- Configuration data (accessed via `self.config`): + - Contains initial settings, that define how the strategy works. + - Example: `self.config.trade_size`, `self.config.instrument_id` + +- Strategy state variables (as direct attributes): + - Track any custom state of the strategy. + - Example: `self.time_started`, `self.count_of_processed_bars` + +This separation makes code easier to understand and maintain. + +:::note +Even though it often makes sense to define a strategy which will trade a single +instrument. The number of instruments a single strategy can work with is only limited by machine resources. +::: + +### Managed GTD expiry + +It's possible for the strategy to manage expiry for orders with a time in force of GTD (*Good 'till Date*). +This may be desirable if the exchange/broker does not support this time in force option, or for any +reason you prefer the strategy to manage this. + +To use this option, pass `manage_gtd_expiry=True` to your `StrategyConfig`. When an order is submitted with +a time in force of GTD, the strategy will automatically start an internal time alert. +Once the internal GTD time alert is reached, the order will be canceled (if not already *closed*). + +Some venues (such as Binance Futures) support the GTD time in force, so to avoid conflicts when using +`managed_gtd_expiry` you should set `use_gtd=False` for your execution client config. + +### Multiple strategies + +If you intend running multiple instances of the same strategy, with different +configurations (such as trading different instruments), then you will need to define +a unique `order_id_tag` for each of these strategies (as shown above). + +:::note +The platform has built-in safety measures: if two strategies share a duplicated strategy ID, +a `RuntimeError` is raised during registration indicating the strategy ID is already registered. +::: + +The reason for this is that the system must be able to identify which strategy +various commands and events belong to. A strategy ID is made up of the +strategy class name, and the strategies `order_id_tag` separated by a hyphen. For +example the above config would result in a strategy ID of `MyStrategy-001`. + +See the [`StrategyId` API Reference](/docs/python-api-latest/model/identifiers.html) for further details. + +## Related guides + +- [Actors](actors.md) - Base class that strategies extend. +- [Orders](orders.md) - Order types and management from strategies. +- [Backtesting](backtesting.md) - Test strategies with historical data. diff --git a/nautilus-docs/docs/concepts/value_types.md b/nautilus-docs/docs/concepts/value_types.md new file mode 100644 index 0000000..306eaae --- /dev/null +++ b/nautilus-docs/docs/concepts/value_types.md @@ -0,0 +1,303 @@ +# Value Types + +NautilusTrader provides specialized value types for representing core trading concepts: +`Price`, `Quantity`, and `Money`. These types use fixed-point arithmetic internally +for performant, deterministic calculations across different platforms +and environments. + +## Overview + +| Type | Purpose | Signed | Currency | +|------------|------------------------------------------|--------|----------| +| `Quantity` | Trade sizes, order amounts, positions. | No | - | +| `Price` | Market prices, quotes, price levels. | Yes | - | +| `Money` | Monetary amounts, P&L, account balances. | Yes | Yes | + +## Immutability + +All value types are **immutable**. Once a value is constructed, it cannot be changed. +Operations do not mutate the original object. + +```python +from nautilus_trader.model.objects import Quantity + +qty1 = Quantity(100, precision=0) +qty2 = Quantity(50, precision=0) + +# This creates a NEW Quantity; qty1 and qty2 are unchanged +result = qty1 + qty2 + +print(qty1) # 100 +print(qty2) # 50 +print(result) # 150 +``` + +This design provides several benefits: + +- **Thread safety**: Immutable values can be safely shared across threads without synchronization. +- **Predictability**: Values never change unexpectedly, making debugging easier. +- **Hashability**: Immutable types can be used as dictionary keys and in sets. + +## Arithmetic operations + +Value types support standard arithmetic operators (`+`, `-`, `*`, `/`, `%`, `//`) +and unary operators (`-`, `+`, `abs`). The return type depends on the operator +and the operand types. + +### Same-type binary operations + +Addition and subtraction of the same value type return that type, preserving +domain meaning (a price plus a price is still a price): + +| Operation | Result | +|-----------------------|------------| +| `Quantity + Quantity` | `Quantity` | +| `Quantity - Quantity` | `Quantity` | +| `Price + Price` | `Price` | +| `Price - Price` | `Price` | +| `Money + Money` | `Money` | +| `Money - Money` | `Money` | + +```python +from nautilus_trader.model.objects import Price + +price1 = Price(100.50, precision=2) +price2 = Price(0.25, precision=2) + +result = price1 + price2 # Returns Price(100.75, precision=2) +print(type(result)) # +``` + +Multiplication, division, floor division, and modulo between two values of the +same type return `Decimal`: + +| Operation | Result | +|-----------------------|-----------| +| `Price * Price` | `Decimal` | +| `Price / Price` | `Decimal` | +| `Price // Price` | `Decimal` | +| `Price % Price` | `Decimal` | + +The same pattern applies to `Quantity` and `Money`. + +These operations do not return the original type because the result has different +dimensional meaning. Multiplying a price by a price produces "price squared", not +a price. Dividing a quantity by a quantity produces a dimensionless ratio, not a +quantity. Returning `Decimal` makes the unit change explicit and prevents +misinterpretation of the result as a value with the original unit. + +### Unary operations + +Unary operators preserve the value type where the result is valid for that type: + +| Operation | `Price` | `Quantity` | `Money` | +|--------------|-----------|------------|-----------| +| `-x` (neg) | `Price` | `Decimal` | `Money` | +| `+x` (pos) | `Price` | `Quantity` | `Money` | +| `abs(x)` | `Price` | `Quantity` | `Money` | +| `int(x)` | `int` | `int` | `int` | +| `float(x)` | `float` | `float` | `float` | +| `round(x)` | `Decimal` | `Decimal` | `Decimal` | + +`Quantity.__neg__` returns `Decimal` rather than `Quantity` because `Quantity` is +unsigned and cannot represent a negative value. + +```python +from nautilus_trader.model.objects import Price, Quantity, Money +from nautilus_trader.model.currencies import USD + +price = Price(100.50, precision=2) +print(-price) # -100.50 +print(type(-price)) # + +money = Money(-50.00, USD) +print(abs(money)) # 50.00 USD +print(type(abs(money))) # + +qty = Quantity(10, precision=0) +print(+qty) # 10 +print(type(+qty)) # +``` + +### Mixed-type operations + +When operating with other numeric types, the result type follows Python's +[numeric tower](https://docs.python.org/3/library/numbers.html) conventions. The general +principle is that operations widen to the more general type: `float` operations return +`float`, while `int` and `Decimal` operations return `Decimal` for precision preservation. + +This applies to all six binary operators (`+`, `-`, `*`, `/`, `//`, `%`) and works +in both directions (`value op scalar` and `scalar op value`): + +| Left operand | Right operand | Result type | +|--------------|---------------|-------------| +| Value type | `int` | `Decimal` | +| Value type | `float` | `float` | +| Value type | `Decimal` | `Decimal` | +| `int` | Value type | `Decimal` | +| `float` | Value type | `float` | +| `Decimal` | Value type | `Decimal` | + +```python +from decimal import Decimal +from nautilus_trader.model.objects import Quantity + +qty = Quantity(100, precision=0) + +# Quantity + int → Decimal +result1 = qty + 50 +print(type(result1)) # + +# Quantity + float → float +result2 = qty + 50.5 +print(type(result2)) # + +# Quantity + Decimal → Decimal +result3 = qty + Decimal("50") +print(type(result3)) # +``` + +## Precision handling + +Each value type stores a precision field indicating the number of decimal places. +Precision is set at construction and is immutable. There is no "unspecified" precision. + +### Fixed-point representation + +Value types are stored internally as integers scaled to a global fixed precision +(e.g., 10^16 in high-precision mode), not floating-point numbers. The `precision` +field tracks the number of decimal places used at construction, controlling display +formatting and serialization, but the underlying raw value always uses the global scale. + +```python +from nautilus_trader.model.objects import Price + +p1 = Price(1.23, precision=2) # displays as "1.23" +p2 = Price(1.230, precision=3) # displays as "1.230" + +p1 == p2 # True: same underlying value +str(p1) # "1.23" +str(p2) # "1.230" +``` + +**Precision controls display, not identity.** Two prices with the same decimal value but +different precisions are equal. The `precision` field determines string formatting and +how many decimal places are shown, but equality is based on the underlying numeric value. + +**Market data serialization uses precision metadata.** When market data types (quotes, +trades, order book deltas) are written to Parquet or Arrow format, precision is stored in +the file metadata so that values can be correctly decoded. All market data values within +a single file must share the same precision. + +:::note +If a venue changes an instrument's tick size (and thus its precision), data files written +before and after the change will have different precision metadata and should not be +consolidated into a single file. +::: + +For how instrument-level precision constrains valid prices and quantities, see the +[Precision](instruments.md#precision) section of the Instruments guide. + +### Arithmetic precision + +When performing arithmetic between values with different precisions, the result +uses the maximum precision of the operands. + +```python +from nautilus_trader.model.objects import Price + +price1 = Price(100.5, precision=1) # 1 decimal place +price2 = Price(0.125, precision=3) # 3 decimal places + +result = price1 + price2 +print(result) # 100.625 +print(result.precision) # 3 (max of 1 and 3) +``` + +## Type-specific constraints + +### Quantity + +`Quantity` represents non-negative amounts. Attempting to create a negative quantity +or subtract a larger quantity from a smaller one raises an error: + +```python +from nautilus_trader.model.objects import Quantity + +# This raises ValueError: Quantity cannot be negative +qty = Quantity(-100, precision=0) + +# This also raises ValueError +qty1 = Quantity(50, precision=0) +qty2 = Quantity(100, precision=0) +result = qty1 - qty2 # Would be -50, which is invalid +``` + +### Money + +`Money` values include a currency. Addition and subtraction between `Money` values +require matching currencies: + +```python +from nautilus_trader.model.objects import Money +from nautilus_trader.model.currencies import USD, EUR + +usd_amount = Money(100.00, USD) +eur_amount = Money(50.00, EUR) + +# This works - same currency +result = usd_amount + Money(25.00, USD) + +# This raises ValueError - currency mismatch +result = usd_amount + eur_amount +``` + +## Common patterns + +### Accumulating values + +Since value types are immutable, accumulate by reassigning: + +```python +from nautilus_trader.model.objects import Money +from nautilus_trader.model.currencies import USD + +total = Money(0.00, USD) +amounts = [Money(100.00, USD), Money(50.00, USD), Money(25.00, USD)] + +for amount in amounts: + total = total + amount # Reassign to new Money instance + +print(total) # 175.00 USD +``` + +### Converting to other types + +Value types provide conversion methods: + +```python +from nautilus_trader.model.objects import Price + +price = Price(123.456, precision=3) + +# Convert to Decimal (preserves precision) +decimal_value = price.as_decimal() + +# Convert to float +float_value = price.as_double() + +# Convert to string +string_value = str(price) # "123.456" +``` + +### Creating from strings + +Parse value types from string representations: + +```python +from nautilus_trader.model.objects import Quantity, Price, Money + +qty = Quantity.from_str("100.5") +price = Price.from_str("99.95") +money = Money.from_str("1000.00 USD") +``` diff --git a/nautilus-docs/docs/concepts/visualization.md b/nautilus-docs/docs/concepts/visualization.md new file mode 100644 index 0000000..2212b5c --- /dev/null +++ b/nautilus-docs/docs/concepts/visualization.md @@ -0,0 +1,571 @@ +# Visualization + +NautilusTrader provides interactive HTML tearsheets for analyzing backtest results through +an extensible visualization system built on Plotly. You can generate reports with minimal +code and add custom charts and themes. + +## Overview + +The visualization system has three parts: + +1. **Chart Registry** - Decoupled chart definitions that can be extended with custom visualizations. +2. **Theme System** - Consistent styling with built-in and custom themes. +3. **Configuration** - Declarative specification of what to render and how to display it. + +All visualization outputs are self-contained HTML files that can be viewed in any modern +browser, shared with stakeholders, or archived for future reference. + +:::note +The visualization system requires `plotly>=6.3.1`. Install it with: + +```bash +uv pip install "nautilus_trader[visualization]" +``` + +or + +```bash +uv pip install "plotly>=6.3.1" +``` + +::: + +## Tearsheets + +A tearsheet is a performance report that combines multiple charts and +statistics into a single interactive visualization. Tearsheets are generated after +completing a backtest run and provide immediate visual feedback on strategy performance. + +### Quick start + +Generate a tearsheet with default settings: + +```python +from nautilus_trader.analysis import create_tearsheet +from nautilus_trader.backtest.engine import BacktestEngine + +# After running your backtest +engine.run() + +# Generate tearsheet +create_tearsheet( + engine=engine, + output_path="backtest_results.html", +) +``` + +This produces an HTML file with all default charts, using the light theme and automatic +layout. Open `backtest_results.html` in your browser to view the interactive tearsheet. + +### Customization + +Control which charts appear and how they're styled: + +```python +from nautilus_trader.analysis import TearsheetConfig +from nautilus_trader.analysis import TearsheetDrawdownChart +from nautilus_trader.analysis import TearsheetEquityChart +from nautilus_trader.analysis import TearsheetRunInfoChart +from nautilus_trader.analysis import TearsheetStatsTableChart + +config = TearsheetConfig( + charts=[ + TearsheetRunInfoChart(), + TearsheetStatsTableChart(), + TearsheetEquityChart(), + TearsheetDrawdownChart(), + ], + theme="nautilus_dark", + height=2000, +) + +create_tearsheet( + engine=engine, + output_path="custom_tearsheet.html", + config=config, +) +``` + +### Currency filtering + +For multi-currency backtests, filter statistics to a specific currency: + +```python +from nautilus_trader.model.currencies import USD + +create_tearsheet( + engine=engine, + output_path="usd_only.html", + currency=USD, # Currency object, shows only USD statistics +) +``` + +When `currency` is `None` (default), statistics for all currencies are displayed separately in the tearsheet. + +## Available charts + +The tearsheet can include any combination of the following built-in charts: + +| Chart Name | Type | Description | +|--------------------|--------------|----------------------------------------------------------| +| `run_info` | Table | Run metadata and account balances. | +| `stats_table` | Table | Performance statistics (PnL, returns, general metrics). | +| `equity` | Line | Cumulative returns over time with optional benchmark. | +| `drawdown` | Area | Drawdown percentage from peak equity. | +| `monthly_returns` | Heatmap | Monthly return percentages organized by year. | +| `distribution` | Histogram | Distribution of individual return values. | +| `rolling_sharpe` | Line | 60-day rolling Sharpe ratio. | +| `yearly_returns` | Bar | Annual return percentages. | +| `bars_with_fills` | Candlestick | Price bars (OHLC) with order fills overlaid as markers. | + +All charts are registered in the chart registry and are configured via chart objects in +`TearsheetConfig.charts` (each chart object maps to a built-in chart name). + +### Run information table + +The `run_info` chart displays key metadata about the backtest run: + +- Run ID, start time, finish time +- Backtest period (start/end dates) +- Total iterations processed +- Event, order, and position counts +- Account starting and ending balances (per currency) + +This table appears in the top-left position by default. + +### Performance statistics table + +The `stats_table` chart displays performance metrics organized into sections: + +- **PnL Statistics** (per currency): Total PnL, win rate, profit factor, etc. +- **Returns Statistics**: Sharpe ratio, Sortino ratio, max drawdown, etc. +- **General Statistics**: Total trades, average trade duration, etc. + +This table appears in the top-right position by default. + +### Equity curve + +The `equity` chart plots cumulative returns over the backtest period. When `benchmark_returns` +is provided to `create_tearsheet()`, the benchmark is overlaid for comparison. + +```python +import pandas as pd + +# Load benchmark returns (e.g., from a market index) +# Index should be datetime, aligned with strategy returns timeframe +benchmark_returns = pd.read_csv("sp500_returns.csv", index_col=0, parse_dates=True)["return"] + +create_tearsheet( + engine=engine, + output_path="with_benchmark.html", + benchmark_returns=benchmark_returns, + benchmark_name="S&P 500", +) +``` + +The benchmark series is plotted as-is; ensure the index aligns with your strategy's return dates for accurate comparison. + +## Themes + +Themes control the visual styling of charts including colors, fonts, and backgrounds. +NautilusTrader provides four built-in themes: + +| Theme Name | Description | Use Case | +|-----------------|------------------------------------------------|-------------------------------| +| `plotly_white` | Clean light theme with dark gray headers. | Default, professional reports.| +| `plotly_dark` | Dark background with standard Plotly colors. | Low-light environments. | +| `nautilus` | Light theme with NautilusTrader brand colors. | Official light mode. | +| `nautilus_dark` | Dark theme with teal/cyan signature colors. | Official dark mode. | + +### Selecting a theme + +Specify the theme in `TearsheetConfig`: + +```python +config = TearsheetConfig(theme="nautilus_dark") +create_tearsheet(engine=engine, config=config) +``` + +### Custom themes + +Register a custom theme for consistent branding across all visualizations: + +```python +from nautilus_trader.analysis import register_theme + +register_theme( + name="corporate", + template="plotly_white", # Base Plotly template + colors={ + "primary": "#003366", # Navy blue + "positive": "#2e8b57", # Sea green + "negative": "#c41e3a", # Cardinal red + "neutral": "#808080", # Gray + "background": "#ffffff", # White + "grid": "#e5e5e5", # Light gray + # Optional table colors (defaults will be provided if omitted) + "table_section": "#e5e5e5", + "table_row_odd": "#f8f8f8", + "table_row_even": "#ffffff", + "table_text": "#000000", + } +) + +# Use the custom theme +config = TearsheetConfig(theme="corporate") +``` + +The theme system automatically provides sensible defaults for `table_*` colors based on +the `background` and `grid` colors, ensuring backward compatibility with themes registered +before table-specific colors were introduced. + +## Configuration + +The `TearsheetConfig` class provides declarative control over tearsheet generation: + +```python +from nautilus_trader.analysis import GridLayout +from nautilus_trader.analysis import TearsheetConfig +from nautilus_trader.analysis import TearsheetDrawdownChart +from nautilus_trader.analysis import TearsheetEquityChart +from nautilus_trader.analysis import TearsheetStatsTableChart + +config = TearsheetConfig( + charts=[ + TearsheetEquityChart(), + TearsheetDrawdownChart(), + TearsheetStatsTableChart(), + ], + theme="nautilus_dark", + title="Q4 2024 Strategy Performance", + height=1800, + include_benchmark=True, + benchmark_name="SPY", + layout=GridLayout( + rows=2, + cols=2, + heights=[0.60, 0.40], + vertical_spacing=0.08, + horizontal_spacing=0.12, + ), +) +``` + +### Configuration parameters + +| Parameter | Type | Default | Description | +|---------------------|-------------------------------|-----------------------------------|-----------------------------------------------| +| `charts` | `list[TearsheetChart]` | All built-in charts | List of chart objects to include (in order). | +| `theme` | `str` | `"plotly_white"` | Theme name for styling. | +| `layout` | `GridLayout` | `None` (auto-calculated) | Custom subplot grid layout. | +| `title` | `str` | Auto-generated with strategy/time | Tearsheet title. | +| `include_benchmark` | `bool` | `True` | Show benchmark when provided. | +| `benchmark_name` | `str` | `"Benchmark"` | Display name for benchmark. | +| `height` | `int` | `1500` | Total height in pixels. | +| `show_logo` | `bool` | `True` | Display NautilusTrader logo (reserved for future use).| + +When `layout` is `None`, the grid dimensions and row heights are automatically calculated +based on the number of charts. For 8 charts (the default), a 4×2 grid is used with +heights `[0.50, 0.22, 0.16, 0.12]` to give more space to the top row tables. + +## Custom charts + +The registry pattern lets you add custom charts. Charts are functions that +render traces onto a Plotly figure object. + +### Registering a custom chart + +```python +from nautilus_trader.analysis.tearsheet import register_chart +import plotly.graph_objects as go + +def my_custom_chart(returns, output_path=None, title="Custom Chart", theme="plotly_white"): + """ + Create a custom visualization. + + This function signature matches the built-in chart functions for consistency. + """ + from nautilus_trader.analysis.themes import get_theme + + theme_config = get_theme(theme) + + # Create your visualization + fig = go.Figure() + fig.add_trace(go.Scatter( + x=returns.index, + y=returns.cumsum(), + mode="lines", + name="Custom Metric", + line={"color": theme_config["colors"]["primary"]}, + )) + + fig.update_layout( + title=title, + template=theme_config["template"], + xaxis_title="Date", + yaxis_title="Value", + ) + + if output_path: + fig.write_html(output_path) + + return fig + +# Register the chart for standalone use (via `get_chart()` / `list_charts()`) +register_chart("my_custom", my_custom_chart) +``` + +### Tearsheet integration + +For full tearsheet integration with proper grid placement, use the lower-level registration. + +:::warning +The `_register_tearsheet_chart` function is internal API and may change between releases. +For most use cases, prefer `register_chart` for standalone charts or contribute new built-in +charts upstream. +::: + +```python +from nautilus_trader.analysis import TearsheetConfig +from nautilus_trader.analysis import TearsheetCustomChart +from nautilus_trader.analysis import TearsheetEquityChart +from nautilus_trader.analysis import TearsheetStatsTableChart +from nautilus_trader.analysis.tearsheet import _register_tearsheet_chart + +def _render_my_metric(fig, row, col, returns, theme_config, **kwargs): + """ + Render custom metric directly onto a subplot. + + Parameters + ---------- + fig : go.Figure + The figure to add traces to. + row : int + Subplot row position. + col : int + Subplot column position. + returns : pd.Series + Strategy returns from analyzer. + theme_config : dict + Theme configuration dictionary. + **kwargs : dict + Additional parameters (stats_pnls, stats_returns, benchmark_returns, etc.). + """ + metric_values = returns.rolling(30).std() * 100 # Example metric + + fig.add_trace( + go.Scatter( + x=returns.index, + y=metric_values, + mode="lines", + name="30-Day Volatility", + line={"color": theme_config["colors"]["neutral"]}, + ), + row=row, + col=col, + ) + + fig.update_xaxes(title_text="Date", row=row, col=col) + fig.update_yaxes(title_text="Volatility (%)", row=row, col=col) + +# Register for tearsheet use +_register_tearsheet_chart( + name="volatility", + subplot_type="scatter", + title="Rolling Volatility (30-day)", + renderer=_render_my_metric, +) + +# Now "volatility" can be used in TearsheetConfig.charts: +config = TearsheetConfig( + charts=[ + TearsheetStatsTableChart(), + TearsheetEquityChart(), + TearsheetCustomChart(chart="volatility"), + ], +) +``` + +The renderer function receives all necessary data (returns, statistics, theme configuration) +and renders directly onto the specified subplot position. + +## Offline analysis + +For situations where you have precomputed statistics but not a `BacktestEngine` instance, +use the lower-level API: + +```python +from nautilus_trader.analysis.tearsheet import create_tearsheet_from_stats + +# Load precomputed data (structure matches PortfolioAnalyzer output) +stats_pnls = {"USD": {"PnL (total)": 1500.0, "Win Rate": 0.55, ...}} # Per-currency +stats_returns = {"Sharpe Ratio (252 days)": 1.2, "Max Drawdown": -0.15, ...} +stats_general = {"Avg Winner": 100.0, "Avg Loser": -50.0, ...} +returns = pd.Series(...) # Daily returns with datetime index + +create_tearsheet_from_stats( + stats_pnls=stats_pnls, + stats_returns=stats_returns, + stats_general=stats_general, + returns=returns, + output_path="offline_analysis.html", +) +``` + +The dictionary keys should match those returned by `PortfolioAnalyzer.get_performance_stats_*()`. + +This approach is useful for: + +- Analyzing results from multiple backtest runs stored separately. +- Comparing strategies using precomputed metrics. +- Integrating with external analysis pipelines. + +## Best practices + +### Chart selection + +- Use default charts for exploratory analysis to see all available metrics. +- Customize charts when you know which metrics matter for your strategy. +- Remove irrelevant charts to reduce visual clutter and file size. + +### Theme usage + +- Use `plotly_white` for professional reports and presentations. +- Use `nautilus_dark` for official materials or low-light viewing. +- Create custom themes to match internal guidelines or personal preferences. + +### Performance considerations + +- Tearsheet HTML files contain all data inline and can be several megabytes for long backtests. +- Consider generating separate tearsheets for different analysis timeframes. +- For very large datasets, use the individual chart functions instead of full tearsheets. + +### Custom statistics integration + +Custom charts work best when paired with [custom statistics](reports.md) registered in the +`PortfolioAnalyzer`. This ensures your visualizations display metrics computed consistently +with the rest of the system: + +```python +from nautilus_trader.analysis.statistic import PortfolioStatistic + +class MyCustomStatistic(PortfolioStatistic): + """Custom metric for specialized strategy analysis.""" + + def calculate_from_returns(self, returns): + # Your calculation logic + return custom_metric_value + +# Register with analyzer +analyzer.register_statistic(MyCustomStatistic()) + +# Now available in stats_returns for custom charts +``` + +## API levels + +The visualization system provides two API levels: + +### High-level API + +Recommended for most use cases: + +```python +create_tearsheet(engine=engine, config=config) +``` + +Automatically extracts data from the `BacktestEngine`, generates all configured charts, +and produces a complete HTML tearsheet. + +### Low-level API + +For advanced customization or offline analysis: + +```python +create_tearsheet_from_stats( + stats_pnls=stats_pnls, + stats_returns=stats_returns, + stats_general=stats_general, + returns=returns, + run_info=run_info, + account_info=account_info, + config=config, +) +``` + +Provides fine-grained control over data inputs and allows analysis of precomputed statistics. + +### Standalone chart functions + +Individual chart functions can be used independently to generate single-purpose HTML visualizations +or Plotly figures for custom analysis workflows. + +#### Price bars with fills + +The `create_bars_with_fills` function generates a candlestick chart with order fills overlaid, +useful for visually analyzing strategy execution within price action. It can be used standalone +or included in tearsheets: + +```python +from nautilus_trader.analysis import create_bars_with_fills +from nautilus_trader.analysis import create_tearsheet +from nautilus_trader.analysis import TearsheetBarsWithFillsChart +from nautilus_trader.analysis import TearsheetConfig +from nautilus_trader.analysis import TearsheetEquityChart +from nautilus_trader.analysis import TearsheetStatsTableChart +from nautilus_trader.model.data import BarType + +# Standalone usage +bar_type = BarType.from_str("ESM4.XCME-1-MINUTE-LAST-EXTERNAL") +fig = create_bars_with_fills( + engine=engine, + bar_type=bar_type, + title="ES Futures - Entry/Exit Analysis", +) +fig.show() # Display in Jupyter +fig.write_html("bars_with_fills.html") # Or save to file + +# Include in tearsheet +config = TearsheetConfig( + charts=[ + TearsheetStatsTableChart(), + TearsheetEquityChart(), + TearsheetBarsWithFillsChart( + bar_type="ESM4.XCME-1-MINUTE-LAST-EXTERNAL", + title="Bars with Fills", + ), + ], +) +create_tearsheet(engine=engine, config=config) + +# Multiple bars-with-fills charts in one tearsheet +config = TearsheetConfig( + charts=[ + TearsheetStatsTableChart(), + TearsheetEquityChart(), + TearsheetBarsWithFillsChart( + bar_type=f"{instrument.id}-5-MINUTE-MID-INTERNAL", + title=f"Bars with Order Fills - {instrument.id}", + ), + TearsheetBarsWithFillsChart( + bar_type=f"{other_instrument.id}-5-MINUTE-MID-INTERNAL", + title=f"Bars with Order Fills - {other_instrument.id}", + ), + ], +) +create_tearsheet(engine=engine, config=config) +``` + +The visualization shows candlesticks for OHLC price action with triangle markers representing order +fills (green up-triangles for buys, red down-triangles for sells). Charts that need extra +configuration (like `bar_type`) take those parameters directly on the chart object +(e.g. `TearsheetBarsWithFillsChart(bar_type=...)`). + +Other individual chart functions include `create_equity_curve`, `create_drawdown_chart`, +`create_monthly_returns_heatmap`, and more. See the API reference for the complete list. + +## Related guides + +- [Backtesting](backtesting.md) - Learn how to run backtests that generate tearsheets. +- [Reports](reports.md) - Understand the underlying statistics displayed in tearsheets. +- [Portfolio](portfolio.md) - Explore portfolio tracking and performance metrics. diff --git a/nautilus-docs/docs/dev_templates/criterion_template.rs b/nautilus-docs/docs/dev_templates/criterion_template.rs new file mode 100644 index 0000000..b6ac217 --- /dev/null +++ b/nautilus-docs/docs/dev_templates/criterion_template.rs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +//! Criterion benchmark template. +//! +//! Copy this file into `crates//benches/` and adjust the names and +//! imports. Compile with +//! +//! ```bash +//! cargo bench -p --bench +//! ``` + +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, Criterion}; + +// ----------------------------------------------------------------------------- +// Replace `my_function` and set-up code with your real workload. +// ----------------------------------------------------------------------------- + +fn my_function(input: &[u8]) -> usize { + input.iter().map(|b| *b as usize).sum() +} + +fn prepare_data() -> Vec { + (0u8..=255).collect() +} + +fn bench_my_function(c: &mut Criterion) { + let data = prepare_data(); + + c.bench_function("my_function", |b| { + b.iter(|| { + let _ = black_box(my_function(&data)); + }); + }); +} + +criterion_group!(benches, bench_my_function); +criterion_main!(benches); diff --git a/nautilus-docs/docs/dev_templates/iai_template.rs b/nautilus-docs/docs/dev_templates/iai_template.rs new file mode 100644 index 0000000..741a856 --- /dev/null +++ b/nautilus-docs/docs/dev_templates/iai_template.rs @@ -0,0 +1,33 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +//! iai benchmark template. +//! +//! Copy this file into `crates//benches/` and adjust names and +//! imports. + +use std::hint::black_box; + +// ----------------------------------------------------------------------------- +// Replace `fast_add` with the real function you want to measure. +// ----------------------------------------------------------------------------- + +fn fast_add() -> i32 { + let a = black_box(1); + let b = black_box(2); + a + b +} + +iai::main!(fast_add); diff --git a/nautilus-docs/docs/developer_guide/adapters.md b/nautilus-docs/docs/developer_guide/adapters.md new file mode 100644 index 0000000..c4e8e82 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/adapters.md @@ -0,0 +1,2131 @@ +# Adapters + +## Introduction + +This developer guide provides specifications for how to build an integration adapter for the NautilusTrader platform. + +Adapters connect to trading venues and data providers, translating their native APIs into the platform’s unified interface and normalized domain model. + +## Structure of an adapter + +NautilusTrader adapters follow a layered architecture pattern with: + +- **Rust core** for networking clients and performance-sensitive operations. +- **Python layer** for integrating Rust clients into the platform's data and execution engines. + +### Rust core (`crates/adapters/your_adapter/`) + +The Rust layer handles: + +- **HTTP client**: Raw API communication, request signing, rate limiting. +- **WebSocket client**: Low-latency streaming connections, message parsing. +- **Parsing**: Fast conversion of venue data to Nautilus domain models. +- **Python bindings**: PyO3 exports to make Rust functionality available to Python. + +Typical Rust structure: + +``` +crates/adapters/your_adapter/ +├── src/ +│ ├── common/ # Shared types and utilities +│ │ ├── consts.rs # Venue constants / broker IDs +│ │ ├── credential.rs # API key storage and signing helpers +│ │ ├── enums.rs # Venue enums mirrored in REST/WS payloads +│ │ ├── error.rs # Adapter-level error aggregation (when applicable) +│ │ ├── models.rs # Shared model types +│ │ ├── parse.rs # Shared parsing helpers +│ │ ├── retry.rs # Retry classification (when applicable) +│ │ ├── urls.rs # Environment & product aware base-url resolvers +│ │ └── testing.rs # Fixtures reused across unit tests +│ ├── http/ # HTTP client implementation +│ │ ├── client.rs # HTTP client with authentication +│ │ ├── error.rs # HTTP-specific error types +│ │ ├── models.rs # Structs for REST payloads +│ │ ├── parse.rs # Response parsing functions +│ │ └── query.rs # Request and query builders +│ ├── websocket/ # WebSocket implementation +│ │ ├── client.rs # WebSocket client +│ │ ├── dispatch.rs # Execution event dispatch and order routing +│ │ ├── enums.rs # WebSocket-specific enums +│ │ ├── error.rs # WebSocket-specific error types +│ │ ├── handler.rs # Feed handler (I/O boundary) +│ │ ├── messages.rs # Frame and message enums +│ │ ├── parse.rs # Message parsing functions +│ │ └── subscription.rs # Subscription topic helpers (optional) +│ ├── python/ # PyO3 Python bindings +│ │ ├── enums.rs # Python-exposed enums +│ │ ├── http.rs # Python HTTP client bindings +│ │ ├── urls.rs # Python URL helpers +│ │ ├── websocket.rs # Python WebSocket client bindings +│ │ └── mod.rs # Module exports +│ ├── config.rs # Configuration structures +│ ├── data.rs # Data client implementation +│ ├── execution.rs # Execution client implementation +│ ├── factories.rs # Factory functions +│ └── lib.rs # Library entry point +├── tests/ # Integration tests with mock servers +│ ├── data_client.rs # Data client integration tests +│ ├── exec_client.rs # Execution client integration tests +│ ├── http.rs # HTTP client integration tests +│ └── websocket.rs # WebSocket client integration tests +└── test_data/ # Canonical venue payloads +``` + +### Python layer (`nautilus_trader/adapters/your_adapter`) + +The Python layer provides the integration interface through these components: + +1. **Instrument Provider**: Supplies instrument definitions via `InstrumentProvider`. +2. **Data Client**: Handles market data feeds and historical data requests via `LiveDataClient` and `LiveMarketDataClient`. +3. **Execution Client**: Manages order execution via `LiveExecutionClient`. +4. **Factories**: Converts venue-specific data to Nautilus domain models. +5. **Configuration**: User-facing configuration classes for client settings. + +Typical Python structure: + +``` +nautilus_trader/adapters/your_adapter/ +├── config.py # Configuration classes +├── constants.py # Adapter constants +├── data.py # LiveDataClient/LiveMarketDataClient +├── execution.py # LiveExecutionClient +├── factories.py # Instrument factories +├── providers.py # InstrumentProvider +└── __init__.py # Package initialization +``` + +## Adapter implementation sequence + +Follow this dependency-driven order when building an adapter. Each phase +builds on the previous one. Implement the Rust core before any Python layer. + +### Phase 1: Rust core infrastructure + +Build the low-level networking and parsing foundation. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 1.1 | HTTP error types | Define HTTP-specific error enum with retryable/non-retryable variants (`http/error.rs`). | +| 1.2 | HTTP client | Implement credentials, request signing, rate limiting, and retry logic. | +| 1.3 | HTTP API models | Define request/response structs for REST endpoints (`http/models.rs`, `http/query.rs`). | +| 1.4 | HTTP parsing | Convert venue responses to Nautilus domain models (`http/parse.rs`, `common/parse.rs`). | +| 1.5 | WebSocket error types | Define WebSocket-specific error enum (`websocket/error.rs`). | +| 1.6 | WebSocket client | Implement connection lifecycle, authentication, heartbeat, and reconnection. | +| 1.7 | WebSocket messages | Define streaming payload types (`websocket/messages.rs`). | +| 1.8 | WebSocket parsing | Convert stream messages to Nautilus domain models (`websocket/parse.rs`). | +| 1.9 | Python bindings | Expose Rust functionality via PyO3 (`python/mod.rs`). | + +**Milestone**: Rust crate compiles, unit tests pass, HTTP/WebSocket clients can authenticate and stream/request raw data. + +### Phase 2: Instrument definitions + +Instruments are the foundation: both data and execution clients depend on them. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 2.1 | Instrument parsing | Parse venue instrument definitions into Nautilus types (spot, perpetual, future, option). | +| 2.2 | Instrument provider | Implement `InstrumentProvider` to load, filter, and cache instruments. | +| 2.3 | Symbol mapping | Handle venue-specific symbol formats and Nautilus `InstrumentId` conversion. | + +**Milestone**: `InstrumentProvider.load_all_async()` returns valid Nautilus instruments. + +### Phase 3: Market data + +Build data subscriptions and historical data requests. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 3.1 | Public WebSocket streams | Subscribe to order books, trades, tickers, and other public channels. | +| 3.2 | Historical data requests | Fetch historical bars, trades, and order book snapshots via HTTP. | +| 3.3 | Data client (Python) | Implement `LiveDataClient` or `LiveMarketDataClient` wiring Rust clients to the data engine. | + +**Milestone**: Data client connects, subscribes to instruments, and emits market data to the platform. + +### Phase 4: Order execution + +Build order management and account state. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 4.1 | Private WebSocket streams | Subscribe to order updates, fills, positions, and account balance changes. | +| 4.2 | Basic order submission | Implement market and limit orders via HTTP or WebSocket. | +| 4.3 | Order modification/cancel | Implement order amendment and cancellation. | +| 4.4 | Execution client (Python) | Implement `LiveExecutionClient` wiring Rust clients to the execution engine. | +| 4.5 | Execution reconciliation | Generate order, fill, and position status reports for startup reconciliation. | + +**Milestone**: Execution client submits orders, receives fills, and reconciles state on connect. + +### Phase 5: Advanced features + +Extend coverage based on venue capabilities. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 5.1 | Advanced order types | Conditional orders, stop-loss, take-profit, trailing stops, iceberg, etc. | +| 5.2 | Batch operations | Batch order submission, batch cancellation, mass cancel. | +| 5.3 | Venue-specific features | Options chains, funding rates, liquidations, or other venue-specific data. | + +### Phase 6: Configuration and factories + +Wire everything together for production usage. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 6.1 | Configuration classes | Create `LiveDataClientConfig` and `LiveExecClientConfig` subclasses. | +| 6.2 | Factory functions | Implement factory functions to instantiate clients from configuration. | +| 6.3 | Environment variables | Support credential resolution from environment variables. | + +### Phase 7: Testing and documentation + +Validate the integration and document usage. + +| Step | Component | Description | +|------|----------------------------|----------------------------------------------------------------------------------------------| +| 7.1 | Rust unit tests | Test parsers, signing helpers, and business logic in `#[cfg(test)]` blocks. | +| 7.2 | Rust integration tests | Test HTTP/WebSocket clients against mock Axum servers in `tests/`. | +| 7.3 | Python integration tests | Test data/execution clients in `tests/integration_tests/adapters//`. | +| 7.4 | Example scripts | Provide runnable examples demonstrating data subscription and order execution. | + +See the [Testing](#testing) section for detailed test organization guidelines. + +--- + +## Rust adapter patterns + +### Common code (`common/`) + +Group venue constants, credential helpers, enums, and reusable parsers under `src/common`. +Adapters such as OKX keep submodules like `consts`, `credential`, `enums`, and `urls` alongside a `testing` module +for fixtures, providing a single place for cross-cutting pieces. +When an adapter has multiple environments or product categories, add a dedicated `common::urls` helper so +REST/WebSocket base URLs stay in sync with the Python layer. + +### URL resolution + +Define URL constants and resolution functions in `common/urls.rs`: + +```rust +const VENUE_WS_URL: &str = "wss://stream.venue.com/ws"; +const VENUE_TESTNET_WS_URL: &str = "wss://testnet-stream.venue.com/ws"; + +pub const fn get_ws_base_url(testnet: bool) -> &'static str { + if testnet { VENUE_TESTNET_WS_URL } else { VENUE_WS_URL } +} +``` + +Config structs should provide override fields (`base_url_http`, `base_url_ws`, etc.) that fall back +to these defaults when unset. + +### Configurations (`config.rs`) + +Expose typed config structs in `src/config.rs` so Python callers toggle venue-specific behaviour +(see how OKX wires demo URLs, retries, and channel flags). +Keep defaults minimal and delegate URL selection to helpers in `common::urls`. + +All config structs (data and execution) must implement `Default`. This enables the +`..Default::default()` pattern in examples and tests, keeping only the fields that differ +from defaults visible: + +```rust +let exec_config = VenueExecClientConfig { + trader_id, + account_id, + environment: VenueEnvironment::Testnet, + ..Default::default() +}; +``` + +Default values should use sensible production defaults: credentials as `None` (resolved +from environment at runtime), mainnet URLs, standard timeouts. For `trader_id` and +`account_id`, use placeholder values like `TraderId::from("TRADER-001")` and +`AccountId::from("VENUE-001")`. + +### Error taxonomy (`common/error.rs`) + +For adapters with multiple client types, define an adapter-level error enum in `common/error.rs` that +aggregates component errors: + +```rust +#[derive(Debug, thiserror::Error)] +pub enum VenueError { + #[error("HTTP error: {0}")] + Http(#[from] VenueHttpError), + + #[error("WebSocket error: {0}")] + WebSocket(#[from] VenueWsError), + + #[error("Build error: {0}")] + Build(#[from] VenueBuildError), +} +``` + +This enables unified error handling at the adapter boundary while preserving component-specific +error details for debugging. + +### Retry classification (`common/retry.rs`) + +When an adapter needs sophisticated retry logic, define a retry classification module in `common/retry.rs` +that distinguishes between retryable, non-retryable, and fatal errors: + +```rust +#[derive(Debug, thiserror::Error)] +pub enum VenueError { + #[error("Retryable error: {source}")] + Retryable { + #[source] + source: VenueRetryableError, + retry_after: Option, + }, + + #[error("Non-retryable error: {source}")] + NonRetryable { + #[source] + source: VenueNonRetryableError, + }, + + #[error("Fatal error: {source}")] + Fatal { + #[source] + source: VenueFatalError, + }, +} +``` + +Include helper methods like `from_http_status()`, `from_rate_limit_headers()`, `is_retryable()`, +`is_fatal()`, and `retry_after()` to enable consistent error classification across the adapter. +See BitMEX and Bybit adapters for reference implementations. + +### Python exports (`python/mod.rs`) + +Mirror the Rust surface area through PyO3 modules by re-exporting clients, enums, and helper functions. +When new functionality lands in Rust, add it to `python/mod.rs` so the Python layer stays in sync +(the OKX adapter is a good reference). + +### Python bindings (`python/`) + +Expose Rust functionality to Python through PyO3. +Mark venue-specific structs that need Python access with `#[pyclass]` and implement `#[pymethods]` blocks with +`#[getter]` attributes for field access. + +For async methods in the HTTP client, use `pyo3_async_runtimes::tokio::future_into_py` to convert Rust futures +into Python awaitables. +When returning lists of custom types, map each item with `Py::new(py, item)` before constructing the Python list. +Register all exported classes and enums in `python/mod.rs` using `m.add_class::()` so they're available +to Python code. + +Follow the pattern established in other adapters: prefixing Python-facing methods with `py_*` in Rust while using +`#[pyo3(name = "method_name")]` to expose them without the prefix. + +When delivering instruments from WebSocket to Python, use `instrument_any_to_pyobject()` which returns PyO3 types +for caching. +For the reverse direction (Python→Rust), use `pyobject_to_instrument_any()` in `cache_instrument()` methods. +Never call `.into_py_any()` directly on `InstrumentAny` as it doesn't implement the required trait. + +### Type qualification + +Adapter-specific types (enums, structs) and Nautilus domain types should not be fully qualified. +Import them at the module level and use short names (e.g., `OKXContractType` instead of +`crate::common::enums::OKXContractType`, `InstrumentId` instead of `nautilus_model::identifiers::InstrumentId`). +This keeps code concise and readable. +Only fully qualify types from `anyhow` and `tokio` to avoid ambiguity with similarly-named types from other crates. + +### String interning + +Use `ustr::Ustr` for any non-unique strings the platform stores repeatedly (venues, symbols, instrument IDs) to +minimise allocations and comparisons. + +### Instrument cache standardization + +All clients that cache instruments must implement three methods with standardized names: `cache_instruments()` +(plural, bulk replace), `cache_instrument()` (singular, upsert), and `get_instrument()` (retrieve by symbol). +WebSocket clients store instruments in `Arc>` on the outer client for +thread-safe access across clones. + +### Testing helpers (`common/testing.rs`) + +Store shared fixtures and payload loaders in `src/common/testing.rs` for use across HTTP and WebSocket unit tests. +This keeps `#[cfg(test)]` helpers out of production modules and encourages reuse. + +### Factory module (`factories.rs`) + +Complex adapters may define a `factories.rs` module for converting venue data to Nautilus types. +This centralizes transformation logic that would otherwise be scattered across HTTP and WebSocket +parsers: + +```rust +// factories.rs +pub fn create_instrument( + venue_instrument: &VenueInstrument, + ts_init: UnixNanos, +) -> anyhow::Result { + match venue_instrument.instrument_type { + InstrumentType::Perpetual => parse_perpetual(venue_instrument, ts_init), + InstrumentType::Future => parse_future(venue_instrument, ts_init), + InstrumentType::Option => parse_option(venue_instrument, ts_init), + } +} +``` + +Use this pattern when the same venue data structures are parsed in multiple places (HTTP responses, +WebSocket updates, historical data). + +### Connection lifecycle (`connect`) + +Both data and execution clients follow a strict initialization order during `connect()` to prevent +race conditions with reconciliation and strategy startup. The platform waits for all clients to +signal connected before running reconciliation or starting strategies, so all initialization must +complete within `connect()`. + +#### Data client + +1. **Fetch instruments via REST** - call `bootstrap_instruments()` or equivalent. +2. **Cache locally** - populate the client's internal instrument map and HTTP client cache. +3. **Emit to data engine** - send each instrument as `DataEvent::Instrument` via `data_sender`. + These events are queued during startup and processed before reconciliation runs. +4. **Cache to WebSocket** - call `ws.cache_instruments()` so the handler can parse messages. +5. **Connect WebSocket** - establish the streaming connection. + +```rust +async fn connect(&mut self) -> anyhow::Result<()> { + let instruments = self.bootstrap_instruments().await?; + ws.cache_instruments(instruments); + ws.connect().await?; + ws.wait_until_active(10.0).await?; + // ... +} +``` + +#### Execution client + +1. **Initialize instruments** - fetch via REST if not already cached. Cache to HTTP, WebSocket, + and broadcaster clients. +2. **Connect WebSocket** - establish the private streaming connection. +3. **Subscribe to channels** - orders, executions, positions, wallet/margin. +4. **Start WebSocket stream handler** - begin processing incoming messages. +5. **Fetch account state** - request account state via REST and emit via `emitter.send_account_state()`. +6. **Await account registered** - poll the cache until the account is registered (30s timeout). + This ensures the portfolio can process orders during reconciliation. +7. **Signal connected** - call `self.core.set_connected()`. + +```rust +async fn connect(&mut self) -> anyhow::Result<()> { + self.ensure_instruments_initialized_async().await?; + + self.ws_client.connect().await?; + self.ws_client.wait_until_active(10.0).await?; + // ... subscribe channels, start stream ... + + self.refresh_account_state().await?; + self.await_account_registered(30.0).await?; + + self.core.set_connected(); + Ok(()) +} +``` + +The `await_account_registered` method polls `self.core.cache().account(&account_id)` at 10ms +intervals until the account appears or the timeout expires. + +## HTTP client patterns + +Adapters use a two-layer HTTP client architecture: a raw client for low-level API operations and a domain +client for high-level logic. The split also enables efficient cloning for Python bindings. + +### Client structure + +The architecture consists of two complementary clients: + +1. **Raw client** (`MyRawHttpClient`) - Low-level API methods matching venue endpoints. +2. **Domain client** (`MyHttpClient`) - High-level methods using Nautilus domain types. + +```rust +use std::sync::Arc; +use nautilus_network::http::HttpClient; + +// Raw HTTP client - low-level API methods matching venue endpoints +pub struct MyRawHttpClient { + base_url: String, + client: HttpClient, // Use nautilus_network::http::HttpClient, not reqwest directly + credential: Option, + retry_manager: RetryManager, + cancellation_token: CancellationToken, +} + +// Domain HTTP client - wraps raw client with Arc, provides high-level API +pub struct MyHttpClient { + pub(crate) inner: Arc, + // Additional domain-specific state (e.g., instrument cache) + instruments: DashMap, +} +``` + +**Key points**: + +- **Raw client** (`MyRawHttpClient`) contains low-level HTTP methods named to match venue endpoints + (e.g., `get_instruments`, `get_balance`, `place_order`). These methods take venue-specific query + objects and return venue-specific response types. +- **Domain client** (`MyHttpClient`) wraps the raw client in an `Arc` for efficient cloning (required + for Python bindings). It provides high-level methods that accept Nautilus domain types + (e.g., `InstrumentId`, `ClientOrderId`) and return domain objects. It may also cache instruments + or other venue metadata. +- Use `nautilus_network::http::HttpClient` instead of `reqwest::Client` directly for rate limiting, + retry logic, and consistent error handling. +- Both clients are exposed to Python, but the domain client is the primary interface. + +### Parser functions + +Parser functions convert venue-specific data structures into Nautilus domain objects. Place them in +`common/parse.rs` for cross-cutting conversions (instruments, trades, bars) or `http/parse.rs` for +REST-specific transformations. Each parser takes venue data plus context (account IDs, timestamps, +instrument references) and returns a Nautilus domain type wrapped in `Result`. + +**Standard patterns:** + +- Handle string-to-numeric conversions with proper error context using `.parse::()` and `anyhow::Context`. +- Check for empty strings before parsing optional fields - venues often return `""` instead of omitting fields. +- Map venue enums to Nautilus enums explicitly with `match` statements rather than implementing automatic conversions that could hide mapping errors. +- Accept instrument references when precision or other metadata is required for constructing Nautilus types (quantities, prices). +- Use descriptive function names: `parse_position_status_report`, `parse_order_status_report`, `parse_trade_tick`. + +Place parsing helpers (`parse_price_with_precision`, `parse_timestamp`) in the same module as private functions when they're reused across multiple parsers. + +### Method naming and organization + +The raw client mirrors venue endpoints with venue-specific parameter and response types. The domain +client wraps it and exposes high-level methods that accept Nautilus domain types. + +**Naming conventions:** + +- **Raw client methods**: Named to match venue endpoints as closely as possible (e.g., `get_instruments`, `get_balance`, `place_order`). These methods are internal to the raw client and take venue-specific types (builders, JSON values). +- **Domain client methods**: Named based on operation semantics (e.g., `request_instruments`, `submit_order`, `cancel_order`). These are the methods exposed to Python and take Nautilus domain objects (InstrumentId, ClientOrderId, OrderSide, etc.). + +**Domain method flow:** + +Domain methods follow a three-step pattern: build venue-specific parameters from Nautilus types, call the corresponding raw client method, then parse the response. For endpoints returning domain objects (positions, orders, trades), call parser functions from `common/parse`. For endpoints returning raw venue data (fee rates, balances), extract the result directly from the response envelope. Methods prefixed with `request_*` indicate they return domain data, while methods like `submit_*`, `cancel_*`, or `modify_*` perform actions and return acknowledgments. + +The domain client wraps the raw client in an `Arc` for efficient cloning required by Python bindings. + +### Query parameter builders + +Use the `derive_builder` crate with proper defaults and ergonomic Option handling: + +```rust +use derive_builder::Builder; + +#[derive(Clone, Debug, Deserialize, Serialize, Builder)] +#[serde(rename_all = "camelCase")] +#[builder(setter(into, strip_option), default)] +pub struct InstrumentsInfoParams { + pub category: ProductType, + #[serde(skip_serializing_if = "Option::is_none")] + pub symbol: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub limit: Option, +} + +impl Default for InstrumentsInfoParams { + fn default() -> Self { + Self { + category: ProductType::Linear, + symbol: None, + limit: None, + } + } +} +``` + +**Key attributes:** + +- `#[builder(setter(into, strip_option), default)]` - enables clean API: `.symbol("BTCUSDT")` instead of `.symbol(Some("BTCUSDT".to_string()))`. +- `#[serde(skip_serializing_if = "Option::is_none")]` - omits optional fields from query strings. +- Always implement `Default` for builder parameters. + +### Request signing and authentication + +Keep signing logic in a `Credential` struct under `common/credential.rs`: + +- Store API keys using `Ustr` for efficient comparison, secrets in `Box<[u8]>` with `#[zeroize]`. +- Implement `sign()` and `sign_bytes()` methods that compute HMAC-SHA256 signatures. +- Pass the credential to the raw HTTP client; the domain client delegates signing through the inner client. + +For WebSocket authentication, the handler constructs login messages using the same `Credential::sign()` method with a WebSocket-specific timestamp format. + +### Credential module structure + +Each adapter's `common/credential.rs` must provide two things: + +1. **`credential_env_vars()` free function**: returns environment variable names as a tuple. +2. **`Credential::resolve()` method**: resolves credentials from config values or environment + variables using `resolve_env_var_pair` from `nautilus_core::env`. + +Config structs are DTOs and must not contain credential resolution logic. All resolution +belongs in `credential.rs`. + +**Standard layout:** + +```rust +use nautilus_core::env::resolve_env_var_pair; + +/// Returns the environment variable names for API credentials. +pub fn credential_env_vars(is_testnet: bool) -> (&'static str, &'static str) { + if is_testnet { + ("{VENUE}_TESTNET_API_KEY", "{VENUE}_TESTNET_API_SECRET") + } else { + ("{VENUE}_API_KEY", "{VENUE}_API_SECRET") + } +} + +impl Credential { + /// Resolves credentials from provided values or environment variables. + pub fn resolve( + api_key: Option, + api_secret: Option, + is_testnet: bool, + ) -> Option { + let (key_var, secret_var) = credential_env_vars(is_testnet); + let (k, s) = resolve_env_var_pair(api_key, api_secret, key_var, secret_var)?; + Some(Self::new(k, s)) + } +} +``` + +### Environment variable conventions + +Adapters load API credentials from environment variables when not provided directly, avoiding +hardcoded secrets. + +**Naming conventions:** + +| Environment | API Key Variable | API Secret Variable | +|--------------|---------------------------|---------------------| +| Mainnet/Live | `{VENUE}_API_KEY` | `{VENUE}_API_SECRET` | +| Testnet | `{VENUE}_TESTNET_API_KEY` | `{VENUE}_TESTNET_API_SECRET` | +| Demo | `{VENUE}_DEMO_API_KEY` | `{VENUE}_DEMO_API_SECRET` | + +Some venues require additional credentials: + +- OKX: `OKX_API_PASSPHRASE` + +**Key principles:** + +- Environment variable names must be centralized in `credential_env_vars()`, never + duplicated as string literals across files. +- Environment variable resolution should happen in core Rust code, not Python bindings. +- Use `get_or_env_var_opt` for optional credentials (returns `None` if missing). +- Use `get_or_env_var` when credentials are required (returns error if missing). +- Invalid credentials (e.g. malformed keys) must fail fast with an error, never silently + degrade to unauthenticated mode. + +### Error handling and retry logic + +Use the `RetryManager` from `nautilus_network` for consistent retry behavior. + +### Rate limiting + +Configure rate limiting through `HttpClient` using `LazyLock` static variables. + +**Naming conventions:** + +- REST quotas: `{VENUE}_REST_QUOTA` (e.g., `OKX_REST_QUOTA`, `BYBIT_REST_QUOTA`) +- WebSocket quotas: `{VENUE}_WS_{OPERATION}_QUOTA` (e.g., `OKX_WS_CONNECTION_QUOTA`, `OKX_WS_ORDER_QUOTA`) +- Rate limit keys: `{VENUE}_RATE_LIMIT_KEY_{OPERATION}` (e.g., `OKX_RATE_LIMIT_KEY_SUBSCRIPTION`, `OKX_RATE_LIMIT_KEY_ORDER`) + +**Standard rate limit keys for WebSocket:** + +| Key | Operations | +|---------------------------------|----------------------------------| +| `*_RATE_LIMIT_KEY_SUBSCRIPTION` | Subscribe, unsubscribe, login. | +| `*_RATE_LIMIT_KEY_ORDER` | Place orders (regular and algo). | +| `*_RATE_LIMIT_KEY_CANCEL` | Cancel orders, mass cancel. | +| `*_RATE_LIMIT_KEY_AMEND` | Amend/modify orders. | + +**Example:** + +```rust +pub static OKX_REST_QUOTA: LazyLock = + LazyLock::new(|| Quota::per_second(NonZeroU32::new(250).unwrap())); + +pub static OKX_WS_SUBSCRIPTION_QUOTA: LazyLock = + LazyLock::new(|| Quota::per_hour(NonZeroU32::new(480).unwrap())); + +pub const OKX_RATE_LIMIT_KEY_ORDER: &str = "order"; +``` + +Pass rate limit keys when sending WebSocket messages to enforce per-operation quotas: + +```rust +self.send_with_retry(payload, Some(vec![OKX_RATE_LIMIT_KEY_ORDER.to_string()])).await +``` + +## WebSocket client patterns + +WebSocket clients handle real-time streaming data. They manage connection state, authentication, +subscriptions, and reconnection logic. + +### Client structure + +WebSocket adapters use a **two-layer architecture** to separate Python-accessible state from high-performance async I/O: + +#### Connection state tracking + +Track connection state using `Arc>` to provide lock-free, race-free visibility across all clones: + +```rust +use arc_swap::ArcSwap; + +pub struct MyWebSocketClient { + connection_mode: Arc>, // Shared connection mode (lock-free) + signal: Arc, // Cancellation signal for graceful shutdown + // ... +} +``` + +**Pattern breakdown:** + +- **Outer `Arc`**: Shared across all clones (Python bindings clone clients before async operations). +- **`ArcSwap`**: Enables atomic pointer replacement via `.store()` without replacing the outer Arc. +- **Inner `Arc`**: The actual connection state from `WebSocketClient::connection_mode_atomic()`. + +Initialize with a placeholder atomic (`ConnectionMode::Closed`), then in `connect()` call +`.store(client.connection_mode_atomic())` to atomically swap to the real client's state. +All clones see updates instantly through lock-free `.load()` calls in `is_active()`. + +The underlying `WebSocketClient` sends a `RECONNECTED` sentinel message when reconnection completes, triggering resubscription logic in the handler. + +**Outer client** (`{Venue}WebSocketClient`): + +- Orchestrates connection lifecycle, authentication, subscriptions. +- Maintains state for Python access using `Arc>`. +- Tracks subscription state for reconnection logic. +- Stores instruments cache for replay on reconnect. +- Sends commands to handler via `cmd_tx` channel. +- Receives venue events via `out_rx` channel. + +**Inner handler** (`{Venue}WsFeedHandler`): + +- Runs in dedicated Tokio task as stateless I/O boundary. +- Owns `WebSocketClient` exclusively (no `RwLock` needed). +- Processes commands from `cmd_rx` → serializes to JSON → sends via WebSocket. +- Receives raw WebSocket messages → deserializes into `{Venue}WsFrame` → converts to `{Venue}WsMessage` → emits via `out_tx`. +- Owns pending request state using `AHashMap` (single-threaded, no locking). +- Uses `VecDeque<{Venue}WsMessage>` to buffer multi-message yields from a single frame parse. + +Some venues expose separate WebSocket endpoints for market data and order management +(different URLs, authentication flows, or message protocols). In this case, split into +two client+handler pairs under `websocket/data/` and `websocket/orders/` subdirectories, +each following the same two-layer pattern. Name them `{Venue}MdWebSocketClient` / +`{Venue}MdWsFeedHandler` and `{Venue}OrdersWebSocketClient` / `{Venue}OrdersWsFeedHandler`. + +**Communication pattern:** + +```mermaid +flowchart LR + subgraph client["Client (orchestrator)"] + cmd_tx["cmd_tx
├ Subscribe { args }
├ PlaceOrder { params }
└ MassCancel { id }"] + out_rx["out_rx
← {Venue}WsMessage
← Authenticated
← ChannelData"] + end + + subgraph handler["Handler (I/O boundary)"] + cmd_rx[cmd_rx] + out_tx[out_tx] + ws[WebSocket] + end + + cmd_tx --> cmd_rx + cmd_rx -->|"serialize"| ws + ws -->|"parse → transform"| out_tx + out_tx --> out_rx +``` + +**Key principles:** + +- **No shared locks on hot path**: Handler owns `WebSocketClient`, client sends commands via lock-free mpsc channel. +- **Command pattern for all sends**: Subscriptions, orders, cancellations all route through `HandlerCommand` enum. +- **Event pattern for state**: Handler emits `{Venue}WsMessage` events (including `Authenticated`), client maintains state from events. +- **Pending state ownership**: Handler owns `AHashMap` for matching responses (no `Arc` between layers). +- **Message buffering**: Handler uses `VecDeque<{Venue}WsMessage>` for frames that produce multiple output messages. The `next()` method drains the queue before polling channels. +- **Python constraint**: Client uses `Arc` only for state Python might query; handler uses `AHashMap` for internal matching. + +### Authentication + +Authentication state is managed through events: + +- Handler processes `Login` response → **returns** `{Venue}WsMessage::Authenticated` immediately. +- Client receives event → updates local auth state → proceeds with subscriptions. +- `AuthTracker` may be shared via `Arc` for state queries, but handler returns events directly (no blocking). + +**Note**: The `Authenticated` message is consumed in the client's spawn loop for reconnection +flow coordination and is not forwarded to downstream consumers (data/execution clients). +Downstream consumers can query authentication state via `AuthTracker` if needed. The execution +client's `Authenticated` handler only logs at debug level with no important logic depending +on this event. + +### Subscription management + +#### Shared `SubscriptionState` pattern + +The `SubscriptionState` struct from `nautilus_network::websocket` is shared between client and handler using `Arc>` internally for thread-safe access: + +- **`SubscriptionState` is shared via `Arc`**: Both client and handler receive `.clone()` of the same instance (shallow clone of Arc pointers). +- **Responsibility split**: Client tracks user intent (`mark_subscribe`, `mark_unsubscribe`), handler tracks server confirmations (`confirm_subscribe`, `confirm_unsubscribe`, `mark_failure`). +- **Why both need it**: Single source of truth with lock-free concurrent access, no synchronization overhead. + +#### Subscription lifecycle + +A **subscription** represents any topic in one of two states: + +| State | Description | +|---------------|-------------| +| **Pending** | Subscription request sent to venue, awaiting acknowledgment. | +| **Confirmed** | Venue acknowledged subscription and is actively streaming data. | + +State transitions follow this lifecycle: + +| Trigger | Method Called | From State | To State | Notes | +|-------------------|----------------------|------------|-----------|-------| +| User subscribes | `mark_subscribe()` | — | Pending | Topic added to pending set. | +| Venue confirms | `confirm()` | Pending | Confirmed | Moved from pending to confirmed. | +| Venue rejects | `mark_failure()` | Pending | Pending | Stays pending for retry on reconnect. | +| User unsubscribes | `mark_unsubscribe()` | Confirmed | Pending | Temporarily pending until ack. | +| Unsubscribe ack | `clear_pending()` | Pending | Removed | Topic fully removed. | + +**Key principles**: + +- `subscription_count()` reports **only confirmed subscriptions**, not pending ones. +- Failed subscriptions remain pending and are automatically retried on reconnect. +- Both confirmed and pending subscriptions are restored after reconnection. +- Unsubscribe operations must check the `op` field in acknowledgments to avoid re-confirming topics. + +#### Topic format patterns + +Adapters use venue-specific delimiters to structure subscription topics: + +| Adapter | Delimiter | Example | Pattern | +|------------|-----------|------------------------|------------------------------| +| **BitMEX** | `:` | `trade:XBTUSD` | `{channel}:{symbol}` | +| **OKX** | `:` | `trades:BTC-USDT-SWAP` | `{channel}:{symbol}` | +| **Bybit** | `.` | `orderbook.50.BTCUSDT` | `{channel}.{depth}.{symbol}` | + +Parse topics using `split_once()` with the appropriate delimiter to extract channel and symbol components. + +### Reconnection logic + +On reconnection, restore authentication and subscriptions: + +1. **Track subscriptions**: Preserve original subscription arguments in collections (e.g., `Arc`) to avoid parsing topics back to arguments. + +2. **Reconnection flow**: + - Receive `{Venue}WsMessage::Reconnected` from handler. + - If authenticated: Re-authenticate and wait for confirmation. + - Restore all tracked subscriptions via handler commands. + +**Preserving subscription arguments:** + +Store original subscription arguments in a separate collection to enable deterministic reconnection +replay without parsing topics back into arguments: + +```rust +pub struct MyWebSocketClient { + subscription_state: Arc, + subscription_args: Arc>, // topic → original args + // ... +} + +impl MyWebSocketClient { + async fn subscribe(&self, args: SubscriptionArgs) -> Result<(), Error> { + let topic = args.to_topic(); + self.subscription_state.mark_subscribe(&topic); + self.subscription_args.insert(topic.clone(), args.clone()); + self.send_cmd(HandlerCommand::Subscribe(args)).await + } + + async fn unsubscribe(&self, topic: &str) -> Result<(), Error> { + self.subscription_state.mark_unsubscribe(topic); + self.subscription_args.remove(topic); + self.send_cmd(HandlerCommand::Unsubscribe(topic.to_string())).await + } + + async fn restore_subscriptions(&self) { + for entry in self.subscription_args.iter() { + let _ = self.send_cmd(HandlerCommand::Subscribe(entry.value().clone())).await; + } + } +} +``` + +This avoids complex topic parsing and ensures subscriptions are replayed exactly as originally +requested. + +### Ping/Pong handling + +Support both WebSocket control frame pings and application-level text pings: + +- **Control frame pings**: Handled automatically by `WebSocketClient` via the `PingHandler` callback. +- **Text pings**: Some venues (e.g., OKX) use `"ping"`/`"pong"` text messages. Configure `heartbeat_msg: Some(TEXT_PING.to_string())` in `WebSocketConfig` and respond to incoming `TEXT_PING` with `TEXT_PONG` in the handler. + +The handler should check for ping messages early in the message processing loop and respond immediately to maintain connection health. + +### Disconnection lifecycle (`close`) + +The `close()` method follows a three-step shutdown sequence: signal, command, await. + +```rust +impl MyWebSocketClient { + pub async fn close(&mut self) -> Result<(), MyWsError> { + tracing::debug!("Starting close process"); + + // 1. Send disconnect command so handler can clean up gracefully + if let Err(e) = self.cmd_tx.read().await.send(HandlerCommand::Disconnect) { + tracing::warn!("Failed to send disconnect command to handler: {e}"); + } + + // 2. Set stop signal so handler loop exits after processing disconnect + self.signal.store(true, Ordering::Release); + + // 3. Await task handle with timeout, abort if stuck + if let Some(task_handle) = self.task_handle.take() { + match Arc::try_unwrap(task_handle) { + Ok(handle) => { + let abort_handle = handle.abort_handle(); + match tokio::time::timeout(Duration::from_secs(2), handle).await { + Ok(Ok(())) => tracing::debug!("Handler task completed"), + Ok(Err(e)) => tracing::error!("Handler task error: {e:?}"), + Err(_) => { + tracing::warn!("Timeout waiting for handler task, aborting"); + abort_handle.abort(); + } + } + } + Err(arc_handle) => { + tracing::debug!("Cannot unwrap task handle, aborting"); + arc_handle.abort(); + } + } + } + + Ok(()) + } +} +``` + +**Key points:** + +- Send `Disconnect` before setting the stop signal so the handler processes it before exiting. +- Return `Result<(), {Venue}WsError>` so callers can handle failures. +- Use `Ordering::Release` on the signal store so the handler sees the write. +- Extract `abort_handle` before awaiting so it remains available after timeout. +- When `Arc::try_unwrap` fails (other clones exist), abort directly. + +### Stream consumption (`stream`) + +The outer client exposes a `stream()` method that hands ownership of `out_rx` to the +caller as an async stream. Data and execution clients call this once to drive their +message processing loop: + +```rust +impl MyWebSocketClient { + pub fn stream(&mut self) -> impl Stream + 'static { + let rx = self + .out_rx + .take() + .expect("Stream receiver already taken or not connected"); + let mut rx = Arc::try_unwrap(rx) + .expect("Cannot take ownership - other references exist"); + async_stream::stream! { + while let Some(msg) = rx.recv().await { + yield msg; + } + } + } +} +``` + +The data/execution client consumes the stream in a `tokio::select!` loop with a +cancellation token or stop signal, matching on `{Venue}WsMessage` variants and calling +parse functions to produce Nautilus domain types. + +### Subscription topic helpers (`subscription.rs`) + +When a venue's subscription topics have complex structure (multiple parameter types, +instrument type / family / ID variants, candle width encoding), extract topic building +and parsing into `websocket/subscription.rs`. This keeps `client.rs` focused on +connection lifecycle and `handler.rs` focused on I/O. + +For venues with simple `{channel}:{symbol}` topics, inline helpers in the client are +sufficient and a separate module is not needed. + +### Handler configuration constants + +Define handler-specific tuning constants for consistent behavior: + +| Constant | Purpose | Typical value | +|----------------------------|--------------------------------------------------|---------------| +| `DEFAULT_HEARTBEAT_SECS` | Interval for sending keep-alive messages. | 15-30 | +| `WEBSOCKET_AUTH_WINDOW_MS` | Maximum age for authentication timestamps. | 5000-30000 | +| `BATCH_PROCESSING_LIMIT` | Maximum messages processed per event loop cycle. | 100-1000 | + +Place these in `websocket/handler.rs` or `common/consts.rs` depending on scope. + +### Message routing + +The handler uses two message enums to separate wire deserialization from emitted events. +The data and execution client layers convert emitted events into Nautilus domain types. + +Define two enums: + +1. **`{Venue}WsFrame`**: Serde-deserialized wire frames. Contains every JSON shape the venue + can send (login responses, subscription acks, channel data, order responses, errors, pings). + Typically `pub(super)` since only the handler uses it. + +2. **`{Venue}WsMessage`**: Handler output events emitted on `out_tx`. Contains the subset of + wire data the client needs plus synthetic control variants (`Reconnected`, `Authenticated`, + `SendFailed`) that have no wire representation. This is the `pub` type consumers match on. + +The handler deserializes raw text into `{Venue}WsFrame`, handles control frames internally +(subscription acks, login, pings), and converts relevant frames into `{Venue}WsMessage` events +sent via `out_tx`. The client receives from `out_rx` and routes to data/execution callbacks, +which convert venue types to Nautilus domain types using parse functions. + +#### Message type naming convention + +Types prefixed with the venue name (e.g., `OKX`, `Bitmex`) contain raw exchange-specific types. +Types prefixed with `Nautilus` contain normalized domain types ready for the trading system. + +**Wire frame enum (serde-deserialized, handler-internal):** + +```rust +pub(super) enum MyWsFrame { + Login { event, code, msg, conn_id }, + Subscription { event, arg, conn_id, code, msg }, + OrderResponse { id, op, code, msg, data }, + BookData { arg, action, data: Vec }, + Data { arg, data: Value }, + Error { code, msg }, + Ping, + Reconnected, +} +``` + +**Handler output enum (emitted to client):** + +```rust +pub enum MyWsMessage { + BookData { arg, action, data: Vec }, + ChannelData { channel, inst_id, data: Value }, + Orders(Vec), + OrderResponse { id, op, code, msg, data }, + SendFailed { request_id, client_order_id, op, error }, + Instruments(Vec), + Error(MyWebSocketError), + Reconnected, + Authenticated, +} +``` + +The frame enum includes every wire shape (login acks, subscription acks, pings) for +deserialization. The output enum drops shapes the handler consumes internally and adds synthetic +variants (`Authenticated`, `SendFailed`) that originate in handler logic, not on the wire. + +Include `OrderResponse` for venue acknowledgements (place, cancel, amend) and `SendFailed` for +WebSocket send failures after retries are exhausted. The execution client dispatch layer converts +these into Nautilus rejection events (`OrderRejected`, `OrderCancelRejected`, etc.). + +**Conversion in data/exec client:** + +The data client's message loop matches on `{Venue}WsMessage` variants and calls parse functions +to produce Nautilus domain types (`Data`, `OrderBookDeltas`, etc.). The execution client's +dispatch layer handles `OrderResponse`, `SendFailed`, and `Orders` variants. This keeps +the handler focused on I/O and deserialization while the client layers own domain conversion. + +The execution dispatch converts order and fill messages using a two-tier routing contract: + +1. The handler emits venue-specific order types (e.g., `Orders(Vec)`). +2. The client dispatch layer tracks which orders were submitted through this client. +3. **Tracked order**: convert venue types to order events (`OrderAccepted`, `OrderCanceled`, + `OrderFilled`, etc.) and synthesize any missing lifecycle events (e.g., `OrderAccepted` + before a fast fill). +4. **External/unknown order**: convert to reports (`OrderStatusReport` or `FillReport`) for + downstream reconciliation. + +#### `WsDispatchState` + +Execution dispatch state lives in a `WsDispatchState` struct defined in `websocket/dispatch.rs`. +It tracks which lifecycle events have already been emitted to prevent duplicates across +reconnections and fast-fill races: + +```rust +#[derive(Debug, Default)] +pub struct WsDispatchState { + pub order_identities: DashMap, + pub emitted_accepted: DashSet, + pub triggered_orders: DashSet, + pub filled_orders: DashSet, + clearing: AtomicBool, +} +``` + +| Field | Purpose | +|---------------------|-----------------------------------------------------------------| +| `order_identities` | Maps client order ID to identity metadata set at submission. | +| `emitted_accepted` | Prevents duplicate `OrderAccepted` events. | +| `triggered_orders` | Tracks conditional orders that have triggered. | +| `filled_orders` | Prevents duplicate `OrderFilled` events on reconnect replay. | +| `clearing` | Guards concurrent eviction when sets reach capacity. | + +Each `DashSet` is bounded by a `DEDUP_CAPACITY` constant (typically 10,000). When a set +reaches capacity, `evict_if_full()` clears it atomically using a compare-exchange on the +`clearing` flag to prevent concurrent clears. + +The `dispatch_ws_message()` free function in the same module routes `{Venue}WsMessage` +variants to the appropriate order event builders, using `WsDispatchState` for dedup +and `OrderIdentity` for tracked-vs-external classification. + +### Error handling + +#### Client-side error propagation + +Channel send failures (client → handler) should propagate loudly as `Result<(), Error>`: + +```rust +impl MyWebSocketClient { + async fn send_cmd(&self, cmd: HandlerCommand) -> Result<(), Error> { + self.cmd_tx.read().await.send(cmd) + .map_err(|e| Error::ClientError(format!("Handler not available: {e}"))) + } + + pub async fn submit_order(...) -> Result<(), Error> { + let cmd = HandlerCommand::PlaceOrder { ... }; + self.send_cmd(cmd).await // Propagates channel failures + } +} +``` + +#### Handler-side retry logic + +WebSocket send failures (handler → network) should be retried by the handler using `RetryManager`: + +```rust +pub struct MyWsFeedHandler { + inner: Option, + retry_manager: RetryManager, + // ... +} + +impl MyWsFeedHandler { + async fn send_with_retry(&self, payload: String, rate_limit_keys: Option>) -> Result<(), MyWsError> { + if let Some(client) = &self.inner { + self.retry_manager.execute_with_retry( + "websocket_send", + || async { + client.send_text(payload.clone(), rate_limit_keys.clone()) + .await + .map_err(|e| MyWsError::ClientError(format!("Send failed: {e}"))) + }, + should_retry_error, + create_timeout_error, + ).await + } else { + Err(MyWsError::ClientError("No active WebSocket client".to_string())) + } + } + + async fn handle_place_order(...) -> anyhow::Result<()> { + let payload = serde_json::to_string(&request)?; + + match self.send_with_retry(payload, Some(vec![RATE_LIMIT_KEY])).await { + Ok(()) => Ok(()), + Err(e) => { + // Emit SendFailed so the exec client dispatch can produce OrderRejected + let _ = self.out_tx.send(MyWsMessage::SendFailed { + request_id: request_id.clone(), + client_order_id: Some(client_order_id), + op: Some(MyWsOperation::Order), + error: e.to_string(), + }); + Err(anyhow::anyhow!("Failed to send order: {e}")) + } + } + } +} + +fn should_retry_error(error: &MyWsError) -> bool { + match error { + MyWsError::NetworkError(_) | MyWsError::Timeout(_) => true, + MyWsError::AuthenticationError(_) | MyWsError::ParseError(_) => false, + } +} +``` + +**Key principles:** + +- Client propagates channel failures immediately (handler unavailable). +- Handler retries transient WebSocket failures (network issues, timeouts). +- Handler emits `SendFailed` when retries are exhausted; the exec client dispatch converts + these into Nautilus rejection events (`OrderRejected`, `OrderCancelRejected`). +- Use `RetryManager` from `nautilus_network::retry` for consistent backoff. + +### Naming conventions + +Adapters follow standardized naming conventions for consistency across all venue integrations. + +#### Channel naming: `raw` → `msg` → `out` + +WebSocket message channels follow a two-stage transformation pipeline within the handler: + +| Stage | Type | Description | Example | +|-------|------|-------------|---------| +| `raw` | Raw WebSocket frames | Bytes/text from the network layer. | `raw_rx: UnboundedReceiver` | +| `out` | Venue-specific messages | Parsed venue message types. | `out_tx: UnboundedSender` | + +The handler deserializes raw frames into venue-specific types and emits them on `out_tx`. +The data and execution client layers then convert venue types into Nautilus domain types. + +**Example flow:** + +```rust +// Client creates output channel for venue messages +let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel(); // Venue messages (MyWsMessage) + +// Handler receives raw frames, outputs venue messages +let handler = MyWsFeedHandler::new( + cmd_rx, + raw_rx, // Input: Message (raw WebSocket frames) + out_tx, // Output: MyWsMessage + // ... +); +``` + +Channel names reflect the data transformation stage, not the destination. Use `raw_*` for raw +WebSocket frames (`Message`) and `out_*` for venue-specific message types. + +### Backpressure strategy + +WebSocket channels on latency-sensitive paths are intentionally **unbounded**. The platform +prioritizes latency and prefers an explicit crash (OOM) over delaying or dropping data. + +:::note +Do not add bounded channels, buffering limits, or backpressure unless the latency requirement changes. +::: + +#### Field naming: `inner` and command channels + +Structs holding references to lower-level components follow these conventions: + +| Field | Type | Description | +|---------------|-----------------------------------------------------|-------------| +| `inner` | `Option` | Network-level WebSocket client (handler only, exclusively owned). | +| `cmd_tx` | `Arc>>` | Command channel to handler (client side). | +| `cmd_rx` | `UnboundedReceiver` | Command channel from client (handler side). | +| `out_tx` | `UnboundedSender<{Venue}WsMessage>` | Output channel to client (handler side). | +| `out_rx` | `Option>>` | Output channel from handler (client side). | +| `task_handle` | `Option>>` | Handler task handle. | + +**Example:** + +```rust +// Client struct +pub struct MyWebSocketClient { + cmd_tx: Arc>>, + out_rx: Option>>, + task_handle: Option>>, + connection_mode: Arc>, // Lock-free connection state + // ... +} + +impl MyWebSocketClient { + async fn send_cmd(&self, cmd: HandlerCommand) -> Result<(), Error> { + self.cmd_tx.read().await.send(cmd) + .map_err(|e| Error::ClientError(format!("Handler not available: {e}"))) + } +} + +// Handler struct +pub(super) struct MyWsFeedHandler { + inner: Option, // Exclusively owned - no RwLock + cmd_rx: UnboundedReceiver, + raw_rx: UnboundedReceiver, + out_tx: UnboundedSender, + pending_requests: AHashMap, // Single-threaded - no locks + pending_messages: VecDeque, // Multi-message buffer + // ... +} +``` + +The handler exclusively owns `WebSocketClient` without locks. The client sends commands via +`cmd_tx` (wrapped in `RwLock` to allow reconnection channel replacement) and receives events +via `out_rx`. Use a `send_cmd()` helper to standardize command sending. + +#### Type naming: `{Venue}Ws{TypeSuffix}` + +All WebSocket-related types follow a standardized naming pattern: `{Venue}Ws{TypeSuffix}` + +- `{Venue}`: Capitalized venue name (e.g., `OKX`, `Bybit`, `Bitmex`, `Hyperliquid`). +- `Ws`: Abbreviated "WebSocket" (not fully spelled out). +- `{TypeSuffix}`: Full type descriptor (e.g., `Message`, `Error`, `Request`, `Response`). + +**Examples:** + +```rust +// Correct - abbreviated Ws, full type suffix +pub enum OKXWsMessage { ... } +pub enum BybitWsError { ... } +pub struct HyperliquidWsRequest { ... } +``` + +**Standard type suffixes:** + +- `Message`: WebSocket message enums. +- `Error`: WebSocket error types. +- `Request`: Request message types. +- `Response`: Response message types. + +**Tokio channel qualification:** + +Always fully qualify tokio channel types as `tokio::sync::mpsc::` to avoid ambiguity with +similarly-named types from other crates. Never import `mpsc` directly at module level. + +```rust +// Correct +let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); +``` + +### Split WebSocket architectures + +Some venues expose multiple WebSocket endpoints with distinct protocols or encodings. +When a venue requires separate connections for market data and order management, split +the `websocket/` module into submodules that mirror the connection boundaries: + +``` +src/ +├── websocket/ +│ ├── mod.rs # Re-exports from submodules +│ ├── streams/ # Market data pub/sub connection +│ │ ├── client.rs # Streams client +│ │ ├── handler.rs # Streams feed handler +│ │ ├── messages.rs # Streams message types +│ │ └── mod.rs +│ └── trading/ # Order management + user data (authenticated WS API) +│ ├── client.rs # Trading client +│ ├── handler.rs # Trading handler +│ ├── messages.rs # Trading message types +│ ├── user_data.rs # User data stream venue types (execution reports, etc.) +│ ├── parse.rs # Parse functions for user data -> Nautilus types +│ ├── error.rs # Trading error types +│ └── mod.rs +``` + +Each submodule follows the same two-layer client/handler pattern described above. The +parent `websocket/mod.rs` re-exports the public client types. + +The `trading/` module handles both order operations (place, cancel, modify) and the +user data stream (execution reports, account updates). When the venue's authenticated +WebSocket API supports `session.logon` and inline user data subscriptions, both +concerns share a single authenticated connection. This avoids a separate `execution/` +module and the deprecated REST listenKey lifecycle. + +For venues where user data events arrive on a separate stream connection (e.g., +futures APIs that return a listenKey for a dedicated stream URL), the `streams/` +handler dispatches both market data and user data events from the combined connection. + +#### Naming conventions for split architectures + +Type names include the submodule qualifier to avoid ambiguity: + +| Submodule | Command type | Message type | +|--------------|--------------------------------------|-------------------------------------| +| `streams/` | `{Venue}WsStreamsCommand` | `{Venue}WsMessage` (venue types) | +| `trading/` | `{Venue}WsTradingCommand` | `{Venue}WsTradingMessage` | + +The `{Venue}Ws` prefix follows the standard type naming convention. The qualifier +(`Streams`, `Trading`) distinguishes types that would otherwise collide across +submodules. + +#### When to split + +Split the WebSocket module when the venue has: + +- Different endpoints with different protocols (e.g., SBE binary for market data, JSON + for trading) +- A dedicated order management WebSocket API (`ws-api` style) alongside pub/sub streams +- User data delivered inline on the authenticated trading connection rather than via a + separate listenKey stream + +Do not split when a single connection handles all message types through channel-based +multiplexing (the common pattern for OKX, Bybit, and similar venues). + +## Modeling venue payloads + +Use the following conventions when mirroring upstream schemas in Rust. + +### REST models (`http::models` and `http::query`) + +- Put request and response representations in `src/http/models.rs` and derive `serde::Deserialize` (add `serde::Serialize` when the adapter sends data back). +- Mirror upstream payload names with blanket casing attributes such as `#[serde(rename_all = "camelCase")]` or `#[serde(rename_all = "snake_case")]`; only add per-field renames when the upstream key would be an invalid Rust identifier or collide with a keyword (for example `#[serde(rename = "type")] pub order_type: String`). +- Keep helper structs for query parameters in `src/http/query.rs`, deriving `serde::Serialize` to remain type-safe and reusing constants from `common::consts` instead of duplicating literals. + +### WebSocket messages (`websocket::messages`) + +- Define streaming payload types in `src/websocket/messages.rs`, giving each venue topic a struct or enum that mirrors the upstream JSON. +- Apply the same naming guidance as REST models: rely on blanket casing renames and keep field names aligned with the venue unless syntax forces a change; consider serde helpers such as `#[serde(tag = "op")]` or `#[serde(flatten)]` and document the choice. +- Note any intentional deviations from the upstream schema in code comments and module docs so other contributors can follow the mapping quickly. + +--- + +## Testing + +Adapters should ship two layers of coverage: the Rust crate that talks to the venue and the Python glue that exposes it to the wider platform. +Keep the suites deterministic and colocated with the production code they protect. + +**Key principle:** The `tests/` directory is reserved for integration tests that require external infrastructure (mock Axum servers, simulated network conditions). +Unit tests for parsing, serialization, and business logic belong in `#[cfg(test)]` blocks within source modules. + +### Rust testing + +#### Layout + +``` +crates/adapters/your_adapter/ +├── src/ +│ ├── http/ +│ │ ├── client.rs # HTTP client + unit tests +│ │ └── parse.rs # REST payload parsers + unit tests +│ └── websocket/ +│ ├── client.rs # WebSocket client + unit tests +│ └── parse.rs # Streaming parsers + unit tests +├── tests/ # Integration tests (mock servers) +│ ├── data_client.rs # Data client integration tests +│ ├── exec_client.rs # Execution client integration tests +│ ├── http.rs # HTTP client integration tests +│ └── websocket.rs # WebSocket client integration tests +└── test_data/ # Canonical venue payloads used by the suites + ├── http_{method}_{endpoint}.json # Full venue responses with retCode/result/time + └── ws_{message_type}.json # WebSocket message samples +``` + +#### Test file organization + +| File | Purpose | +|----------------------|---------------------------------------------------------------------------------------------------------------------------| +| `tests/data_client.rs` | Integration tests for the data client. Validates data subscriptions, historical data requests, and market data parsing. | +| `tests/exec_client.rs` | Integration tests for the execution client. Validates order submission, modification, cancellation, and execution reports. | +| `tests/http.rs` | Low-level HTTP client tests. Validates request signing, error handling, and response parsing against mock Axum servers. | +| `tests/websocket.rs` | WebSocket client tests. Validates connection lifecycle, authentication, subscriptions, and message routing. | + +**Guidelines:** + +- Place unit tests next to the module they exercise (`#[cfg(test)]` blocks). Use `src/common/testing.rs` (or an equivalent helper module) for shared fixtures so production files stay tidy. +- Keep Axum-based integration suites under `crates/adapters//tests/`, mirroring the public APIs (HTTP client, WebSocket client, data client, execution client). +- Data and execution client tests (`data_client.rs`, `exec_client.rs`) should focus on higher-level behavior: subscription workflows, order lifecycle, and domain model transformations. HTTP and WebSocket tests (`http.rs`, `websocket.rs`) focus on transport-level concerns. +- Store upstream payload samples (snapshots, REST replies) under `test_data/` and reference them from both unit and integration tests. Name test data files consistently: `http_get_{endpoint_name}.json` for REST responses, `ws_{message_type}.json` for WebSocket messages. Include complete venue response envelopes (status codes, timestamps, result wrappers) rather than just the data payload. Provide multiple realistic examples in each file - for instance, position data should include long, short, and flat positions to exercise all parser branches. +- **Test data sourcing**: Test data must be obtained from either official API documentation examples or directly from the live API via network calls. Never fabricate or generate test data manually, as this risks missing edge cases (e.g., negative precision values, scientific notation, unexpected field types) that only appear in real venue responses. + +#### Unit tests + +Unit tests belong in `#[cfg(test)]` blocks within source modules, not in the `tests/` directory. + +**What to test (in source modules):** + +- Deserialization of venue JSON payloads into Rust structs. +- Parsing functions that convert venue types to Nautilus domain models. +- Request signing and authentication helpers. +- Enum conversions and mapping logic. +- Price, quantity, and precision calculations. + +**What NOT to test:** + +- Standard library behavior (Vec operations, HashMap lookups, string parsing). +- Third-party crate functionality (chrono date arithmetic, serde attributes). +- Test helper code itself (fixture loaders, mock builders). + +Tests should exercise production code paths. If a test only verifies that `Vec::extend()` works or that chrono can parse a date string, it provides no value. + +##### WebSocket unit test coverage + +WebSocket unit tests exercise three areas: message deserialization, parse dispatch, and handler +logic. Each area lives in a `#[cfg(test)]` block within the module it tests. + +**Message types (`messages.rs`):** + +- Deserialize every message variant from fixture JSON files in `test_data/`. +- Round-trip tests: serialize a constructed struct, deserialize the output, and assert equality. + Round-trip tests catch field renames, missing `skip_serializing_if` attributes, and precision + loss that deserialization-only tests miss. +- Cover edge cases in venue payloads: null optional fields, empty arrays, zero quantities. + +**Parse functions (`parse.rs`):** + +- Exercise the fast-path byte scanner for each type tag or discriminant value. +- Exercise the slow-path fallback (fields not at expected byte positions). +- Verify unknown type tags produce a descriptive error, not a panic. + +**Handler logic (`handler.rs`):** + +- Verify the handler filters internal messages (heartbeats, subscription acks, pong frames) + and does not forward them to consumers. +- Verify reconnect signals trigger re-authentication and emit the `Reconnected` variant. +- Verify multi-message buffering: when a single raw frame produces multiple output messages, + all messages appear in the correct order from `next()`. +- Verify pending-order cleanup on error and success responses. + +#### Integration tests + +Integration tests belong in the `tests/` directory and exercise the public API against mock infrastructure. + +**What to test (in tests/ directory):** + +- HTTP client requests against mock Axum servers. +- WebSocket connection lifecycle, authentication, and message routing. +- Data client subscription workflows and historical data requests. +- Execution client order submission, modification, and cancellation flows. +- Error handling and retry behavior with simulated failures. + +At a minimum, review existing adapter test suites for reference patterns and verify every adapter +proves the same core behaviours. + +##### HTTP client integration coverage + +- **Happy paths** – fetch a representative public resource (e.g., instruments or mark price) and verify the + response is converted into Nautilus domain models. +- **Credential guard** – call a private endpoint without credentials and assert a structured error; repeat with + credentials to prove success. +- **Rate limiting / retry mapping** – surface venue-specific rate-limit responses and assert the adapter produces + the correct `OkxError`/`BitmexHttpError` variant so the retry policy can react. +- **Query builders** – exercise builders for paginated/time-bounded endpoints (historical trades, candles) and + assert the emitted query string matches the venue specification (`after`, `before`, `limit`, etc.). +- **Error translation** – verify non-2xx upstream responses map to adapter error enums with the original code/message attached. + +##### WebSocket client integration coverage + +- **Login handshake** – confirm a successful login flips the internal auth state and test failure cases where the + server returns a non-zero code; the client should surface an error and avoid marking itself as authenticated. +- **Ping/Pong** – prove both text-based and control-frame pings trigger immediate pong responses. +- **Subscription lifecycle** – assert subscription requests/acks are emitted for public and private channels, and that + unsubscribe calls remove entries from the cached subscription sets. +- **Reconnect behaviour** – simulate a disconnect and verify the client re-authenticates, restores public channels, + and skips private channels that were explicitly unsubscribed pre-disconnect. +- **Message routing** – feed representative data/ack/error payloads through the socket and assert they arrive on the + public stream as the correct `{Venue}WsMessage` variant. +- **Quota tagging** – (optional but recommended) validate that order/cancel/amend operations are tagged with the + appropriate quota label so rate limiting can be enforced independently of subscription traffic. + +**CI robustness:** + +- Never use bare `tokio::time::sleep()` with arbitrary durations. Tests become flaky under CI load and slower than necessary. +- Use the `wait_until_async` test helper to poll for conditions with timeout. Tests return + immediately when the condition is met and fail deterministically on timeout rather than + relying on arbitrary sleep durations. +- Prefer event-driven assertions with shared state (for example, collect `subscription_events`, track pending/confirmed topics, wait for `connection_count` transitions). +- Use adapter-specific helpers to gate on explicit signals such as "auth confirmed" or "reconnection finished" so suites remain deterministic under load. + +##### Data and execution client integration testing + +Data (`tests/data_client.rs`) and execution (`tests/exec_client.rs`) client integration tests verify the full message flow from WebSocket through parsing to event emission. + +**Test infrastructure:** + +| Component | Purpose | +|------------------------------|------------------------------------------------------------------------------------| +| Mock Axum server | Serves HTTP endpoints (instruments, fee rates, positions) and WebSocket channels. | +| `TestServerState` | Tracks connections, subscriptions, and authentication state for assertions. | +| Thread-local event channels | `set_data_event_sender()` / `set_exec_event_sender()` for capturing emitted events.| +| `wait_until_async` | Polls conditions with timeout for deterministic async assertions. | + +**Data client coverage:** + +| Test scenario | Validates | +|------------------------------|----------------------------------------------------------------| +| Connect/disconnect | Connection lifecycle, WebSocket establishment, clean shutdown. | +| Subscribe trades | Trade tick events emitted to data channel. | +| Subscribe quotes | Quote events from ticker (LINEAR) or orderbook (SPOT). | +| Subscribe book deltas | OrderBookDeltas events from orderbook snapshots/updates. | +| Subscribe mark/index prices | Filtered by subscription state (only emit when subscribed). | +| Reset state | Subscription tracking cleared, connection terminated. | +| Instruments on connect | Instrument events emitted during connection setup. | + +**Execution client coverage:** + +| Test scenario | Validates | +|------------------------------|----------------------------------------------------------------| +| Connect/disconnect | Auth handshake, private + trade WS connections, subscriptions. | +| Demo mode | Only private WS connects (trade WS skipped for HTTP fallback). | +| Order submission | Order accepted/rejected events, venue ID correlation. | +| Order modification/cancel | Update and cancel acknowledgment events. | +| Position/wallet updates | PositionStatusReport and AccountState events. | + +**Key patterns:** + +- Each `#[tokio::test]` runs on a fresh thread, ensuring thread-local channel isolation. +- Use `wait_until_async` for subscription/connection state instead of arbitrary sleeps. +- Drain instrument events before subscription tests to isolate assertions. +- Verify subscription state in `TestServerState` before asserting on emitted events. + +### Python testing + +#### Layout + +``` +tests/integration_tests/adapters/your_adapter/ +├── conftest.py # Shared fixtures (mock clients, test instruments) +├── test_data.py # Data client integration tests +├── test_execution.py # Execution client integration tests +├── test_providers.py # Instrument provider tests +├── test_factories.py # Factory and configuration tests +└── __init__.py # Package initialization +``` + +#### Test file organization + +| File | Purpose | +|---------------------|--------------------------------------------------------------------------------------------------------------------| +| `test_data.py` | Tests for `LiveDataClient` and `LiveMarketDataClient`. Validates subscriptions, data parsing, and message handling. | +| `test_execution.py` | Tests for `LiveExecutionClient`. Validates order submission, modification, cancellation, and execution reports. | +| `test_providers.py` | Tests for `InstrumentProvider`. Validates instrument loading, filtering, and caching behavior. | +| `test_factories.py` | Tests for factory functions. Validates client instantiation and configuration wiring. | + +**Guidelines:** + +- Exercise the adapter's Python surface (instrument providers, data/execution clients, factories) inside `tests/integration_tests/adapters//`. +- Mock the PyO3 boundary (`nautilus_pyo3` shims, stubbed Rust clients) so tests stay fast while verifying that configuration, factory wiring, and error handling match the exported Rust API. +- Mirror the Rust integration coverage: when the Rust suite adds a new behaviour (e.g., reconnection replay, error propagation), assert the Python layer performs the same sequence (connect/disconnect, submit/amend/cancel translations, venue ID hand-off, failure handling). BitMEX's Python tests provide the target level of detail. + +--- + +## Documentation + +All adapter documentation (module-level docs, doc comments, and inline comments) should follow the +[Documentation Style Guide](docs.md). + +### Rust documentation requirements + +Every Rust module, struct, and public method must have documentation comments. +Use third-person declarative voice (e.g., "Returns the account ID" not "Return the account ID"). + +- **Modules**: Use `//!` doc comments at the top of each file (after the license header) to describe the module's purpose. +- **Structs**: Use `///` doc comments above struct definitions. Keep descriptions concise; one sentence is often sufficient. +- **Public methods**: Every `pub fn` and `pub async fn` must have a `///` doc comment describing what the method does. + Do not document individual parameters in a separate `# Arguments` section. The type signatures and names should be self-explanatory. + Parameters may be mentioned in the description when behavior is complex or non-obvious. + +**What NOT to document**: + +- Private methods and fields (unless complex logic warrants it). +- Individual parameters/arguments (use descriptive names instead). +- Implementation details that are obvious from the code. +- Files in the `python/` module (PyO3 bindings). Documentation conventions are TBD (*may* use numpydoc specification). + +--- + +## Python adapter layer + +Step-by-step guide to building the Python layer of an adapter using the provided template. + +### Method ordering convention + +When implementing adapter classes, group methods by category in this order: + +1. **Connection handlers**: `_connect`, `_disconnect` +2. **Subscribe handlers**: `_subscribe`, `_subscribe_*` +3. **Unsubscribe handlers**: `_unsubscribe`, `_unsubscribe_*` +4. **Request handlers**: `_request`, `_request_*` + +This keeps related functionality together rather than interleaving subscribe/unsubscribe pairs. + +### InstrumentProvider + +The `InstrumentProvider` loads instrument definitions from the venue: all instruments, specific +instruments by ID, or a filtered subset. + +```python +from nautilus_trader.common.providers import InstrumentProvider +from nautilus_trader.model import InstrumentId + + +class TemplateInstrumentProvider(InstrumentProvider): + """Example `InstrumentProvider` showing the minimal overrides required for a complete integration.""" + + async def load_all_async(self, filters: dict | None = None) -> None: + raise NotImplementedError("implement `load_all_async` in your adapter subclass") + + async def load_ids_async(self, instrument_ids: list[InstrumentId], filters: dict | None = None) -> None: + raise NotImplementedError("implement `load_ids_async` in your adapter subclass") + + async def load_async(self, instrument_id: InstrumentId, filters: dict | None = None) -> None: + raise NotImplementedError("implement `load_async` in your adapter subclass") +``` + +| Method | Description | +|------------------|----------------------------------------------------------------| +| `load_all_async` | Loads all instruments asynchronously, optionally with filters. | +| `load_ids_async` | Loads specific instruments by their IDs. | +| `load_async` | Loads a single instrument by its ID. | + +### DataClient + +The `LiveDataClient` handles data feeds that are not market data: news feeds, custom data streams, +or other non-market sources. + +```python +from nautilus_trader.data.messages import RequestData +from nautilus_trader.data.messages import SubscribeData +from nautilus_trader.data.messages import UnsubscribeData +from nautilus_trader.live.data_client import LiveDataClient +from nautilus_trader.model import DataType + + +class TemplateLiveDataClient(LiveDataClient): + """Example `LiveDataClient` showing the overridable abstract methods.""" + + async def _connect(self) -> None: + raise NotImplementedError("implement `_connect` in your adapter subclass") + + async def _disconnect(self) -> None: + raise NotImplementedError("implement `_disconnect` in your adapter subclass") + + async def _subscribe(self, command: SubscribeData) -> None: + raise NotImplementedError("implement `_subscribe` in your adapter subclass") + + async def _unsubscribe(self, command: UnsubscribeData) -> None: + raise NotImplementedError("implement `_unsubscribe` in your adapter subclass") + + async def _request(self, request: RequestData) -> None: + raise NotImplementedError("implement `_request` in your adapter subclass") +``` + +| Method | Description | +|----------------|------------------------------------------------| +| `_connect` | Establishes a connection to the data provider. | +| `_disconnect` | Closes the connection to the data provider. | +| `_subscribe` | Subscribes to a specific data type. | +| `_unsubscribe` | Unsubscribes from a specific data type. | +| `_request` | Requests data from the provider. | + +### MarketDataClient + +The `MarketDataClient` handles market-specific data: order books, top-of-book quotes and trades, +instrument status updates, and historical data requests. + +```python +from nautilus_trader.data.messages import RequestBars +from nautilus_trader.data.messages import RequestData +from nautilus_trader.data.messages import RequestInstrument +from nautilus_trader.data.messages import RequestInstruments +from nautilus_trader.data.messages import RequestOrderBookDeltas +from nautilus_trader.data.messages import RequestOrderBookDepth +from nautilus_trader.data.messages import RequestOrderBookSnapshot +from nautilus_trader.data.messages import RequestQuoteTicks +from nautilus_trader.data.messages import RequestTradeTicks +from nautilus_trader.data.messages import SubscribeBars +from nautilus_trader.data.messages import SubscribeData +from nautilus_trader.data.messages import SubscribeFundingRates +from nautilus_trader.data.messages import SubscribeIndexPrices +from nautilus_trader.data.messages import SubscribeInstrument +from nautilus_trader.data.messages import SubscribeInstrumentClose +from nautilus_trader.data.messages import SubscribeInstruments +from nautilus_trader.data.messages import SubscribeInstrumentStatus +from nautilus_trader.data.messages import SubscribeMarkPrices +from nautilus_trader.data.messages import SubscribeOrderBook +from nautilus_trader.data.messages import SubscribeQuoteTicks +from nautilus_trader.data.messages import SubscribeTradeTicks +from nautilus_trader.data.messages import UnsubscribeBars +from nautilus_trader.data.messages import UnsubscribeData +from nautilus_trader.data.messages import UnsubscribeFundingRates +from nautilus_trader.data.messages import UnsubscribeIndexPrices +from nautilus_trader.data.messages import UnsubscribeInstrument +from nautilus_trader.data.messages import UnsubscribeInstrumentClose +from nautilus_trader.data.messages import UnsubscribeInstruments +from nautilus_trader.data.messages import UnsubscribeInstrumentStatus +from nautilus_trader.data.messages import UnsubscribeMarkPrices +from nautilus_trader.data.messages import UnsubscribeOrderBook +from nautilus_trader.data.messages import UnsubscribeQuoteTicks +from nautilus_trader.data.messages import UnsubscribeTradeTicks +from nautilus_trader.live.data_client import LiveMarketDataClient + + +class TemplateLiveMarketDataClient(LiveMarketDataClient): + """Example `LiveMarketDataClient` showing the overridable abstract methods.""" + + async def _connect(self) -> None: + raise NotImplementedError("implement `_connect` in your adapter subclass") + + async def _disconnect(self) -> None: + raise NotImplementedError("implement `_disconnect` in your adapter subclass") + + async def _subscribe(self, command: SubscribeData) -> None: + raise NotImplementedError("implement `_subscribe` in your adapter subclass") + + async def _subscribe_instruments(self, command: SubscribeInstruments) -> None: + raise NotImplementedError("implement `_subscribe_instruments` in your adapter subclass") + + async def _subscribe_instrument(self, command: SubscribeInstrument) -> None: + raise NotImplementedError("implement `_subscribe_instrument` in your adapter subclass") + + async def _subscribe_order_book_deltas(self, command: SubscribeOrderBook) -> None: + raise NotImplementedError("implement `_subscribe_order_book_deltas` in your adapter subclass") + + async def _subscribe_order_book_depth(self, command: SubscribeOrderBook) -> None: + raise NotImplementedError("implement `_subscribe_order_book_depth` in your adapter subclass") + + async def _subscribe_quote_ticks(self, command: SubscribeQuoteTicks) -> None: + raise NotImplementedError("implement `_subscribe_quote_ticks` in your adapter subclass") + + async def _subscribe_trade_ticks(self, command: SubscribeTradeTicks) -> None: + raise NotImplementedError("implement `_subscribe_trade_ticks` in your adapter subclass") + + async def _subscribe_mark_prices(self, command: SubscribeMarkPrices) -> None: + raise NotImplementedError("implement `_subscribe_mark_prices` in your adapter subclass") + + async def _subscribe_index_prices(self, command: SubscribeIndexPrices) -> None: + raise NotImplementedError("implement `_subscribe_index_prices` in your adapter subclass") + + async def _subscribe_bars(self, command: SubscribeBars) -> None: + raise NotImplementedError("implement `_subscribe_bars` in your adapter subclass") + + async def _subscribe_funding_rates(self, command: SubscribeFundingRates) -> None: + raise NotImplementedError("implement `_subscribe_funding_rates` in your adapter subclass") + + async def _subscribe_instrument_status(self, command: SubscribeInstrumentStatus) -> None: + raise NotImplementedError("implement `_subscribe_instrument_status` in your adapter subclass") + + async def _subscribe_instrument_close(self, command: SubscribeInstrumentClose) -> None: + raise NotImplementedError("implement `_subscribe_instrument_close` in your adapter subclass") + + async def _subscribe_option_greeks(self, command: SubscribeOptionGreeks) -> None: + raise NotImplementedError("implement `_subscribe_option_greeks` in your adapter subclass") + + async def _unsubscribe(self, command: UnsubscribeData) -> None: + raise NotImplementedError("implement `_unsubscribe` in your adapter subclass") + + async def _unsubscribe_instruments(self, command: UnsubscribeInstruments) -> None: + raise NotImplementedError("implement `_unsubscribe_instruments` in your adapter subclass") + + async def _unsubscribe_instrument(self, command: UnsubscribeInstrument) -> None: + raise NotImplementedError("implement `_unsubscribe_instrument` in your adapter subclass") + + async def _unsubscribe_order_book_deltas(self, command: UnsubscribeOrderBook) -> None: + raise NotImplementedError("implement `_unsubscribe_order_book_deltas` in your adapter subclass") + + async def _unsubscribe_order_book_depth(self, command: UnsubscribeOrderBook) -> None: + raise NotImplementedError("implement `_unsubscribe_order_book_depth` in your adapter subclass") + + async def _unsubscribe_quote_ticks(self, command: UnsubscribeQuoteTicks) -> None: + raise NotImplementedError("implement `_unsubscribe_quote_ticks` in your adapter subclass") + + async def _unsubscribe_trade_ticks(self, command: UnsubscribeTradeTicks) -> None: + raise NotImplementedError("implement `_unsubscribe_trade_ticks` in your adapter subclass") + + async def _unsubscribe_mark_prices(self, command: UnsubscribeMarkPrices) -> None: + raise NotImplementedError("implement `_unsubscribe_mark_prices` in your adapter subclass") + + async def _unsubscribe_index_prices(self, command: UnsubscribeIndexPrices) -> None: + raise NotImplementedError("implement `_unsubscribe_index_prices` in your adapter subclass") + + async def _unsubscribe_bars(self, command: UnsubscribeBars) -> None: + raise NotImplementedError("implement `_unsubscribe_bars` in your adapter subclass") + + async def _unsubscribe_funding_rates(self, command: UnsubscribeFundingRates) -> None: + raise NotImplementedError("implement `_unsubscribe_funding_rates` in your adapter subclass") + + async def _unsubscribe_instrument_status(self, command: UnsubscribeInstrumentStatus) -> None: + raise NotImplementedError("implement `_unsubscribe_instrument_status` in your adapter subclass") + + async def _unsubscribe_instrument_close(self, command: UnsubscribeInstrumentClose) -> None: + raise NotImplementedError("implement `_unsubscribe_instrument_close` in your adapter subclass") + + async def _unsubscribe_option_greeks(self, command: UnsubscribeOptionGreeks) -> None: + raise NotImplementedError("implement `_unsubscribe_option_greeks` in your adapter subclass") + + async def _request(self, request: RequestData) -> None: + raise NotImplementedError("implement `_request` in your adapter subclass") + + async def _request_instrument(self, request: RequestInstrument) -> None: + raise NotImplementedError("implement `_request_instrument` in your adapter subclass") + + async def _request_instruments(self, request: RequestInstruments) -> None: + raise NotImplementedError("implement `_request_instruments` in your adapter subclass") + + async def _request_order_book_deltas(self, request: RequestOrderBookDeltas) -> None: + raise NotImplementedError("implement `_request_order_book_deltas` in your adapter subclass") + + async def _request_order_book_depth(self, request: RequestOrderBookDepth) -> None: + raise NotImplementedError("implement `_request_order_book_depth` in your adapter subclass") + + async def _request_order_book_snapshot(self, request: RequestOrderBookSnapshot) -> None: + raise NotImplementedError("implement `_request_order_book_snapshot` in your adapter subclass") + + async def _request_quote_ticks(self, request: RequestQuoteTicks) -> None: + raise NotImplementedError("implement `_request_quote_ticks` in your adapter subclass") + + async def _request_trade_ticks(self, request: RequestTradeTicks) -> None: + raise NotImplementedError("implement `_request_trade_ticks` in your adapter subclass") + + async def _request_bars(self, request: RequestBars) -> None: + raise NotImplementedError("implement `_request_bars` in your adapter subclass") + +``` + +| Method | Description | +|------------------------------------|---------------------------------------------------------| +| `_connect` | Establishes a connection to the venue APIs. | +| `_disconnect` | Closes the connection to the venue APIs. | +| `_subscribe` | Subscribes to generic data (base for custom types). | +| `_subscribe_instruments` | Subscribes to market data for multiple instruments. | +| `_subscribe_instrument` | Subscribes to market data for a single instrument. | +| `_subscribe_order_book_deltas` | Subscribes to order book delta updates. | +| `_subscribe_order_book_depth` | Subscribes to order book depth updates. | +| `_subscribe_quote_ticks` | Subscribes to top-of-book quote updates. | +| `_subscribe_trade_ticks` | Subscribes to trade tick updates. | +| `_subscribe_mark_prices` | Subscribes to mark price updates. | +| `_subscribe_index_prices` | Subscribes to index price updates. | +| `_subscribe_bars` | Subscribes to bar/candlestick updates. | +| `_subscribe_funding_rates` | Subscribes to funding rate updates. | +| `_subscribe_instrument_status` | Subscribes to instrument status updates. | +| `_subscribe_instrument_close` | Subscribes to instrument close price updates. | +| `_subscribe_option_greeks` | Subscribes to option greeks updates. | +| `_unsubscribe` | Unsubscribes from generic data (base for custom types). | +| `_unsubscribe_instruments` | Unsubscribes from market data for multiple instruments. | +| `_unsubscribe_instrument` | Unsubscribes from market data for a single instrument. | +| `_unsubscribe_order_book_deltas` | Unsubscribes from order book delta updates. | +| `_unsubscribe_order_book_depth` | Unsubscribes from order book depth updates. | +| `_unsubscribe_quote_ticks` | Unsubscribes from quote tick updates. | +| `_unsubscribe_trade_ticks` | Unsubscribes from trade tick updates. | +| `_unsubscribe_mark_prices` | Unsubscribes from mark price updates. | +| `_unsubscribe_index_prices` | Unsubscribes from index price updates. | +| `_unsubscribe_bars` | Unsubscribes from bar updates. | +| `_unsubscribe_funding_rates` | Unsubscribes from funding rate updates. | +| `_unsubscribe_instrument_status` | Unsubscribes from instrument status updates. | +| `_unsubscribe_instrument_close` | Unsubscribes from instrument close price updates. | +| `_unsubscribe_option_greeks` | Unsubscribes from option greeks updates. | +| `_request` | Requests generic data (base for custom types). | +| `_request_instrument` | Requests historical data for a single instrument. | +| `_request_instruments` | Requests historical data for multiple instruments. | +| `_request_order_book_snapshot` | Requests an order book snapshot. | +| `_request_order_book_depth` | Requests order book depth. | +| `_request_order_book_deltas` | Requests historical order book deltas. | +| `_request_quote_ticks` | Requests historical quote tick data. | +| `_request_trade_ticks` | Requests historical trade tick data. | +| `_request_bars` | Requests historical bar data. | +| `_request_funding_rates` | Requests historical funding rate data. | + +#### Order book delta flag requirements + +When implementing `_subscribe_order_book_deltas` or streaming order book +data, adapters **must** set `RecordFlag` flags correctly on each +`OrderBookDelta`. See also [Delta flags and event boundaries](../concepts/data.md#delta-flags-and-event-boundaries). + +- **`F_LAST`**: Set on the last delta of every logical event group. The + `DataEngine` uses this flag as the flush signal when `buffer_deltas` is + enabled. Without it, deltas accumulate indefinitely and are never + published to subscribers. + +- **`F_SNAPSHOT`**: Set on all deltas that belong to a snapshot sequence + (a `Clear` action followed by `Add` actions reconstructing the book). + +- **Empty book snapshots**: When emitting a snapshot for an empty book, + the `Clear` delta must have `F_SNAPSHOT | F_LAST`. Otherwise buffered + consumers never receive it. + +- **Incremental updates**: Each venue update message ends with a delta + that has `F_LAST` set. If the venue batches multiple updates into one + message, terminate each logical group with `F_LAST`. + +```python +from nautilus_trader.model.enums import RecordFlag + +# Incremental update (single event) +delta = OrderBookDelta( + instrument_id=instrument_id, + action=BookAction.UPDATE, + order=order, + flags=RecordFlag.F_LAST, # Last (and only) delta in this event + sequence=sequence, + ts_event=ts_event, + ts_init=ts_init, +) + +# Snapshot sequence +clear_delta = OrderBookDelta( + instrument_id=instrument_id, + action=BookAction.CLEAR, + order=NULL_ORDER, + flags=RecordFlag.F_SNAPSHOT, # Not the last delta + ... +) + +last_add_delta = OrderBookDelta( + instrument_id=instrument_id, + action=BookAction.ADD, + order=last_order, + flags=RecordFlag.F_SNAPSHOT | RecordFlag.F_LAST, # End of snapshot + ... +) +``` + +:::warning +A missing `F_LAST` is a silent bug: no error is raised, but subscribers +never receive the data when buffering is enabled. +::: + +### ExecutionClient + +The `ExecutionClient` manages order submission, modification, and cancellation against the venue +trading system. + +```python +from nautilus_trader.execution.messages import BatchCancelOrders +from nautilus_trader.execution.messages import CancelAllOrders +from nautilus_trader.execution.messages import CancelOrder +from nautilus_trader.execution.messages import GenerateFillReports +from nautilus_trader.execution.messages import GenerateOrderStatusReport +from nautilus_trader.execution.messages import GenerateOrderStatusReports +from nautilus_trader.execution.messages import GeneratePositionStatusReports +from nautilus_trader.execution.messages import ModifyOrder +from nautilus_trader.execution.messages import SubmitOrder +from nautilus_trader.execution.messages import SubmitOrderList +from nautilus_trader.execution.reports import ExecutionMassStatus +from nautilus_trader.execution.reports import FillReport +from nautilus_trader.execution.reports import OrderStatusReport +from nautilus_trader.execution.reports import PositionStatusReport +from nautilus_trader.live.execution_client import LiveExecutionClient + + +class TemplateLiveExecutionClient(LiveExecutionClient): + """Example `LiveExecutionClient` outlining the required overrides.""" + + async def _connect(self) -> None: + raise NotImplementedError("implement `_connect` in your adapter subclass") + + async def _disconnect(self) -> None: + raise NotImplementedError("implement `_disconnect` in your adapter subclass") + + async def _submit_order(self, command: SubmitOrder) -> None: + raise NotImplementedError("implement `_submit_order` in your adapter subclass") + + async def _submit_order_list(self, command: SubmitOrderList) -> None: + raise NotImplementedError("implement `_submit_order_list` in your adapter subclass") + + async def _modify_order(self, command: ModifyOrder) -> None: + raise NotImplementedError("implement `_modify_order` in your adapter subclass") + + async def _cancel_order(self, command: CancelOrder) -> None: + raise NotImplementedError("implement `_cancel_order` in your adapter subclass") + + async def _cancel_all_orders(self, command: CancelAllOrders) -> None: + raise NotImplementedError("implement `_cancel_all_orders` in your adapter subclass") + + async def _batch_cancel_orders(self, command: BatchCancelOrders) -> None: + raise NotImplementedError("implement `_batch_cancel_orders` in your adapter subclass") + + async def generate_order_status_report( + self, + command: GenerateOrderStatusReport, + ) -> OrderStatusReport | None: + raise NotImplementedError("method `generate_order_status_report` must be implemented in the subclass") + + async def generate_order_status_reports( + self, + command: GenerateOrderStatusReports, + ) -> list[OrderStatusReport]: + raise NotImplementedError("method `generate_order_status_reports` must be implemented in the subclass") + + async def generate_fill_reports( + self, + command: GenerateFillReports, + ) -> list[FillReport]: + raise NotImplementedError("method `generate_fill_reports` must be implemented in the subclass") + + async def generate_position_status_reports( + self, + command: GeneratePositionStatusReports, + ) -> list[PositionStatusReport]: + raise NotImplementedError("method `generate_position_status_reports` must be implemented in the subclass") + + async def generate_mass_status( + self, + lookback_mins: int | None = None, + ) -> ExecutionMassStatus | None: + raise NotImplementedError("method `generate_mass_status` must be implemented in the subclass") +``` + +| Method | Description | +|------------------------------------|-----------------------------------------------------------| +| `_connect` | Establishes a connection to the venue APIs. | +| `_disconnect` | Closes the connection to the venue APIs. | +| `_submit_order` | Submits a new order to the venue. | +| `_submit_order_list` | Submits a list of orders to the venue. | +| `_modify_order` | Modifies an existing order on the venue. | +| `_cancel_order` | Cancels a specific order on the venue. | +| `_cancel_all_orders` | Cancels all orders for an instrument on the venue. | +| `_batch_cancel_orders` | Cancels a batch of orders for an instrument on the venue. | +| `generate_order_status_report` | Generates a report for a specific order on the venue. | +| `generate_order_status_reports` | Generates reports for all orders on the venue. | +| `generate_fill_reports` | Generates reports for filled orders on the venue. | +| `generate_position_status_reports` | Generates reports for position status on the venue. | +| `generate_mass_status` | Generates execution mass status reports. | + +### Configuration + +Configuration classes hold adapter-specific settings like API keys and connection details. + +```python +from nautilus_trader.config import LiveDataClientConfig +from nautilus_trader.config import LiveExecClientConfig + + +class TemplateDataClientConfig(LiveDataClientConfig): + """Configuration for `TemplateDataClient` instances.""" + + api_key: str + api_secret: str + base_url: str + + +class TemplateExecClientConfig(LiveExecClientConfig): + """Configuration for `TemplateExecClient` instances.""" + + api_key: str + api_secret: str + base_url: str +``` + +**Key attributes**: + +- `api_key`: The API key for authenticating with the data provider. +- `api_secret`: The API secret for authenticating with the data provider. +- `base_url`: The base URL for connecting to the data provider's API. + +## Common test scenarios + +Exercise adapters across every venue behaviour they claim to support. Incorporate these scenarios +into the Rust and Python suites. + +### Product coverage + +Test each supported product family. + +- Spot instruments +- Derivatives (perpetuals, futures, swaps) +- Options and structured products + +### Order flow + +- Cover each supported order type (limit, market, stop, conditional, etc.) under every venue time-in-force option, expiries, and rejection handling. +- Submit buy and sell market orders and assert balance, position, and average-price updates align with venue responses. +- Submit representative buy and sell limit orders, verifying acknowledgements, execution reports, full and partial fills, and cancel flows. + +### State management + +- Start sessions with existing open orders to verify the adapter reconciles state on connect before + issuing new commands. +- Seed preloaded positions and confirm position snapshots, valuation, and PnL agree with the venue prior to trading. + +--- + +## Data testing spec + +See the full [Data Testing Spec](spec_data_testing.md) for the `DataTester` test matrix. + +--- + +## Execution testing spec + +See the full [Execution Testing Spec](spec_exec_testing.md) for the `ExecTester` test matrix. diff --git a/nautilus-docs/docs/developer_guide/benchmarking.md b/nautilus-docs/docs/developer_guide/benchmarking.md new file mode 100644 index 0000000..e450a47 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/benchmarking.md @@ -0,0 +1,180 @@ +# Benchmarking + +This guide explains how NautilusTrader measures Rust performance, when to +use each tool and the conventions you should follow when adding new benches. + +--- + +## Tooling overview + +NautilusTrader relies on **two complementary benchmarking frameworks**: + +| Framework | What is it? | What it measures | When to prefer it | +|-----------|-------------|------------------|-------------------| +| [**Criterion**](https://docs.rs/criterion/latest/criterion/) | Statistical benchmark harness that produces detailed HTML reports and performs outlier detection. | Wall-clock run time with confidence intervals. | End-to-end scenarios, anything slower than ≈100 ns, visual comparisons. | +| [**iai**](https://docs.rs/iai/latest/iai/) | Deterministic micro-benchmark harness that counts retired CPU instructions via hardware counters. | Exact instruction counts (noise-free). | Ultra-fast functions, CI gating via instruction diff. | + +Most hot code paths benefit from **both** kinds of measurements. + +:::note +iai is deterministic (immune to system noise) but results are machine-specific. Use it for regression detection within CI, not for cross-machine comparisons. +::: + +--- + +## Directory layout + +Each crate keeps its performance tests in a local `benches/` folder: + +```text +crates// +└── benches/ + ├── foo_criterion.rs # Criterion group(s) + └── foo_iai.rs # iai micro benches +``` + +`Cargo.toml` must list every benchmark explicitly so `cargo bench` discovers +them: + +```toml +[[bench]] +name = "foo_criterion" +path = "benches/foo_criterion.rs" +harness = false + +[[bench]] +name = "foo_iai" +path = "benches/foo_iai.rs" +harness = false +``` + +--- + +## Writing Criterion benchmarks + +1. Perform **all expensive set-up outside** the timing loop (`b.iter`). +2. Wrap inputs/outputs in `black_box` to prevent the optimizer from removing + work. +3. Group related cases with `benchmark_group!` and set `throughput` or + `sample_size` when the defaults aren’t ideal. + +```rust +use std::hint::black_box; + +use criterion::{Criterion, criterion_group, criterion_main}; + +fn bench_my_algo(c: &mut Criterion) { + let data = prepare_data(); // Heavy set-up done once + + c.bench_function("my_algo", |b| { + b.iter(|| my_algo(black_box(&data))); + }); +} + +criterion_group!(benches, bench_my_algo); +criterion_main!(benches); +``` + +--- + +## Writing iai benchmarks + +`iai` requires functions that take **no parameters** and return a value (which +can be ignored). Keep them as small as possible so the measured instruction +count is meaningful. + +```rust +use std::hint::black_box; + +fn bench_add() -> i64 { + let a = black_box(123); + let b = black_box(456); + a + b +} + +iai::main!(bench_add); +``` + +--- + +## Running benches locally + +- **Single crate**: `cargo bench -p nautilus-core`. +- **Single benchmark module**: `cargo bench -p nautilus-core --bench time`. +- **CI performance benches**: `make cargo-ci-benches` (runs the crates included + in the CI performance workflow one at a time to avoid the mixed-panic-strategy + linker issue). + +Criterion writes HTML reports to `target/criterion/`; open `target/criterion/report/index.html` in your browser. + +### Generating a flamegraph + +`cargo-flamegraph` lets you see a sampled call-stack profile of a single +benchmark. On Linux it uses `perf`, and on macOS it uses `DTrace`. + +1. Install `cargo-flamegraph` once per machine (it installs a `cargo flamegraph` + subcommand automatically). + + ```bash + cargo install flamegraph + ``` + +2. Run a specific bench with the symbol-rich `bench` profile. + + ```bash + # example: the matching benchmark in nautilus-common + cargo flamegraph --bench matching -p nautilus-common --profile bench + ``` + +3. Open the generated `flamegraph.svg` in your browser and zoom into hot paths. + +#### Linux + +On Linux, `perf` must be available. On Debian/Ubuntu, you can install it with: + +```bash +sudo apt install linux-tools-common linux-tools-$(uname -r) +``` + +If you see an error mentioning `perf_event_paranoid` you need to relax the +kernel’s perf restrictions for the current session (root required): + +```bash +sudo sh -c 'echo 1 > /proc/sys/kernel/perf_event_paranoid' +``` + +A value of `1` is typically enough; set it back to `2` (default) or make +the change permanent via `/etc/sysctl.conf` if desired. + +#### macOS + +On macOS, `DTrace` requires root permissions, so you must run `cargo flamegraph` +with `sudo`. + +:::warning +Running with `sudo` creates files in `target/` owned by root, causing permission errors with subsequent `cargo` commands. You may need to remove root-owned files manually or run `sudo cargo clean`. +::: + +```bash +sudo cargo flamegraph --bench matching -p nautilus-common --profile bench +``` + +Because `[profile.bench]` keeps full debug symbols the SVG will show readable +function names without bloating production binaries (which still use +`panic = "abort"` and are built via `[profile.release]`). + +> **Note** Benchmark binaries are compiled with the custom `[profile.bench]` +> defined in the workspace `Cargo.toml`. That profile inherits from +> `release-debugging`, preserving full optimisation *and* debug symbols so that +> tools like `cargo flamegraph` or `perf` produce human-readable stack traces. + +--- + +## Templates + +Ready-to-copy starter files live in [`docs/dev_templates/`](../dev_templates/): + +- **Criterion**: [`criterion_template.rs`](../dev_templates/criterion_template.rs) +- **iai**: [`iai_template.rs`](../dev_templates/iai_template.rs) + +Copy the template into `benches/`, adjust imports and names, and start measuring! diff --git a/nautilus-docs/docs/developer_guide/coding_standards.md b/nautilus-docs/docs/developer_guide/coding_standards.md new file mode 100644 index 0000000..58456d4 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/coding_standards.md @@ -0,0 +1,141 @@ +# Coding Standards + +## Code Style + +The current codebase can be used as a guide for formatting conventions. +Additional guidelines are provided below. + +### Universal formatting rules + +The following applies to **all** source files (Rust, Python, Cython, shell, etc.): + +- Use **spaces only**, never hard tab characters. +- Lines should generally stay below **100 characters**; wrap thoughtfully when necessary. +- Prefer American English spelling (`color`, `serialize`, `behavior`). + +### Shell script portability + +Shell scripts in this repository use **bash** (not POSIX sh) and must be portable across **Linux** and **macOS**. User-facing scripts (e.g., `scripts/cli/install.sh`) must also work on **Windows** via Git Bash or WSL. + +**Shebang**: Always use `#!/usr/bin/env bash` for portability. + +**Common pitfalls**: GNU and BSD utilities differ between Linux and macOS: + +| Command | Linux (GNU) | macOS (BSD) | Portable solution | +|----------------------|-------------------|-------------------|------------------------------------------| +| `sed -i` | `sed -i 's/…'` | `sed -i '' 's/…'` | Use backup extension: `sed -i.bak 's/…'` | +| `stat` (file size) | `stat -c%s file` | `stat -f%z file` | Detect with `stat --version` | +| `sha256sum` | `sha256sum file` | N/A | Use `shasum -a 256` or detect | +| `readlink -f` | Works | N/A | Avoid, or use `realpath` | +| `grep -P` (PCRE) | Works | N/A | Use `-E` (extended regex) instead | +| `date` (nanoseconds) | `date +%N` | N/A | Use `$RANDOM` for cache-busting | + +**Bash version**: macOS ships with bash 3.2; avoid bash 4+ features in user-facing scripts: + +| Feature | Bash version | Alternative | +|-----------------------------------|--------------|----------------------------------| +| Associative arrays (`declare -A`) | 4.0+ | Use files or simple arrays | +| `readarray` / `mapfile` | 4.0+ | Use `while read` loops | +| `${var,,}` / `${var^^}` (case) | 4.0+ | Use `tr '[:upper:]' '[:lower:]'` | + +**CI scripts** (`scripts/ci/*`) run on Linux runners, so bash 4+ and GNU tools are acceptable there. + +### Comment conventions + +1. Generally leave **one blank line above** every comment block or docstring so it is visually separated from code. +2. Use *sentence case* – capitalize the first letter, keep the rest lowercase unless proper nouns or acronyms. +3. Do not use double spaces after periods. +4. **Single-line comments** *must not* end with a period *unless* the line ends with a URL or inline Markdown link – in those cases leave the punctuation exactly as the link requires. +5. **Multi-line comments** should separate sentences with commas (not period-per-line). The final line *should* end with a period. +6. Keep comments concise; favor clarity and only explain the non-obvious – *less is more*. +7. Avoid emoji symbols in text. + +### Doc comment mood + +**Rust** doc comments should be written in the **indicative mood** – e.g. *"Returns a cached client."* + +This convention aligns with the prevailing style of the Rust ecosystem and makes generated +documentation feel natural to end-users. + +### Terminology and phrasing + +1. **Error messages**: Avoid using ", got" in error messages. Use more descriptive alternatives like ", was", ", received", or ", found" depending on context. + - ❌ `"Expected string, got {type(value)}"` + - ✅ `"Expected string, was {type(value)}"` + +2. **Spelling**: Use "hardcoded" (single word) rather than "hard-coded" or "hard coded" – this is the more modern and accepted spelling. + +3. **Error variable naming**: Use single-letter `e` for caught errors/exceptions: + - Rust: `Err(e)` not `Err(err)` or `Err(error)`, and `|e|` not `|err|` in closures + - Python: `except SomeError as e:` not `as err:` or `as error:` + +### Naming conventions + +1. **Internal fields**: Abbreviations are acceptable for private/internal fields (e.g., `_price_prec`, `_size_prec`) to keep hot-path code concise. + +2. **User-facing API**: Use full, descriptive names for public properties, function parameters, return types, and metric names/labels (e.g., `price_precision`, `size_precision`). This prevents abbreviated terminology from leaking into dashboards or alerts. + +3. **Error messages and logs**: Use full words for clarity (e.g., "price precision" not "price prec"). The user should never see abbreviated terminology. + +### Formatting + +1. For longer lines of code, and when passing more than a couple of arguments, you should take a new line which aligns at the next logical indent (rather than attempting a hanging 'vanity' alignment off an opening parenthesis). This practice conserves space to the right, keeps important code more central in view, and survives function/method name changes. + +2. The closing parenthesis should be located on a new line, aligned at the logical indent. + +3. Multiple hanging parameters or arguments should end with a trailing comma: + +```python +long_method_with_many_params( + some_arg1, + some_arg2, + some_arg3, # <-- trailing comma +) +``` + +## Commit messages + +Here are some guidelines for the style of your commit messages: + +1. Limit subject titles to 60 characters or fewer. Capitalize subject line and do not end with period. + +2. Use 'imperative voice', i.e. the message should describe what the commit will do if applied. + +3. Optional: Use the body to explain change. Separate from subject with a blank line. Keep under 100 character width. You can use bullet points with or without terminating periods. + +4. Optional: Provide # references to relevant issues or tickets. + +5. Optional: Provide any hyperlinks which are informative. + +### Gitlint (optional) + +Gitlint is available to help enforce commit message standards automatically. It checks that commit messages follow the guidelines above (character limits, formatting, etc.). This is **opt-in** and not enforced in CI. + +**Benefits**: Encourages concise yet expressive commit messages, helps develop clear explanations of changes. + +**Installation**: First install gitlint to run it locally: + +```bash +uv pip install gitlint +``` + +To enable gitlint as an automatic commit-msg hook: + +```bash +pre-commit install --hook-type commit-msg +``` + +**Manual usage**: Check your last commit message: + +```bash +gitlint +``` + +Configuration is in `.gitlint` at the repository root: + +- **60-character title limit**: Ensures clear rendering on GitHub and encourages brevity while remaining descriptive. +- **79-character body width**: Aligns with Python's PEP 8 conventions and the traditional limit for git tooling. + +:::note +Gitlint may be enforced in CI in the future, so adopting these practices early eases the transition. +::: diff --git a/nautilus-docs/docs/developer_guide/docs.md b/nautilus-docs/docs/developer_guide/docs.md new file mode 100644 index 0000000..5f3d83c --- /dev/null +++ b/nautilus-docs/docs/developer_guide/docs.md @@ -0,0 +1,134 @@ +# Docs Style + +This guide outlines the style conventions and best practices for writing documentation for NautilusTrader. + +## General principles + +- We favor simplicity over complexity, less is more. +- We favor concise yet readable prose and documentation. +- We value standardization in conventions, style, patterns, etc. +- Documentation should be accessible to users of varying technical backgrounds. + +## Language and tone + +- Use active voice when possible ("Configure the adapter" vs "The adapter should be configured"). +- Write in present tense for describing current functionality. +- Use future tense only for planned features. +- Avoid unnecessary jargon; define technical terms on first use. +- Be direct and concise; avoid filler words like "basically", "simply", "just". +- Use parallel structure in lists; keep grammatical patterns consistent across items. + +## Markdown tables + +### Column alignment and spacing + +- Use symmetrical column widths based on the space dictated by the widest content in each column. +- Align column separators (`|`) vertically for better readability. +- Use consistent spacing around cell content. + +### Notes and descriptions + +- All notes and descriptions should have terminating periods. +- Keep notes concise but informative. +- Use sentence case (capitalize only the first letter and proper nouns). + +### Example + +```markdown +| Order Type | Spot | Margin | USDT Futures | Coin Futures | Notes | +|------------------------|------|--------|--------------|--------------|-------------------------| +| `MARKET` | ✓ | ✓ | ✓ | ✓ | | +| `STOP_MARKET` | - | ✓ | ✓ | ✓ | Not supported for Spot. | +| `MARKET_IF_TOUCHED` | - | - | ✓ | ✓ | Futures only. | +``` + +### Support indicators + +- Use `✓` for supported features. +- Use `-` for unsupported features (not `✗` or other symbols). +- When adding notes for unsupported features, emphasize with italics: `*Not supported*`. +- Leave cells empty when no content is needed. + +## Code references + +- Use backticks for inline code, method names, class names, and configuration options. +- Use code blocks for multi-line examples. +- When referencing code locations, use `file_path::function_name` or `file_path::ClassName` rather than line numbers, which become stale as code changes. + +## Headings + +We follow modern documentation conventions that prioritize readability and accessibility: + +- Use title case for the main page heading (# Level 1 only). +- Use sentence case for all subheadings (## Level 2 and below). +- Always capitalize proper nouns regardless of heading level (product names, technologies, companies, acronyms). +- Use proper heading hierarchy (don't skip levels). + +This convention aligns with industry standards used by major technology companies including Google Developer Documentation, Microsoft Docs, and Anthropic's documentation. +It improves readability, reduces cognitive load, and is more accessible for international users and screen readers. + +### Examples + +```markdown +# NautilusTrader Developer Guide + +## Getting started with Python +## Using the Binance adapter +## REST API implementation +## WebSocket data streaming +## Testing with pytest +``` + +## Lists + +- Use hyphens (`-`) for unordered list bullets; avoid `*` or `+` to keep the Markdown style consistent across the project. +- Use numbered lists only when order matters. +- Maintain consistent indentation for nested lists. +- End list items with periods when they are complete sentences. + +## Links and references + +- Use descriptive link text (avoid "click here" or "this link"). +- Reference external documentation when appropriate. +- Keep all internal links relative and accurate. + +## Technical terminology + +- Base capability matrices on the Nautilus domain model, not exchange-specific terminology. +- Mention exchange-specific terms in parentheses or notes when necessary for clarity. +- Use consistent terminology throughout the documentation. + +## Examples and code samples + +- Provide practical, working examples. +- Include necessary imports and context. +- Use realistic variable names and values. +- Add comments to explain non-obvious parts of examples. + +## Admonitions + +Use admonition blocks to highlight important information: + +| Admonition | Purpose | +|--------------|---------------------------------------------------------------| +| `:::note` | Supplementary context that clarifies but isn't essential. | +| `:::info` | Important information the reader should be aware of. | +| `:::tip` | Helpful suggestions or best practices. | +| `:::warning` | Potential pitfalls or important caveats. | +| `:::danger` | Critical issues that could cause data loss or system failure. | + +Avoid overusing admonitions; too many diminish their impact. + +## Line length and wrapping + +- Wrap lines at no more than ~100-120 characters for better readability and diff reviews. +- Break long sentences at natural points (after commas, conjunctions, or phrases). +- Avoid orphaned words on new lines when possible. +- Code blocks and URLs can exceed the line limit when necessary. + +## API documentation + +- Document parameters and return types clearly. +- Include usage examples for complex APIs. +- Explain any side effects or important behavior. +- Keep parameter descriptions concise but complete. diff --git a/nautilus-docs/docs/developer_guide/environment_setup.md b/nautilus-docs/docs/developer_guide/environment_setup.md new file mode 100644 index 0000000..ce9709f --- /dev/null +++ b/nautilus-docs/docs/developer_guide/environment_setup.md @@ -0,0 +1,399 @@ +# Environment Setup + +For development we recommend using the PyCharm *Professional* edition IDE, as it interprets Cython syntax. Alternatively, you could use Visual Studio Code with a Cython extension. + +[uv](https://docs.astral.sh/uv) is the preferred tool for handling all Python virtual environments and dependencies. + +[pre-commit](https://pre-commit.com/) is used to automatically run various checks, auto-formatters and linting tools at commit. + +NautilusTrader uses increasingly more [Rust](https://www.rust-lang.org), so Rust should be installed on your system as well +([installation guide](https://www.rust-lang.org/tools/install)). + +[Cap'n Proto](https://capnproto.org/) is required for serialization schema compilation. The required +version is specified in the `capnp-version` file in the repository root. Ubuntu's default package +is typically too old, so you may need to install from source (see below). + +:::info +NautilusTrader *must* compile and run on **Linux, macOS, and Windows**. Please keep portability in +mind (use `std::path::Path`, avoid Bash-isms in shell scripts, etc.). +::: + +## Setup + +The following steps are for UNIX-like systems, and only need to be completed once. + +1. Follow the [installation guide](../getting_started/installation.md) to set up the project with a modification to the final command to install development and test dependencies: + +```bash +uv sync --active --all-groups --all-extras +``` + +or + +```bash +make install +``` + +If you're developing and iterating frequently, then compiling in debug mode is often sufficient and *significantly* faster than a fully optimized build. +To install in debug mode, use: + +```bash +make install-debug +``` + +2. Set up the pre-commit hook which will then run automatically at commit: + +```bash +pre-commit install +``` + +Before opening a pull-request run the formatting and lint suite locally so that CI passes on the +first attempt: + +```bash +make format +make pre-commit +``` + +Make sure the Rust compiler reports **zero errors** – broken builds slow everyone down. + +3. **Required for Rust/PyO3 (Linux and macOS)**: When using Python installed via `uv` on Linux or macOS, set the following environment variables: + +```bash +# Add to your shell configuration (e.g., ~/.zshrc or ~/.bashrc) + +# Linux only: Set the library path for the Python interpreter +export LD_LIBRARY_PATH="$(python -c 'import sys; print(sys.base_prefix)')/lib:$LD_LIBRARY_PATH" + +# Set the Python executable path for PyO3 +export PYO3_PYTHON=$(pwd)/.venv/bin/python + +# Set the Python home path (required for Rust tests) +export PYTHONHOME=$(python -c "import sys; print(sys.base_prefix)") +``` + +:::note +The `LD_LIBRARY_PATH` export is Linux-specific and not needed on macOS or Windows. + +- `PYO3_PYTHON` tells PyO3 which Python interpreter to use, reducing unnecessary recompilation. +- `PYTHONHOME` is required when running `make cargo-test` with a `uv`-installed Python. + Without it, tests that depend on PyO3 may fail to locate the Python runtime. + +::: + +To verify your environment is configured correctly: + +```bash +python -c "import sys; print('Python:', sys.executable, sys.version)" +echo "PYO3_PYTHON: $PYO3_PYTHON" +echo "PYTHONHOME: $PYTHONHOME" +``` + +## Builds + +Following any changes to `.rs`, `.pyx` or `.pxd` files, you can re-compile by running: + +```bash +uv run --no-sync python build.py +``` + +or + +```bash +make build +``` + +If you're developing and iterating frequently, then compiling in debug mode is often sufficient and *significantly* faster than a fully optimized build. +To compile in debug mode, use: + +```bash +make build-debug +``` + +## Cap'n Proto + +[Cap'n Proto](https://capnproto.org/) is required for serialization schema compilation. +The required version is defined in the `capnp-version` file in the repository root. + +The recommended way to install the correct version on **Linux** or **macOS** is: + +```bash +./scripts/install-capnp.sh +``` + +This script ensures the pinned version is installed. Alternatively, you can install manually: + +On **macOS**, install via Homebrew: + +```bash +brew install capnp +``` + +Verify the installed version matches `capnp-version`. If Homebrew provides an older version, +install from source using the Linux instructions below. + +On **Ubuntu/Linux**, the default package is typically too old. Install from source: + +```bash +CAPNP_VERSION=$(cat capnp-version) +cd ~ +wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz +tar xzf capnproto-c++-${CAPNP_VERSION}.tar.gz +cd capnproto-c++-${CAPNP_VERSION} +./configure +make -j$(nproc) +sudo make install +sudo ldconfig +``` + +Verify installation: + +```bash +capnp --version +``` + +On **Windows**, install via Chocolatey: + +```bash +choco install capnproto +``` + +Verify the installed version matches `capnp-version`. If Chocolatey provides an older version, +see the [Cap'n Proto installation guide](https://capnproto.org/install.html) for alternative +installation methods. + +## Faster builds + +The cranelift backends reduces build time significantly for dev, testing and IDE checks. However, cranelift is available on the nightly toolchain and needs extra configuration. Install the nightly toolchain + +``` +rustup install nightly +rustup override set nightly +rustup component add rust-analyzer # install nightly lsp +rustup override set stable # reset to stable +``` + +Activate the nightly feature and use "cranelift" backend for dev and testing profiles in workspace `Cargo.toml`. You can apply the below patch using `git apply `. You can remove it using `git apply -R ` before pushing changes. + +:::warning +Do not commit these changes. The cranelift patch is for local development only and will break CI if pushed. +::: + +``` +diff --git a/Cargo.toml b/Cargo.toml +index 62b78cd8d0..beb0800211 100644 +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -1,3 +1,6 @@ ++# This line needs to come before anything else in Cargo.toml ++cargo-features = ["codegen-backend"] ++ + [workspace] + resolver = "2" + members = [ +@@ -140,6 +143,7 @@ lto = false + panic = "unwind" + incremental = true + codegen-units = 256 ++codegen-backend = "cranelift" + + [profile.test] + opt-level = 0 +@@ -150,11 +154,13 @@ strip = false + lto = false + incremental = true + codegen-units = 256 ++codegen-backend = "cranelift" + + [profile.nextest] + inherits = "test" + debug = false # Improves compile times + strip = "debuginfo" # Improves compile times ++codegen-backend = "cranelift" + + [profile.release] + opt-level = 3 +``` + +Pass `RUSTUP_TOOLCHAIN=nightly` when running `make build-debug` like commands and include it in all [rust analyzer settings](#rust-analyzer-settings) for faster builds and IDE checks. + +## Services + +You can use `docker-compose.yml` file located in `.docker` directory +to bootstrap the Nautilus working environment. This will start the following services: + +```bash +docker-compose up -d +``` + +If you only want specific services running (like `postgres` for example), you can start them with command: + +```bash +docker-compose up -d postgres +``` + +Used services are: + +- `postgres`: Postgres database with root user `POSTGRES_USER` which defaults to `postgres`, `POSTGRES_PASSWORD` which defaults to `pass` and `POSTGRES_DB` which defaults to `postgres`. +- `redis`: Redis server. +- `pgadmin`: PgAdmin4 for database management and administration. + +:::info +Please use this as development environment only. For production, use a proper and more secure setup. +::: + +After the services has been started, you must log in with `psql` cli to create `nautilus` Postgres database. +To do that you can run, and type `POSTGRES_PASSWORD` from docker service setup + +```bash +psql -h localhost -p 5432 -U postgres +``` + +After you have logged in as `postgres` administrator, run `CREATE DATABASE` command with target db name (we use `nautilus`): + +``` +psql (16.2, server 15.2 (Debian 15.2-1.pgdg110+1)) +Type "help" for help. + +postgres=# CREATE DATABASE nautilus; +CREATE DATABASE + +``` + +## Nautilus CLI developer guide + +## Introduction + +The Nautilus CLI is a command-line interface tool for interacting with the NautilusTrader ecosystem. +It offers commands for managing the PostgreSQL database and handling various trading operations. + +:::warning +On Linux systems with GNOME desktop, the `nautilus` command typically refers to the GNOME file manager (`/usr/bin/nautilus`). +After installing the NautilusTrader CLI, you may need to ensure the Cargo binary takes precedence by either: + +- Adding an alias to your shell config: `alias nautilus="$HOME/.cargo/bin/nautilus"` +- Using the full path: `~/.cargo/bin/nautilus` +- Ensuring `~/.cargo/bin` appears before `/usr/bin` in your `PATH` + +::: + +:::note +The Nautilus CLI command is only supported on UNIX-like systems. +::: + +## Install + +You can install the Nautilus CLI using the below Makefile target, which uses `cargo install` under the hood. +This will place the nautilus binary in your system's PATH, assuming Rust's `cargo` is properly configured. + +```bash +make install-cli +``` + +## Commands + +You can run `nautilus --help` to view the CLI structure and available command groups: + +### Database + +These commands handle bootstrapping the PostgreSQL database. +To use them, you need to provide the correct connection configuration, +either through command-line arguments or a `.env` file located in the root directory or the current working directory. + +- `--host` or `POSTGRES_HOST` for the database host +- `--port` or `POSTGRES_PORT` for the database port +- `--user` or `POSTGRES_USERNAME` for the root administrator (typically the postgres user) +- `--password` or `POSTGRES_PASSWORD` for the root administrator's password +- `--database` or `POSTGRES_DATABASE` for both the database **name and the new user** with privileges to that database + (e.g., if you provide `nautilus` as the value, a new user named nautilus will be created with the password from `POSTGRES_PASSWORD`, and the `nautilus` database will be bootstrapped with this user as the owner). + +Example of `.env` file + +``` +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_USERNAME=postgres +POSTGRES_PASSWORD=pass +POSTGRES_DATABASE=nautilus +``` + +List of commands are: + +1. `nautilus database init`: Will bootstrap schema, roles and all sql files located in `schema` root directory (like `tables.sql`). +2. `nautilus database drop`: Will drop all tables, roles and data in target Postgres database. + +## Rust analyzer settings + +Rust analyzer is a popular language server for Rust and has integrations for many IDEs. It is recommended to configure rust analyzer to have same environment variables as `make build-debug` for faster compile times. Below tested configurations for VSCode and Astro Nvim are provided. For more information see [PR](https://github.com/nautechsystems/nautilus_trader/pull/2524) or rust analyzer [config docs](https://rust-analyzer.github.io/book/configuration.html). + +### VSCode + +You can add the following settings to your VSCode `settings.json` file: + +``` + "rust-analyzer.restartServerOnConfigChange": true, + "rust-analyzer.linkedProjects": [ + "Cargo.toml" + ], + "rust-analyzer.cargo.features": "all", + "rust-analyzer.check.workspace": false, + "rust-analyzer.check.extraEnv": { + "VIRTUAL_ENV": "/.venv", + "CC": "clang", + "CXX": "clang++" + }, + "rust-analyzer.cargo.extraEnv": { + "VIRTUAL_ENV": "/.venv", + "CC": "clang", + "CXX": "clang++" + }, + "rust-analyzer.runnables.extraEnv": { + "VIRTUAL_ENV": "/.venv", + "CC": "clang", + "CXX": "clang++" + }, + "rust-analyzer.check.features": "all", + "rust-analyzer.testExplorer": true +``` + +### Astro Nvim (Neovim + AstroLSP) + +You can add the following to your astro lsp config file: + +``` + config = { + rust_analyzer = { + settings = { + ["rust-analyzer"] = { + restartServerOnConfigChange = true, + linkedProjects = { "Cargo.toml" }, + cargo = { + features = "all", + extraEnv = { + VIRTUAL_ENV = "/.venv", + CC = "clang", + CXX = "clang++", + }, + }, + check = { + workspace = false, + command = "check", + features = "all", + extraEnv = { + VIRTUAL_ENV = "/.venv", + CC = "clang", + CXX = "clang++", + }, + }, + runnables = { + extraEnv = { + VIRTUAL_ENV = "/.venv", + CC = "clang", + CXX = "clang++", + }, + }, + testExplorer = true, + }, + }, + }, +``` diff --git a/nautilus-docs/docs/developer_guide/ffi.md b/nautilus-docs/docs/developer_guide/ffi.md new file mode 100644 index 0000000..6366708 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/ffi.md @@ -0,0 +1,142 @@ +# FFI Memory Contract + +NautilusTrader exposes several **C-compatible** types so that compiled Rust code can be +consumed from C-extensions generated by Cython or by other native languages. The most +important of these is `CVec` – a *thin* wrapper around a Rust `Vec` that is passed across +the FFI boundary **by value**. + +The rules below are *strict*; violating them results in undefined behaviour (usually a double-free or a memory leak). + +## Fail-fast panics at the FFI boundary + +Rust panics must never unwind across `extern "C"` functions. Unwinding into C or Python is +undefined behaviour and can corrupt the foreign stack or leave partially-dropped resources +behind. To enforce the fail-fast architecture we wrap every exported symbol in +`crate::ffi::abort_on_panic`, which executes the body and calls `process::abort()` if a panic +occurs. The panic message is still logged before the abort, so debugging output is preserved +while avoiding undefined behaviour. + +When adding new FFI functions, call `abort_on_panic(|| { … })` around the implementation (or +use a helper that does so) to maintain this guarantee. + +## CVec lifecycle + +| Step | Owner | Action | +|-------|-------------------------------|--------| +| **1** | Rust | Build a `Vec` and convert it with `into()` – this *leaks* the vector and transfers ownership of the raw allocation to foreign code. | +| **2** | Foreign (Python / Cython / C) | Use the data while the `CVec` value is in scope. **Do not modify the fields `ptr`, `len`, `cap`.** | +| **3** | Foreign | Exactly once, call the *type-specific* drop helper exported by Rust (for example `vec_drop_book_levels`, `vec_drop_book_orders`, `vec_time_event_handlers_drop`). The helper reconstructs the original `Vec` with `Vec::from_raw_parts` and lets it drop, freeing the memory. | + +:::warning +If step **3** is forgotten the allocation is leaked for the remainder of the process; if it +is performed **twice** the program will double-free and likely crash. +::: + +## Capsules created on the Python side + +Several Cython helpers allocate temporary C buffers with `PyMem_Malloc`, wrap them into a +`CVec`, and return the address inside a `PyCapsule`. **Every such capsule is created with a +destructor** (`capsule_destructor` or `capsule_destructor_deltas`) that frees both the buffer +and the `CVec`. Callers must therefore *not* free the memory manually – doing so would double +free. + +## Capsules created on the Rust side *(PyO3 bindings)* + +When Rust code pushes a heap-allocated value into Python it **must** use +`PyCapsule::new_with_destructor` so that Python knows how to free the allocation +once the capsule becomes unreachable. The closure/destructor is responsible +for reconstructing the original `Box` or `Vec` and letting it drop. + +```rust +use pyo3::types::PyCapsule; + +Python::attach(|py| { + // Allocate the value on the heap + let my_data = Box::new(MyStruct::new()); + let ptr = Box::into_raw(my_data); + + // Move it into the capsule and register a destructor that frees the memory + let capsule = PyCapsule::new_with_destructor( + py, + ptr, + None, + |ptr, _| { + // Reconstruct the Box and let it drop, freeing the allocation + let _ = unsafe { Box::from_raw(ptr) }; + }, + ) + .expect("capsule creation failed"); + + // ... pass `capsule` back to Python ... +}); +``` + +Do **not** use `PyCapsule::new(…, None)`; that variant registers *no* destructor +and will leak memory unless the recipient manually extracts and frees the +pointer (something we never rely on). The codebase has been updated to follow +this rule everywhere – adding new FFI modules must follow the same pattern. + +## Why there is no generic `cvec_drop` anymore + +Earlier versions of the codebase shipped a generic `cvec_drop` function that always treated the +buffer as `Vec`. Using it with any other element type causes a size-mismatch during +deallocation and corrupts the allocator's bookkeeping. Because the helper was not referenced +anywhere inside the project it has been removed to avoid accidental misuse. + +Instead, use the **type-specific** drop helper for your element type (e.g., `vec_drop_book_levels`, +`vec_drop_book_orders`). If no helper exists for your type, add one following the pattern in +`crates/core/src/ffi/cvec.rs`. + +## Box-backed `*_API` wrappers (owned Rust objects) + +When the Rust core needs to hand a *complex* value (for example an +`OrderBook`, `SyntheticInstrument`, or `TimeEventAccumulator`) to foreign +code it allocates the value on the heap with `Box::new` and returns a +small `repr(C)` wrapper whose only field is that `Box`. + +```rust +#[repr(C)] +pub struct OrderBook_API(Box); + +#[unsafe(no_mangle)] +pub extern "C" fn orderbook_new(id: InstrumentId, book_type: BookType) -> OrderBook_API { + OrderBook_API(Box::new(OrderBook::new(id, book_type))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn orderbook_drop(book: OrderBook_API) { + drop(book); // frees the heap allocation +} +``` + +Memory-safety requirements are therefore: + +1. Every constructor (`*_new`) **must** have a matching `*_drop` exported + next to it. +2. Validate parameters before heap allocation to fail fast and avoid allocating invalid objects. +3. The *Python/Cython* binding must guarantee that `*_drop` is invoked + exactly once. Two approaches exist: + + • **Preferred for new code**: Wrap the pointer in a `PyCapsule` created with + `PyCapsule::new_with_destructor`, passing a destructor that calls + the drop helper. + + • **Legacy pattern** (v1 Cython modules only): Call the helper explicitly in + `__del__`/`__dealloc__` on the Python side: + + ```python + cdef class OrderBook: + cdef OrderBook_API _mem + + def __cinit__(self, ...): + self._mem = orderbook_new(...) + + def __del__(self): + if self._mem._0 != NULL: + orderbook_drop(self._mem) + ``` + +Whichever style is used, remember: **forgetting the drop call leaks the +entire structure**, while calling it twice will double-free and crash. + +New FFI code must use `PyCapsule` with destructors and follow this template before it can be merged. diff --git a/nautilus-docs/docs/developer_guide/index.md b/nautilus-docs/docs/developer_guide/index.md new file mode 100644 index 0000000..6bf1c4f --- /dev/null +++ b/nautilus-docs/docs/developer_guide/index.md @@ -0,0 +1,27 @@ +# Developer Guide + +Guidance on developing and extending NautilusTrader, or contributing back to the project. + +NautilusTrader uses a **Rust core with Python bindings** architecture: + +- **Rust** handles networking, data parsing, order matching, and other performance-critical operations. +- **Python** provides the user-facing API for strategy development, configuration, and system integration. +- **PyO3** bridges the two, exposing Rust functionality to Python with minimal overhead. + +This approach combines Python's simplicity and ecosystem with Rust's performance and memory safety. + +## Contents + +- [Environment Setup](environment_setup.md) +- [Coding Standards](coding_standards.md) +- [Rust](rust.md) +- [Python](python.md) +- [Testing](testing.md) +- [Test Datasets](test_datasets.md) +- [Docs Style](docs.md) +- [Release Notes](releases.md) +- [Adapters](adapters.md) +- [Data Testing Spec](spec_data_testing.md) +- [Execution Testing Spec](spec_exec_testing.md) +- [Benchmarking](benchmarking.md) +- [FFI Memory Contract](ffi.md) diff --git a/nautilus-docs/docs/developer_guide/python.md b/nautilus-docs/docs/developer_guide/python.md new file mode 100644 index 0000000..466a2b3 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/python.md @@ -0,0 +1,98 @@ +# Python + +The [Python](https://www.python.org/) programming language is used for the majority of user-facing code in NautilusTrader. +Python provides a rich ecosystem of libraries and frameworks, making it ideal for strategy development, data analysis, and system integration. + +## Code style + +### PEP-8 + +The codebase generally follows the PEP-8 style guide. +One notable departure is that Python truthiness is not always taken advantage of to check if an argument is `None` for everything other than collections. + +As per the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), it's discouraged to use truthiness to check if an argument is/is not `None`, when there is a chance an unexpected object could be passed into the function or method which will yield an unexpected truthiness evaluation (which could result in a logical error type bug). + +*"Always use if foo is None: (or is not None) to check for a None value. E.g., when testing whether a variable or argument that defaults to None was set to some other value. The other value might be a value that's false in a boolean context!"* + +:::note +Use truthiness to check for empty collections (e.g., `if not my_list:`) rather than comparing explicitly to `None` or empty. +::: + +We welcome all feedback on where the codebase departs from PEP-8 for no apparent reason. + +### Type hints + +All function and method signatures *must* include type annotations: + +```python +def __init__(self, config: EMACrossConfig) -> None: +def on_bar(self, bar: Bar) -> None: +def on_save(self) -> dict[str, bytes]: +def on_load(self, state: dict[str, bytes]) -> None: +``` + +**Union syntax**: Use PEP 604 union syntax for optional types: + +```python +# Preferred +def get_instrument(self, id: InstrumentId) -> Instrument | None: + +# Avoid +def get_instrument(self, id: InstrumentId) -> Optional[Instrument]: +``` + +**Generic types**: Use `TypeVar` for reusable components: + +```python +T = TypeVar("T") +class ThrottledEnqueuer(Generic[T]): +``` + +### Docstrings + +The [NumPy docstring spec](https://numpydoc.readthedocs.io/en/latest/format.html) is used throughout the codebase. +This needs to be followed consistently so the docs build correctly. + +**Python** docstrings should be written in the **imperative mood** – e.g. *"Return a cached client."* + +This convention aligns with the prevailing style of the Python ecosystem and makes generated +documentation feel natural to end-users. + +#### Private methods + +Do not add docstrings to private methods (prefixed with `_`): + +- Docstrings generate public-facing API documentation. +- Docstrings on private methods incorrectly imply they are part of the public API. +- Private methods are implementation details not intended for end-users. + +Exceptions where docstrings are acceptable: + +- Very complex methods with non-trivial logic, multiple steps, or important edge cases. +- Methods requiring detailed parameter or return value documentation due to complexity. + +When a private method needs context (such as a tricky precondition or side effect), prefer a short inline comment (`#`) near the relevant logic rather than a docstring. + +### Test naming + +Descriptive names explaining the scenario: + +```python +def test_currency_with_negative_precision_raises_overflow_error(self): +def test_sma_with_no_inputs_returns_zero_count(self): +def test_sma_with_single_input_returns_expected_value(self): +``` + +### Ruff + +[ruff](https://astral.sh/ruff) is used to lint the codebase. Ruff rules can be found in the top-level `pyproject.toml`, with ignore justifications typically commented. + +## Cython (legacy) + +:::warning[Deprecation notice] +Cython is being phased out in favor of Rust implementations. New code should use Rust. This section documents legacy Cython code only. +::: + +For legacy `.pyx` and `.pxd` files, make sure all functions and methods returning `void` or a primitive C type (such as `bint`, `int`, `double`) include the `except *` keyword in the signature. Without it, Python exceptions are silently ignored. + +For more information, see the [Cython docs](https://cython.readthedocs.io/en/latest/index.html). diff --git a/nautilus-docs/docs/developer_guide/releases.md b/nautilus-docs/docs/developer_guide/releases.md new file mode 100644 index 0000000..df89d9a --- /dev/null +++ b/nautilus-docs/docs/developer_guide/releases.md @@ -0,0 +1,271 @@ +# Releases + +This guide covers the release process and the standards for writing release notes. + +## Overview + +NautilusTrader uses a three-branch model: + +- **`develop`**: active development; publishes dev wheels to Cloudflare R2 on every push. +- **`nightly`**: pre-release testing; publishes alpha wheels and CLI binaries. +- **`master`**: stable releases; triggers the full release pipeline. + +Pushing to `master` automatically tags the version from `pyproject.toml`, creates a GitHub +release, publishes wheels and sdist to PyPI, builds Docker images, and triggers a docs rebuild. + +## Versioning + +The project maintains two version numbers: + +| File | Scope | Example | +|--------------------------|----------------|-----------| +| `pyproject.toml` | Python package | `1.223.0` | +| `Cargo.toml` (workspace) | Rust crates | `0.53.0` | + +These are bumped independently. The Python version drives the release tag (`v1.223.0`). + +## Release checklist + +### Pre-release (on `develop`) + +- [ ] Finalize `RELEASES.md`: review all items, remove empty sections +- [ ] Ensure versions are set in `pyproject.toml` and `Cargo.toml` workspace +- [ ] Ensure all CI checks pass on `develop` + +### Release + +- [ ] Merge `develop` into `nightly`, verify nightly CI passes +- [ ] Merge `nightly` into `master` +- [ ] Verify the `build` workflow completes: + - Wheels built for Linux x86/ARM, macOS, Windows + - `cargo-deny` and `cargo-vet` pass + - Tag created and GitHub release published + - Wheels and sdist published to PyPI +- [ ] Verify the `docker` workflow completes (images built and pushed) +- [ ] Verify the `build-docs` workflow completes (docs rebuild triggered) + +### Post-release (on `develop`) + +- [ ] Update the release date in `RELEASES.md` for the published version +- [ ] Add horizontal separator `---` below the completed release +- [ ] Add the next version template at the top of `RELEASES.md` (see below) +- [ ] Bump `pyproject.toml` version to the next release number + +## Release notes + +This section documents the standards for writing release notes in `RELEASES.md`. + +### Sections + +Use the following sections in this order: + +1. Enhancements +2. Breaking Changes +3. Security +4. Fixes +5. Internal Improvements +6. Documentation Updates +7. Deprecations + +Omit sections that have no items for a given release. + +### Enhancements + +New features and user-visible improvements. + +**Format**: + +```markdown +- Added `subscribe_order_fills(...)` and `unsubscribe_order_fills(...)` for `Actor` +- Added BitMEX conditional orders support +- Added support for `OrderBookDepth10` requests (#2955), thanks @faysou +``` + +**Guidelines**: + +- Start with "Added". +- Use backticks for code elements. +- Be specific about what was added, not how. + +### Breaking Changes + +Changes that may break existing code. + +**Format**: + +```markdown +- Removed `nautilus_trader.analysis.statistics` subpackage - must import from `nautilus_trader.analysis` +- Renamed `BinanceAccountType.USDT_FUTURE` to `USDT_FUTURES` +- Changed `start` parameter to required for `Actor` data request methods +``` + +**Guidelines**: + +- Start with "Removed", "Renamed", or "Changed". +- Explain migration path briefly. + +### Security + +Security hardening and fixes that prevent crashes, undefined behavior, or data corruption. +Includes significant hardening improvements elevated from Internal Improvements. + +**Format**: + +```markdown +- Fixed non-executable stack for Cython extensions to support hardened Linux systems +- Fixed divide-by-zero and overflow bugs in model crate that could cause crashes +- Fixed core arithmetic operations to reject NaN/Infinity values and improve overflow handling +``` + +**Guidelines**: + +- Include overflow/underflow fixes, memory safety improvements, FFI guards, data integrity fixes. +- Focus on user impact: what could have happened. +- Exclude routine dependency updates, minor hardening, or test-only fixes. +- Omit this section entirely if there are no security items for the release. + +### Fixes + +Bug fixes that improve correctness but don't qualify as security issues. + +**Format**: + +```markdown +- Fixed reduce-only order panic when quantity exceeds position +- Fixed Binance order status parsing for external orders (#3006), thanks for reporting @bmlquant +``` + +**Guidelines**: + +- Start with "Fixed". + +### Internal Improvements + +Implementation details and infrastructure changes. + +**Format**: + +```markdown +- Added ARM64 support to Docker builds +- Ported `PortfolioAnalyzer` to Rust +- Improved clock and timer thread safety +- Upgraded Rust (MSRV) to 1.90.0 +- Upgraded `pyo3` crates to v0.26.0 +``` + +**Guidelines**: + +- Use "Added", "Implemented", "Improved", "Optimized", "Upgraded", "Refined", "Standardized". +- Include version numbers for dependency upgrades. + +### Documentation Updates + +Changes to guides and examples. + +**Format**: + +```markdown +- Added rate limit tables with links to official docs +- Improved dark and light themes for readability +- Fixed broken links +``` + +### Deprecations + +Features marked for removal. + +**Format**: + +```markdown +- Deprecated `convert_quote_qty_to_base`; disable (`False`) to maintain consistent behaviour. Will be removed in future version +``` + +**Guidelines**: + +- Explain migration path and provide alternatives. + +## Attribution + +- Credit external contributors: `thanks @username` or `thanks for reporting @username`. +- Include issue/PR numbers for community contributions and complex features: `(#1234)`. + +## Style + +- Use sentence case (capitalize first word only). +- Do not end with periods. +- Use backticks for code elements. +- Focus on **what** changed, not how. + +**Be specific**: + +```markdown +❌ Improved Binance adapter +✅ Improved Binance fill handling when instrument not cached +``` + +## Security classification + +Include in Security if the change addresses: + +- Memory safety (overflow, underflow, divide-by-zero that threatens stability). +- Undefined behavior or crashes that could corrupt state. +- Data integrity (NaN/Infinity propagation, race conditions leading to corruption). +- Input validation preventing injection or exploitation (SQL injection, command injection, path traversal). +- Build hardening (non-exec stack, FFI guards). +- Significant hardening that users should know about. + +Otherwise use Fixes (for logic bugs and panics) or Internal Improvements (for minor hardening). + +Note: Plain logic panics belong in Fixes unless they threaten system stability or data corruption. + +## Examples + +**Security** (could cause crashes/corruption): + +```markdown +- Fixed divide-by-zero in margin calculations that could crash the engine +- Fixed non-executable stack for Cython extensions to support hardened systems +``` + +**Fixes** (incorrect but safe): + +```markdown +- Fixed Binance order status parsing for external orders +- Fixed position purge logic to prevent purging re-opened position +``` + +**Enhancements** (user-facing): + +```markdown +- Added BitMEX conditional orders support +``` + +**Internal** (implementation): + +```markdown +- Implemented BitMEX ping/pong handling +``` + +## Release notes template + +```markdown +# NautilusTrader Beta + +Released on TBD (UTC). + +### Enhancements + +### Breaking Changes + +### Security + +### Fixes + +### Internal Improvements + +### Documentation Updates + +### Deprecations + +--- +``` diff --git a/nautilus-docs/docs/developer_guide/rust.md b/nautilus-docs/docs/developer_guide/rust.md new file mode 100644 index 0000000..1da78a3 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/rust.md @@ -0,0 +1,1265 @@ +# Rust + +The [Rust](https://www.rust-lang.org/learn) programming language is an ideal fit for implementing the mission-critical core of the platform and systems. +Its strong type system, ownership model, and compile-time checks eliminate memory errors and data races by construction, +while zero-cost abstractions and the absence of a garbage collector deliver C-like performance, important for high-frequency trading workloads. + +## Cargo manifest conventions + +- In `[dependencies]`, list internal crates (`nautilus-*`) first in alphabetical order, insert a blank line, then external required dependencies alphabetically, followed by another blank line and the optional dependencies (those with `optional = true`) in alphabetical order. Preserve inline comments with their dependency. +- Add `"python"` to every `extension-module` feature list that builds a Python artefact, keeping it adjacent to `"pyo3/extension-module"` so the full Python stack is obvious. +- When a manifest groups adapters separately (for example `crates/pyo3`), keep the `# Adapters` block immediately below the internal crate list so downstream consumers can scan adapter coverage quickly. +- Always include a blank line before `[dev-dependencies]` and `[build-dependencies]` sections. +- Apply the same layout across related manifests when the feature or dependency sets change to avoid drift between crates. +- Use snake_case filenames for `bin/` sources (for example `bin/ws_data.rs`) and reflect those paths in each `[[bin]]` section. +- Keep `[[bin]] name` entries in kebab-case (for example `name = "hyperliquid-ws-data"`) so the compiled binaries retain their intended CLI names. + +## Versioning guidance + +- Use workspace inheritance for shared dependencies (for example `serde = { workspace = true }`). +- Only pin versions directly for crate-specific dependencies that are not part of the workspace. +- Group workspace-provided dependencies before crate-only dependencies so the inheritance is easy to audit. +- Keep related dependencies aligned: `capnp`/`capnpc` (exact), `arrow`/`parquet` (major.minor), + `datafusion`/`object_store`, and `dydx-proto`/`prost`/`tonic`. Pre-commit enforces this. +- Adapter-only dependencies belong in the "Adapter dependencies" section of the workspace + `Cargo.toml`. Pre-commit prevents core crates from using them. + +## Feature flag conventions + +- Prefer additive feature flags. Enabling a feature must not break existing functionality. +- Use descriptive flag names that explain what capability is enabled. +- Document every feature in the crate-level documentation so consumers know what they toggle. +- Common patterns: + - `high-precision`: switches the value-type backing (64-bit or 128-bit integers) to support domains that require extra precision. + - `default = []`: keep defaults minimal. + - `python`: enables Python bindings. + - `extension-module`: builds a Python extension module (always include `python`). + - `ffi`: enables C FFI bindings. + - `stubs`: exposes testing stubs. + +## Build configurations + +To avoid unnecessary rebuilds during development, align cargo features, profiles, and flags across different build targets. +Cargo's build cache is keyed by the exact combination of features, profiles, and flags. Any mismatch triggers a full rebuild. + +### Aligned targets (testing and linting) + +| Target | Features | Profile | `--all-targets` | `--no-deps` | Purpose | +|-----------------------------|----------------------------------|-----------|-----------------|-------------|----------------| +| `cargo-test` | `ffi,python,high-precision,defi` | `nextest` | ✓ (implicit) | n/a | Run tests. | +| `cargo-clippy` (pre-commit) | `ffi,python,high-precision,defi` | `nextest` | ✓ | n/a | Lint all code. | + +These targets share the same feature set and profile, allowing cargo to reuse compiled artifacts between linting and testing without rebuilds. +The `nextest` profile is used to align with the workflow of the majority of core maintainers who use cargo-nextest for running tests. + +### Documentation builds + +Documentation is built separately using `make docs-rust`, which runs: + +```bash +cargo +nightly doc --all-features --no-deps --workspace +``` + +This uses the nightly toolchain and `--all-features` rather than the aligned feature set above, so it does not share build artifacts with testing/linting. + +### Separate target (Python extension building) + +| Target | Features | Profile | Notes | +|---------------|--------------------------------------|-----------|-------| +| `build` | Includes `extension-module` + subset | `release` | Requires different features for PyO3 extension module. | +| `build-debug` | Includes `extension-module` + subset | `dev` | Requires different features for PyO3 extension module. | + +Python extension building intentionally uses different features (`extension-module` is required) and will trigger rebuilds. This is expected and unavoidable. + +### Rebuild triggers to avoid + +Mismatches in any of these cause full rebuilds: + +- Different feature combinations (e.g., `--features "a,b"` vs `--features "a,c"`). +- Different `--no-default-features` usage (enables/disables default features). +- Different profiles (e.g., `dev` vs `nextest` vs `release`). + +When adding new build targets or modifying existing ones, maintain alignment with the testing/linting group to preserve fast incremental builds. + +## Module organization + +- Keep modules focused on a single responsibility. +- Use `mod.rs` as the module root when defining submodules. +- Prefer relatively flat hierarchies over deep nesting to keep paths manageable. +- Re-export commonly used items from the crate root for convenience. + +## Code style and conventions + +### File header requirements + +All Rust files must include the standardized copyright header: + +```rust +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- +``` + +:::info[Automated enforcement] +The `check_copyright_year.sh` pre-commit hook verifies copyright headers include the current year. +::: + +### Code formatting + +Import formatting is automatically handled by rustfmt when running `make format`. +The tool organizes imports into groups (standard library, external crates, local imports) and sorts them alphabetically within each group. + +Within this section, follow these spacing rules: + +- Leave **one blank line between functions** (including tests) – this improves readability and +mirrors the default behavior of `rustfmt`. +- Leave **one blank line above every doc comment** (`///` or `//!`) so that the comment is clearly + detached from the previous code block. + +#### String formatting + +Prefer inline format strings over positional arguments: + +```rust +// Preferred - inline format with variable names +anyhow::bail!("Failed to subtract {n} months from {datetime}"); + +// Instead of - positional arguments +anyhow::bail!("Failed to subtract {} months from {}", n, datetime); +``` + +This makes messages more readable and self-documenting, especially when there are multiple variables. + +### Type qualification + +Follow these conventions for qualifying types in code: + +- **anyhow**: Always fully qualify `anyhow` macros (`anyhow::bail!`, `anyhow::anyhow!`) and the Result type (`anyhow::Result`). +- **Nautilus domain types**: Do not fully qualify Nautilus domain types. Use them directly after importing (e.g., `Symbol`, `InstrumentId`, `Price`). +- **tokio**: Generally fully qualify `tokio` types as they can have equivalents in std library and other crates (e.g., `tokio::spawn`, `tokio::time::timeout`). + +```rust +use nautilus_model::identifiers::Symbol; + +pub fn process_symbol(symbol: Symbol) -> anyhow::Result<()> { + if !symbol.is_valid() { + anyhow::bail!("Invalid symbol: {symbol}"); + } + + tokio::spawn(async move { + // Process symbol asynchronously + }); + + Ok(()) +} +``` + +:::info[Automated enforcement] +The `check_anyhow_usage.sh` pre-commit hook enforces these anyhow conventions automatically. +::: + +### Logging + +- Fully qualify logging macros so the backend is explicit: + - Use `log::…` (`log::debug!`, `log::info!`, `log::warn!`, etc.) for all Rust components. +- Start messages with a capitalised word, prefer complete sentences, and omit terminal periods (e.g. `"Processing batch"`, not `"Processing batch."`). + +:::info[Automated enforcement] +The `check_logging_macro_usage.sh` pre-commit hook enforces fully qualified logging macros. +::: + +### Error handling + +Use structured error handling patterns consistently: + +1. **Primary Pattern**: Use `anyhow::Result` for fallible functions: + + ```rust + pub fn calculate_balance(&mut self) -> anyhow::Result { + // Implementation + } + ``` + +2. **Custom Error Types**: Use `thiserror` for domain-specific errors: + + ```rust + #[derive(Error, Debug)] + pub enum NetworkError { + #[error("Connection failed: {0}")] + ConnectionFailed(String), + #[error("Timeout occurred")] + Timeout, + } + ``` + +3. **Error Propagation**: Use the `?` operator for clean error propagation. + +4. **Error Creation**: Prefer `anyhow::bail!` for early returns with errors: + + ```rust + // Preferred - using bail! for early returns + pub fn process_value(value: i32) -> anyhow::Result { + if value < 0 { + anyhow::bail!("Value cannot be negative: {value}"); + } + Ok(value * 2) + } + + // Instead of - verbose return statement + if value < 0 { + return Err(anyhow::anyhow!("Value cannot be negative: {value}")); + } + ``` + + **Note**: Use `anyhow::bail!` for early returns, but `anyhow::anyhow!` in closure contexts like `ok_or_else()` where early returns aren't possible. + +5. **Error Context**: Use lowercase for `.context()` messages to support error chaining (except proper nouns/acronyms): + + ```rust + // Good - lowercase chains naturally + parse_timestamp(value).context("failed to parse timestamp")?; + + // Exception - proper nouns stay capitalized + connect().context("BitMEX websocket did not become active")?; + ``` + +:::info[Automated enforcement] +The `check_error_conventions.sh` and `check_anyhow_usage.sh` pre-commit hooks enforce these error handling patterns. +::: + +### Async patterns + +Use consistent async/await patterns: + +1. **Async function naming**: No special suffix is required; prefer natural names. +2. **Tokio usage**: Fully qualify tokio types (e.g., `tokio::time::timeout`). See [Adapter runtime patterns](#adapter-runtime-patterns) for spawn rules. +3. **Error handling**: Return `anyhow::Result` from async functions to match the synchronous conventions. +4. **Cancellation safety**: Call out whether the function is cancellation-safe and what invariants still hold when it is cancelled. +5. **Stream handling**: Use `tokio_stream` (or `futures::Stream`) for async iterators to make back-pressure explicit. +6. **Timeout patterns**: Wrap network or long-running awaits with timeouts (`tokio::time::timeout`) and propagate or handle the timeout error. + +### Adapter runtime patterns + +Adapter crates (under `crates/adapters/`) require special handling for spawning async tasks due to Python FFI compatibility: + +1. **Use `get_runtime().spawn()` instead of `tokio::spawn()`**: When called from Python threads (which have no Tokio context), `tokio::spawn()` panics because it relies on thread-local storage. The global runtime pattern provides an explicit reference accessible from any thread. + + ```rust + use nautilus_common::live::get_runtime; + + // Correct - works from Python threads + get_runtime().spawn(async move { + // async work + }); + + // Incorrect - panics from Python threads + tokio::spawn(async move { + // async work + }); + ``` + +2. **Use the shorter import path**: Import `get_runtime` from the `live` module re-export, not the full path: + + ```rust + // Preferred - shorter path via re-export + use nautilus_common::live::get_runtime; + + // Avoid - unnecessarily verbose + use nautilus_common::live::runtime::get_runtime; + ``` + +3. **Use `get_runtime().block_on()` for sync-to-async bridges**: When synchronous code needs to call async functions in adapters: + + ```rust + fn sync_method(&self) -> anyhow::Result<()> { + get_runtime().block_on(self.async_implementation()) + } + ``` + +4. **Tests are exempt**: Test code using `#[tokio::test]` creates its own runtime context, so `tokio::spawn()` works correctly. The enforcement hook skips test files and test modules. + +:::info[Automated enforcement] +The `check_tokio_usage.sh` pre-commit hook enforces these adapter runtime patterns automatically. +::: + +### Attribute patterns + +Consistent attribute usage and ordering: + +```rust +#[repr(C)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr( + feature = "python", + pyo3::pyclass(module = "nautilus_trader.model") +)] +#[cfg_attr( + feature = "python", + pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.model") +)] +pub struct Symbol(Ustr); +``` + +For enums with extensive derive attributes: + +```rust +#[repr(C)] +#[derive( + Copy, + Clone, + Debug, + Display, + Hash, + PartialEq, + Eq, + PartialOrd, + Ord, + AsRefStr, + FromRepr, + EnumIter, + EnumString, +)] +#[strum(ascii_case_insensitive)] +#[strum(serialize_all = "SCREAMING_SNAKE_CASE")] +#[cfg_attr( + feature = "python", + pyo3::pyclass( + frozen, + eq, + eq_int, + module = "nautilus_trader.model", + from_py_object, + rename_all = "SCREAMING_SNAKE_CASE", + ) +)] +#[cfg_attr( + feature = "python", + pyo3_stub_gen::derive::gen_stub_pyclass_enum(module = "nautilus_trader.model") +)] +pub enum AccountType { + /// An account with unleveraged cash assets only. + Cash = 1, + /// An account which facilitates trading on margin, using account assets as collateral. + Margin = 2, +} +``` + +### Type stub annotations + +Python type stubs (`.pyi` files) are generated from Rust source using +[pyo3-stub-gen](https://github.com/Jij-Inc/pyo3-stub-gen). Every type and function +exposed to Python needs a matching stub annotation so the generated stubs stay in sync +with the bindings. + +**Annotation types:** + +| PyO3 construct | Stub annotation | +| ----------------- | ------------------------------------------------ | +| `#[pyclass]` | `pyo3_stub_gen::derive::gen_stub_pyclass` | +| enum `#[pyclass]` | `pyo3_stub_gen::derive::gen_stub_pyclass_enum` | +| `#[pymethods]` | `pyo3_stub_gen::derive::gen_stub_pymethods` | +| `#[pyfunction]` | `pyo3_stub_gen::derive::gen_stub_pyfunction` | + +**Placement rules:** + +- On structs and enums, use `#[cfg_attr(feature = "python", ...)]` and place the stub + annotation directly below the `pyo3::pyclass` attribute. +- On `#[pymethods]` impl blocks, place `#[pyo3_stub_gen::derive::gen_stub_pymethods]` + directly below `#[pymethods]`. +- On functions, place the stub annotation directly above `#[pyfunction]`, after any doc + comments. Fully qualify the path rather than importing it. + +```rust +/// Converts a list of `Bar` into Arrow IPC bytes. +#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")] +#[pyfunction(name = "bars_to_arrow")] +pub fn py_bars_to_arrow(data: Vec) -> PyResult> { + // ... +} +``` + +```rust +#[pymethods] +#[pyo3_stub_gen::derive::gen_stub_pymethods] +impl AccountState { + #[staticmethod] + #[pyo3(name = "from_dict")] + pub fn py_from_dict(values: &Bound<'_, PyDict>) -> PyResult { + // ... + } +} +``` + +**Module parameter:** set `module = "nautilus_trader."` to match the Python +package where the type is imported. For example, model types use +`nautilus_trader.model` and serialization functions use +`nautilus_trader.serialization`. + +**Cargo.toml:** add `pyo3-stub-gen` as an optional dependency and include it in the +`python` feature list: + +```toml +[features] +python = ["pyo3", "pyo3-stub-gen"] + +[dependencies] +pyo3-stub-gen = { workspace = true, optional = true } +``` + +**Regenerating stubs:** run `make py-stubs-v2` (or `python python/generate_stubs.py`) +after changing annotations. The post-processor handles `py_` prefix stripping, +`@property`/`@staticmethod`/`@classmethod` decoration, keyword escaping, deduplication, +and ruff formatting. + +### Constructor patterns + +Use the `new()` vs `new_checked()` convention consistently: + +```rust +/// Creates a new [`Symbol`] instance with correctness checking. +/// +/// # Errors +/// +/// Returns an error if `value` is not a valid string. +/// +/// # Notes +/// +/// PyO3 requires a `Result` type for proper error handling and stacktrace printing in Python. +pub fn new_checked>(value: T) -> anyhow::Result { + // Implementation +} + +/// Creates a new [`Symbol`] instance. +/// +/// # Panics +/// +/// Panics if `value` is not a valid string. +pub fn new>(value: T) -> Self { + Self::new_checked(value).expect(FAILED) +} +``` + +Always use the `FAILED` constant for `.expect()` messages related to correctness checks: + +```rust +use nautilus_core::correctness::FAILED; +``` + +### Type conversion patterns + +For types that parse from strings, provide both fallible and infallible conversions: + +1. **`FromStr`**: Fallible parsing via `.parse()` or `from_str()`. Returns `Result`. + +2. **`From>`**: Ergonomic infallible conversion that accepts `&str`, `String`, `Cow`, etc. directly without requiring `.as_str()`. + +```rust +impl FromStr for Symbol { + type Err = SymbolParseError; + + fn from_str(s: &str) -> Result { + // parsing logic + } +} + +impl> From for Symbol { + fn from(value: T) -> Self { + Self::from_str(value.as_ref()).expect(FAILED) + } +} +``` + +**Design note**: The `From` impl may panic on invalid input. This is intentional for API ergonomics. Use `FromStr` / `.parse()` when error handling is needed. The `From` impl provides convenience for cases where the input is known to be valid. + +**Constraint**: This pattern cannot be used for types that implement `AsRef` themselves (e.g., string wrapper types), as it would conflict with the blanket `impl From for T`. For such types, provide separate `From<&str>` and `From` impls instead. + +### Constants and naming conventions + +Use SCREAMING_SNAKE_CASE for constants with descriptive names: + +```rust +/// Number of nanoseconds in one second. +pub const NANOSECONDS_IN_SECOND: u64 = 1_000_000_000; + +/// Bar specification for 1-minute last price bars. +pub const BAR_SPEC_1_MINUTE_LAST: BarSpecification = BarSpecification { + step: NonZero::new(1).unwrap(), + aggregation: BarAggregation::Minute, + price_type: PriceType::Last, +}; +``` + +### Hash collections + +Use `AHashMap` and `AHashSet` from the `ahash` crate for performance-critical hot paths. +For non-performance-critical code, standard `HashMap`/`HashSet` are preferred for simplicity: + +```rust +// For hot paths - using AHashMap/AHashSet +use ahash::{AHashMap, AHashSet}; + +let mut symbols: AHashSet = AHashSet::new(); +let mut prices: AHashMap = AHashMap::new(); + +// For non-hot paths - standard library HashMap/HashSet +use std::collections::{HashMap, HashSet}; + +let mut symbols: HashSet = HashSet::new(); +let mut prices: HashMap = HashMap::new(); +``` + +**Why use `ahash`?** + +- **Superior performance**: AHash uses AES-NI hardware instructions when available, providing 2-3x faster hashing compared to the default SipHash. +- **Low collision rates**: Despite being non-cryptographic, AHash provides excellent distribution and low collision rates for typical data. +- **Drop-in replacement**: Fully compatible API with standard library collections. + +**When to use standard `HashMap`/`HashSet`:** + +- **Non-performance-critical code**: For simple cases where performance is not critical (e.g., factory registries, configuration maps, test fixtures), standard `HashMap`/`HashSet` are acceptable and even preferred for simplicity. +- **Cryptographic security required**: Use standard `HashMap` when hash flooding attacks are a concern (e.g., handling untrusted user input in network protocols). +- **Network clients**: Prefer standard `HashMap` for network-facing components where security considerations outweigh performance benefits. +- **External library boundaries**: Use standard `HashMap` when interfacing with external libraries that expect it (e.g., Arrow serialization metadata). + +### Thread-safe hash map patterns + +`AHashMap` is not thread-safe. Wrapping it in `Arc` only enables sharing the pointer across threads but does not coordinate mutation. Use `Arc` only when the map is immutable after construction, otherwise add proper synchronization. + +```rust +// Avoid: Data races when multiple threads mutate +let cache = Arc::new(AHashMap::new()); +let cache_clone = Arc::clone(&cache); +tokio::spawn(async move { + cache_clone.insert(key, value); // Data race +}); +cache.insert(other_key, other_value); // Data race +``` + +**Patterns:** + +1. **Immutable after construction** – Build the map once, then share it read-only: + + ```rust + let mut map = AHashMap::new(); + map.insert(key1, value1); + map.insert(key2, value2); + let shared_map = Arc::new(map); // Now immutable + + // Multiple threads can safely read + let map_clone = Arc::clone(&shared_map); + tokio::spawn(async move { + if let Some(value) = map_clone.get(&key1) { + // Safe read-only access + } + }); + ``` + +2. **Concurrent reads and writes** – Use `DashMap`: + + ```rust + use dashmap::DashMap; + + let cache: Arc> = Arc::new(DashMap::new()); + + // Multiple threads can safely read and write concurrently + cache.insert(key, value); + if let Some(entry) = cache.get(&key) { + // Safe concurrent access + } + ``` + + `DashMap` internally uses sharding and fine-grained locking for efficient concurrent access. + +3. **Single-threaded hot paths** – Use plain `AHashMap` in single-threaded contexts: + + ```rust + struct Handler { + instruments: AHashMap, + } + + impl Handler { + async fn next(&mut self) -> Option<()> { + // Handler runs on a single task, no concurrent access + self.instruments.insert(key, value); + Ok(()) + } + } + ``` + +**Decision tree:** + +- Immutable after construction → Use `Arc>` +- Concurrent access needed → Use `Arc>` +- Single-threaded access → Use plain `AHashMap` + +### Re-export patterns + +Organize re-exports alphabetically and place at the end of lib.rs files: + +```rust +// Re-exports +pub use crate::{ + nanos::UnixNanos, + time::AtomicTime, + uuid::UUID4, +}; + +// Module-level re-exports +pub use crate::identifiers::{ + account_id::AccountId, + actor_id::ActorId, + client_id::ClientId, +}; +``` + +### Documentation standards + +Use third-person declarative voice for all doc comments (e.g., "Returns the account ID" not "Return the account ID"). + +#### Section header casing + +Rustdoc section headers use Title Case, matching the Rust standard library convention: + +- `# Examples` +- `# Errors` +- `# Panics` +- `# Safety` +- `# Notes` +- `# Thread Safety` +- `# Feature Flags` + +#### Module-Level documentation + +All modules must have module-level documentation starting with a brief description: + +```rust +//! Functions for correctness checks similar to the *design by contract* philosophy. +//! +//! This module provides validation checking of function or method conditions. +//! +//! A condition is a predicate which must be true just prior to the execution of +//! some section of code - for correct behavior as per the design specification. +``` + +For modules with feature flags, document them clearly: + +```rust +//! # Feature flags +//! +//! This crate provides feature flags to control source code inclusion during compilation, +//! depending on the intended use case: +//! +//! - `ffi`: Enables the C foreign function interface (FFI) from [cbindgen](https://github.com/mozilla/cbindgen). +//! - `python`: Enables Python bindings from [PyO3](https://pyo3.rs). +//! - `extension-module`: Builds as a Python extension module (used with `python`). +//! - `stubs`: Enables type stubs for use in testing scenarios. +``` + +#### Field documentation + +All struct and enum fields must have documentation with terminating periods: + +```rust +pub struct Currency { + /// The currency code as an alpha-3 string (e.g., "USD", "EUR"). + pub code: Ustr, + /// The currency decimal precision. + pub precision: u8, + /// The ISO 4217 currency code. + pub iso4217: u16, + /// The full name of the currency. + pub name: Ustr, + /// The currency type, indicating its category (e.g. Fiat, Crypto). + pub currency_type: CurrencyType, +} +``` + +#### Function documentation + +Document all public functions with: + +- Purpose and behavior +- Explanation of input argument usage +- Error conditions (if applicable) +- Panic conditions (if applicable) + +```rust +/// Returns a reference to the `AccountBalance` for the specified currency, or `None` if absent. +/// +/// # Panics +/// +/// Panics if `currency` is `None` and `self.base_currency` is `None`. +pub fn base_balance(&self, currency: Option) -> Option<&AccountBalance> { + // Implementation +} +``` + +#### Errors and panics documentation format + +For single line errors and panics documentation, use sentence case with the following convention: + +```rust +/// Returns a reference to the `AccountBalance` for the specified currency, or `None` if absent. +/// +/// # Errors +/// +/// Returns an error if the currency conversion fails. +/// +/// # Panics +/// +/// Panics if `currency` is `None` and `self.base_currency` is `None`. +pub fn base_balance(&self, currency: Option) -> anyhow::Result> { + // Implementation +} +``` + +For multi-line errors and panics documentation, use sentence case with bullets and terminating periods: + +```rust +/// Calculates the unrealized profit and loss for the position. +/// +/// # Errors +/// +/// Returns an error if: +/// - The market price for the instrument cannot be found. +/// - The conversion rate calculation fails. +/// - Invalid position state is encountered. +/// +/// # Panics +/// +/// This function panics if: +/// - The instrument ID is invalid or uninitialized. +/// - Required market data is missing from the cache. +/// - Internal state consistency checks fail. +pub fn calculate_unrealized_pnl(&self, market_price: Price) -> anyhow::Result { + // Implementation +} +``` + +#### Safety documentation format + +For Safety documentation, use the `SAFETY:` prefix followed by a short description explaining why the unsafe operation is valid: + +```rust +/// Creates a new instance from raw components without validation. +/// +/// # Safety +/// +/// The caller must ensure that all input parameters are valid and properly initialized. +pub unsafe fn from_raw_parts(ptr: *const u8, len: usize) -> Self { + // SAFETY: Caller guarantees ptr is valid and len is correct + Self { + data: std::slice::from_raw_parts(ptr, len), + } +} +``` + +For inline unsafe blocks, use the `SAFETY:` comment directly above the unsafe code: + +```rust +impl Send for MessageBus { + fn send(&self) { + // SAFETY: Message bus is not meant to be passed between threads + unsafe { + // unsafe operation here + } + } +} +``` + +## Python bindings + +Python bindings are provided via [PyO3](https://pyo3.rs), allowing users to import NautilusTrader crates directly in Python without a Rust toolchain. + +### PyO3 naming conventions + +When exposing Rust functions to Python **via PyO3**: + +1. The Rust symbol **must** be prefixed with `py_*` to make its purpose explicit inside the Rust + codebase. +2. Use the `#[pyo3(name = "…")]` attribute to publish the *Python* name **without** the `py_` + prefix so the Python API remains clean. + +```rust +#[pyo3(name = "do_something")] +pub fn py_do_something() -> PyResult<()> { + // … +} +``` + +:::info[Automated enforcement] +The `check_pyo3_conventions.sh` pre-commit hook enforces the `py_` prefix for PyO3 functions. +::: + +### PyO3 enum conventions + +Enums exposed to Python should use the following `pyclass` attributes: + +- `frozen`: enums are immutable value types. +- `eq, eq_int`: enables equality with other enum instances and integer discriminants. +- `rename_all = "SCREAMING_SNAKE_CASE"`: standardizes Python variant names. +- `from_py_object`: enables conversion from Python objects. + +:::warning[Do not use the `hash` pyclass attribute with `eq_int` enums] +PyO3's auto-generated `__hash__` uses Rust's `DefaultHasher`, which produces different values +than Python's `hash()` on the equivalent integer. Since `eq_int` makes `MyEnum.VARIANT == 1` +true, the hash contract (`a == b` implies `hash(a) == hash(b)`) would be violated. Instead, +provide a manual `__hash__` returning the discriminant directly: +::: + +```rust +#[pymethods] +impl MyEnum { + const fn __hash__(&self) -> isize { + *self as isize + } +} +``` + +### Testing conventions + +- Use `mod tests` as the standard test module name unless you need to specifically compartmentalize. +- Use `#[rstest]` attributes consistently, this standardization reduces cognitive overhead. +- Do *not* use Arrange, Act, Assert separator comments in Rust tests. + +:::info[Automated enforcement] +The `check_testing_conventions.sh` pre-commit hook enforces the use of `#[rstest]` over `#[test]`. +::: + +#### Parameterized testing + +Use the `rstest` attribute consistently, and for parameterized tests: + +```rust +#[rstest] +#[case("AUDUSD", false)] +#[case("AUD/USD", false)] +#[case("CL.FUT", true)] +fn test_symbol_is_composite(#[case] input: &str, #[case] expected: bool) { + let symbol = Symbol::new(input); + assert_eq!(symbol.is_composite(), expected); +} +``` + +#### Property-based testing + +Use the `proptest` crate for property-based tests. Place these in a separate +`property_tests` module (not inside `mod tests`) to keep deterministic unit +tests separate from randomized property tests: + +```rust +#[cfg(test)] +mod property_tests { + use proptest::prelude::*; + use rstest::rstest; + + use super::*; + + // Define strategies for generating test inputs + fn my_strategy() -> impl Strategy { + prop_oneof![ + Just(MyType::VariantA), + Just(MyType::VariantB), + ] + } + + fn value_strategy() -> impl Strategy { + prop_oneof![ + -1000.0..1000.0, + Just(0.0), + ] + } + + // Group all property tests inside the proptest! macro + proptest! { + #[rstest] + fn prop_construction_roundtrip( + value in value_strategy(), + variant in my_strategy() + ) { + // Test invariants that should hold for all generated inputs + } + } +} +``` + +Conventions: + +- Name the module `property_tests`, separate from `mod tests`. +- Import `proptest::prelude::*` and `rstest::rstest`. +- Define strategy functions returning `impl Strategy`. +- Combine value ranges with edge cases using `prop_oneof!`. +- Filter invalid combinations with `prop_filter_map`. +- Prefix test names with `prop_`. +- Mark each test inside `proptest!` with `#[rstest]`. + +#### Test naming + +Use descriptive test names that explain the scenario: + +```rust +fn test_sma_with_no_inputs() +fn test_sma_with_single_input() +fn test_symbol_is_composite() +``` + +### Box-style banner comments + +Do not use box-style banner or separator comments. If code requires visual +separation, consider splitting it into separate modules or files. Instead use: + +- Clear function names that convey purpose. +- Module structure for logical groupings (`mod tests { mod fixtures { } }`). +- Impl blocks to group related methods. +- Doc comments (`///`) for semantic documentation. +- IDE navigation and code folding. + +Patterns to avoid: + +```rust +// ============================================================================ +// Some Section +// ============================================================================ + +// ========== Test Fixtures ========== +``` + +## Rust-Python memory management + +When working with PyO3 bindings, it's critical to understand and avoid reference cycles between Rust's `Arc` reference counting and Python's garbage collector. +This section documents best practices for handling Python objects in Rust callback-holding structures. + +### The reference cycle problem + +**Problem**: Using `Arc` in callback-holding structs creates circular references: + +1. **Rust `Arc` holds Python objects** → increases Python reference count. +2. **Python objects might reference Rust objects** → creates cycles. +3. **Neither side can be garbage collected** → memory leak. + +**Example of problematic pattern**: + +```rust +// AVOID: This creates reference cycles +struct CallbackHolder { + handler: Option>, // ❌ Arc wrapper causes cycles +} +``` + +### The solution: GIL-based cloning + +**Solution**: Use plain `PyObject` with proper GIL-based cloning via `clone_py_object()`: + +```rust +use nautilus_core::python::clone_py_object; + +// CORRECT: Use plain PyObject without Arc wrapper +struct CallbackHolder { + handler: Option, // ✅ No Arc wrapper +} + +// Manual Clone implementation using clone_py_object +impl Clone for CallbackHolder { + fn clone(&self) -> Self { + Self { + handler: self.handler.as_ref().map(clone_py_object), + } + } +} +``` + +### Best practices + +#### 1. Use `clone_py_object()` for Python object cloning + +```rust +// When cloning Python callbacks +let cloned_callback = clone_py_object(&original_callback); + +// In manual Clone implementations +self.py_handler.as_ref().map(clone_py_object) +``` + +#### 2. Remove `#[derive(Clone)]` from callback-holding structs + +```rust +// BEFORE: Automatic derive causes issues with PyObject +#[derive(Clone)] // ❌ Remove this +struct Config { + handler: Option, +} + +// AFTER: Manual implementation with proper cloning +struct Config { + handler: Option, +} + +impl Clone for Config { + fn clone(&self) -> Self { + Self { + // Clone regular fields normally + url: self.url.clone(), + // Use clone_py_object for Python objects + handler: self.handler.as_ref().map(clone_py_object), + } + } +} +``` + +#### 3. Update function signatures to accept `PyObject` + +```rust +// BEFORE: Arc wrapper in function signatures +fn spawn_task(handler: Arc) { ... } // ❌ + +// AFTER: Plain PyObject +fn spawn_task(handler: PyObject) { ... } // ✅ +``` + +#### 4. Avoid `Arc::new()` when creating Python callbacks + +```rust +// BEFORE: Wrapping in Arc +let callback = Arc::new(py_function); // ❌ + +// AFTER: Use directly +let callback = py_function; // ✅ +``` + +### Why this works + +The `clone_py_object()` function: + +- **Acquires the Python GIL** before performing clone operations. +- **Uses Python's native reference counting** via `clone_ref()`. +- **Avoids Rust Arc wrappers** that interfere with Python GC. +- **Maintains thread safety** through proper GIL management. + +This approach allows both Rust and Python garbage collectors to work correctly, eliminating memory leaks from reference cycles. + +## Common anti-patterns + +1. **Avoid `.clone()` in hot paths** – favour borrowing or shared ownership via `Arc`. +2. **Avoid `.unwrap()` in production code** – generally propagate errors with `?` or map them into domain errors, but unwrapping lock poisoning is acceptable because it signals a severe program state that should abort fast. +3. **Avoid `String` when `&str` suffices** – minimise allocations on tight loops. +4. **Avoid exposing interior mutability** – hide mutexes/`RefCell` behind safe APIs. +5. **Avoid large structs in `Result`** – box large error payloads (`Box`). + +## Unsafe Rust + +It will be necessary to write `unsafe` Rust code to be able to achieve the value +of interoperating between Cython and Rust. The ability to step outside the boundaries of safe Rust is what makes it possible to +implement many of the most fundamental features of the Rust language itself, just as C and C++ are used to implement +their own standard libraries. + +Great care will be taken with the use of Rusts `unsafe` facility - which enables a small set of additional language features, thereby changing +the contract between the interface and caller, shifting some responsibility for guaranteeing correctness +from the Rust compiler, and onto us. The goal is to realize the advantages of the `unsafe` facility, whilst avoiding *any* undefined behavior. +The definition for what the Rust language designers consider undefined behavior can be found in the [language reference](https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html). + +### Safety policy + +To maintain correctness, any use of `unsafe` Rust must follow our policy: + +- If a function is `unsafe` to call, there *must* be a `Safety` section in the documentation explaining why the function is `unsafe`, + covering the invariants which the function expects the callers to uphold, and how to meet their obligations in that contract. +- Document why each function is `unsafe` in its doc comment's Safety section, and cover all `unsafe` blocks with unit tests. +- Always include a `SAFETY:` comment explaining why the unsafe operation is valid. +- **Crate-level lint** – every crate that exposes FFI symbols enables + `#![deny(unsafe_op_in_unsafe_fn)]`. Even inside an `unsafe fn`, each pointer dereference or + other dangerous operation must be wrapped in its own `unsafe { … }` block. +- **CVec contract** – for raw vectors that cross the FFI boundary read the + [FFI Memory Contract](ffi.md). Foreign code becomes the owner of the allocation and **must** + call the matching `vec_drop_*` function exactly once. + +### Categories of unsafe code + +The codebase uses unsafe Rust in these categories: + +1. **FFI boundaries** – Raw pointer operations for C interop. See [FFI documentation](ffi.md). +2. **Interior mutability** – `UnsafeCell` for thread-local registries with controlled access patterns. +3. **Unsafe Send/Sync** – Types that are not inherently thread-safe but satisfy trait bounds + through runtime invariants (e.g., single-threaded access guaranteed by architecture). + +### Unsafe Send/Sync requirements + +When implementing `Send` or `Sync` unsafely: + +1. Document exactly which fields violate the trait requirements. +2. Explain the runtime mechanism that ensures safety (e.g., single-threaded event loop). +3. Include a `WARNING` stating that violating the invariant is undefined behavior. +4. Prefer runtime enforcement (assertions, `Result` returns) over documentation-only guarantees. + +```rust +// SAFETY: Contains Rc> which is not thread-safe. +// Single-threaded access guaranteed by the backtest engine architecture. +// WARNING: Actually sending across threads is undefined behavior. +#[allow(unsafe_code)] +unsafe impl Send for BacktestDataClient {} +``` + +### Defense in depth + +Where unsafe code relies on invariants, add defense mechanisms: + +- **Type verification**: Check types at runtime before casting (e.g., `TypeId` comparison). +- **Debug assertions**: Catch memory corruption early in debug builds. +- **RAII guards**: Ensure cleanup on both normal return and panic paths. +- **Runtime checks**: Fail fast when invariants are violated rather than proceeding unsafely. + +## Tooling configuration + +The project uses several tools for code quality: + +- **rustfmt**: Automatic code formatting (see `rustfmt.toml`). +- **clippy**: Linting and best practices (see `clippy.toml`). +- **cbindgen**: C header generation for FFI. + +## Rust version management + +The project pins to a specific Rust version via `rust-toolchain.toml`. + +**Keep your toolchain synchronized with CI:** + +```bash +rustup update # Update to latest stable Rust +rustup show # Verify correct toolchain is active +``` + +If pre-commit passes locally but fails in CI, clear the pre-commit cache and re-run: + +```bash +pre-commit clean # Clear cached environments +make pre-commit # Re-run all checks +``` + +This ensures you're using the same Rust and clippy versions as CI. + +## Resources + +- [The Rustonomicon](https://doc.rust-lang.org/nomicon/) – The Dark Arts of Unsafe Rust. +- [The Rust Reference – Unsafety](https://doc.rust-lang.org/stable/reference/unsafety.html). +- [Safe Bindings in Rust – Russell Johnston](https://www.abubalay.com/blog/2020/08/22/safe-bindings-in-rust). +- [Google – Rust and C interoperability](https://www.chromium.org/Home/chromium-security/memory-safety/rust-and-c-interoperability/). + +## Cap'n Proto serialization + +The `nautilus-serialization` crate provides optional Cap'n Proto serialization support for efficient data interchange. +This feature is opt-in to avoid requiring the Cap'n Proto compiler for standard builds. + +### Installing Cap'n Proto + +Install the Cap'n Proto compiler before working with schemas. The required version is +specified in the `capnp-version` file in the repository root. + +See the [Environment Setup](environment_setup.md#capn-proto) guide for detailed installation +instructions for each platform. + +:::warning +Ubuntu's default `capnproto` package is too old. Linux users must install from source. +::: + +Verify installation: + +```bash +capnp --version # Should match the version in capnp-version +``` + +### Schema development workflow + +Schema files live in `crates/serialization/schemas/capnp/`: + +- `common/` - Base types, identifiers, enums. +- `commands/` - Trading commands. +- `events/` - Order and position events. +- `data/` - Market data types. + +When modifying schemas: + +1. Edit the `.capnp` schema file in the appropriate subdirectory. +2. Regenerate Rust bindings: + + ```bash + make regen-capnp + # or + ./scripts/regen_capnp.sh + ``` + +3. Review changes: + + ```bash + git diff crates/serialization/generated/capnp + ``` + +4. Update conversions in `crates/serialization/src/capnp/conversions.rs` if needed. +5. Run tests: + + ```bash + make cargo-test EXTRA_FEATURES="capnp" + ``` + +### Generated code + +Generated Rust files are checked into `crates/serialization/generated/capnp/` for these reasons: + +- **docs.rs compatibility**: The documentation build environment lacks the Cap'n Proto + compiler. +- **Contributor convenience**: Most developers don't need to install capnp for standard + development. +- **Build reproducibility**: Ensures consistent code generation across environments. + +The generated files are automatically created during builds via `build.rs` when the `capnp` +feature is enabled, but we commit them to the repository to support builds without the +compiler installed. + +### Verifying schema consistency + +Before committing schema changes, ensure generated files are up-to-date: + +```bash +make check-capnp-schemas +``` + +This target: + +1. Skips with a warning if `capnp` is not installed (acceptable for local development). +2. Fails if regeneration errors occur (e.g., version mismatch). +3. Regenerates schemas and fails if generated files differ from committed versions. + +CI runs this check automatically to catch drift (capnp is always installed in CI). + +### Testing with capnp feature + +```bash +# Run workspace tests with capnp +make cargo-test EXTRA_FEATURES="capnp" + +# Run specific crate tests with capnp +make cargo-test-crate-nautilus-serialization FEATURES="capnp" + +# Run specific test +cargo test -p nautilus-serialization --features capnp test_price_roundtrip +``` + +### Schema evolution guidelines + +When evolving schemas: + +- **Additive changes only**: Add new fields at the end. +- **Never remove fields**: Mark deprecated fields in comments. +- **Never reuse field numbers**: Even after deprecation. +- **Test roundtrip compatibility**: Ensure old and new versions interoperate. + +Cap'n Proto's evolution rules allow schema changes without breaking binary compatibility, but +you must follow these constraints to maintain forward/backward compatibility. diff --git a/nautilus-docs/docs/developer_guide/spec_data_testing.md b/nautilus-docs/docs/developer_guide/spec_data_testing.md new file mode 100644 index 0000000..d73f183 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/spec_data_testing.md @@ -0,0 +1,854 @@ +# Data Testing Spec + +This section defines a rigorous test matrix for validating adapter data +functionality using the `DataTester` actor. Both Python +(`nautilus_trader.test_kit.strategies.tester_data`) and Rust +(`nautilus_testkit::testers`) provide the `DataTester`. Each test case is +identified by a prefixed ID (e.g. TC-D01) and grouped by functionality. + +**Each adapter must pass the subset of tests matching its supported data types.** + +Test groups are ordered from least derived to most derived data: instruments +and raw book data first, then quotes, trades, bars, and derivatives data. +An adapter that passes groups 1–4 is considered baseline data compliant. + +Document adapter-specific data behavior (custom channels, throttling, +snapshot semantics, etc.) in the adapter's own guide, not here. + +## Prerequisites + +Before running data tests: + +- Target instrument available and loadable via the instrument provider. +- API credentials set via environment variables (`{VENUE}_API_KEY`, `{VENUE}_API_SECRET`) when + the venue requires authentication for the data being tested. +- If the venue offers a demo/testnet mode (e.g. `is_demo=True`), use credentials created + for that environment. Demo and production API keys are typically separate and not + interchangeable; using the wrong credentials produces authentication errors (e.g. HTTP 401). + +**Python node setup** (reference: `examples/live/{adapter}/{adapter}_data_tester.py`): + +```python +from nautilus_trader.live.node import TradingNode +from nautilus_trader.test_kit.strategies.tester_data import DataTester, DataTesterConfig + +node = TradingNode(config=config_node) +tester = DataTester(config=config_tester) +node.trader.add_actor(tester) +# Register adapter factories, build, and run +``` + +**Rust node setup** (reference: `crates/adapters/{adapter}/examples/node_data_tester.rs`): + +```rust +use nautilus_testkit::testers::{DataTester, DataTesterConfig}; + +let tester_config = DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_quotes(true); +let tester = DataTester::new(tester_config); +node.add_actor(tester)?; +node.run().await?; +``` + +Each group below begins with a summary table, followed by detailed test cards. +Test IDs use spaced numbering to allow insertion without renumbering. + +--- + +## Group 1: Instruments + +Verify instrument loading and subscription before testing market data streams. + +| TC | Name | Description | Skip when | +|---------|-----------------------------|------------------------------------------------------|----------------------| +| TC-D01 | Request instruments | Load all instruments for a venue. | Never. | +| TC-D02 | Subscribe instrument | Subscribe to instrument updates. | No instrument sub. | +| TC-D03 | Load specific instrument | Load a single instrument by ID. | Never. | + +### TC-D01: Request instruments + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected. | +| **Action** | DataTester requests all instruments for the venue on start. | +| **Event sequence** | `on_instruments` callback receives instrument list. | +| **Pass criteria** | At least one instrument received; each has valid symbol, price precision, and size increment. | +| **Skip when** | Never. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + request_instruments=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_request_instruments(true) +``` + +### TC-D02: Subscribe instrument + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to instrument updates. | +| **Event sequence** | `on_instrument` callback receives instrument. | +| **Pass criteria** | Instrument received with correct `instrument_id`, valid fields. | +| **Skip when** | Adapter does not support instrument subscriptions. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_instrument=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_instrument(true) +``` + +### TC-D03: Load specific instrument + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected. | +| **Action** | Load a specific instrument by `InstrumentId` via the instrument provider. | +| **Event sequence** | Instrument available in cache after load. | +| **Pass criteria** | Instrument loaded with correct ID, price precision, size increment, and trading rules. | +| **Skip when** | Never. | + +**Considerations:** + +- This tests the instrument provider's `load` / `load_async` method directly. +- Verify the instrument is cached and available via `self.cache.instrument(instrument_id)`. + +--- + +## Group 2: Order book + +Test order book subscription modes and snapshot requests. + +| TC | Name | Description | Skip when | +|---------|--------------------------------|----------------------------------------------------|------------------------| +| TC-D10 | Subscribe book deltas | Stream `OrderBookDeltas` updates. | No book support. | +| TC-D11 | Subscribe book at interval | Periodic `OrderBook` snapshots. | No book support. | +| TC-D12 | Subscribe book depth | `OrderBookDepth10` snapshots. | No book depth. | +| TC-D13 | Request book snapshot | One-time book snapshot request. | No book snapshot. | +| TC-D14 | Managed book from deltas | Build local book from delta stream. | No book support. | +| TC-D15 | Request historical book deltas | Historical book deltas request. | No historical deltas. | + +### TC-D10: Subscribe book deltas + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to order book deltas. | +| **Event sequence** | `OrderBookDeltas` events received in `on_order_book_deltas`. | +| **Pass criteria** | Deltas received with valid instrument ID; at least one delta contains bid/ask updates. | +| **Skip when** | Adapter does not support order book data. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_book_deltas=True, + book_type=BookType.L2_MBP, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_book_deltas(true) + .with_book_type(BookType::L2_MBP) +``` + +### TC-D11: Subscribe book at interval + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to periodic order book snapshots. | +| **Event sequence** | `OrderBook` events received in `on_order_book` at configured interval. | +| **Pass criteria** | Book snapshots received with bid/ask levels; updates arrive at approximately the configured interval. | +| **Skip when** | Adapter does not support order book data. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_book_at_interval=True, + book_type=BookType.L2_MBP, + book_depth=10, + book_interval_ms=1000, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_book_at_interval(true) + .with_book_type(BookType::L2_MBP) + .with_book_depth(Some(NonZeroUsize::new(10).unwrap())) + .with_book_interval_ms(NonZeroUsize::new(1000).unwrap()) +``` + +### TC-D12: Subscribe book depth + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to `OrderBookDepth10` snapshots. | +| **Event sequence** | `OrderBookDepth10` events received in `on_order_book_depth`. | +| **Pass criteria** | Depth snapshots received with up to 10 bid/ask levels; prices are correctly ordered. | +| **Skip when** | Adapter does not support book depth subscriptions. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_book_depth=True, + book_type=BookType.L2_MBP, + book_depth=10, +) +``` + +**Rust config:** Not yet supported. Book depth subscription is TODO in the Rust `DataTester`. + +### TC-D13: Request book snapshot + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester requests a one-time order book snapshot. | +| **Event sequence** | Book snapshot received via historical data callback. | +| **Pass criteria** | Snapshot contains bid/ask levels with valid prices and sizes. | +| **Skip when** | Adapter does not support book snapshot requests. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + request_book_snapshot=True, + book_depth=10, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_request_book_snapshot(true) + .with_book_depth(Some(NonZeroUsize::new(10).unwrap())) +``` + +### TC-D14: Managed book from deltas + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, book deltas streaming. | +| **Action** | DataTester subscribes to deltas with `manage_book=True`; builds local order book from the delta stream. | +| **Event sequence** | `OrderBookDeltas` applied to local `OrderBook`; book logged with configured depth. | +| **Pass criteria** | Local book builds correctly from deltas; bid levels descend, ask levels ascend; book is not empty after initial snapshot. | +| **Skip when** | Adapter does not support order book data. | + +**Considerations:** + +- The managed book applies each delta to an `OrderBook` instance maintained by the actor. +- Use `book_levels_to_print` to control logging verbosity. + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_book_deltas=True, + manage_book=True, + book_type=BookType.L2_MBP, + book_levels_to_print=10, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_book_deltas(true) + .with_manage_book(true) + .with_book_type(BookType::L2_MBP) +``` + +### TC-D15: Request historical book deltas + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester requests historical order book deltas. | +| **Event sequence** | Historical deltas received via callback. | +| **Pass criteria** | Deltas received with valid timestamps and book actions. | +| **Skip when** | Adapter does not support historical book delta requests. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + request_book_deltas=True, +) +``` + +**Rust config:** Not yet supported. Historical book delta requests are TODO in the Rust `DataTester`. + +--- + +## Group 3: Quotes + +Test quote tick subscriptions and historical requests. + +| TC | Name | Description | Skip when | +|---------|---------------------------|-------------------------------------------------|------------------------| +| TC-D20 | Subscribe quotes | Verify `QuoteTick` events flow after start. | Never. | +| TC-D21 | Request historical quotes | Request historical quote ticks. | No historical quotes. | + +### TC-D20: Subscribe quotes + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to quotes on start. | +| **Event sequence** | `QuoteTick` events received in `on_quote_tick`. | +| **Pass criteria** | At least one `QuoteTick` received with valid bid/ask prices and sizes; bid < ask. | +| **Skip when** | Never. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_quotes=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_quotes(true) +``` + +### TC-D21: Request historical quotes + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester requests historical quote ticks. | +| **Event sequence** | Historical quotes received via `on_historical_data` callback. | +| **Pass criteria** | Quotes received with valid timestamps, bid/ask prices and sizes. | +| **Skip when** | Adapter does not support historical quote requests. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + request_quotes=True, + requests_start_delta=pd.Timedelta(hours=1), +) +``` + +--- + +## Group 4: Trades + +Test trade tick subscriptions and historical requests. + +| TC | Name | Description | Skip when | +|--------|---------------------------|-------------------------------------------------|------------------------| +| TC-D30 | Subscribe trades | Verify `TradeTick` events flow after start. | Never. | +| TC-D31 | Request historical trades | Request historical trade ticks. | No historical trades. | + +### TC-D30: Subscribe trades + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to trades on start. | +| **Event sequence** | `TradeTick` events received in `on_trade_tick`. | +| **Pass criteria** | At least one `TradeTick` received with valid price, size, and aggressor side. | +| **Skip when** | Never. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_trades=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_trades(true) +``` + +### TC-D31: Request historical trades + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester requests historical trade ticks. | +| **Event sequence** | Historical trades received via `on_historical_data` callback. | +| **Pass criteria** | Trades received with valid timestamps, prices, sizes, and trade IDs. | +| **Skip when** | Adapter does not support historical trade requests. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + request_trades=True, + requests_start_delta=pd.Timedelta(hours=1), +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_request_trades(true) +``` + +--- + +## Group 5: Bars + +Test bar subscriptions and historical requests. + +| TC | Name | Description | Skip when | +|---------|-------------------------|---------------------------------------------------|---------------------| +| TC-D40 | Subscribe bars | Verify `Bar` events flow after start. | No bar support. | +| TC-D41 | Request historical bars | Request historical OHLCV bars. | No historical bars. | + +### TC-D40: Subscribe bars + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, bar type configured. | +| **Action** | DataTester subscribes to bars for a configured `BarType`. | +| **Event sequence** | `Bar` events received in `on_bar`. | +| **Pass criteria** | At least one `Bar` received with valid OHLCV values; high >= low, high >= open, high >= close. | +| **Skip when** | Adapter does not support bar subscriptions. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + bar_types=[BarType.from_str("BTCUSDT-PERP.VENUE-1-MINUTE-LAST-EXTERNAL")], + subscribe_bars=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_bar_types(vec![bar_type]) + .with_subscribe_bars(true) +``` + +### TC-D41: Request historical bars + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, bar type configured. | +| **Action** | DataTester requests historical bars for a configured `BarType`. | +| **Event sequence** | Historical bars received via callback. | +| **Pass criteria** | Bars received with valid OHLCV values and ascending timestamps. | +| **Skip when** | Adapter does not support historical bar requests. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + bar_types=[BarType.from_str("BTCUSDT-PERP.VENUE-1-MINUTE-LAST-EXTERNAL")], + request_bars=True, + requests_start_delta=pd.Timedelta(hours=1), +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_bar_types(vec![bar_type]) + .with_request_bars(true) +``` + +--- + +## Group 6: Derivatives data + +Test derivatives-specific data streams: mark prices, index prices, and funding rates. + +| TC | Name | Description | Skip when | +|--------|----------------------------------|---------------------------------------------|-----------------------| +| TC-D50 | Subscribe mark prices | `MarkPriceUpdate` events. | Not a derivative. | +| TC-D51 | Subscribe index prices | `IndexPriceUpdate` events. | Not a derivative. | +| TC-D52 | Subscribe funding rates | `FundingRateUpdate` events. | Not a perpetual. | +| TC-D53 | Request historical funding rates | Historical funding rate data. | Not a perpetual. | + +### TC-D50: Subscribe mark prices + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, derivative instrument loaded. | +| **Action** | DataTester subscribes to mark price updates. | +| **Event sequence** | `MarkPriceUpdate` events received in `on_mark_price`. | +| **Pass criteria** | At least one `MarkPriceUpdate` received with valid instrument ID and mark price. | +| **Skip when** | Instrument is not a derivative, or adapter does not provide mark prices. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_mark_prices=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_mark_prices(true) +``` + +### TC-D51: Subscribe index prices + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, derivative instrument loaded. | +| **Action** | DataTester subscribes to index price updates. | +| **Event sequence** | `IndexPriceUpdate` events received in `on_index_price`. | +| **Pass criteria** | At least one `IndexPriceUpdate` received with valid instrument ID and index price. | +| **Skip when** | Instrument is not a derivative, or adapter does not provide index prices. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_index_prices=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_index_prices(true) +``` + +### TC-D52: Subscribe funding rates + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, perpetual instrument loaded. | +| **Action** | DataTester subscribes to funding rate updates. | +| **Event sequence** | `FundingRateUpdate` events received in `on_funding_rate`. | +| **Pass criteria** | At least one `FundingRateUpdate` received with valid instrument ID and rate. | +| **Skip when** | Instrument is not a perpetual, or adapter does not provide funding rates. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_funding_rates=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_funding_rates(true) +``` + +### TC-D53: Request historical funding rates + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, perpetual instrument loaded. | +| **Action** | DataTester requests historical funding rates (default 7-day lookback). | +| **Event sequence** | Historical funding rates received via callback. | +| **Pass criteria** | Funding rates received with valid timestamps and rate values. | +| **Skip when** | Instrument is not a perpetual, or adapter does not support historical funding rate requests. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + request_funding_rates=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_request_funding_rates(true) +``` + +--- + +## Group 7: Instrument status + +Test instrument status and close event subscriptions. + +| TC | Name | Description | Skip when | +|--------|-----------------------------|------------------------------------------------|-----------------------| +| TC-D60 | Subscribe instrument status | `InstrumentStatus` events. | No status support. | +| TC-D61 | Subscribe instrument close | `InstrumentClose` events. | No close support. | + +### TC-D60: Subscribe instrument status + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to instrument status updates. | +| **Event sequence** | `InstrumentStatus` events received in `on_instrument_status`. | +| **Pass criteria** | Status events received with valid `MarketStatusAction` (e.g. `Trading`). | +| **Skip when** | Adapter does not support instrument status subscriptions. | + +**Considerations:** + +- Status events may only fire on state changes (e.g. trading halt → resume). +- During normal trading hours, a `Trading` status may be received on subscribe. + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_instrument_status=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_instrument_status(true) +``` + +### TC-D61: Subscribe instrument close + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded. | +| **Action** | DataTester subscribes to instrument close events. | +| **Event sequence** | `InstrumentClose` events received in `on_instrument_close`. | +| **Pass criteria** | Close event received with valid close price and close type. | +| **Skip when** | Adapter does not support instrument close subscriptions. | + +**Considerations:** + +- Close events typically fire at end-of-session for traditional markets. +- May not fire for 24/7 crypto venues unless the adapter synthesizes a daily close. + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_instrument_close=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_instrument_close(true) +``` + +--- + +## Group 8: Option greeks + +Test option greeks and option chain subscriptions. + +| TC | Name | Description | Skip when | +|--------|-----------------------------|------------------------------------------------|------------------------| +| TC-D62 | Subscribe option greeks | `OptionGreeks` data for a single instrument. | No greeks support. | +| TC-D63 | Subscribe option chain | `OptionChainSlice` snapshots for a series. | No chain support. | + +### TC-D62: Subscribe option greeks + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, option instrument loaded. | +| **Action** | DataTester subscribes to option greeks updates. | +| **Event sequence** | `OptionGreeks` events received in `on_option_greeks`. | +| **Pass criteria** | Greeks received with valid delta, gamma, vega, theta values. | +| **Skip when** | Adapter does not support option greeks subscriptions. | + +**Considerations:** + +- Greeks are only available for option instruments. +- Values depend on the venue's pricing model and may update on every quote change. + +### TC-D63: Subscribe option chain + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, option instruments loaded for the series. | +| **Action** | DataTester subscribes to option chain snapshots for a series. | +| **Event sequence** | `OptionChainSlice` snapshots received in `on_option_chain`. | +| **Pass criteria** | Chain snapshot contains greeks for instruments matching the series. | +| **Skip when** | Adapter does not support option chain subscriptions. | + +**Considerations:** + +- Option chain subscriptions are managed by the DataEngine, which creates per-instrument + quote and greeks subscriptions internally. +- ATM-relative strike ranges require a forward price bootstrap before subscriptions begin. + +--- + +## Group 9: Lifecycle + +Test actor lifecycle behavior: unsubscribe handling and custom parameters. + +| TC | Name | Description | Skip when | +|--------|-------------------------|----------------------------------------------------|----------------------| +| TC-D70 | Unsubscribe on stop | Unsubscribe from data feeds on actor stop. | No unsub support. | +| TC-D71 | Custom subscribe params | Adapter-specific subscription parameters. | N/A. | +| TC-D72 | Custom request params | Adapter-specific request parameters. | N/A. | + +### TC-D70: Unsubscribe on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Active data subscriptions (quotes, trades, book). | +| **Action** | Stop the actor with `can_unsubscribe=True` (default). | +| **Event sequence** | Data subscriptions removed; no further data events received. | +| **Pass criteria** | Clean unsubscribe; no errors in logs; no data events after stop. | +| **Skip when** | Adapter does not support unsubscribe. | + +**Python config:** + +```python +DataTesterConfig( + instrument_ids=[instrument_id], + subscribe_quotes=True, + subscribe_trades=True, + can_unsubscribe=True, +) +``` + +**Rust config:** + +```rust +DataTesterConfig::new(client_id, vec![instrument_id]) + .with_subscribe_quotes(true) + .with_subscribe_trades(true) + .with_can_unsubscribe(true) +``` + +### TC-D71: Custom subscribe params + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, adapter accepts additional subscription parameters. | +| **Action** | Subscribe with `subscribe_params` dict containing adapter-specific parameters. | +| **Event sequence** | Subscription established with custom parameters applied. | +| **Pass criteria** | Data flows with adapter-specific parameters in effect. | +| **Skip when** | N/A (adapter-specific). | + +**Considerations:** + +- The `subscribe_params` dict is opaque to the DataTester and passed through to the adapter. +- Consult the adapter's guide for supported parameters. + +### TC-D72: Custom request params + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, adapter accepts additional request parameters. | +| **Action** | Request data with `request_params` dict containing adapter-specific parameters. | +| **Event sequence** | Request fulfilled with custom parameters applied. | +| **Pass criteria** | Historical data received with adapter-specific parameters in effect. | +| **Skip when** | N/A (adapter-specific). | + +**Considerations:** + +- The `request_params` dict is opaque to the DataTester and passed through to the adapter. +- Consult the adapter's guide for supported parameters. + +--- + +## DataTester configuration reference + +Quick reference for all `DataTesterConfig` parameters. Defaults shown are for the Python config. +Note: Rust `DataTesterConfig::new` sets `manage_book=true`, while Python defaults it to `False`. + +| Parameter | Type | Default | Affects groups | +|------------------------------|-------------------|-----------------|----------------| +| `instrument_ids` | list[InstrumentId]| *required* | All | +| `client_id` | ClientId? | None | All | +| `bar_types` | list[BarType]? | None | 5 | +| `subscribe_book_deltas` | bool | False | 2 | +| `subscribe_book_depth` | bool | False | 2 | +| `subscribe_book_at_interval` | bool | False | 2 | +| `subscribe_quotes` | bool | False | 3 | +| `subscribe_trades` | bool | False | 4 | +| `subscribe_mark_prices` | bool | False | 6 | +| `subscribe_index_prices` | bool | False | 6 | +| `subscribe_funding_rates` | bool | False | 6 | +| `subscribe_bars` | bool | False | 5 | +| `subscribe_instrument` | bool | False | 1 | +| `subscribe_instrument_status`| bool | False | 7 | +| `subscribe_instrument_close` | bool | False | 7 | +| `subscribe_params` | dict? | None | 8 | +| `can_unsubscribe` | bool | True | 8 | +| `request_instruments` | bool | False | 1 | +| `request_book_snapshot` | bool | False | 2 | +| `request_book_deltas` | bool | False | 2 | +| `request_quotes` | bool | False | 3 | +| `request_trades` | bool | False | 4 | +| `request_bars` | bool | False | 5 | +| `request_funding_rates` | bool | False | 6 | +| `request_params` | dict? | None | 8 | +| `requests_start_delta` | Timedelta? | 1 hour | 3, 4, 5 | +| `book_type` | BookType | L2_MBP | 2 | +| `book_depth` | PositiveInt? | None | 2 | +| `book_interval_ms` | PositiveInt | 1000 | 2 | +| `book_levels_to_print` | PositiveInt | 10 | 2 | +| `manage_book` | bool | False | 2 | +| `use_pyo3_book` | bool | False | 2 | +| `log_data` | bool | True | All | + +--- diff --git a/nautilus-docs/docs/developer_guide/spec_exec_testing.md b/nautilus-docs/docs/developer_guide/spec_exec_testing.md new file mode 100644 index 0000000..d38e701 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/spec_exec_testing.md @@ -0,0 +1,1690 @@ +# Execution Testing Spec + +This section defines a rigorous test matrix for validating adapter execution +functionality using the `ExecTester` strategy. Both Python +(`nautilus_trader.test_kit.strategies.tester_exec`) and Rust +(`nautilus_testkit::testers`) provide the `ExecTester`. Each test case is +identified by a prefixed ID (e.g. TC-E01) and grouped by functionality. + +**Each adapter must pass the subset of tests matching its supported capabilities.** + +Tests progress from simple (single market order) to complex (brackets, +modification chains, rejection handling). An adapter that passes groups 1–5 is +considered baseline compliant. Data connectivity should be verified first using +the [Data Testing Spec](spec_data_testing.md). + +Document adapter-specific behavior (how a venue simulates market orders, +handles TIF options, etc.) in the adapter's own guide, not here. Each adapter +guide should include a capability matrix showing which order types, time-in-force +options, actions, and flags it supports. + +## Prerequisites + +Before running execution tests: + +- Demo/testnet account with valid API credentials (preferred, not required). +- Account funded with sufficient margin for the test instrument and quantities. +- Target instrument available and loadable via the instrument provider. +- Environment variables set: `{VENUE}_API_KEY`, `{VENUE}_API_SECRET` (or sandbox variants). +- If the venue offers a demo/testnet mode (e.g. `is_demo=True`), use credentials created + for that environment. Demo and production API keys are typically separate and not + interchangeable; using the wrong credentials produces authentication errors (e.g. HTTP 401). +- Risk engine bypassed (`LiveRiskEngineConfig(bypass=True)`) to avoid interference. +- Reconciliation enabled to verify state consistency. + +**Python node setup** (reference: `examples/live/{adapter}/{adapter}_exec_tester.py`): + +```python +from nautilus_trader.live.node import TradingNode +from nautilus_trader.test_kit.strategies.tester_exec import ExecTester, ExecTesterConfig + +node = TradingNode(config=config_node) +strategy = ExecTester(config=config_tester) +node.trader.add_strategy(strategy) +# Register adapter factories, build, and run +``` + +**Rust node setup** (reference: `crates/adapters/{adapter}/examples/node_exec_tester.rs`): + +```rust +use nautilus_testkit::testers::{ExecTester, ExecTesterConfig}; + +let tester_config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, order_qty); +let tester = ExecTester::new(tester_config); +node.add_strategy(tester)?; +node.run().await?; +``` + +## Basic smoke test + +A quick sanity check that can run at any time, for example after adapter changes or between +development iterations. The tester opens a position with a market order on start, places a +buy and sell post-only limit order, waits 30 seconds, then stops (cancelling open orders and +closing the position). + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.001"), + open_position_on_start_qty=Decimal("0.001"), + enable_limit_buys=True, + enable_limit_sells=True, + use_post_only=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, dec!(0.001)) + .with_open_position_on_start_qty(Some(dec!(0.001))) + .with_enable_limit_buys(true) + .with_enable_limit_sells(true) + .with_use_post_only(true) +``` + +**Expected behavior:** + +1. On start: market order fills, opening a position. +2. Two limit orders placed at `tob_offset_ticks` away from best bid/ask (default 500 ticks). +3. Strategy idles for 30 seconds. Check logs for errors, rejected orders, or disconnections. +4. On stop: open limit orders cancelled, position closed with a market order. + +**Pass criteria:** No errors in logs, position opened and closed cleanly, limit orders +acknowledged by the venue. + +--- + +Each group below begins with a summary table, followed by detailed test cards. +Test IDs use spaced numbering to allow insertion without renumbering. + +--- + +## Group 1: Market orders + +Test market order submission and fills. Market orders should execute immediately. + +| TC | Name | Description | Skip when | +|--------|-------------------------------|-----------------------------------------------------|---------------------| +| TC-E01 | Market BUY - submit and fill | Open long position via market buy. | No market orders. | +| TC-E02 | Market SELL - submit and fill | Open short position via market sell. | No market orders. | +| TC-E03 | Market order with IOC TIF | Market order explicitly using IOC time in force. | No IOC. | +| TC-E04 | Market order with FOK TIF | Market order explicitly using FOK time in force. | No FOK. | +| TC-E05 | Market order with quote qty | Market order using quote currency quantity. | No quote quantity. | +| TC-E06 | Close position via market | Close an open position with a market order on stop. | No market orders. | + +### TC-E01: Market BUY - submit and fill + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, market data flowing, no open position. | +| **Action** | ExecTester opens a long position via `open_position_on_start_qty`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Position opened with side=LONG, quantity matches config, fill price within market range, `AccountState` updated. | +| **Skip when** | Adapter does not support market orders. | + +**Considerations:** + +- Some adapters simulate market orders as aggressive limit IOC orders (check adapter guide). +- The event sequence from the strategy's perspective should be identical regardless of the venue mechanism. +- Fill price should be within the recent bid/ask spread. +- Partial fills are valid; verify the cumulative filled quantity matches the order quantity. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) +``` + +### TC-E02: Market SELL - submit and fill + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, market data flowing, no open position. | +| **Action** | ExecTester opens a short position via negative `open_position_on_start_qty`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Position opened with side=SHORT, quantity matches config, fill price within market range. | +| **Skip when** | Adapter does not support market orders or short selling. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("-0.01"), + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(-1, 2))) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) +``` + +### TC-E03: Market order with IOC TIF + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, market data flowing. | +| **Action** | Open position with `open_position_time_in_force=IOC`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Same as TC-E01; the IOC TIF is explicitly set on the order. | +| **Skip when** | No IOC support. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), + open_position_time_in_force=TimeInForce.IOC, + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false); +config.open_position_time_in_force = TimeInForce::Ioc; +``` + +### TC-E04: Market order with FOK TIF + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, market data flowing. | +| **Action** | Open position with `open_position_time_in_force=FOK`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Same as TC-E01; the FOK TIF is explicitly set on the order. | +| **Skip when** | No FOK support. | + +**Considerations:** + +- FOK requires the entire quantity to be fillable immediately or the order is canceled. +- Use small test quantities so book depth is sufficient for a complete fill. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), + open_position_time_in_force=TimeInForce.FOK, + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false); +config.open_position_time_in_force = TimeInForce::Fok; +``` + +### TC-E05: Market order with quote quantity + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, adapter supports quote quantity. | +| **Action** | Open position with `use_quote_quantity=True`, quantity in quote currency. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Order submitted with quote currency quantity; fill quantity is in base currency. | +| **Skip when** | Adapter does not support quote quantity orders. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("100.0"), # Quote currency amount + open_position_on_start_qty=Decimal("100.0"), + use_quote_quantity=True, + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("100")) + .with_open_position_on_start(Some(Decimal::from(100))) + .with_use_quote_quantity(true) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) +``` + +### TC-E06: Close position via market order on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open position from TC-E01 or TC-E02. | +| **Action** | Stop the strategy; ExecTester closes position via market order. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled` (closing order). | +| **Pass criteria** | Position closed (net quantity = 0), no open orders remaining. | +| **Skip when** | Adapter does not support market orders. | + +**Considerations:** + +- This test naturally follows TC-E01 or TC-E02 as part of the same session. +- `close_positions_on_stop=True` is the default. +- The closing order should be on the opposite side of the position. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), + close_positions_on_stop=True, + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) + .with_close_positions_on_stop(true) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) +``` + +--- + +## Group 2: Limit orders + +Test limit order submission, acceptance, and behavior across time-in-force options. + +| TC | Name | Description | Skip when | +|--------|----------------------------|--------------------------------------------------|--------------------| +| TC-E10 | Limit BUY GTC | Place GTC limit buy below TOB, verify accepted. | Never. | +| TC-E11 | Limit SELL GTC | Place GTC limit sell above TOB, verify accepted. | Never. | +| TC-E12 | Limit BUY and SELL pair | Both sides simultaneously, verify both accepted. | Never. | +| TC-E13 | Limit IOC aggressive fill | Limit IOC at aggressive price, expect fill. | No IOC. | +| TC-E14 | Limit IOC passive no fill | Limit IOC away from market, expect cancel. | No IOC. | +| TC-E15 | Limit FOK fill | Limit FOK at aggressive price, expect fill. | No FOK. | +| TC-E16 | Limit FOK no fill | Limit FOK away from market, expect cancel. | No FOK. | +| TC-E17 | Limit GTD | Limit with expiry time, verify accepted. | No GTD. | +| TC-E18 | Limit GTD expiry | Wait for GTD expiry, verify `OrderExpired`. | No GTD. | +| TC-E19 | Limit DAY | Limit with DAY TIF, verify accepted. | No DAY. | + +### TC-E10: Limit BUY GTC - submit and accept + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places a limit buy at `best_bid - tob_offset_ticks`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order is open on the venue with correct price, quantity, side=BUY, TIF=GTC. | +| **Skip when** | Never. | + +**Considerations:** + +- The `tob_offset_ticks` (default 500) places the order well away from the market to avoid accidental fills. +- Verify the order appears in the cache with `OrderStatus.ACCEPTED`. +- The order should remain open until explicitly canceled. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false) +``` + +### TC-E11: Limit SELL GTC - submit and accept + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places a limit sell at `best_ask + tob_offset_ticks`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order is open on the venue with correct price, quantity, side=SELL, TIF=GTC. | +| **Skip when** | Never. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(true) +``` + +### TC-E12: Limit BUY and SELL pair + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places both a limit buy and limit sell. | +| **Event sequence** | Two independent sequences: each `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Both orders open on venue, buy below bid, sell above ask. | +| **Skip when** | Never. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(true) +``` + +### TC-E13: Limit IOC aggressive fill + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Submit a limit buy IOC at or above the best ask (aggressive price). | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Order fills immediately; position opened. | +| **Skip when** | Adapter does not support IOC TIF. | + +**Considerations:** + +- This test requires manual order creation or adapter-specific configuration, as the ExecTester's + default limit order placement uses GTC TIF. +- IOC orders that don't fill immediately are canceled by the venue. + +### TC-E14: Limit IOC passive - no fill + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Submit a limit buy IOC well below the market (passive price). | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderCanceled`. | +| **Pass criteria** | Order is immediately canceled by venue with no fill. | +| **Skip when** | Adapter does not support IOC TIF. | + +**Considerations:** + +- The venue should cancel the unfilled IOC order; verify `OrderCanceled` event (not `OrderExpired`). + +### TC-E15: Limit FOK fill + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing, sufficient book depth. | +| **Action** | Submit a limit buy FOK at aggressive price with quantity within top-of-book depth. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Order fills completely in a single fill event. | +| **Skip when** | Adapter does not support FOK TIF. | + +**Considerations:** + +- FOK requires the entire quantity to be fillable; use small quantities so book depth is sufficient. + +### TC-E16: Limit FOK no fill + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Submit a limit buy FOK at passive price (well below market). | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderCanceled`. | +| **Pass criteria** | Order is immediately canceled by venue with no fill. | +| **Skip when** | Adapter does not support FOK TIF. | + +### TC-E17: Limit GTD - submit and accept + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Place limit buy with `order_expire_time_delta_mins` set (e.g., 60 minutes). | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order accepted with GTD TIF and correct expiry timestamp. | +| **Skip when** | Adapter does not support GTD TIF. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + order_expire_time_delta_mins=60, + enable_limit_buys=True, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false); +config.order_expire_time_delta_mins = Some(60); +``` + +### TC-E18: Limit GTD expiry + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open GTD limit order from TC-E17 (or use a very short expiry). | +| **Action** | Wait for the GTD expiry time to elapse. | +| **Event sequence** | `OrderExpired`. | +| **Pass criteria** | Order transitions to expired status; `OrderExpired` event received. | +| **Skip when** | Adapter does not support GTD TIF. | + +**Considerations:** + +- Use a short `order_expire_time_delta_mins` (e.g., 1–2 minutes) to avoid long waits. +- Some venues may report expiry as a cancel; verify the adapter maps this to `OrderExpired`. + +### TC-E19: Limit DAY - submit and accept + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, market is in trading hours. | +| **Action** | Submit limit buy with DAY TIF. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order accepted with DAY TIF; will be automatically canceled at end of trading day. | +| **Skip when** | Adapter does not support DAY TIF. | + +**Considerations:** + +- DAY orders may behave differently on 24/7 crypto venues vs traditional markets. +- Verify behavior when submitted outside trading hours (if applicable). + +--- + +## Group 3: Stop and conditional orders + +Test stop and conditional order types. These orders rest on the venue until a trigger condition is met. + +| TC | Name | Description | Skip when | +|--------|------------------------|-------------------------------------------------------|---------------------| +| TC-E20 | StopMarket BUY | Stop buy above ask, verify accepted. | No `STOP_MARKET`. | +| TC-E21 | StopMarket SELL | Stop sell below bid, verify accepted. | No `STOP_MARKET`. | +| TC-E22 | StopLimit BUY | Stop-limit buy with trigger + limit price. | No `STOP_LIMIT`. | +| TC-E23 | StopLimit SELL | Stop-limit sell with trigger + limit price. | No `STOP_LIMIT`. | +| TC-E24 | MarketIfTouched BUY | MIT buy below bid. | No `MIT`. | +| TC-E25 | MarketIfTouched SELL | MIT sell above ask. | No `MIT`. | +| TC-E26 | LimitIfTouched BUY | LIT buy with trigger + limit price. | No `LIT`. | +| TC-E27 | LimitIfTouched SELL | LIT sell with trigger + limit price. | No `LIT`. | + +### TC-E20: StopMarket BUY + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places a stop-market buy above the current ask. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Stop order accepted on venue with correct trigger price and side=BUY. | +| **Skip when** | Adapter does not support `StopMarket` orders. | + +**Considerations:** + +- The trigger price should be above the current ask by `stop_offset_ticks`. +- The order should NOT trigger immediately (trigger price is above market). +- Verifying trigger and fill requires the market to move, which may not happen during the test. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=False, + enable_stop_buys=True, + enable_stop_sells=False, + stop_order_type=OrderType.STOP_MARKET, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) + .with_enable_stop_buys(true) + .with_enable_stop_sells(false) + .with_stop_order_type(OrderType::StopMarket) +``` + +### TC-E21: StopMarket SELL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places a stop-market sell below the current bid. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Stop order accepted on venue with correct trigger price and side=SELL. | +| **Skip when** | Adapter does not support `StopMarket` orders. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=False, + enable_stop_buys=False, + enable_stop_sells=True, + stop_order_type=OrderType.STOP_MARKET, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) + .with_enable_stop_buys(false) + .with_enable_stop_sells(true) + .with_stop_order_type(OrderType::StopMarket) +``` + +### TC-E22: StopLimit BUY + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places a stop-limit buy with trigger price above ask and limit offset. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Stop-limit order accepted with correct trigger price, limit price, and side=BUY. | +| **Skip when** | Adapter does not support `StopLimit` orders. | + +**Considerations:** + +- Requires `stop_limit_offset_ticks` to be set for the limit price offset from the trigger price. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=False, + enable_stop_buys=True, + enable_stop_sells=False, + stop_order_type=OrderType.STOP_LIMIT, + stop_limit_offset_ticks=50, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) + .with_enable_stop_buys(true) + .with_enable_stop_sells(false) + .with_stop_order_type(OrderType::StopLimit); +config.stop_limit_offset_ticks = Some(50); +``` + +### TC-E23: StopLimit SELL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places a stop-limit sell with trigger price below bid. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Stop-limit order accepted with correct trigger price, limit price, and side=SELL. | +| **Skip when** | Adapter does not support `StopLimit` orders. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=False, + enable_stop_buys=False, + enable_stop_sells=True, + stop_order_type=OrderType.STOP_LIMIT, + stop_limit_offset_ticks=50, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) + .with_enable_stop_buys(false) + .with_enable_stop_sells(true) + .with_stop_order_type(OrderType::StopLimit); +config.stop_limit_offset_ticks = Some(50); +``` + +### TC-E24: MarketIfTouched BUY + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Place MIT buy with trigger below current bid (buy on dip). | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | MIT order accepted on venue with correct trigger price. | +| **Skip when** | Adapter does not support `MarketIfTouched` orders. | + +### TC-E25: MarketIfTouched SELL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Place MIT sell with trigger above current ask (sell on rally). | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | MIT order accepted on venue with correct trigger price. | +| **Skip when** | Adapter does not support `MarketIfTouched` orders. | + +### TC-E26: LimitIfTouched BUY + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Place LIT buy with trigger below bid and limit price offset. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | LIT order accepted with correct trigger price and limit price. | +| **Skip when** | Adapter does not support `LimitIfTouched` orders. | + +### TC-E27: LimitIfTouched SELL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | Place LIT sell with trigger above ask and limit price offset. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | LIT order accepted with correct trigger price and limit price. | +| **Skip when** | Adapter does not support `LimitIfTouched` orders. | + +--- + +## Group 4: Order modification + +Test order modification (amend) and cancel-replace workflows. + +| TC | Name | Description | Skip when | +|-------|------------------------------|-----------------------------------------------------|-----------------------------| +| TC-E30 | Modify limit BUY price | Amend open limit buy to new price. | No modify support. | +| TC-E31 | Modify limit SELL price | Amend open limit sell to new price. | No modify support. | +| TC-E32 | Cancel-replace limit BUY | Cancel and resubmit limit buy at new price. | Never. | +| TC-E33 | Cancel-replace limit SELL | Cancel and resubmit limit sell at new price. | Never. | +| TC-E34 | Modify stop trigger price | Amend stop order trigger price. | No modify or no stop. | +| TC-E35 | Cancel-replace stop order | Cancel and resubmit stop at new trigger price. | No stop orders. | +| TC-E36 | Modify rejected | Modify on unsupported adapter. | Adapter supports modify. | + +### TC-E30: Modify limit BUY price + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open GTC limit buy from TC-E10. | +| **Action** | ExecTester modifies limit buy to a new price as market moves (`modify_orders_to_maintain_tob_offset=True`). | +| **Event sequence** | `OrderPendingUpdate` → `OrderUpdated`. | +| **Pass criteria** | Order price updated on venue; `OrderUpdated` event contains new price. | +| **Skip when** | Adapter does not support order modification. | + +**Considerations:** + +- Requires market movement to trigger the ExecTester's order maintenance logic. +- The modify is triggered when the order price drifts from the target TOB offset. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=False, + modify_orders_to_maintain_tob_offset=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false); +config.modify_orders_to_maintain_tob_offset = true; +``` + +### TC-E31: Modify limit SELL price + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open GTC limit sell from TC-E11. | +| **Action** | ExecTester modifies limit sell to new price as market moves. | +| **Event sequence** | `OrderPendingUpdate` → `OrderUpdated`. | +| **Pass criteria** | Order price updated on venue; `OrderUpdated` event contains new price. | +| **Skip when** | Adapter does not support order modification. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=True, + modify_orders_to_maintain_tob_offset=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(true); +config.modify_orders_to_maintain_tob_offset = true; +``` + +### TC-E32: Cancel-replace limit BUY + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open GTC limit buy. | +| **Action** | ExecTester cancels and resubmits limit buy at new price as market moves. | +| **Event sequence** | `OrderPendingCancel` → `OrderCanceled` → `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Original order canceled, new order accepted at updated price. | +| **Skip when** | Never (cancel-replace is always available). | + +**Considerations:** + +- This is the universal alternative when the adapter does not support native modify. +- Two distinct orders in the cache: the canceled original and the new replacement. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=False, + cancel_replace_orders_to_maintain_tob_offset=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false); +config.cancel_replace_orders_to_maintain_tob_offset = true; +``` + +### TC-E33: Cancel-replace limit SELL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open GTC limit sell. | +| **Action** | ExecTester cancels and resubmits limit sell at new price. | +| **Event sequence** | `OrderPendingCancel` → `OrderCanceled` → `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Original order canceled, new order accepted at updated price. | +| **Skip when** | Never. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=False, + enable_limit_sells=True, + cancel_replace_orders_to_maintain_tob_offset=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(false) + .with_enable_limit_sells(true); +config.cancel_replace_orders_to_maintain_tob_offset = true; +``` + +### TC-E34: Modify stop trigger price + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open stop order from TC-E20 or TC-E22. | +| **Action** | ExecTester modifies stop trigger price as market moves (`modify_stop_orders_to_maintain_offset=True`). | +| **Event sequence** | `OrderPendingUpdate` → `OrderUpdated`. | +| **Pass criteria** | Stop order trigger price updated on venue. | +| **Skip when** | Adapter does not support modify, or no stop order support. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_stop_buys=True, + modify_stop_orders_to_maintain_offset=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_stop_buys(true); +config.modify_stop_orders_to_maintain_offset = true; +``` + +### TC-E35: Cancel-replace stop order + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open stop order. | +| **Action** | ExecTester cancels and resubmits stop at new trigger price. | +| **Event sequence** | `OrderPendingCancel` → `OrderCanceled` → `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Original stop canceled, new stop accepted at updated trigger price. | +| **Skip when** | No stop order support. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_stop_buys=True, + cancel_replace_stop_orders_to_maintain_offset=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_stop_buys(true); +config.cancel_replace_stop_orders_to_maintain_offset = true; +``` + +### TC-E36: Modify rejected + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open limit order, adapter does NOT support modify. | +| **Action** | Attempt to modify the order (programmatically, not via ExecTester auto-maintain). | +| **Event sequence** | `OrderModifyRejected`. | +| **Pass criteria** | Modify attempt results in `OrderModifyRejected` event with reason; original order remains unchanged. | +| **Skip when** | Adapter supports order modification. | + +**Considerations:** + +- This tests the adapter's rejection path, not the ExecTester's cancel-replace logic. +- The rejection reason should indicate that modification is not supported. + +--- + +## Group 5: Order cancellation + +Test order cancellation workflows. + +| TC | Name | Description | Skip when | +|-------|----------------------------|------------------------------------------------------|----------------------| +| TC-E40 | Cancel single limit order | Cancel an open limit order. | Never. | +| TC-E41 | Cancel all on stop | Strategy stop cancels all open orders (default). | Never. | +| TC-E42 | Individual cancels on stop | Cancel orders one-by-one on stop. | Never. | +| TC-E43 | Batch cancel on stop | Cancel orders via batch API on stop. | No batch cancel. | +| TC-E44 | Cancel already-canceled | Cancel a non-open order. | Never. | + +### TC-E40: Cancel single limit order + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open GTC limit order from TC-E10 or TC-E11. | +| **Action** | Stop the strategy; ExecTester cancels the open limit order. | +| **Event sequence** | `OrderPendingCancel` → `OrderCanceled`. | +| **Pass criteria** | Order status transitions to CANCELED; no open orders remaining. | +| **Skip when** | Never. | + +**Considerations:** + +- `cancel_orders_on_stop=True` (default) triggers cancellation when the strategy stops. +- Verify the `OrderCanceled` event contains the correct `venue_order_id`. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=False, + cancel_orders_on_stop=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false) + .with_cancel_orders_on_stop(true) +``` + +### TC-E41: Cancel all on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Multiple open orders (limit buy + limit sell from TC-E12). | +| **Action** | Stop the strategy with `cancel_orders_on_stop=True` (default). | +| **Event sequence** | For each order: `OrderPendingCancel` → `OrderCanceled`. | +| **Pass criteria** | All open orders canceled; no open orders remaining. | +| **Skip when** | Never. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=True, + cancel_orders_on_stop=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(true) + .with_cancel_orders_on_stop(true) +``` + +### TC-E42: Individual cancels on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Multiple open orders. | +| **Action** | Stop with `use_individual_cancels_on_stop=True`. | +| **Event sequence** | Individual `OrderPendingCancel` → `OrderCanceled` for each order. | +| **Pass criteria** | Each order canceled individually; all orders reach CANCELED status. | +| **Skip when** | Never. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=True, + use_individual_cancels_on_stop=True, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(true); +config.use_individual_cancels_on_stop = true; +``` + +### TC-E43: Batch cancel on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Multiple open orders, adapter supports batch cancel. | +| **Action** | Stop with `use_batch_cancel_on_stop=True`. | +| **Event sequence** | Batch `OrderPendingCancel` → `OrderCanceled` for all orders. | +| **Pass criteria** | All orders canceled via single batch request; all reach CANCELED status. | +| **Skip when** | Adapter does not support batch cancel. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=True, + use_batch_cancel_on_stop=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(true) + .with_use_batch_cancel_on_stop(true) +``` + +### TC-E44: Cancel already-canceled order + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | A previously canceled order (from TC-E40). | +| **Action** | Attempt to cancel the same order again. | +| **Event sequence** | `OrderCancelRejected`. | +| **Pass criteria** | Cancel attempt is rejected; `OrderCancelRejected` event received with reason. | +| **Skip when** | Never. | + +**Considerations:** + +- This tests the adapter's error handling for invalid cancel requests. +- The rejection reason should indicate the order is not in a cancelable state. + +--- + +## Group 6: Bracket orders + +Test bracket order submission (entry + take-profit + stop-loss). + +| TC | Name | Description | Skip when | +|-------|-------------------------------|---------------------------------------------------|----------------------| +| TC-E50 | Bracket BUY | Entry limit buy + TP limit sell + SL stop sell. | No bracket support. | +| TC-E51 | Bracket SELL | Entry limit sell + TP limit buy + SL stop buy. | No bracket support. | +| TC-E52 | Bracket entry fill activates | Verify TP/SL become active after entry fill. | No bracket support. | +| TC-E53 | Bracket with post-only entry | Entry order uses post-only flag. | No bracket or PO. | + +### TC-E50: Bracket BUY + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester submits a bracket order: limit buy entry + take-profit sell + stop-loss sell. | +| **Event sequence** | Entry: `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`; TP and SL: `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Three orders created and accepted: entry below bid, TP above ask, SL below entry. | +| **Skip when** | Adapter does not support bracket orders. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_brackets=True, + bracket_entry_order_type=OrderType.LIMIT, + bracket_offset_ticks=500, + enable_limit_buys=True, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_brackets(true) + .with_bracket_entry_order_type(OrderType::Limit) + .with_bracket_offset_ticks(500) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false) +``` + +### TC-E51: Bracket SELL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester submits bracket: limit sell entry + TP buy + SL buy. | +| **Event sequence** | Same pattern as TC-E50 but for sell side. | +| **Pass criteria** | Three orders created and accepted on sell side. | +| **Skip when** | Adapter does not support bracket orders. | + +### TC-E52: Bracket entry fill activates TP/SL + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Bracket order from TC-E50 where entry order fills. | +| **Action** | Entry order fills; verify contingent TP and SL orders activate. | +| **Event sequence** | Entry: `OrderFilled`; TP and SL transition from contingent to active. | +| **Pass criteria** | After entry fill, TP and SL orders are live on the venue. | +| **Skip when** | Adapter does not support bracket orders. | + +**Considerations:** + +- This requires the entry order to actually fill, which may need aggressive pricing. +- The TP/SL activation mechanism varies by venue (some activate immediately, some are OCA groups). + +### TC-E53: Bracket with post-only entry + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter supports brackets and post-only. | +| **Action** | Submit bracket with `use_post_only=True` (applied to entry and TP). | +| **Event sequence** | Same as TC-E50 with post-only flag on entry. | +| **Pass criteria** | Entry and TP orders accepted as post-only (maker); SL is not post-only. | +| **Skip when** | No bracket support or no post-only support. | + +--- + +## Group 7: Order flags + +Test order-level flags and special parameters. + +| TC | Name | Description | Skip when | +|-------|----------------------|--------------------------------------------------------|----------------------| +| TC-E60 | PostOnly accepted | Limit with post-only, placed away from TOB. | No post-only. | +| TC-E61 | ReduceOnly on close | Close position with reduce-only flag. | No reduce-only. | +| TC-E62 | Display quantity | Iceberg order with visible quantity < total. | No display quantity. | +| TC-E63 | Custom order params | Adapter-specific params via `order_params`. | N/A. | + +### TC-E60: PostOnly accepted + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places limit buy with `use_post_only=True` at passive price. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order accepted as a maker order; post-only flag acknowledged by venue. | +| **Skip when** | Adapter does not support post-only flag. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=False, + use_post_only=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false) + .with_use_post_only(true) +``` + +### TC-E61: ReduceOnly on close + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open position (from TC-E01). | +| **Action** | Stop strategy with `reduce_only_on_stop=True`; closing order uses reduce-only flag. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled` (with reduce-only). | +| **Pass criteria** | Closing order has reduce-only flag; position fully closed. | +| **Skip when** | Adapter does not support reduce-only flag. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), + reduce_only_on_stop=True, + close_positions_on_stop=True, + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) + .with_reduce_only_on_stop(true) + .with_close_positions_on_stop(true) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) +``` + +### TC-E62: Display quantity (iceberg) + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, adapter supports display quantity. | +| **Action** | Place limit order with `order_display_qty` < `order_qty`. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order accepted with display quantity set; only display qty visible on the book. | +| **Skip when** | Adapter does not support display quantity / iceberg orders. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("1.0"), + order_display_qty=Decimal("0.1"), + enable_limit_buys=True, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("1.0")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false); +config.order_display_qty = Some(Quantity::from("0.1")); +``` + +### TC-E63: Custom order params + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, adapter accepts additional parameters. | +| **Action** | Place order with `order_params` dict containing adapter-specific parameters. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | +| **Pass criteria** | Order accepted; adapter-specific parameters passed through to venue. | +| **Skip when** | N/A (adapter-specific). | + +**Considerations:** + +- The `order_params` dict is opaque to the ExecTester and passed through to the adapter. +- Consult the adapter's guide for supported parameters. + +--- + +## Group 8: Rejection handling + +Test that the adapter correctly handles and reports order rejections. + +| TC | Name | Description | Skip when | +|-------|-------------------------|------------------------------------------------------|-------------------------| +| TC-E70 | PostOnly rejection | Post-only order that would cross the spread. | No post-only. | +| TC-E71 | ReduceOnly rejection | Reduce-only order with no position to reduce. | No reduce-only. | +| TC-E72 | Unsupported order type | Submit order type not supported by adapter. | Never. | +| TC-E73 | Unsupported TIF | Submit order with unsupported time in force. | Never. | + +### TC-E70: PostOnly rejection + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | +| **Action** | ExecTester places post-only order on the wrong side of the book (`test_reject_post_only=True`), causing it to cross the spread. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderRejected`. | +| **Pass criteria** | Order rejected by venue; `OrderRejected` event received with reason indicating post-only violation. | +| **Skip when** | Adapter does not support post-only flag. | + +**Considerations:** + +- The ExecTester's `test_reject_post_only` mode intentionally prices the order to cross. +- Some venues may partially fill instead of rejecting; behavior is venue-specific. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + enable_limit_buys=True, + enable_limit_sells=False, + use_post_only=True, + test_reject_post_only=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_enable_limit_buys(true) + .with_enable_limit_sells(false) + .with_use_post_only(true) + .with_test_reject_post_only(true) +``` + +### TC-E71: ReduceOnly rejection + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, no open position for the instrument. | +| **Action** | ExecTester opens a market position with `reduce_only=True` via `test_reject_reduce_only=True` and `open_position_on_start_qty`, when no position exists to reduce. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderRejected`. | +| **Pass criteria** | Order rejected; `OrderRejected` event with reason indicating reduce-only violation. | +| **Skip when** | Adapter does not support reduce-only flag. | + +**Considerations:** + +- The `test_reject_reduce_only` flag only applies to the opening market order submitted via + `open_position_on_start_qty`. +- Verify no prior position exists for the instrument before running this test. + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), + test_reject_reduce_only=True, + enable_limit_buys=False, + enable_limit_sells=False, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) + .with_test_reject_reduce_only(true) + .with_enable_limit_buys(false) + .with_enable_limit_sells(false) +``` + +### TC-E72: Unsupported order type + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, order type not in adapter's supported set. | +| **Action** | Submit an order type the adapter does not support. | +| **Event sequence** | `OrderDenied` (pre-submission rejection by adapter). | +| **Pass criteria** | Order denied before reaching venue; `OrderDenied` event with reason. | +| **Skip when** | Never (every adapter has unsupported order types to test). | + +**Considerations:** + +- `OrderDenied` occurs at the adapter level before the order reaches the venue. +- This differs from `OrderRejected` which comes from the venue. +- Test by configuring a stop order type that the adapter does not support. + +### TC-E73: Unsupported TIF + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, TIF not in adapter's supported set. | +| **Action** | Submit an order with a TIF the adapter does not support. | +| **Event sequence** | `OrderDenied` (pre-submission rejection by adapter). | +| **Pass criteria** | Order denied before reaching venue; `OrderDenied` event with reason. | +| **Skip when** | Never (every adapter has unsupported TIF options to test). | + +**Considerations:** + +- Similar to TC-E72 but for time-in-force options. +- Test with TIF values from the Nautilus enum that the adapter does not map. + +--- + +## Group 9: Lifecycle (start/stop) + +Test strategy lifecycle behavior and state management on start and stop. + +| TC | Name | Description | Skip when | +|--------|-----------------------------|--------------------------------------------------------|----------------------| +| TC-E80 | Open position on start | Open a position immediately when strategy starts. | No market orders. | +| TC-E81 | Cancel orders on stop | Cancel all open orders when strategy stops. | Never. | +| TC-E82 | Close positions on stop | Close open positions when strategy stops. | No market orders. | +| TC-E83 | Unsubscribe on stop | Unsubscribe from data feeds on strategy stop. | No unsub support. | +| TC-E84 | Reconcile open orders | Reconcile existing open orders from a prior session. | Never. | +| TC-E85 | Reconcile filled orders | Reconcile previously filled orders from a prior session.| Never. | +| TC-E86 | Reconcile open long | Reconcile existing open long position. | Never. | +| TC-E87 | Reconcile open short | Reconcile existing open short position. | Never. | + +### TC-E80: Open position on start + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Adapter connected, instrument loaded, no existing position. | +| **Action** | Strategy starts with `open_position_on_start_qty` set. | +| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | Position opened on start; market order submitted and filled before limit order maintenance begins. | +| **Skip when** | Adapter does not support market orders. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + open_position_on_start_qty=Decimal("0.01"), +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_open_position_on_start(Some(Decimal::new(1, 2))) +``` + +### TC-E81: Cancel orders on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open limit orders from the strategy session. | +| **Action** | Stop the strategy with `cancel_orders_on_stop=True` (default). | +| **Event sequence** | For each open order: `OrderPendingCancel` → `OrderCanceled`. | +| **Pass criteria** | All strategy-owned open orders canceled on stop. | +| **Skip when** | Never. | + +### TC-E82: Close positions on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Open position from the strategy session. | +| **Action** | Stop the strategy with `close_positions_on_stop=True` (default). | +| **Event sequence** | Closing order: `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | +| **Pass criteria** | All strategy-owned positions closed; net position = 0. | +| **Skip when** | Adapter does not support market orders. | + +### TC-E83: Unsubscribe on stop + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | Active data subscriptions (quotes, trades, book). | +| **Action** | Stop the strategy with `can_unsubscribe=True` (default). | +| **Event sequence** | Data subscriptions removed. | +| **Pass criteria** | No further data events received after stop; clean disconnection. | +| **Skip when** | Adapter does not support unsubscribe. | + +**Python config:** + +```python +ExecTesterConfig( + instrument_id=instrument_id, + order_qty=Decimal("0.01"), + can_unsubscribe=True, +) +``` + +**Rust config:** + +```rust +ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) + .with_can_unsubscribe(true) +``` + +### TC-E84: Reconcile open orders + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | One or more open limit orders on the venue from a prior session. | +| **Action** | Start the node with `reconciliation=True`. | +| **Event sequence** | `OrderStatusReport` generated for each open order. | +| **Pass criteria** | Each open order is loaded into the cache with correct `venue_order_id`, status=ACCEPTED, price, quantity, side, and order type. | +| **Skip when** | Never. | + +**Considerations:** + +- Leave limit orders open from a prior test session (do not cancel on stop). +- Use `external_order_claims` to claim the instrument so the adapter reconciles orders for it. +- Verify that the reconciled order count matches the venue-reported count. + +### TC-E85: Reconcile filled orders + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | One or more filled orders on the venue from a prior session. | +| **Action** | Start the node with `reconciliation=True`. | +| **Event sequence** | `FillReport` generated for each historical fill. | +| **Pass criteria** | Each filled order is loaded into the cache with correct `venue_order_id`, status=FILLED, fill price, fill quantity, and commission. | +| **Skip when** | Never. | + +**Considerations:** + +- Requires orders that filled in a prior session. +- Verify fill price, quantity, and commission match the venue's reported values. +- Some adapters may only report fills within a lookback window. + +### TC-E86: Reconcile open long position + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | An open long position on the venue from a prior session. | +| **Action** | Start the node with `reconciliation=True`. | +| **Event sequence** | `PositionStatusReport` generated for the long position. | +| **Pass criteria** | Position loaded into cache with correct instrument, side=LONG, quantity, and entry price matching the venue. | +| **Skip when** | Never. | + +**Considerations:** + +- Open a long position in a prior session and stop the strategy without closing it + (`close_positions_on_stop=False`). +- Verify the reconciled position quantity and average entry price match the venue. +- After reconciliation, the strategy should be able to manage or close this position. + +### TC-E87: Reconcile open short position + +| Field | Value | +|--------------------|------------------------------------------------------------------------| +| **Prerequisite** | An open short position on the venue from a prior session. | +| **Action** | Start the node with `reconciliation=True`. | +| **Event sequence** | `PositionStatusReport` generated for the short position. | +| **Pass criteria** | Position loaded into cache with correct instrument, side=SHORT, quantity, and entry price matching the venue. | +| **Skip when** | Never. | + +**Considerations:** + +- Open a short position in a prior session and stop the strategy without closing it + (`close_positions_on_stop=False`). +- Verify the reconciled position quantity and average entry price match the venue. +- After reconciliation, the strategy should be able to manage or close this position. + +--- + +## ExecTester configuration reference + +Quick reference for all `ExecTesterConfig` parameters. Defaults shown are for the Python config; +the Rust builder uses equivalent defaults. + +| Parameter | Type | Default | Affects groups | +|-------------------------------------------------|-------------------|-----------------|----------------| +| `instrument_id` | InstrumentId | *required* | All | +| `order_qty` | Decimal | *required* | All | +| `order_display_qty` | Decimal? | None | 2, 7 | +| `order_expire_time_delta_mins` | PositiveInt? | None | 2 | +| `order_params` | dict? | None | 7 | +| `client_id` | ClientId? | None | All | +| `subscribe_quotes` | bool | True | — | +| `subscribe_trades` | bool | True | — | +| `subscribe_book` | bool | False | — | +| `book_type` | BookType | L2_MBP | — | +| `book_depth` | PositiveInt? | None | — | +| `book_interval_ms` | PositiveInt | 1000 | — | +| `book_levels_to_print` | PositiveInt | 10 | — | +| `open_position_on_start_qty` | Decimal? | None | 1, 9 | +| `open_position_time_in_force` | TimeInForce | GTC | 1 | +| `enable_limit_buys` | bool | True | 2, 4, 5, 6 | +| `enable_limit_sells` | bool | True | 2, 4, 5, 6 | +| `enable_stop_buys` | bool | False | 3, 4 | +| `enable_stop_sells` | bool | False | 3, 4 | +| `limit_time_in_force` | TimeInForce? | None | 2, 6 | +| `tob_offset_ticks` | PositiveInt | 500 | 2, 4 | +| `stop_order_type` | OrderType | STOP_MARKET | 3 | +| `stop_offset_ticks` | PositiveInt | 100 | 3 | +| `stop_limit_offset_ticks` | PositiveInt? | None | 3 | +| `stop_time_in_force` | TimeInForce? | None | 3 | +| `stop_trigger_type` | TriggerType? | None | 3 | +| `enable_brackets` | bool | False | 6 | +| `bracket_entry_order_type` | OrderType | LIMIT | 6 | +| `bracket_offset_ticks` | PositiveInt | 500 | 6 | +| `modify_orders_to_maintain_tob_offset` | bool | False | 4 | +| `modify_stop_orders_to_maintain_offset` | bool | False | 4 | +| `cancel_replace_orders_to_maintain_tob_offset` | bool | False | 4 | +| `cancel_replace_stop_orders_to_maintain_offset` | bool | False | 4 | +| `use_post_only` | bool | False | 2, 6, 7, 8 | +| `use_quote_quantity` | bool | False | 1, 7 | +| `emulation_trigger` | TriggerType? | None | 2, 3 | +| `cancel_orders_on_stop` | bool | True | 5, 9 | +| `close_positions_on_stop` | bool | True | 9 | +| `close_positions_time_in_force` | TimeInForce? | None | 9 | +| `reduce_only_on_stop` | bool | True | 7, 9 | +| `use_individual_cancels_on_stop` | bool | False | 5 | +| `use_batch_cancel_on_stop` | bool | False | 5 | +| `dry_run` | bool | False | — | +| `log_data` | bool | True | — | +| `test_reject_post_only` | bool | False | 8 | +| `test_reject_reduce_only` | bool | False | 8 | +| `can_unsubscribe` | bool | True | 9 | diff --git a/nautilus-docs/docs/developer_guide/test_datasets.md b/nautilus-docs/docs/developer_guide/test_datasets.md new file mode 100644 index 0000000..debabc3 --- /dev/null +++ b/nautilus-docs/docs/developer_guide/test_datasets.md @@ -0,0 +1,282 @@ +# Test Datasets + +Target standards for curating, storing, and consuming external datasets used as test fixtures. +New datasets should follow these standards. Existing datasets are documented under +[legacy datasets](#legacy-datasets) and will be migrated incrementally. + +## Dataset categories + +**Small data** (< 1 MB) is checked directly into `tests/test_data//` +alongside a `metadata.json` file. These files are always available without network access. + +**Large data** (> 1 MB) is hosted as Parquet in the R2 test-data bucket. +A SHA-256 checksum is recorded in `tests/test_data/large/checksums.json`. +The `ensure_test_data_exists()` helper downloads the file on first use and verifies integrity. + +**User-fetched data** is used when a vendor license, entitlement model, or access control does not +allow NautilusTrader to redistribute the data through the public repo or the public R2 bucket. +In this model, the repo stores only a manifest, fetch instructions, and the transform code. Each +user downloads the source data with their own vendor account and converts it locally. + +Use the user-fetched model when any of the following apply: + +- The vendor requires each user to hold their own account, API key, or historical-data license. +- The license allows internal use but does not clearly allow redistribution of derived fixtures. +- The dataset is suitable for examples or opt-in integration tests, but not for default CI. + +## Required metadata + +Every curated dataset that stores or redistributes a concrete artifact must include a +`metadata.json` with at minimum: + +| Field | Description | +|----------------|-------------------------------------------------------------------| +| `file` | Filename of the dataset. | +| `sha256` | SHA-256 hash of the file. | +| `size_bytes` | File size in bytes. | +| `original_url` | Download URL of the original source data. | +| `licence` | License terms and any redistribution constraints. | +| `added_at` | ISO 8601 timestamp when the dataset was curated. | + +These fields match the output of `scripts/curate-dataset.sh`. Additional recommended +fields for richer provenance: + +| Field | Description | +|-----------------|------------------------------------------------------------------| +| `instrument` | Instrument symbol(s) covered. | +| `date` | Trading date(s) covered. | +| `format` | Storage format (e.g., "Nautilus OrderBookDelta Parquet"). | +| `original_file` | Original vendor filename before transformation. | +| `parser` | Parser used for transformation (e.g., "itchy 0.3.4"). | + +User-fetched datasets use the same metadata fields where they apply. They should also include: + +| Field | Description | +|-----------------------|-----------------------------------------------------------------------| +| `distribution` | Must be `"user-fetch"`. | +| `fetch_method` | How the user acquires the source data (API, web portal, CLI, etc.). | +| `fetch_reference` | URL or document reference for the user-facing download flow. | +| `auth` | Required credentials or entitlements, if any. | +| `transform_version` | Version of the local transform pipeline that builds the final files. | +| `redistribution` | Short note describing redistribution limits for the dataset. | +| `public_mirror` | Must be `false` for restricted vendor datasets. | + +For user-fetched datasets without a single committed or mirrored artifact, `file`, `sha256`, and +`size_bytes` may be omitted from `metadata.json`. In that case, `target_files` in `manifest.json` +is authoritative for the local output files. For user-fetched datasets, `original_url` may point +to the vendor download entry point rather than the exact file URL when the exact file is generated +per user account or per request. + +Other metadata fields remain recommended where they apply. In particular, `licence` and `added_at` +should still be recorded for user-fetched datasets. + +## Storage format + +New datasets should be stored as **Nautilus Parquet** (not raw vendor formats). +This ensures: + +- Consistent data types across all test datasets. +- No vendor format parsing at test time. +- Clear derivative-work status for licensing. + +Use ZSTD compression (level 3) with 1M row groups. + +User-fetched datasets should also end up as Nautilus Parquet after the local transform step. +Raw vendor files should stay outside the repo and outside the public R2 bucket. + +## Naming convention + +``` +___.parquet +``` + +Examples: + +- `itch_AAPL_2019-01-30_deltas.parquet` +- `tardis_BTCUSDT_2020-09-01_depth10.parquet` + +## Curation workflow + +### Simple files (single download) + +Use `scripts/curate-dataset.sh`: + +```bash +scripts/curate-dataset.sh +``` + +This creates a versioned directory (`v1//`) with the file, +`LICENSE.txt`, and `metadata.json` containing the required fields above. + +### Complex pipelines (parse + transform) + +For datasets requiring format conversion (e.g., binary ITCH to Parquet): + +1. Write a curation function in `crates/testkit/src//` gated behind + `#[cfg(test)]` or an `#[ignore]` test. +2. The function should: download, parse, filter, convert to NautilusTrader types, write Parquet. +3. Output the Parquet file and `metadata.json` to a local directory. +4. Upload to R2 manually, then add the checksum to `checksums.json`. + +### User-fetched pipelines (restricted redistribution) + +For datasets that NautilusTrader cannot redistribute: + +1. Commit a manifest and `metadata.json`, but do not commit the real vendor data or derived + Parquet output. +2. Provide a local fetch command or helper that uses the user's own vendor credentials, + entitlements, or purchased historical files. +3. Convert the vendor data locally into Nautilus Parquet. +4. Store the resulting files in a local cache path that is ignored by git. +5. Make tests and examples opt in. They should skip cleanly when the dataset is missing. + +The default distribution order for new datasets is: + +1. Checked in small data. +2. Public R2 large data. +3. User-fetched data. + +Choose user-fetched only when the first two options are not acceptable under the vendor's terms. + +Do not: + +- Upload restricted vendor datasets to the public R2 bucket. +- Commit real vendor-derived Parquet files to the repo when redistribution rights are unclear. +- Make default CI depend on vendor credentials or paid historical-data access. + +You may maintain a private mirror for internal CI or employees when the license permits internal +sharing. Treat this as a separate operational path, not as part of the public test-data standard. + +## Adding a new dataset + +1. Curate the data following the workflow above. +2. Write `metadata.json` with all required fields. +3. For small data: commit to `tests/test_data//`. +4. For large data: upload Parquet to R2, add checksum to `tests/test_data/large/checksums.json`. +5. For user-fetched data: commit the manifest and fetch instructions only. Keep the source and + derived data out of the repo and out of the public R2 bucket. +6. Add path helper functions to `crates/testkit/src/common.rs` when shared testkit access is needed. +7. Write tests that consume the dataset. + +For user-fetched data, prefer this layout: + +```text +tests/test_data/// + metadata.json + manifest.json + README.md +``` + +Use `tests/test_data/local///` as the standard local cache path for generated +artifacts. Keep raw vendor downloads in a sibling `vendor/` directory under the same cache path +when local retention is needed. + +The manifest should be machine-readable and stable. It should capture the minimum information needed +to reproduce the fetch and transform steps on another machine. + +`metadata.json` is authoritative for provenance, licensing, and redistribution rules. +`manifest.json` is authoritative for fetch inputs, commands, cache locations, and output files. + +Recommended manifest fields: + +| Field | Description | +|---------------------|--------------------------------------------------------------------| +| `slug` | Stable dataset identifier. | +| `vendor` | Vendor or venue name. | +| `source_type` | `api`, `portal-download`, `purchased-archive`, etc. | +| `source_filters` | Symbols, event IDs, market IDs, date ranges, or file names. | +| `target_files` | Output Nautilus Parquet files expected after conversion. | +| `cache_dir` | Local output location relative to `tests/test_data/local/`. | +| `fetch_command` | Suggested command or script entry point. | +| `transform_command` | Suggested local conversion command. | +| `env` | Required environment variables. | +| `notes` | Short operational notes for users. | + +Tests that rely on user-fetched data should: + +- Be marked or grouped separately from default CI tests. +- Skip with a clear message when the local dataset is absent. +- Avoid network access unless the user explicitly opts in. +- Reuse a stable local cache path so the fetch happens once per machine. + +For pytest-based tests, prefer a guard like: + +```python +if not filepath.exists(): + pytest.skip(f"User-fetched test data not found: {filepath}") +``` + +For Rust tests that require manual dataset preparation, prefer `#[ignore]` when the test is not +expected to run in default CI. + +## Test runner serialization + +Tests that download large data files share target paths across test binaries. +Because `nextest` runs each binary in a separate process, concurrent downloads +to the same path can race. The nextest config at `.config/nextest.toml` defines +a `large-data-tests` group with `max-threads = 1` to serialize these binaries. + +When adding a new test binary that downloads large shared files, add it to the +group filter: + +```toml +[[profile.default.overrides]] +filter = 'binary(grid_mm_itch) | binary(orderbook_integration) | binary(your_new_binary)' +test-group = 'large-data-tests' +``` + +## Regenerating datasets + +When a schema change invalidates a large Parquet file, regenerate it from the +original source data using the curation tests below. After regenerating: + +1. `sha256sum /tmp/.parquet` +2. Update `tests/test_data/large/checksums.json` with the new hash. +3. Update the corresponding `metadata.json` (sha256, size_bytes). +4. Upload the Parquet file to R2. +5. Commit `checksums.json` and `metadata.json` (this also busts the CI cache). + +### ITCH AAPL L3 deltas + +Source: `01302019.NASDAQ_ITCH50.gz` (~4.4 GB) from NASDAQ EMI. + +```bash +# Download source (keep a local copy, this is a large file) +wget -O ~/Downloads/01302019.NASDAQ_ITCH50.gz \ + "https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/01302019.NASDAQ_ITCH50.gz" + +# Curation test expects source at /tmp +ln -sf ~/Downloads/01302019.NASDAQ_ITCH50.gz /tmp/01302019.NASDAQ_ITCH50.gz + +# Regenerate parquet (output: /tmp/itch_AAPL.XNAS_2019-01-30_deltas.parquet) +cargo test -p nautilus-testkit --lib test_curate_aapl_itch -- --ignored --nocapture +``` + +### Tardis Deribit BTC-PERPETUAL L2 deltas + +Source: `tardis_deribit_incremental_book_L2_2020-04-01_BTC-PERPETUAL.csv.gz` from +[Tardis](https://tardis.dev/). First-of-month data is available as free samples +(no API key required). + +```bash +# Download source (free sample, no API key needed) +wget -O tests/test_data/large/tardis_deribit_incremental_book_L2_2020-04-01_BTC-PERPETUAL.csv.gz \ + "https://datasets.tardis.dev/v1/deribit/incremental_book_L2/2020/04/01/BTC-PERPETUAL.csv.gz" + +# Regenerate parquet (output: /tmp/tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet) +cargo test -p nautilus-tardis test_curate_deribit_deltas -- --ignored --nocapture +``` + +## Legacy datasets + +These datasets predate this policy and use raw vendor formats (CSV/CSV.gz) +without `metadata.json`. They remain valid for existing tests. New datasets +should follow the Parquet standard above. + +| Dataset | Source | Format | Location | Status | +|--------------------------|--------|------------------|---------------------------|---------| +| Tardis Deribit L2 deltas | Tardis | Parquet (large) | `tests/test_data/large/` | Curated | +| ITCH AAPL L3 deltas | NASDAQ | Parquet (large) | `tests/test_data/large/` | Curated | +| Tardis Deribit L2 | Tardis | CSV (checked in) | `tests/test_data/tardis/` | Legacy | +| Tardis Binance snapshots | Tardis | CSV.gz (large) | `tests/test_data/large/` | Legacy | +| Tardis Bitmex trades | Tardis | CSV.gz (large) | `tests/test_data/large/` | Legacy | diff --git a/nautilus-docs/docs/developer_guide/testing.md b/nautilus-docs/docs/developer_guide/testing.md new file mode 100644 index 0000000..bbf767d --- /dev/null +++ b/nautilus-docs/docs/developer_guide/testing.md @@ -0,0 +1,289 @@ +# Testing + +Our automated tests serve as executable specifications for the trading platform. +A healthy suite documents intended behaviour, gives contributors confidence to refactor, and catches regressions before they reach production. +Tests also double as living examples that clarify complex flows and provide rapid CI feedback so issues surface early. + +The suite covers these categories: + +- Unit tests +- Integration tests +- Acceptance tests +- Performance tests +- Property-based tests +- Fuzzing +- Memory leak tests + +## Property-based testing + +Property testing verifies that logic holds for *all* valid inputs, not just hand-picked examples. +We use [`proptest`](https://altsysrq.github.io/proptest-book/intro.html) in Rust to enforce invariants. + +- **Use cases:** Core domain types (`Price`, `Quantity`, `UnixNanos`), accounting engines, matching engines, and state machines. +- **Example invariants:** + - Round-trip serialization: `parse(to_string(value)) == value` + - Inverse operations: `(A + B) - B == A` + - Transitivity: `If A < B and B < C, then A < C` + +## Fuzzing + +Fuzzing introduces unstructured or malicious data to the system to verify it fails gracefully. + +- **Use cases:** Network boundaries, exchange data parsers (JSON, FIX, WebSocket feeds), and complex state machines. +- **Goal:** The system returns a `Result::Err` and never panics, hangs, or leaks memory when encountering malformed data. + +When building or modifying core types, write property tests to cover the mathematical boundaries. + +Performance tests help evolve performance-critical components. + +Run tests with [pytest](https://docs.pytest.org), our primary test runner. +Use parametrized tests and fixtures (e.g., `@pytest.mark.parametrize`) to avoid repetitive code and improve clarity. + +## Running tests + +### Python tests + +From the repository root: + +```bash +make pytest +# or +uv run --active --no-sync pytest --new-first --failed-first +# or +pytest +``` + +For performance tests: + +```bash +make test-performance +# or +uv run --active --no-sync pytest tests/performance_tests --benchmark-disable-gc --codspeed +``` + +The `--benchmark-disable-gc` flag prevents garbage collection from skewing results. Run performance tests in isolation (not with unit tests) to avoid interference. + +### Rust tests + +```bash +make cargo-test +# or +cargo nextest run --workspace --features "python,ffi,high-precision,defi" --cargo-profile nextest +``` + +#### Testing with optional features + +Use `EXTRA_FEATURES` to include optional features like `capnp` or `hypersync`: + +```bash +# Test with capnp feature +make cargo-test EXTRA_FEATURES="capnp" + +# Test with multiple features +make cargo-test EXTRA_FEATURES="capnp hypersync" + +# Legacy shorthand for hypersync +make cargo-test HYPERSYNC=true + +# Test specific crate with features +make cargo-test-crate-nautilus-serialization FEATURES="capnp" +``` + +### IDE integration + +- **PyCharm**: Right-click the tests folder or file → "Run pytest". +- **VS Code**: Use the Python Test Explorer extension. + +## Test style + +- Name test functions after what they exercise; you do not need to encode the expected assertions in the name. +- Add docstrings when they clarify setup, scenarios, or expectations. +- Prefer pytest-style free functions for Python tests instead of test classes with setup methods. +- **Group assertions** when possible: perform all setup/act steps first, then assert together to avoid the act-assert-act smell. +- Use `unwrap`, `expect`, or direct `panic!`/`assert` calls inside tests; clarity and conciseness matter more than defensive error handling here. + +For Rust-specific test conventions (module structure, `#[rstest]`, parameterization), see the [Rust guide](rust.md#testing-conventions). + +## Waiting for asynchronous effects + +When waiting for background work to complete, prefer the polling helpers `await eventually(...)` from `nautilus_trader.test_kit.functions` and `wait_until_async(...)` from `nautilus_common::testing` instead of arbitrary sleeps. They surface failures faster and reduce flakiness in CI because they stop as soon as the condition is satisfied or time out with a useful error. + +## Mocks + +Prefer hand-written stubs that return fixed values over mocking frameworks. Use `MagicMock` only when you need to assert call counts/arguments or simulate complex state changes. Avoid mocking the objects you're actually testing. + +## Code coverage + +We generate coverage reports with `coverage` and publish them to [codecov](https://about.codecov.io/). + +Aim for high coverage without sacrificing appropriate error handling or causing "test induced damage" to the architecture. + +Some branches remain untestable without modifying production behaviour. +For example, a final condition in a defensive if-else block may only trigger for unexpected values; leave these checks in place so future changes can exercise them if needed. + +Design-time exceptions can also be impractical to test, so 100% coverage is not the target. + +## Excluded code coverage + +We use `pragma: no cover` comments to [exclude code from coverage](https://coverage.readthedocs.io/en/coverage-4.3.3/excluding.html) when tests would be redundant. +Typical examples include: + +- Asserting an abstract method raises `NotImplementedError` when called. +- Asserting the final condition check of an if-else block when impossible to test (as above). + +Such tests are expensive to maintain because they must track refactors while providing little value. +Keep concrete implementations of abstract methods fully covered. +Remove `pragma: no cover` when it no longer applies and restrict its use to the cases above. + +## Debugging Rust tests + +Use the default test configuration to debug Rust tests. + +To run the full suite with debug symbols for later, run `make cargo-test-debug` instead of `make cargo-test`. + +In IntelliJ IDEA, adjust the run configuration for parametrised `#[rstest]` cases so it reads `test --package nautilus-model --lib data::bar::tests::test_get_time_bar_start::case_1` +(remove `-- --exact` and append `::case_n` where `n` starts at 1). This workaround matches the behaviour explained [here](https://github.com/rust-lang/rust-analyzer/issues/8964#issuecomment-871592851). + +In VS Code you can pick the specific test case to debug directly. + +## Python + Rust Mixed Debugging + +This workflow lets you debug Python and Rust code simultaneously from a Jupyter notebook inside VS Code. + +### Setup + +Install these VS Code extensions: Rust Analyzer, CodeLLDB, Python, Jupyter. + +### Step 0: Compile `nautilus_trader` with debug symbols + + ```bash + cd nautilus_trader && make build-debug-pyo3 + ``` + +### Step 1: Set up debugging configuration + +```python +from nautilus_trader.test_kit.debug_helpers import setup_debugging + +setup_debugging() +``` + +This command creates the required VS Code debugging configurations and starts a `debugpy` server for the Python debugger. + +By default `setup_debugging()` expects the `.vscode` folder one level above the `nautilus_trader` root directory. +Adjust the target location if your workspace layout differs. + +### Step 2: Set breakpoints + +- **Python breakpoints:** Set in VS Code in the Python source files. +- **Rust breakpoints:** Set in VS Code in the Rust source files. + +### Step 3: Start mixed debugging + +1. In VS Code select the **"Debug Jupyter + Rust (Mixed)"** configuration. +2. Start debugging (F5) or press the green run arrow. +3. Both Python and Rust debuggers attach to your Jupyter session. + +### Step 4: Execute code + +Run Jupyter notebook cells that call Rust functions. The debugger stops at breakpoints in both Python and Rust code. + +### Available configurations + +`setup_debugging()` creates these VS Code configurations: + +- **`Debug Jupyter + Rust (Mixed)`** - Mixed debugging for Jupyter notebooks. +- **`Jupyter Mixed Debugging (Python)`** - Python-only debugging for notebooks. +- **`Rust Debugger (for Jupyter debugging)`** - Rust-only debugging for notebooks. + +### Example + +Open and run the example notebook: `debug_mixed_jupyter.ipynb`. + +### Reference + +- [PyO3 debugging](https://pyo3.rs/v0.25.1/debugging.html?highlight=deb#debugging-from-jupyter-notebooks) + +## Data type testing + +Each data type flows through multiple layers of the platform. The table below shows where +existing types are tested, so new types can follow the same pattern. + +### Test layer matrix + +| Layer | Location | What it covers | +|------------------------|---------------------------------------------|------------------------------------------------------------| +| DataEngine subscribe | `crates/data/tests/engine.rs` | Engine processes subscribe/unsubscribe commands correctly. | +| DataEngine publish | `crates/data/tests/engine.rs` | Engine routes published data to the message bus. | +| DataActor subscribe | `crates/common/src/actor/tests.rs` | Actor subscribes and receives data via typed publish. | +| DataActor unsubscribe | `crates/common/src/actor/tests.rs` | Actor stops receiving data after unsubscribe. | +| PyO3 actor dispatch | `crates/common/src/python/actor.rs` | Rust handler dispatches to Python `on_*` method. | +| Python Actor subscribe | `tests/unit_tests/common/test_actor.py` | Python actor subscribes; command count increments. | +| Python Actor unsub | `tests/unit_tests/common/test_actor.py` | Python actor unsubscribes; subscription list clears. | +| Backtest client | `nautilus_trader/backtest/data_client.pyx` | Backtest client overrides base subscribe/unsubscribe. | +| Adapter live tests | `docs/developer_guide/spec_data_testing.md` | Live data acceptance tests (DataTester). | + +### Coverage per data type + +The following table shows which layers have test coverage for each data type. +Use this as a checklist when adding a new type. + +| Data type | Engine | Actor (Rust) | PyO3 dispatch | Actor (Python) | Backtest client | Adapter spec | +|---------------------|--------|--------------|---------------|----------------|-----------------|--------------| +| `InstrumentAny` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `OrderBookDeltas` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `OrderBook` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `QuoteTick` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `TradeTick` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `Bar` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `MarkPriceUpdate` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `IndexPriceUpdate` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `FundingRateUpdate` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `InstrumentStatus` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `InstrumentClose` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `OptionGreeks` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| `OptionChainSlice` | - | ✓ | ✓ | ✓ | - | ✓ | +| `CustomData` | ✓ | ✓ | ✓ | ✓ | ✓ | - | + +`OptionChainSlice` is assembled by the DataEngine's `OptionChainManager` from per-instrument +greeks and quote subscriptions. It does not have its own engine subscribe command or +backtest client override. + +### Adding a new data type + +When introducing a new data type, add tests at each layer: + +1. **DataEngine** (`crates/data/tests/engine.rs`): Add `test_execute_subscribe_` and + `test_execute_unsubscribe_` tests. Follow the pattern in existing subscribe tests: + register client, build command, call `engine.execute`, assert subscription list. + +2. **DataActor Rust** (`crates/common/src/actor/tests.rs`): + - Add `received_: Vec` field to `TestDataActor`. + - Implement the `on_` handler in the `DataActor` trait impl. + - Add `test_subscribe_and_receive_` and `test_unsubscribe_` tests. + - Use the typed publish function (`msgbus::publish_`), not `publish_any`, + for types that use `TypedHandler` routing. + +3. **PyO3 actor dispatch** (`crates/common/src/python/actor.rs`): + - Add `dispatch_on_` method that calls `py_self.call_method1("on_", ...)`. + - Add `on_` in the `DataActor` trait impl that calls the dispatch method. + - Add `#[pyo3(name = "on_")]` method in the `#[pymethods]` block. + - Add `on_` to `RustTestDataActor` wrapper and the inline Python test class. + - Add handler test and dispatch test. + +4. **Python Actor** (`tests/unit_tests/common/test_actor.py`): + - Add `test_subscribe_` and `test_unsubscribe_` tests. + - Assert `actor.subscribed_()` returns expected entries after subscribe and + is empty after unsubscribe. + +5. **Backtest client** (`nautilus_trader/backtest/data_client.pyx`): Override + `subscribe_` and `unsubscribe_` if the base `MarketDataClient` raises + `NotImplementedError` for the method. + +6. **Documentation**: Add entries to `actors.md` callback table, `strategies.md` handler + signatures, `adapters.md` subscribe method stubs, and `spec_data_testing.md` test cards. + +:::tip +Search for an existing type like `instrument_close` or `funding_rate` across all six layers +to find concrete examples of the patterns described above. +::: diff --git a/nautilus-docs/docs/getting_started/backtest_high_level.py b/nautilus-docs/docs/getting_started/backtest_high_level.py new file mode 100644 index 0000000..9ba0e88 --- /dev/null +++ b/nautilus-docs/docs/getting_started/backtest_high_level.py @@ -0,0 +1,251 @@ +# %% [markdown] +# # Backtest (high-level API) +# +# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/getting_started/backtest_high_level.py). + +# %% [markdown] +# ## Overview +# +# This tutorial walks through how to use a `BacktestNode` to backtest a simple EMA cross strategy +# on a simulated FX ECN venue using historical quote tick data. +# +# The following points will be covered: +# - Load raw data (external to Nautilus) into the data catalog. +# - Set up configuration objects for a `BacktestNode`. +# - Run backtests with a `BacktestNode`. +# + +# %% [markdown] +# ## Prerequisites +# - Python 3.12+ installed. +# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`). + +# %% [markdown] +# ## Imports +# +# We'll start with all of our imports for the remainder of this tutorial. + +# %% +import shutil +from decimal import Decimal +from pathlib import Path + +import pandas as pd + +from nautilus_trader.backtest.node import BacktestDataConfig +from nautilus_trader.backtest.node import BacktestEngineConfig +from nautilus_trader.backtest.node import BacktestNode +from nautilus_trader.backtest.node import BacktestRunConfig +from nautilus_trader.backtest.node import BacktestVenueConfig +from nautilus_trader.config import ImportableStrategyConfig +from nautilus_trader.core.datetime import dt_to_unix_nanos +from nautilus_trader.model import QuoteTick +from nautilus_trader.persistence.catalog import ParquetDataCatalog +from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler +from nautilus_trader.test_kit.providers import CSVTickDataLoader +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# %% [markdown] +# Before starting, download some sample data for backtesting. +# +# For this example we will use FX data from `histdata.com`. Go to https://www.histdata.com/download-free-forex-historical-data/?/ascii/tick-data-quotes/ and select an FX pair, then select one or more months of data to download. +# +# Examples of downloaded files: +# +# - `DAT_ASCII_EURUSD_T_202410.csv` (EUR/USD data for month 2024-10) +# - `DAT_ASCII_EURUSD_T_202411.csv` (EUR/USD data for month 2024-11) +# +# Once you have downloaded the data: +# +# 1. Extract the CSV files and copy them into a folder, for example `~/Downloads/Data/HISTDATA/`. +# 2. Set the `DATA_DIR` variable below to the directory containing the CSV files. + +# %% +DATA_DIR = "~/Downloads/Data/HISTDATA/" + +# %% +path = Path(DATA_DIR).expanduser() +raw_files = [ + f for f in path.iterdir() if f.is_file() and (f.suffix == ".csv" or f.name.endswith(".csv.gz")) +] +assert raw_files, f"Unable to find any CSV files in directory {path}" +raw_files + +# %% [markdown] +# ## Loading data into the Parquet data catalog +# +# Histdata stores the FX data in CSV/text format with fields `timestamp, bid_price, ask_price`. +# First, load this raw data into a `pandas.DataFrame` with a schema compatible with Nautilus quotes. +# +# Then create Nautilus `QuoteTick` objects by processing the DataFrame with a `QuoteTickDataWrangler`. +# + +# %% +# Load the first CSV file into a pandas DataFrame +df = CSVTickDataLoader.load( + file_path=raw_files[0], + index_col=0, + header=None, + names=["timestamp", "bid_price", "ask_price", "volume"], + usecols=["timestamp", "bid_price", "ask_price"], + parse_dates=["timestamp"], + date_format="%Y%m%d %H%M%S%f", +) + +df = df.sort_index() +df.head(2) + +# %% +# Process quotes using a wrangler +EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD") +wrangler = QuoteTickDataWrangler(EURUSD) + +ticks = wrangler.process(df) + +# Preview: see first 2 ticks +ticks[0:2] + +# %% [markdown] +# See the [Loading data](../concepts/data) guide for more details. +# +# Instantiate a `ParquetDataCatalog` with a storage directory (here we use the current directory). +# Write the instrument and tick data to the catalog. +# + +# %% +CATALOG_PATH = Path.cwd() / "catalog" + +# Clear if it already exists, then create fresh +if CATALOG_PATH.exists(): + shutil.rmtree(CATALOG_PATH) +CATALOG_PATH.mkdir(parents=True) + +# Create a catalog instance +catalog = ParquetDataCatalog(CATALOG_PATH) + +# Write instrument to the catalog +catalog.write_data([EURUSD]) + +# Write ticks to catalog +catalog.write_data(ticks) + +# %% [markdown] +# ## Using the Data Catalog +# +# The catalog provides methods like `.instruments(...)` and `.quote_ticks(...)` to query stored data. +# + +# %% +# Get list of all instruments in catalog +catalog.instruments() + +# %% +# See 1st instrument from catalog +instrument = catalog.instruments()[0] +instrument + +# %% +# Query quote ticks from catalog to determine the data range +all_ticks = catalog.quote_ticks(instrument_ids=[EURUSD.id.value]) +print(f"Total ticks in catalog: {len(all_ticks)}") + +if all_ticks: + # Get timestamps from the data + first_tick_time = pd.Timestamp(all_ticks[0].ts_init, unit="ns", tz="UTC") + last_tick_time = pd.Timestamp(all_ticks[-1].ts_init, unit="ns", tz="UTC") + print(f"Data range: {first_tick_time} to {last_tick_time}") + + # Set backtest range to first 2 weeks of data (as ISO strings for BacktestDataConfig) + start_time = first_tick_time.isoformat() + end_time = (first_tick_time + pd.Timedelta(days=14)).isoformat() + print(f"Backtest range: {start_time} to {end_time}") + + # Preview selected data + start_ns = all_ticks[0].ts_init + end_ns = dt_to_unix_nanos(first_tick_time + pd.Timedelta(days=14)) + selected_quote_ticks = catalog.quote_ticks( + instrument_ids=[EURUSD.id.value], + start=start_ns, + end=end_ns, + ) + print(f"Selected ticks for backtest: {len(selected_quote_ticks)}") + selected_quote_ticks[:2] +else: + raise ValueError("No ticks found in catalog") + +# %% [markdown] +# ## Add venues + +# %% +venue_configs = [ + BacktestVenueConfig( + name="SIM", + oms_type="HEDGING", + account_type="MARGIN", + base_currency="USD", + starting_balances=["1_000_000 USD"], + ), +] + +# %% [markdown] +# ## Add data + +# %% +str(CATALOG_PATH) + +# %% +data_configs = [ + BacktestDataConfig( + catalog_path=str(CATALOG_PATH), + data_cls=QuoteTick, + instrument_id=instrument.id, + start_time=start_time, + end_time=end_time, + ), +] + +# %% [markdown] +# ## Add strategies + +# %% +strategies = [ + ImportableStrategyConfig( + strategy_path="nautilus_trader.examples.strategies.ema_cross:EMACross", + config_path="nautilus_trader.examples.strategies.ema_cross:EMACrossConfig", + config={ + "instrument_id": instrument.id, + "bar_type": "EUR/USD.SIM-15-MINUTE-BID-INTERNAL", + "fast_ema_period": 10, + "slow_ema_period": 20, + "trade_size": Decimal(1_000_000), + }, + ), +] + +# %% [markdown] +# ## Configure backtest +# +# Nautilus uses a `BacktestRunConfig` object to centralize backtest configuration. +# The `BacktestRunConfig` is Partialable, so you can configure it in stages. +# This design reduces boilerplate when you create multiple backtest runs (for example when performing a parameter grid search). +# + +# %% +config = BacktestRunConfig( + engine=BacktestEngineConfig(strategies=strategies), + data=data_configs, + venues=venue_configs, +) + +# %% [markdown] +# ## Run backtest +# +# Now we can run the backtest node, which will simulate trading across the entire data stream. + +# %% +node = BacktestNode(configs=[config]) + +results = node.run() +results diff --git a/nautilus-docs/docs/getting_started/backtest_low_level.py b/nautilus-docs/docs/getting_started/backtest_low_level.py new file mode 100644 index 0000000..32069e1 --- /dev/null +++ b/nautilus-docs/docs/getting_started/backtest_low_level.py @@ -0,0 +1,220 @@ +# %% [markdown] +# # Backtest (low-level API) +# +# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/getting_started/backtest_low_level.py). + +# %% [markdown] +# ## Overview +# +# This tutorial walks through how to use a `BacktestEngine` to backtest a simple EMA cross strategy +# with a TWAP execution algorithm on a simulated Binance Spot exchange using historical trade tick data. +# +# The following points will be covered: +# - Load raw data (external to Nautilus) using data loaders and wranglers. +# - Add this data to a `BacktestEngine`. +# - Add venues, strategies, and execution algorithms to a `BacktestEngine`. +# - Run backtests with a `BacktestEngine`. +# - Perform post-run analysis and repeated runs. +# + +# %% [markdown] +# ## Prerequisites +# - Python 3.12+ installed. +# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`). + +# %% [markdown] +# ## Imports +# +# We'll start with all of our imports for the remainder of this tutorial. + +# %% +from decimal import Decimal + +from nautilus_trader.backtest.config import BacktestEngineConfig +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.examples.algorithms.twap import TWAPExecAlgorithm +from nautilus_trader.examples.strategies.ema_cross_twap import EMACrossTWAP +from nautilus_trader.examples.strategies.ema_cross_twap import EMACrossTWAPConfig +from nautilus_trader.model import BarType +from nautilus_trader.model import Money +from nautilus_trader.model import TraderId +from nautilus_trader.model import Venue +from nautilus_trader.model.currencies import ETH +from nautilus_trader.model.currencies import USDT +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.persistence.wranglers import TradeTickDataWrangler +from nautilus_trader.test_kit.providers import TestDataProvider +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# %% [markdown] +# ## Loading data +# +# For this tutorial we use stub test data from the NautilusTrader repository (the automated test suite also uses this data to verify platform correctness). +# +# First, instantiate a data provider to read raw CSV trade tick data into a `pd.DataFrame`. +# Next, initialize the matching instrument (`ETHUSDT` spot on Binance). +# Then wrangle the data into Nautilus `TradeTick` objects to add to the `BacktestEngine`. +# + +# %% +# Load stub test data +provider = TestDataProvider() +trades_df = provider.read_csv_ticks("binance/ethusdt-trades.csv") + +# Initialize the instrument which matches the data +ETHUSDT_BINANCE = TestInstrumentProvider.ethusdt_binance() + +# Process into Nautilus objects +wrangler = TradeTickDataWrangler(instrument=ETHUSDT_BINANCE) +ticks = wrangler.process(trades_df) + +# %% [markdown] +# See the [Loading External Data](https://nautilustrader.io/docs/latest/concepts/data#loading-data) guide for details on the data processing pipeline. + +# %% [markdown] +# ## Initialize a backtest engine +# +# Create a `BacktestEngine`. Here we pass a `BacktestEngineConfig` with a custom `trader_id` to show the configuration pattern. +# +# See the [Configuration](https://nautilustrader.io/docs/api_reference/config) API reference for all available options. +# + +# %% +# Configure backtest engine +config = BacktestEngineConfig(trader_id=TraderId("BACKTESTER-001")) + +# Build the backtest engine +engine = BacktestEngine(config=config) + +# %% [markdown] +# ## Add venues +# +# Create a venue to trade on that matches the market data you add to the engine. +# +# In this case we set up a simulated Binance Spot exchange. +# + +# %% +# Add a trading venue (multiple venues possible) +BINANCE = Venue("BINANCE") +engine.add_venue( + venue=BINANCE, + oms_type=OmsType.NETTING, + account_type=AccountType.CASH, # Spot CASH account (not for perpetuals or futures) + base_currency=None, # Multi-currency account + starting_balances=[Money(1_000_000.0, USDT), Money(10.0, ETH)], +) + +# %% [markdown] +# ## Add data +# +# Add data to the backtest engine. Start by adding the `Instrument` object we initialized earlier to match the data. +# +# Then add the trades we wrangled earlier. +# + +# %% +# Add instrument(s) +engine.add_instrument(ETHUSDT_BINANCE) + +# Add data +engine.add_data(ticks) + +# %% [markdown] +# :::note +# You can add multiple data types (including custom types) and backtest across multiple venues. +# ::: +# + +# %% [markdown] +# ## Add strategies +# +# Add the trading strategies you plan to run as part of the system. +# +# Initialize a strategy configuration, then create and add the strategy: +# + +# %% +# Configure your strategy +strategy_config = EMACrossTWAPConfig( + instrument_id=ETHUSDT_BINANCE.id, + bar_type=BarType.from_str("ETHUSDT.BINANCE-250-TICK-LAST-INTERNAL"), + trade_size=Decimal("0.10"), + fast_ema_period=10, + slow_ema_period=20, + twap_horizon_secs=10.0, + twap_interval_secs=2.5, +) + +# Instantiate and add your strategy +strategy = EMACrossTWAP(config=strategy_config) +engine.add_strategy(strategy=strategy) + +# %% [markdown] +# The strategy config above includes TWAP parameters, but we still need to add the `ExecAlgorithm` component. +# +# ## Add execution algorithms +# +# Add a TWAP execution algorithm to the engine, following the same pattern as strategies. +# + +# %% +# Instantiate and add your execution algorithm +exec_algorithm = TWAPExecAlgorithm() # Using defaults +engine.add_exec_algorithm(exec_algorithm) + +# %% [markdown] +# ## Run backtest +# +# After configuring the data, venues, and trading system, run a backtest. +# Call the `.run(...)` method to process all available data by default. +# +# See the [BacktestEngineConfig](https://nautilustrader.io/docs/latest/api_reference/config) API reference for all available options. +# + +# %% +# Run the engine (from start to end of data) +engine.run() + +# %% [markdown] +# ## Post-run and analysis +# +# The engine logs a post-run tearsheet with default statistics. You can load custom statistics too; see the [Portfolio statistics](../concepts/portfolio.md#portfolio-statistics) guide. +# +# The engine retains data and execution objects in memory for generating reports. +# + +# %% +engine.trader.generate_account_report(BINANCE) + +# %% +engine.trader.generate_order_fills_report() + +# %% +engine.trader.generate_positions_report() + +# %% [markdown] +# ## Repeated runs +# +# You can reset the engine for repeated runs with different strategy and component configurations. +# +# Instruments and data persist across resets by default, so you don't need to reload them. + +# %% +# For repeated backtest runs, reset the engine +engine.reset() + +# Instruments and data persist, just add new components and run again + +# %% [markdown] +# Remove and add individual components (actors, strategies, execution algorithms) as required. +# +# See the [Trader](../api_reference/trading.md) API reference for a description of all methods available to achieve this. +# + +# %% +# Once done, good practice to dispose of the object if the script continues +engine.dispose() diff --git a/nautilus-docs/docs/getting_started/index.md b/nautilus-docs/docs/getting_started/index.md new file mode 100644 index 0000000..249dba6 --- /dev/null +++ b/nautilus-docs/docs/getting_started/index.md @@ -0,0 +1,75 @@ +# Getting Started + +To get started with NautilusTrader, you will need: + +- A Python 3.12–3.14 environment with the `nautilus_trader` package installed. +- A way to run Python scripts or Jupyter notebooks for backtesting and/or live trading. + +## Installation + +How to install NautilusTrader on your machine. + +## Quickstart + +Run your first backtest in five minutes. No data downloads, no catalog setup. + +## Examples in repository + +The [online documentation](https://nautilustrader.io/docs/latest/) shows just a subset of examples. For the full set, see this repository on GitHub. + +The following table lists example locations ordered by recommended learning progression: + +| Directory | Contains | +|:----------------------------|:----------------------------------------------------------------------------------------------------------------------------| +| [examples/](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples) | Fully runnable, self-contained Python examples. | +| [docs/tutorials/](../tutorials/) | Tutorials demonstrating common workflows. | +| [docs/concepts/](../concepts/) | Concept guides with concise code snippets illustrating key features. | +| [nautilus_trader/examples/](../../nautilus_trader/examples/) | Pure-Python examples of basic strategies, indicators, and execution algorithms. | +| [tests/unit_tests/](../../tests/unit_tests/) | Unit tests covering core functionality and edge cases. | + +## Backtesting API levels + +NautilusTrader provides two different API levels for backtesting: + +| API Level | Description | Characteristics | +|:---------------|:--------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| High-Level API | Uses `BacktestNode` and `TradingNode` | Recommended for production: easier transition to live trading; requires a Parquet-based data catalog. | +| Low-Level API | Uses `BacktestEngine` | Intended for library development: no live-trading path; direct component access; may encourage non-live-compatible patterns. | + +:::warning[One node per process] +Running multiple `BacktestNode` or `TradingNode` instances concurrently in the same process is not supported due to global singleton state. +Sequential execution with proper disposal between runs is supported. + +See [Processes and threads](../concepts/architecture.md#processes-and-threads) for details. +::: + +See the [Backtesting](../concepts/backtesting.md) guide for help choosing an API level. + +### Backtest (low-level API) + +This tutorial runs through how to load raw data (external to Nautilus) using data loaders and wranglers, +and then use this data with a `BacktestEngine` to run a single backtest. + +### Backtest (high-level API) + +This tutorial runs through how to load raw data (external to Nautilus) into the data catalog, +and then use this data with a `BacktestNode` to run a single backtest. + +## Running in docker + +Alternatively, you can download a self-contained dockerized Jupyter notebook server, which requires no setup or +installation. This is the fastest way to get up and running to try out NautilusTrader. Note that deleting the container will also delete any data. + +- To get started, install docker: + - Go to [Docker installation guide](https://docs.docker.com/get-docker/) and follow the instructions. +- From a terminal, download the latest image: + - `docker pull ghcr.io/nautechsystems/jupyterlab:nightly --platform linux/amd64` +- Run the docker container, exposing the Jupyter port: + - `docker run -p 8888:8888 ghcr.io/nautechsystems/jupyterlab:nightly` +- Open your web browser to `localhost:{port}`: + - + +:::warning +Examples use `log_level="ERROR"` because Nautilus logging exceeds Jupyter's stdout rate limit, +causing notebooks to hang at lower log levels. +::: diff --git a/nautilus-docs/docs/getting_started/installation.md b/nautilus-docs/docs/getting_started/installation.md new file mode 100644 index 0000000..c956313 --- /dev/null +++ b/nautilus-docs/docs/getting_started/installation.md @@ -0,0 +1,361 @@ +# Installation + +NautilusTrader is officially supported for Python 3.12-3.14 on the following 64-bit platforms: + +| Operating System | Supported Versions | CPU Architecture | +|------------------------|--------------------|-------------------| +| Linux (Ubuntu) | 22.04 and later | x86_64 | +| Linux (Ubuntu) | 22.04 and later | ARM64 | +| macOS | 15.0 and later | ARM64 | +| Windows Server | 2022 and later | x86_64 | + +:::note +NautilusTrader may work on other platforms, but only those listed above are regularly used by developers and tested in CI. +::: + +Continuous CI coverage comes from the GitHub Actions runners we build on: + +- `Linux (Ubuntu)` builds currently pin to `ubuntu-22.04` to keep glibc 2.35 compatibility even as `ubuntu-latest` moves ahead. +- `macOS (ARM64)` builds run on `macos-latest`, so support tracks that runner image as it moves ahead. +- `Windows (x86_64)` builds run on `windows-latest`, so support tracks that runner image as it moves ahead. + +On Linux, confirm your glibc version with `ldd --version` and ensure it reports 2.35 or newer before proceeding. + +We recommend using the latest supported version of Python and installing [nautilus_trader](https://pypi.org/project/nautilus_trader/) inside a virtual environment to isolate dependencies. + +**There are two supported ways to install**: + +1. Pre-built binary wheel from PyPI *or* the Nautech Systems package index. +2. Build from source. + +:::tip +We highly recommend installing using the [uv](https://docs.astral.sh/uv) package manager with a "vanilla" CPython. + +Conda and other Python distributions *may* work but aren’t officially supported. +::: + +## From PyPI + +To install the latest [nautilus_trader](https://pypi.org/project/nautilus_trader/) binary wheel (or sdist package) from PyPI: + +```bash +uv pip install nautilus_trader +``` + +## Extras + +Install optional dependencies as 'extras' for specific integrations: + +- `betfair`: Betfair adapter (integration) dependencies. +- `docker`: Needed for Docker when using the IB gateway (with the Interactive Brokers adapter). +- `ib`: Interactive Brokers adapter (integration) dependencies. +- `polymarket`: Polymarket adapter (integration) dependencies. +- `visualization`: Plotly-based interactive tearsheets and charts. + +To install with specific extras: + +```bash +uv pip install "nautilus_trader[docker,ib]" +``` + +## From the Nautech Systems package index + +The Nautech Systems package index (`packages.nautechsystems.io`) complies with [PEP-503](https://peps.python.org/pep-0503/) and hosts both stable and development binary wheels for `nautilus_trader`. +This enables users to install either the latest stable release or pre-release versions for testing. + +### Stable wheels + +Stable wheels correspond to official releases of `nautilus_trader` on PyPI, and use standard versioning. + +To install the latest stable release: + +```bash +uv pip install nautilus_trader --index-url=https://packages.nautechsystems.io/simple +``` + +:::tip +Use `--extra-index-url` instead of `--index-url` if you want uv to fall back to PyPI automatically: + +::: + +### Development wheels + +Development wheels are published from both the `nightly` and `develop` branches, +allowing users to test features and fixes ahead of stable releases. + +This process also helps preserve compute resources and provides easy access to the exact binaries tested in CI pipelines, +while adhering to [PEP-440](https://peps.python.org/pep-0440/) versioning standards: + +- `develop` wheels use the version format `dev{date}+{build_number}` (e.g., `1.208.0.dev20241212+7001`). +- `nightly` wheels use the version format `a{date}` (alpha) (e.g., `1.208.0a20241212`). + +| Platform | Nightly | Develop | +| :----------------- | :------ | :------ | +| `Linux (x86_64)` | ✓ | ✓ | +| `Linux (ARM64)` | ✓ | - | +| `macOS (ARM64)` | ✓ | ✓ | +| `Windows (x86_64)` | ✓ | ✓ | + +**Note**: Development wheels from the `develop` branch publish for every supported platform except Linux ARM64. +Skipping that target keeps CI feedback fast while avoiding unnecessary build resource usage. + +:::warning +We do not recommend using development wheels in production environments, such as live trading controlling real capital. +::: + +### Installation commands + +By default, uv will install the latest stable release. Adding the `--pre` flag ensures that pre-release versions, including development wheels, are considered. + +To install the latest available pre-release (including development wheels): + +```bash +uv pip install nautilus_trader --pre --index-url=https://packages.nautechsystems.io/simple +``` + +To install a specific development wheel (e.g., `1.221.0a20250912` for September 12, 2025): + +```bash +uv pip install nautilus_trader==1.221.0a20250912 --index-url=https://packages.nautechsystems.io/simple +``` + +### Available versions + +You can view all available versions of `nautilus_trader` on the [package index](https://packages.nautechsystems.io/simple/nautilus-trader/index.html). + +To programmatically request and list available versions: + +```bash +curl -s https://packages.nautechsystems.io/simple/nautilus-trader/index.html | grep -oP '(?<=.whl +``` + +## Versioning and releases + +NautilusTrader is still under active development. Some features may be incomplete, and while +the API is becoming more stable, breaking changes can occur between releases. +We strive to document these changes in the release notes on a **best-effort basis**. + +We aim to follow a **bi-weekly release schedule**, though experimental or larger features may cause delays. + +Use NautilusTrader only if you are prepared to adapt to these changes. + +## Redis + +Using [Redis](https://redis.io) with NautilusTrader is **optional** and only required if configured as the backend for a cache database or [message bus](../concepts/message_bus.md). + +:::info +The minimum supported Redis version is 6.2 (required for [streams](https://redis.io/docs/latest/develop/data-types/streams/) functionality). +::: + +For a quick setup, we recommend using a [Redis Docker container](https://hub.docker.com/_/redis/). You can find an example setup in the `.docker` directory, +or run the following command to start a container: + +```bash +docker run -d --name redis -p 6379:6379 redis:latest +``` + +This command will: + +- Pull the latest version of Redis from Docker Hub if it's not already downloaded. +- Run the container in detached mode (`-d`). +- Name the container `redis` for easy reference. +- Expose Redis on the default port 6379, making it accessible to NautilusTrader on your machine. + +To manage the Redis container: + +- Start it with `docker start redis` +- Stop it with `docker stop redis` + +:::tip +We recommend using [Redis Insight](https://redis.io/insight/) as a GUI to visualize and debug Redis data efficiently. +::: + +## Precision mode + +NautilusTrader supports two precision modes for its core value types (`Price`, `Quantity`, `Money`), +which differ in their internal bit-width and maximum decimal precision. + +- **High-precision**: 128-bit integers with up to 16 decimals of precision, and a larger value range. +- **Standard-precision**: 64-bit integers with up to 9 decimals of precision, and a smaller value range. + +:::note +By default, the official Python wheels ship in high-precision (128-bit) mode on Linux and macOS. +On Windows, only standard-precision (64-bit) Python wheels are available because MSVC's C/C++ frontend +does not support `__int128`, preventing the Cython/FFI layer from handling 128-bit integers. + +For pure Rust crates, high-precision works on all platforms (including Windows) since Rust handles +`i128`/`u128` via software emulation. The default is standard-precision unless you explicitly enable +the `high-precision` feature flag. +::: + +The performance tradeoff is that standard-precision is ~3–5% faster in typical backtests, +but has lower decimal precision and a smaller representable value range. + +:::note +Performance benchmarks comparing the modes are pending. +::: + +### Build configuration + +The precision mode is determined by: + +- Setting the `HIGH_PRECISION` environment variable during compilation, **and/or** +- Enabling the `high-precision` Rust feature flag explicitly. + +#### High-precision mode (128-bit) + +```bash +export HIGH_PRECISION=true +make install-debug +``` + +#### Standard-precision mode (64-bit) + +```bash +export HIGH_PRECISION=false +make install-debug +``` + +### Rust feature flag + +To enable high-precision (128-bit) mode in Rust, add the `high-precision` feature to your `Cargo.toml`: + +```toml +[dependencies] +nautilus_core = { version = "*", features = ["high-precision"] } +``` + +:::info +See the [Value Types](../concepts/overview.md#value-types) specifications for more details. +::: diff --git a/nautilus-docs/docs/getting_started/quickstart.py b/nautilus-docs/docs/getting_started/quickstart.py new file mode 100644 index 0000000..474b45e --- /dev/null +++ b/nautilus-docs/docs/getting_started/quickstart.py @@ -0,0 +1,211 @@ +# %% [markdown] +# # Quickstart +# +# Run your first backtest in under five minutes. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/getting_started/quickstart.py). + +# %% [markdown] +# ## Prerequisites +# +# - Python 3.12+ +# - `pip install nautilus_trader` + +# %% [markdown] +# ## Write a strategy +# +# A strategy extends the `Strategy` base class and overrides event handlers to +# react to market data. This one trades an EMA crossover: buy when a fast +# exponential moving average crosses above a slow one, sell when it crosses below. + +# %% +from decimal import Decimal + +from nautilus_trader.config import StrategyConfig +from nautilus_trader.indicators import ExponentialMovingAverage +from nautilus_trader.model.data import Bar +from nautilus_trader.model.data import BarType +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.trading.strategy import Strategy + + +class EMACrossConfig(StrategyConfig, frozen=True): + instrument_id: InstrumentId + bar_type: BarType + trade_size: Decimal + fast_ema_period: int = 10 + slow_ema_period: int = 20 + + +class EMACross(Strategy): + def __init__(self, config: EMACrossConfig): + super().__init__(config) + self.fast_ema = ExponentialMovingAverage(config.fast_ema_period) + self.slow_ema = ExponentialMovingAverage(config.slow_ema_period) + + def on_start(self): + self.register_indicator_for_bars(self.config.bar_type, self.fast_ema) + self.register_indicator_for_bars(self.config.bar_type, self.slow_ema) + self.subscribe_bars(self.config.bar_type) + + def on_bar(self, bar: Bar): + if not self.indicators_initialized(): + return + + if self.fast_ema.value >= self.slow_ema.value: + if self.portfolio.is_flat(self.config.instrument_id): + self.buy() + elif self.portfolio.is_net_short(self.config.instrument_id): + self.close_all_positions(self.config.instrument_id) + self.buy() + elif self.fast_ema.value < self.slow_ema.value: + if self.portfolio.is_flat(self.config.instrument_id): + self.sell() + elif self.portfolio.is_net_long(self.config.instrument_id): + self.close_all_positions(self.config.instrument_id) + self.sell() + + def buy(self): + instrument = self.cache.instrument(self.config.instrument_id) + order = self.order_factory.market( + self.config.instrument_id, + OrderSide.BUY, + instrument.make_qty(self.config.trade_size), + ) + self.submit_order(order) + + def sell(self): + instrument = self.cache.instrument(self.config.instrument_id) + order = self.order_factory.market( + self.config.instrument_id, + OrderSide.SELL, + instrument.make_qty(self.config.trade_size), + ) + self.submit_order(order) + + def on_stop(self): + self.close_all_positions(self.config.instrument_id) + + +# %% [markdown] +# `on_start` registers the two EMA indicators so the engine updates them +# automatically with each new bar. `on_bar` waits for the indicators to warm up, +# then enters or reverses a position based on the crossover signal. + +# %% [markdown] +# ## Set up the backtest +# +# Generate synthetic EUR/USD 1-minute bars, configure a `BacktestEngine`, and run. + +# %% +import numpy as np +import pandas as pd + +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.config import BacktestEngineConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.model.identifiers import Venue +from nautilus_trader.model.objects import Money +from nautilus_trader.persistence.wranglers import BarDataWrangler +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# Create a EUR/USD instrument on the SIM venue +EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD") + +# Generate synthetic 1-minute bars (random walk around 1.10) +rng = np.random.default_rng(42) +n = 10_000 +price = 1.10 + np.cumsum(rng.normal(0, 0.0002, n)) +spread = np.abs(rng.normal(0, 0.0003, n)) +bars_df = pd.DataFrame( + { + "open": price, + "high": price + spread, + "low": price - spread, + "close": price + rng.normal(0, 0.00005, n), + }, + index=pd.date_range("2024-01-01", periods=n, freq="1min", tz="UTC"), +) +bars_df["high"] = bars_df[["open", "high", "close"]].max(axis=1) +bars_df["low"] = bars_df[["open", "low", "close"]].min(axis=1) + +bar_type = BarType.from_str("EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL") +bars = BarDataWrangler(bar_type, EURUSD).process(bars_df) + +# %% [markdown] +# `BarDataWrangler` converts a pandas DataFrame with OHLCV columns into Nautilus +# `Bar` objects. The bar type string encodes the instrument, aggregation period, +# price source, and data origin. + +# %% +# Configure the engine +engine = BacktestEngine( + config=BacktestEngineConfig( + logging=LoggingConfig(log_level="ERROR"), + ), +) + +# Add a simulated FX venue +SIM = Venue("SIM") +engine.add_venue( + venue=SIM, + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, + starting_balances=[Money(1_000_000, USD)], + base_currency=USD, + default_leverage=Decimal(1), +) + +# Add instrument, data, and strategy +engine.add_instrument(EURUSD) +engine.add_data(bars) + +strategy = EMACross( + EMACrossConfig( + instrument_id=EURUSD.id, + bar_type=bar_type, + trade_size=Decimal(100000), + ), +) +engine.add_strategy(strategy) + +# Run the backtest +engine.run() + +# %% [markdown] +# The engine processes all 10,000 bars in timestamp order. Each bar updates the +# registered indicators, then triggers `on_bar`. The simulated exchange fills +# market orders at the current price. + +# %% [markdown] +# ## See results +# +# The engine generates reports from the completed backtest. The account report +# shows balance changes over time. The positions report lists each round-trip +# trade with its realized PnL. + +# %% +engine.trader.generate_account_report(SIM) + +# %% +engine.trader.generate_positions_report() + +# %% +engine.trader.generate_order_fills_report() + +# %% [markdown] +# ## Next steps +# +# - [Backtest (low-level API)](backtest_low_level) for direct `BacktestEngine` usage +# with real market data and execution algorithms. +# - [Backtest (high-level API)](backtest_high_level) for config-driven backtesting +# with `BacktestNode` and the Parquet data catalog. +# - [Tutorials](../tutorials/) for venue-specific walkthroughs covering Binance, +# Bybit, Databento, and more. + +# %% +engine.dispose() diff --git a/nautilus-docs/docs/integrations/architect_ax.md b/nautilus-docs/docs/integrations/architect_ax.md new file mode 100644 index 0000000..40a754a --- /dev/null +++ b/nautilus-docs/docs/integrations/architect_ax.md @@ -0,0 +1,400 @@ +# AX Exchange + +[AX Exchange](https://architect.exchange) is the world's first centralized and regulated exchange +for perpetual futures on traditional underlying asset classes. Operated by Architect Bermuda Ltd. +and licensed by the [Bermuda Monetary Authority (BMA)](https://www.bma.bm/), AX brings crypto-style +perpetual contracts to traditional financial markets including foreign exchange, metals, energy, +equity indices, and interest rates. + +This integration supports live market data ingest and order execution with AX Exchange. + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/architect_ax/). + +## Overview + +This guide assumes a trader is setting up for both live market data feeds, and trade execution. +The AX Exchange adapter includes multiple components, which can be used together or separately +depending on the use case. + +- `AxHttpClient`: Low-level HTTP API connectivity. +- `AxMdWebSocketClient`: Market data WebSocket connectivity. +- `AxOrdersWebSocketClient`: Orders WebSocket connectivity. +- `AxInstrumentProvider`: Instrument parsing and loading functionality. +- `AxDataClient`: A market data feed manager. +- `AxExecutionClient`: An account management and trade execution gateway. +- `AxLiveDataClientFactory`: Factory for AX data clients (used by the trading node builder). +- `AxLiveExecClientFactory`: Factory for AX execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), +and won't need to necessarily work with these lower level components directly. +::: + +## AX Exchange documentation + +AX Exchange provides documentation for users which can be found at the +[Architect documentation site](https://docs.architect.exchange/). +It's recommended you also refer to the AX Exchange documentation in conjunction with this +NautilusTrader integration guide. + +## Products + +AX Exchange specializes in perpetual futures contracts on traditional asset classes. Perpetual +contracts never expire, eliminating rollover costs associated with standard futures. + +| Asset Class | Examples | Notes | +|------------------|-------------------------------------|------------------------------| +| Foreign exchange | GBPUSD-PERP, EURUSD-PERP. | Major and minor FX pairs. | +| Stock indices | Equity index perpetuals. | | +| Metals | XAU-PERP (gold), XAG-PERP (silver). | Precious metals perpetuals. | +| Energy | Crude oil, natural gas. | Energy commodity perpetuals. | +| Interest rates | SOFR, treasury yields. | Rate perpetuals. | + +### Perpetual contracts + +A perpetual contract (perpetual swap) is a derivative that tracks the price of an underlying +asset without expiring. Unlike standard futures, there is no settlement date, which eliminates +rollover costs and simplifies position management. A funding rate mechanism keeps the contract +price aligned with the underlying index price through periodic payments between long and short +holders. See the [Architect documentation](https://docs.architect.exchange/) for details on +funding rate mechanics and contract specifications. + +Characteristics of AX perpetual contracts: + +- **Cash-settled in USD**: No physical delivery. All profit and loss is settled in USD. +- **Funding rates**: Periodic payments keep the contract price aligned with the underlying. +- **Multiplier of 1**: Each contract represents one unit of exposure to the underlying. +- **Whole contracts only**: Fractional quantities are not supported. +- **Margin**: Initial margin is required to open a position; maintenance margin to keep it open. + +In NautilusTrader, all AX instruments are represented as `PerpetualContract`, an asset-class +agnostic perpetual swap type. The asset class (FX, commodity, equity, etc.) is inferred +automatically from the underlying. The adapter uses `MARGIN` account type and `NETTING` order +management. + +## Symbology + +AX Exchange uses a straightforward naming convention. All instruments are perpetual futures +identified by the `-PERP` suffix appended to the underlying asset symbol. + +**Format**: `{SYMBOL}-PERP` + +| Underlying | AX Symbol | Nautilus InstrumentId | +|----------------|----------------|-----------------------| +| GBP/USD | `GBPUSD-PERP` | `GBPUSD-PERP.AX` | +| EUR/USD | `EURUSD-PERP` | `EURUSD-PERP.AX` | +| Gold | `XAU-PERP` | `XAU-PERP.AX` | +| Silver | `XAG-PERP` | `XAG-PERP.AX` | + +The venue identifier is `AX`. To construct a Nautilus `InstrumentId`: + +```python +from nautilus_trader.model.identifiers import InstrumentId + +instrument_id = InstrumentId.from_str("GBPUSD-PERP.AX") +``` + +## Environments + +AX Exchange provides two trading environments. Configure the appropriate environment using the +`environment` parameter in your client configuration. + +| Environment | Config | Description | +|----------------|----------------------------------------|----------------------------------------| +| **Sandbox** | `environment=AxEnvironment.SANDBOX` | Test environment with simulated funds. | +| **Production** | `environment=AxEnvironment.PRODUCTION` | Live trading with real funds. | + +### Sandbox + +The sandbox is the default environment for development and testing with simulated funds. +All sandbox endpoints are resolved automatically when `environment=AxEnvironment.SANDBOX`. + +#### 1. Create a sandbox account + +Follow the [Architect documentation](https://docs.architect.exchange/) to create a sandbox +account. An invite code is required during registration. + +#### 2. Create API keys and fund the account + +Use the AX sandbox UI to generate API keys and deposit simulated funds into your account. +Store the `api_key` and `api_secret` securely. + +#### 3. Set environment variables + +```bash +export AX_API_KEY="your-sandbox-api-key" +export AX_API_SECRET="your-sandbox-api-secret" +``` + +#### 4. Configure the trading node + +```python +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + AX: AxDataClientConfig( + environment=AxEnvironment.SANDBOX, + instrument_provider=InstrumentProviderConfig(load_all=True), + ), + }, + exec_clients={ + AX: AxExecClientConfig( + environment=AxEnvironment.SANDBOX, + instrument_provider=InstrumentProviderConfig(load_all=True), + ), + }, +) +``` + +### Production + +For live trading with real funds. Requires a verified AX Exchange account. + +```python +config = AxExecClientConfig( + environment=AxEnvironment.PRODUCTION, +) +``` + +:::warning +Ensure you are using the correct environment before placing orders. +The sandbox environment is the default to prevent accidental live trading. +::: + +## Market data + +The adapter provides real-time market data via WebSocket subscriptions, with HTTP endpoints +for historical data backfill. + +### Data types + +| AX Data | Nautilus Data Type | Notes | +|-----------------|---------------------|-------------------------------------------------------------| +| Order book (L1) | `QuoteTick` | Best bid/ask top-of-book from L1 book subscription. | +| Order book (L2) | `OrderBookDelta` | Aggregated price levels. | +| Order book (L3) | `OrderBookDelta` | Individual order quantities. | +| Trades | `TradeTick` | Real-time trade events from L1 subscription. | +| Bars/candles | `Bar` | OHLCV data (total volume only, no buy/sell breakdown). | +| Funding rates | `FundingRateUpdate` | Polled via HTTP (not real-time WebSocket); interval configurable. | + +:::note +Historical quote tick requests are not supported by AX Exchange. Only real-time quote +data is available via WebSocket L1 book subscriptions. +::: + +### Bar intervals + +| Interval | Description | +|----------|-------------| +| `1s` | 1-second | +| `5s` | 5-second | +| `1m` | 1-minute | +| `5m` | 5-minute | +| `15m` | 15-minute | +| `1h` | 1-hour | +| `1d` | 1-day | + +## Orders capability + +AX Exchange supports market and limit order types with stop triggers. + +### Order types + +| Order Type | Supported | Notes | +|------------------------|-----------|----------------------------------------------------| +| `MARKET` | ✓ | Execute immediately at best available price. | +| `LIMIT` | ✓ | Execute at specified price or better. | +| `STOP_LIMIT` | ✓ | Trigger a limit order when stop price is breached. | +| `LIMIT_IF_TOUCHED` | - | *Not currently implemented by AX Exchange*. | +| `STOP_MARKET` | - | *Not supported*. | +| `MARKET_IF_TOUCHED` | - | *Not supported*. | +| `TRAILING_STOP_MARKET` | - | *Not supported*. | + +### Execution instructions + +| Instruction | Supported | Notes | +|---------------|-----------|-----------------------------------------------------| +| `post_only` | ✓ | Maker-only; rejected if order would take liquidity. | +| `reduce_only` | - | *Not supported*. | + +### Time in force + +| Time in Force | Supported | Notes | +|---------------|-----------|----------------------------------------------| +| `GTC` | ✓ | Good Till Canceled. | +| `GTD` | ✓ | Good Till Date. | +| `DAY` | ✓ | Valid until end of trading day. | +| `IOC` | ✓ | Immediate or Cancel. | +| `FOK` | ✓ | Fill or Kill. | +| `AT_THE_OPEN` | ✓ | Execute at market open or expire. | +| `AT_THE_CLOSE`| ✓ | Execute at market close or expire. | + +### Advanced order features + +| Feature | Supported | Notes | +|--------------------|-----------|--------------------------------------------------------------------| +| Order modification | - | *Not supported by AX*. Cancel and resubmit instead. | +| Cancel order | ✓ | Single order cancellation. | +| Cancel all orders | ✓ | Cancel all open orders for an instrument. | +| Batch cancel | ✓ | Cancel multiple specified orders. | +| Order lists | ✓ | Sequential submission (orders submitted individually, non-atomic). | + +### Position management + +| Feature | Supported | Notes | +|------------------|-----------|--------------------------------------| +| Query positions | ✓ | Real-time position updates. | +| Position mode | - | Netting mode only. | +| Cross margin | ✓ | Cross-margin across all instruments. | + +### Order querying + +| Feature | Supported | Notes | +|----------------------|-----------|---------------------------------------------------------| +| Query open orders | ✓ | List all active orders. | +| Query single order | ✓ | By venue order ID or client order ID (any order state). | +| Order status reports | ✓ | Reconciliation from open orders; see note below. | +| Fill reports | ✓ | Execution and fill history. | + +:::note +Order status reports for reconciliation are generated from the open orders endpoint. +Filled or canceled orders are not included in the reconciliation snapshot. Single-order +queries via `query_order` use the dedicated `/order-status` endpoint which works for +any order state. +::: + +## Authentication + +AX Exchange uses bearer token authentication: + +1. API key and secret obtain a session token via `/authenticate`. +2. The session token is used as a bearer token for subsequent REST and WebSocket requests. +3. Session tokens expire after a configurable period (default: 86400 seconds). + +## Configuration + +### Environments and endpoints + +| Environment | HTTP API (market data) | HTTP API (orders) | Market Data WS | Orders WS | +|-------------|--------------------------------------------------|-----------------------------------------------------|--------------------------------------------------|------------------------------------------------------| +| Sandbox | `https://gateway.sandbox.architect.exchange/api` | `https://gateway.sandbox.architect.exchange/orders` | `wss://gateway.sandbox.architect.exchange/md/ws` | `wss://gateway.sandbox.architect.exchange/orders/ws` | +| Production | `https://gateway.architect.exchange/api` | `https://gateway.architect.exchange/orders` | `wss://gateway.architect.exchange/md/ws` | `wss://gateway.architect.exchange/orders/ws` | + +:::info +Order management HTTP endpoints (place, cancel, order status) use a separate base URL +from market data endpoints. This is handled automatically by the adapter configuration. +::: + +### Data client configuration options + +| Option | Default | Description | +|------------------------------------|-----------|---------------------------------------------------------------------| +| `api_key` | `None` | API key; loaded from `AX_API_KEY` env var when omitted. | +| `api_secret` | `None` | API secret; loaded from `AX_API_SECRET` env var when omitted. | +| `environment` | `SANDBOX` | Trading environment (`SANDBOX` or `PRODUCTION`). | +| `base_url_http` | `None` | Override for the REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket URL. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `http_timeout_secs` | `60` | Timeout (seconds) for REST requests. | +| `max_retries` | `3` | Maximum retry attempts for REST requests. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries (exponential backoff). | +| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalog refreshes. | +| `funding_rate_poll_interval_mins` | `15` | Interval (minutes) between funding rate poll requests. | + +### Execution client configuration options + +| Option | Default | Description | +|--------------------------|-----------|---------------------------------------------------------------------| +| `api_key` | `None` | API key; loaded from `AX_API_KEY` env var when omitted. | +| `api_secret` | `None` | API secret; loaded from `AX_API_SECRET` env var when omitted. | +| `environment` | `SANDBOX` | Trading environment (`SANDBOX` or `PRODUCTION`). | +| `base_url_http` | `None` | Override for the REST base URL. | +| `base_url_ws` | `None` | Override for the orders WebSocket URL. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `http_timeout_secs` | `60` | Timeout (seconds) for REST requests. | +| `max_retries` | `3` | Maximum retry attempts for REST requests. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries (exponential backoff). | + +The most common use case is to configure a live `TradingNode` to include AX Exchange +data and execution clients. To achieve this, add an `AX` section to your client +configuration(s): + +```python +from nautilus_trader.adapters.architect_ax import AX +from nautilus_trader.adapters.architect_ax import AxDataClientConfig +from nautilus_trader.adapters.architect_ax import AxEnvironment +from nautilus_trader.adapters.architect_ax import AxExecClientConfig +from nautilus_trader.config import InstrumentProviderConfig +from nautilus_trader.config import TradingNodeConfig + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + AX: AxDataClientConfig( + environment=AxEnvironment.SANDBOX, + instrument_provider=InstrumentProviderConfig(load_all=True), + ), + }, + exec_clients={ + AX: AxExecClientConfig( + environment=AxEnvironment.SANDBOX, + instrument_provider=InstrumentProviderConfig(load_all=True), + ), + }, +) +``` + +Then, create a `TradingNode` and add the client factories: + +```python +from nautilus_trader.adapters.architect_ax import AX +from nautilus_trader.adapters.architect_ax import AxLiveDataClientFactory +from nautilus_trader.adapters.architect_ax import AxLiveExecClientFactory +from nautilus_trader.live.node import TradingNode + +# Instantiate the live trading node with a configuration +node = TradingNode(config=config) + +# Register the client factories with the node +node.add_data_client_factory(AX, AxLiveDataClientFactory) +node.add_exec_client_factory(AX, AxLiveExecClientFactory) + +# Finally build the node +node.build() +``` + +### API credentials + +There are two options for supplying your credentials to the AX Exchange clients. +Either pass the corresponding `api_key` and `api_secret` values to the configuration objects, or +set the following environment variables: + +- `AX_API_KEY` +- `AX_API_SECRET` + +:::tip +We recommend using environment variables to manage your credentials. +::: + +When starting the trading node, you'll receive immediate confirmation of whether your +credentials are valid and have trading permissions. + +## Implementation notes + +- **Whole contracts only**: AX Exchange uses integer contract quantities. Fractional quantities + are not supported and will be rejected. +- **Rate limiting**: The adapter applies a conservative rate limit of 10 requests/second with + automatic exponential backoff on rate limit responses. +- **Market orders**: AX does not support native market orders. The adapter uses a preview endpoint + to determine the take-through price and submits an aggressive IOC limit order. + +## Contributing + +:::info +For additional features or to contribute to the AX Exchange adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/betfair.md b/nautilus-docs/docs/integrations/betfair.md new file mode 100644 index 0000000..3859907 --- /dev/null +++ b/nautilus-docs/docs/integrations/betfair.md @@ -0,0 +1,574 @@ +# Betfair + +Founded in 2000, Betfair operates the world’s largest online betting exchange, +with its headquarters in London and satellite offices across the globe. + +NautilusTrader provides an adapter for integrating with the Betfair REST API and +Exchange Streaming API. + +## Installation + +Install NautilusTrader with Betfair support: + +```bash +uv pip install "nautilus_trader[betfair]" +``` + +To build from source with Betfair extras: + +```bash +uv sync --all-extras +``` + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/betfair/). + +## Betfair documentation + +Betfair provides documentation for developers: + +- [Betfair Developer Portal](https://developer.betfair.com/): Main entry point for API access and documentation. +- [Exchange API Guide](https://developer.betfair.com/exchange-api/): Overview of the Betting, Accounts, and Streaming APIs. + +## Application keys + +Betfair requires an Application Key to authenticate API requests. After registering and funding your account, +obtain your key using the [API-NG Developer AppKeys Tool](https://apps.betfair.com/visualisers/api-ng-account-operations/). + +Two App Keys are assigned per account: a **Live** key (requires a one-time activation fee) and a +**Delayed** key for development and testing. + +:::info +See the [Application Keys](https://betfair-developer-docs.atlassian.net/wiki/spaces/1smk3cen4v3lu3yomq5qye0ni/pages/2687105/Application+Keys) documentation for detailed setup instructions. +::: + +## API credentials + +Supply your Betfair credentials via environment variables or client configuration: + +```bash +export BETFAIR_USERNAME= +export BETFAIR_PASSWORD= +export BETFAIR_APP_KEY= +export BETFAIR_CERTS_DIR= +``` + +:::tip +We recommend using environment variables to manage your credentials. +::: + +## SSL Certificates + +Betfair recommends [non-interactive (bot) login](https://betfair-developer-docs.atlassian.net/wiki/spaces/1smk3cen4v3lu3yomq5qye0ni/pages/2687915/Non-Interactive+bot+login) +with SSL certificates for automated trading systems. The `certs_dir` configuration is optional, +but certificates are recommended for production deployments. + +### Generating certificates + +Create a 2048-bit RSA certificate using OpenSSL: + +```bash +# Generate private key and certificate signing request +openssl genrsa -out client-2048.key 2048 +openssl req -new -key client-2048.key -out client-2048.csr + +# Self-sign the certificate (valid for 365 days) +openssl x509 -req -days 365 -in client-2048.csr -signkey client-2048.key -out client-2048.crt +``` + +### Uploading to Betfair + +Before using the certificate, attach it to your Betfair account: + +1. Navigate to [My Betfair Account Security](https://myaccount.betfair.com/accountdetails/mysecurity?showAPI=1). +2. Scroll to **Automated Betting Program Access** and click **Edit**. +3. Upload your `client-2048.crt` file. + +### Directory structure + +Place your certificate files in a directory and set `BETFAIR_CERTS_DIR` to that path: + +``` +/path/to/certs/ +├── client-2048.crt +└── client-2048.key +``` + +:::info +SSL certificates are used for the Exchange Streaming API connection. The REST API uses +username/password authentication with your Application Key. +::: + +:::warning +Enabling 2-Step Authentication on the Betfair website does not affect API access. +Certificate-based login remains functional regardless of 2FA settings. +::: + +## Overview + +The Betfair adapter provides three primary components: + +- `BetfairInstrumentProvider`: loads Betfair markets and converts them into Nautilus instruments. +- `BetfairDataClient`: streams real-time market data from the Exchange Streaming API. +- `BetfairExecutionClient`: submits orders (bets) and tracks execution status via the REST API. + +## Orders capability + +Betfair operates as a betting exchange with unique characteristics compared to traditional financial exchanges: + +### Order types + +| Order Type | Supported | Notes | +|------------------------|-----------|-------------------------------------| +| `MARKET` | - | Not applicable to betting exchange. | +| `LIMIT` | ✓ | Orders placed at specific odds. | +| `STOP_MARKET` | - | *Not supported*. | +| `STOP_LIMIT` | - | *Not supported*. | +| `MARKET_IF_TOUCHED` | - | *Not supported*. | +| `LIMIT_IF_TOUCHED` | - | *Not supported*. | +| `TRAILING_STOP_MARKET` | - | *Not supported*. | + +### Execution instructions + +| Instruction | Supported | Notes | +|---------------|-----------|-------------------------------------| +| `post_only` | - | Not applicable to betting exchange. | +| `reduce_only` | - | Not applicable to betting exchange. | + +### Time in force options + +| Time in force | Supported | Notes | +|---------------|-----------|--------------------------------------------| +| `GTC` | ✓ | Maps to Betfair `PERSIST` persistence. | +| `GTD` | - | *Not supported*. | +| `DAY` | ✓ | Maps to Betfair `LAPSE` persistence. | +| `FOK` | ✓ | Maps to Betfair `FILL_OR_KILL`. | +| `IOC` | ✓ | Maps to `FILL_OR_KILL` with partial fills. | + +:::note +Betfair uses a persistence model rather than traditional time-in-force. The adapter maps `FOK` to +Betfair's `FILL_OR_KILL`, while `IOC` uses `FILL_OR_KILL` with `min_fill_size=0` to allow partial fills. +::: + +### Advanced order features + +| Feature | Supported | Notes | +|--------------------|-----------|------------------------------------------| +| Order Modification | ✓ | Limited to non-exposure changing fields. | +| Bracket/OCO Orders | - | *Not supported*. | +| Iceberg Orders | - | *Not supported*. | + +### Batch operations + +| Operation | Supported | Notes | +|--------------------|-----------|----------------------| +| Batch Submit | - | *Not supported*. | +| Batch Modify | - | *Not supported*. | +| Batch Cancel | - | *Not supported*. | + +### Position management + +| Feature | Supported | Notes | +|---------------------|-----------|-----------------------------------------| +| Query positions | - | Betting exchange model differs. | +| Position mode | - | Not applicable to betting exchange. | +| Leverage control | - | No leverage in betting exchange. | +| Margin mode | - | No margin in betting exchange. | + +### Order querying + +| Feature | Supported | Notes | +|----------------------|-----------|----------------------------------------| +| Query open orders | ✓ | List all active bets. | +| Query order history | ✓ | Historical betting data. | +| Order status updates | ✓ | Real-time bet state changes. | +| Trade history | ✓ | Bet matching and settlement reports. | + +### Contingent orders + +| Feature | Supported | Notes | +|---------------------|-----------|-----------------------------------------| +| Order lists | - | *Not supported*. | +| OCO orders | - | *Not supported*. | +| Bracket orders | - | *Not supported*. | +| Conditional orders | - | Basic bet conditions only. | + +## Tick scheme and pricing + +Betfair uses a tiered tick scheme with varying increments across price ranges: + +| Price Range | Tick Size | +|---------------|-----------| +| 1.01 - 2.00 | 0.01 | +| 2.00 - 3.00 | 0.02 | +| 3.00 - 4.00 | 0.05 | +| 4.00 - 6.00 | 0.10 | +| 6.00 - 10.00 | 0.20 | +| 10.00 - 20.00 | 0.50 | +| 20.00 - 30.00 | 1.00 | +| 30.00 - 50.00 | 2.00 | +| 50.00 - 100.00 | 5.00 | +| 100.00 - 1000.00 | 10.00 | + +The minimum price is 1.01 and the maximum is 1000.00. + +## Order modification + +Order modification on Betfair has specific constraints: + +- **Price and size cannot be changed atomically** - these require separate operations. +- **Price modification** uses `ReplaceOrders` (cancel + new order at new price). +- **Size reduction** uses `CancelOrders` with a `size_reduction` parameter. +- **Size increase** is not supported - submit a new order instead. + +:::warning +A replace operation generates both a cancel event for the original order and an accepted event +for the replacement order. The adapter tracks pending replacements to suppress synthetic cancel events. +::: + +## Order stream fill handling + +The execution client processes order updates from the Betfair Exchange Streaming API. +Two configuration options control how updates are filtered: + +- **`stream_market_ids_filter`**: Filters at the market level (early exit, silent skip). +- **`ignore_external_orders`**: Filters at the order level (controls warning vs debug logging). + +Note that `stream_market_ids_filter` is independent of reconciliation scope (`reconcile_market_ids_only`). +Stream filtering affects live updates only; reconciliation uses its own market filter. + +```mermaid +flowchart TD + A[Stream update arrives] --> B{Market in
stream_market_ids_filter?} + B -->|No filter set| C{Instrument loaded?} + B -->|Yes| C + B -->|No| D[Skip silently] + C -->|No| E[Warning: Instrument not loaded] + C -->|Yes| F{Known order?
rfo or cache} + F -->|Yes| G[Process order update] + F -->|No| H{ignore_external_orders?} + H -->|True| I[Debug log, skip] + H -->|False| J[Warning log, skip] +``` + +The same `stream_market_ids_filter` is also applied during full-image reconciliation in `check_cache_against_order_image`. + +When `ignore_external_orders=True`, the client silently skips orders and fills not found in cache: + +| Scenario | Description | +|--------------------------------|-----------------------------------------------------| +| Unknown order in stream update | No venue-to-client order ID mapping exists. | +| Unknown order in full image | Order not found in cache during image sync. | +| Unknown fill in full image | Fill does not match any known order during sync. | + +:::info +For multi-node setups sharing a Betfair account, set both `stream_market_ids_filter` (your markets only) +and `ignore_external_orders=True` to avoid warnings about orders managed by other nodes. +::: + +### Fill handling + +The adapter handles several edge cases when processing fills from the stream: + +- **Incremental fills**: Betfair reports cumulative matched sizes. The adapter calculates incremental + fills by tracking the last known filled quantity per order. +- **Overfill protection**: Fills that would exceed the order quantity are rejected. +- **Deduplication**: A cache of published trade IDs prevents duplicate fill events from late messages + or stream reconnection replays. +- **Race conditions**: When stream fills arrive before the HTTP order response, the adapter caches + the venue order ID immediately to ensure correct order matching. +- **Network error recovery**: When an HTTP order submission fails with a network error (timeout, + connection reset), the order may still have been placed on the venue. The adapter leaves the + order in SUBMITTED status and retains the customer order reference so the stream can confirm + the order when it reconnects. API errors (where Betfair explicitly rejected) still reject + immediately. + +## Rate limiting + +The adapter uses separate rate limit buckets so that account state polling and +reconciliation do not throttle order placement: + +| Bucket | Default | Endpoints | Configurable | +|---------|---------|------------------------------------------------------|----------------------------------| +| General | 5/s | Account state, reconciliation, keep-alive. | | +| Orders | 20/s | `placeOrders`, `replaceOrders`, `cancelOrders`. | `order_request_rate_per_second`. | + +Order status and fill report queries retry once on `TOO_MANY_REQUESTS` errors +after a 1-second delay; order operations reject with the error message. + +Betfair's actual API limits are more nuanced: + +| Category | Limit | Notes | +|--------------------------|----------------------|------------------------------------------------------| +| Order operations | 1,000 transactions/s | Total instructions across `placeOrders`, `cancelOrders`, `replaceOrders`. | +| Order projection queries | 3 concurrent | `listMarketBook` (with `OrderProjection`), `listCurrentOrders`, `listMarketProfitAndLoss`. | +| Best practice | 5 requests/s | Recommended for `listMarketBook` per market. | + +:::info +For details on rate limits, see [Why am I receiving the TOO_MANY_REQUESTS error?](https://support.developer.betfair.com/hc/en-us/articles/360000406111) +and [Market Data Request Limits](https://docs.developer.betfair.com/display/1smk3cen4v3lu3yomq5qye0ni/Market+Data+Request+Limits). +::: + +## Custom data types + +The Betfair adapter provides several custom data types that flow through the market stream. +All custom data is delivered automatically when subscribed to markets - no explicit subscription is required, +though strategies can register handlers for specific data types. + +### BetfairTicker + +Real-time ticker data for a betting selection. + +| Field | Type | Description | +|-----------------------|---------|---------------------------------| +| `instrument_id` | str | Nautilus instrument identifier. | +| `last_traded_price` | float | Last matched price (odds). | +| `traded_volume` | float | Total matched volume. | +| `starting_price_near` | float | Near-side BSP indicator. | +| `starting_price_far` | float | Far-side BSP indicator. | + +### BetfairStartingPrice + +The realized Betfair Starting Price (BSP) after market close. + +| Field | Type | Description | +|-----------------|-------|---------------------------------| +| `instrument_id` | str | Nautilus instrument identifier. | +| `bsp` | float | Final starting price (odds). | + +### BetfairRaceRunnerData + +Live GPS tracking data for individual horses (Total Performance Data). +Available for supported UK and Irish races. + +| Field | Type | Description | +|--------------------|-------|-----------------------------------------| +| `race_id` | str | Betfair race identifier. | +| `market_id` | str | Betfair market identifier. | +| `selection_id` | int | Betfair selection (runner) identifier. | +| `latitude` | float | GPS latitude. | +| `longitude` | float | GPS longitude. | +| `speed` | float | Current speed in m/s (Doppler-derived). | +| `progress` | float | Distance to finish line in meters. | +| `stride_frequency` | float | Stride frequency in Hz. | + +### BetfairRaceProgress + +Race summary data with sectional times and running order. + +| Field | Type | Description | +|------------------|------------|-----------------------------------------------| +| `race_id` | str | Betfair race identifier. | +| `market_id` | str | Betfair market identifier. | +| `gate_name` | str | Timing gate (e.g., "1f", "2f", "Finish"). | +| `sectional_time` | float | Time for this section in seconds. | +| `running_time` | float | Total time since race start in seconds. | +| `speed` | float | Lead horse speed in m/s. | +| `progress` | float | Lead horse distance to finish in meters. | +| `order` | list[int] | Selection IDs in current race position order. | +| `jumps` | list[dict] | Jump obstacle data for National Hunt races. | + +### Subscribing to custom data + +Custom data flows automatically through the Betfair market stream when you subscribe to markets. +To receive custom data in your strategy or actor, register a handler with the Betfair client ID: + +```python +from nautilus_trader.adapters.betfair.constants import BETFAIR_CLIENT_ID +from nautilus_trader.adapters.betfair.data_types import BetfairRaceRunnerData +from nautilus_trader.adapters.betfair.data_types import BetfairRaceProgress +from nautilus_trader.adapters.betfair.data_types import BetfairTicker +from nautilus_trader.model.data import DataType + +class MyStrategy(Strategy): + def on_start(self): + # Subscribe to ticker data + self.subscribe_data(DataType(BetfairTicker), client_id=BETFAIR_CLIENT_ID) + + # Subscribe to ALL race runner data (wildcard) + self.subscribe_data(DataType(BetfairRaceRunnerData), client_id=BETFAIR_CLIENT_ID) + + # Or subscribe to a specific runner by selection_id + self.subscribe_data( + DataType(BetfairRaceRunnerData, metadata={"selection_id": 49411491}), + client_id=BETFAIR_CLIENT_ID, + ) + + # Subscribe to ALL race progress updates (wildcard) + self.subscribe_data(DataType(BetfairRaceProgress), client_id=BETFAIR_CLIENT_ID) + + # Or subscribe to a specific race by race_id + self.subscribe_data( + DataType(BetfairRaceProgress, metadata={"race_id": "35278018.1617"}), + client_id=BETFAIR_CLIENT_ID, + ) + + def on_data(self, data): + if isinstance(data, BetfairRaceRunnerData): + self.log.info( + f"Runner {data.selection_id}: speed={data.speed} m/s, " + f"progress={data.progress}m to finish" + ) + elif isinstance(data, BetfairRaceProgress): + self.log.info(f"Race order: {data.order}") + elif isinstance(data, BetfairTicker): + self.log.info(f"LTP: {data.last_traded_price}") +``` + +:::info +Subscribing with `DataType(BetfairRaceRunnerData)` (no metadata) receives data for +**all** runners. Adding `metadata={"selection_id": }` filters to a specific runner. +Similarly, `DataType(BetfairRaceProgress)` receives progress for all races, while +`metadata={"race_id": }` filters to a specific race. + +Race data (RCM messages) requires Total Performance Data (TPD) coverage and a Betfair +API key with TPD access. Not all races have GPS tracking enabled. +::: + +### Loading race data from files + +For backtesting with recorded race data, use the file parser: + +```python +from nautilus_trader.adapters.betfair.parsing.core import parse_betfair_rcm_file + +for data in parse_betfair_rcm_file("path/to/rcm_data.json"): + if isinstance(data, BetfairRaceRunnerData): + print(f"Runner {data.selection_id} at {data.latitude}, {data.longitude}") +``` + +## Configuration + +### Data client configuration options + +| Option | Default | Description | +|---------------------------|-----------|-------------| +| `account_currency` | Required | Betfair account currency for data and price feeds. | +| `username` | `None` | Betfair account username; taken from environment when omitted. | +| `password` | `None` | Betfair account password; taken from environment when omitted. | +| `app_key` | `None` | Betfair application key used for API authentication. | +| `certs_dir` | `None` | Directory containing Betfair SSL certificates for login. | +| `instrument_config` | `None` | Optional `BetfairInstrumentProviderConfig` to scope available markets. | +| `subscription_delay_secs` | `3` | Delay (seconds) before initial market subscription request is sent. | +| `keep_alive_secs` | `36,000` | Keep-alive interval (seconds) for the Betfair session. | +| `subscribe_race_data` | `False` | When `True`, subscribe to Race Change Messages (RCM) for live GPS tracking data. | +| `stream_conflate_ms` | `None` | Explicit stream conflation interval in milliseconds (`0` disables conflation). | +| `stream_heartbeat_ms` | `5,000` | Stream heartbeat interval in milliseconds (500-5000). `None` to omit. | +| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | + +:::warning +When `stream_conflate_ms` is `None`, Betfair applies its default conflation behavior (typically enabled). +Set `stream_conflate_ms=0` explicitly to guarantee no conflation and receive every price update. +::: + +### Execution client configuration options + +| Option | Default | Description | +|------------------------------|----------|-------------| +| `account_currency` | Required | Betfair account currency for order placement and balances. | +| `username` | `None` | Betfair account username; taken from environment when omitted. | +| `password` | `None` | Betfair account password; taken from environment when omitted. | +| `app_key` | `None` | Betfair application key used for API authentication. | +| `certs_dir` | `None` | Directory containing Betfair SSL certificates for login. | +| `instrument_config` | `None` | Optional `BetfairInstrumentProviderConfig` to scope reconciliation. | +| `calculate_account_state` | `True` | Calculate account state locally from events when `True`. | +| `request_account_state_secs` | `300` | Interval (seconds) to poll Betfair for account state (`0` disables). | +| `reconcile_market_ids_only` | `False` | When `True`, reconciliation only covers `instrument_config.market_ids` (no effect if unset). | +| `stream_market_ids_filter` | `None` | List of market IDs to process from stream; others are silently skipped. | +| `ignore_external_orders` | `False` | When `True`, ignore stream orders missing from the local cache. | +| `use_market_version` | `False` | When `True`, attach the latest market version to order requests for price protection. | +| `order_request_rate_per_second` | `20` | Rate limit (requests/second) for order endpoints, separate from general API endpoints. | +| `stream_heartbeat_ms` | `5,000` | Order stream heartbeat interval in milliseconds (500-5000). `None` to omit. | +| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | + +:::warning +If you set `stream_market_ids_filter`, ensure it includes all markets you trade. Orders placed on +markets excluded from this filter will miss live fill and cancel updates from the stream. +::: + +## Session management + +Betfair sessions typically expire every 12-24 hours. The adapter automatically handles session +reconnection when `NO_SESSION` or `INVALID_SESSION_INFORMATION` errors occur: + +- The HTTP client reconnects and obtains a new session token. +- The streaming client re-authenticates and resubscribes to markets. +- The keep-alive mechanism (`keep_alive_secs`, default 10 hours) proactively extends sessions. + +:::info +Session errors during account state polling or keep-alive trigger automatic reconnection. +No manual intervention is required for normal session expiry. +::: + +## Market version price protection + +Betfair markets have a `version` number that increments whenever the market book changes +(e.g., a new price level appears, a bet is matched). The adapter can attach this version +to `placeOrders` and `replaceOrders` requests, providing price protection against stale orders. + +When `use_market_version=True`, each order request includes the market version last seen +by the adapter. If the market has advanced beyond that version by the time Betfair processes +the order, Betfair **lapses** the bet rather than matching it against a changed book. + +```python +from nautilus_trader.adapters.betfair.config import BetfairExecClientConfig + +exec_config = BetfairExecClientConfig( + account_currency="GBP", + use_market_version=True, +) +``` + +The adapter reads the market version from the instrument's `info` dictionary, which +the Exchange Streaming API's `MarketDefinition` updates populate. This means: + +- The version reflects the most recent stream update, not the HTTP API snapshot. +- There is inherent latency between a market change and the adapter receiving the updated version. +- Orders submitted before the first stream `MarketDefinition` is received will not include a version. + +:::warning +Market version protection is conservative. In fast-moving markets, the version may advance +between your order signal and submission, causing the bet to lapse even though the price +is still acceptable. Consider this trade-off between protection and fill rate. +::: + +## Multi-node deployment + +When multiple trading nodes share a single Betfair account across different markets, configure +each node to avoid interference: + +1. Set `stream_market_ids_filter` to include only that node's markets. +2. Set `ignore_external_orders=True` to suppress warnings about orders from other nodes. +3. Set `reconcile_market_ids_only=True` to limit reconciliation scope. + +This prevents warning spam and ensures each node processes only its own orders and fills. + +Here is a minimal example showing how to configure a live `TradingNode` with Betfair clients: + +```python +from nautilus_trader.adapters.betfair import BETFAIR +from nautilus_trader.adapters.betfair import BetfairLiveDataClientFactory +from nautilus_trader.adapters.betfair import BetfairLiveExecClientFactory +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.live.node import TradingNode + +# Configure Betfair data and execution clients (using AUD account currency) +config = TradingNodeConfig( + data_clients={BETFAIR: {"account_currency": "AUD"}}, + exec_clients={BETFAIR: {"account_currency": "AUD"}}, +) + +# Build the TradingNode with Betfair adapter factories +node = TradingNode(config) +node.add_data_client_factory(BETFAIR, BetfairLiveDataClientFactory) +node.add_exec_client_factory(BETFAIR, BetfairLiveExecClientFactory) +node.build() +``` + +## Contributing + +:::info +For additional features or to contribute to the Betfair adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/binance.md b/nautilus-docs/docs/integrations/binance.md new file mode 100644 index 0000000..bc91f88 --- /dev/null +++ b/nautilus-docs/docs/integrations/binance.md @@ -0,0 +1,857 @@ +# Binance + +Founded in 2017, Binance is one of the largest cryptocurrency exchanges in terms +of daily trading volume, and open interest of crypto assets and crypto +derivative products. + +This integration supports live market data ingest and order execution for: + +- **Binance Spot** (including Binance US) +- **Binance USDT-Margined Futures** (perpetuals and delivery contracts) +- **Binance Coin-Margined Futures** + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/binance/). + +## Overview + +This guide assumes a trader is setting up for both live market data feeds, and trade execution. +The Binance adapter includes multiple components, which can be used together or separately depending +on the use case. + +- `BinanceHttpClient`: Low-level HTTP API connectivity. +- `BinanceWebSocketClient`: Low-level WebSocket API connectivity. +- `BinanceInstrumentProvider`: Instrument parsing and loading functionality. +- `BinanceSpotDataClient`/`BinanceFuturesDataClient`: A market data feed manager. +- `BinanceSpotExecutionClient`/`BinanceFuturesExecutionClient`: An account management and trade execution gateway. +- `BinanceLiveDataClientFactory`: Factory for Binance data clients (used by the trading node builder). +- `BinanceLiveExecClientFactory`: Factory for Binance execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), +and won't need to necessarily work with these lower level components directly. +::: + +### Product support + +| Product Type | Supported | Notes | +|------------------------------------------|-----------|-------------------------------------| +| Spot Markets (incl. Binance US) | ✓ | | +| Margin Accounts (Cross & Isolated) | - | Margin trading not implemented. | +| USDT-Margined Futures (PERP & Delivery) | ✓ | | +| Coin-Margined Futures | ✓ | | + +:::note +Margin trading (cross & isolated) is not implemented at this time. +Contributions via [GitHub issue #2631](https://github.com/nautechsystems/nautilus_trader/issues/2631) +or pull requests to add margin trading functionality are welcome. +::: + +## Data types + +To provide complete API functionality to traders, the integration includes several +custom data types: + +- `BinanceTicker`: Represents data returned for Binance 24-hour ticker subscriptions, including price and statistical information. +- `BinanceBar`: Represents data for historical requests or real-time subscriptions to Binance bars, with additional volume metrics. +- `BinanceFuturesMarkPriceUpdate`: Represents mark price updates for Binance Futures subscriptions. + +See the Binance [API Reference](/docs/python-api-latest/adapters/binance.html) for full definitions. + +## Symbology + +As per the Nautilus unification policy for symbols, the native Binance symbols are used where possible including for +spot assets and futures contracts. Because NautilusTrader is capable of multi-venue + multi-account +trading, it's necessary to explicitly clarify the difference between `BTCUSDT` as the spot and margin traded +pair, and the `BTCUSDT` perpetual futures contract (this symbol is used for *both* natively by Binance). + +Therefore, Nautilus appends the suffix `-PERP` to all perpetual symbols. +E.g. for Binance Futures, the `BTCUSDT` perpetual futures contract symbol would be `BTCUSDT-PERP` within the Nautilus system boundary. + +## Order capability + +The following tables detail the order types, execution instructions, and time-in-force options supported across different Binance account types: + +### Order types + +| Order Type | Spot | Margin | USDT Futures | Coin Futures | Notes | +|------------------------|------|--------|--------------|--------------|-------------------------| +| `MARKET` | ✓ | - | ✓ | ✓ | Quote quantity support: Spot only. | +| `LIMIT` | ✓ | - | ✓ | ✓ | | +| `STOP_MARKET` | - | - | ✓ | ✓ | Futures only. | +| `STOP_LIMIT` | ✓ | - | ✓ | ✓ | | +| `MARKET_IF_TOUCHED` | - | - | ✓ | ✓ | Futures only. | +| `LIMIT_IF_TOUCHED` | ✓ | - | ✓ | ✓ | | +| `TRAILING_STOP_MARKET` | - | - | ✓ | ✓ | Futures only. | + +### Execution instructions + +| Instruction | Spot | Margin | USDT Futures | Coin Futures | Notes | +|---------------|------|--------|--------------|--------------|---------------------------------------| +| `post_only` | ✓ | - | ✓ | ✓ | See restrictions below. | +| `reduce_only` | - | - | ✓ | ✓ | Futures only; disabled in Hedge Mode. | + +#### Post-only restrictions + +Only *limit* order types support `post_only`. + +| Order Type | Spot | Margin | USDT Futures | Coin Futures | Notes | +|--------------------------|------|--------|--------------|--------------|-----------------------------------------------------| +| `LIMIT` | ✓ | - | ✓ | ✓ | Uses `LIMIT_MAKER` for Spot, `GTX` TIF for Futures. | +| `STOP_LIMIT` | - | - | ✓ | ✓ | Futures only. | + +### Time in force + +| Time in force | Spot | Margin | USDT Futures | Coin Futures | Notes | +|---------------|------|--------|--------------|--------------|--------------------------------------------| +| `GTC` | ✓ | - | ✓ | ✓ | Good Till Canceled. | +| `GTD` | ✓* | - | ✓ | ✓ | *Converted to GTC for Spot with warning. | +| `FOK` | ✓ | - | ✓ | ✓ | Fill or Kill. | +| `IOC` | ✓ | - | ✓ | ✓ | Immediate or Cancel. | + +### Advanced order features + +| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | +|--------------------|------|--------|--------------|--------------|----------------------------------------------| +| Order Modification | ✓ | - | ✓ | ✓ | Price and quantity for `LIMIT` orders only. | +| Bracket/OCO Orders | - | - | - | - | *Planned*. Currently denied at submission. | +| Iceberg Orders | ✓ | - | ✓ | ✓ | Large orders split into visible portions. | + +### Batch operations + +| Operation | Spot | Margin | USDT Futures | Coin Futures | Notes | +|--------------------|------|--------|--------------|--------------|----------------------------------------------| +| Batch Submit | ✓ | - | ✓ | ✓ | Orders submitted individually (no batch API call). | +| Batch Modify | - | - | - | - | Not implemented. | +| Batch Cancel | -* | - | ✓ | ✓ | *Spot falls back to individual cancels. | + +#### Cancel all orders behavior + +When calling `cancel_all_orders()` from a strategy, the adapter includes orders in both open and inflight (SUBMITTED) states. +This ensures that orders submitted but not yet acknowledged by Binance are also canceled. + +**Multi-strategy safety**: When multiple strategies trade the same instrument, the adapter compares orders owned by the requesting strategy against all orders for that instrument. If the strategy owns all orders, a single cancel-all API call is used. Otherwise, per-strategy cancels are sent (batch for regular orders, individual for algo orders) to avoid affecting other strategies' orders. + +**Futures algo orders**: For Binance Futures, conditional order types (STOP_MARKET, STOP_LIMIT, TAKE_PROFIT, TAKE_PROFIT_MARKET, TRAILING_STOP_MARKET) require a different cancel endpoint than regular orders. +The adapter automatically routes these "algo" orders through the correct endpoint. Once an algo order triggers and becomes a regular order, it uses the standard cancel endpoint. + +**Endpoints used**: + +| Account Type | Regular Orders | Algo Orders (batch) | Algo Orders (individual) | +|--------------|---------------------------------|----------------------------------|-----------------------------| +| Spot/Margin | `DELETE /api/v3/openOrders` | N/A | N/A | +| USDT Futures | `DELETE /fapi/v1/allOpenOrders` | `DELETE /fapi/v1/algoOpenOrders` | `DELETE /fapi/v1/algoOrder` | +| Coin Futures | `DELETE /dapi/v1/allOpenOrders` | `DELETE /dapi/v1/algoOpenOrders` | `DELETE /dapi/v1/algoOrder` | + +### Position management + +| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | +|---------------------|------|--------|--------------|--------------|---------------------------------------------| +| Query positions | - | - | ✓ | ✓ | Real-time position updates. | +| Position mode | - | - | ✓ | ✓ | One-Way vs Hedge mode (position IDs). | +| Leverage control | - | - | ✓ | ✓ | Dynamic leverage adjustment per symbol. | +| Margin mode | - | - | ✓ | ✓ | Cross vs Isolated margin per symbol. | + +### Risk events + +| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | +|----------------------|------|--------|--------------|--------------|---------------------------------------------| +| Liquidation handling | - | - | ✓ | ✓ | Exchange-forced position closures. | +| ADL handling | - | - | ✓ | ✓ | Auto-Deleveraging events. | + +Binance Futures can trigger exchange-generated orders in response to risk events: + +- **Liquidations**: When insufficient margin exists to maintain a position, Binance forcibly closes it at the bankruptcy price. These orders have client IDs starting with `autoclose-`. +- **ADL (Auto-Deleveraging)**: When the insurance fund is depleted, Binance closes profitable positions to cover losses. These orders use client ID `adl_autoclose`. +- **Settlements**: Quarterly contract deliveries use client IDs starting with `settlement_autoclose-`. + +The adapter detects these special order types via their client ID patterns and execution type (`CALCULATED`), then: + +1. Logs a warning with order details for monitoring. +2. Generates an `OrderStatusReport` to seed the cache. +3. Generates a `FillReport` with correct fill details and TAKER liquidity side. + +This ensures liquidation and ADL events are properly reflected in portfolio state and PnL calculations. + +### Order querying + +| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | +|---------------------|------|--------|--------------|--------------|---------------------------------------------| +| Query open orders | ✓ | ✓ | ✓ | ✓ | List all active orders. | +| Query order history | ✓ | ✓ | ✓ | ✓ | Historical order data. | +| Order status updates| ✓ | ✓ | ✓ | ✓ | Real-time order state changes. | +| Trade history | ✓ | ✓ | ✓ | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | +|---------------------|------|--------|--------------|--------------|----------------------------------------------| +| Order lists | - | - | - | - | *Not supported*. | +| OCO orders | - | - | - | - | *Planned*. Currently denied at submission. | +| Bracket orders | - | - | - | - | *Planned*. Currently denied at submission. | +| Conditional orders | ✓ | ✓ | ✓ | ✓ | Stop and market-if-touched orders. | + +### Order parameters + +Customize individual orders by supplying a `params` dictionary when calling `Strategy.submit_order`. The Binance execution clients currently recognise: + +| Parameter | Type | Account types | Description | +|-----------------|--------|-------------------|-------------| +| `price_match` | `str` | USDT/COIN Futures | Set one of Binance's `priceMatch` modes (see Price match section below) to delegate price selection to the exchange. Cannot be combined with `post_only` or iceberg (`display_qty`) instructions. | + +### Price match + +Binance Futures supports BBO (Best Bid/Offer) price matching via the `priceMatch` parameter, which delegates price selection to the exchange. This feature allows limit orders to dynamically join the order book at optimal prices without manually specifying the exact price level. + +When using `price_match`, you submit a limit order with a reference price (for local risk checks), but Binance determines the actual working price based on the current market state and the selected price match mode. + +#### Valid price match values + +Valid `priceMatch` values for Binance Futures: + +| Value | Behaviour | +|---------------|----------------------------------------------------------------| +| `OPPONENT` | Join the best price on the opposing side of the book. | +| `OPPONENT_5` | Join the opposing side price but allow up to a 5-tick offset. | +| `OPPONENT_10` | Join the opposing side price but allow up to a 10-tick offset. | +| `OPPONENT_20` | Join the opposing side price but allow up to a 20-tick offset. | +| `QUEUE` | Join the best price on the same side (stay maker). | +| `QUEUE_5` | Join the same-side queue but offset up to 5 ticks. | +| `QUEUE_10` | Join the same-side queue but offset up to 10 ticks. | +| `QUEUE_20` | Join the same-side queue but offset up to 20 ticks. | + +:::info +For more details, see the [official documentation](https://developers.binance.com/docs/derivatives/usds-margined-futures/trade/rest-api). +::: + +#### Event sequence + +When an order is submitted with `price_match`, the following sequence of events occurs: + +1. **Order submission**: Nautilus sends the order to Binance with the `priceMatch` parameter but omits the limit price in the API request. +2. **Order acceptance**: Binance accepts the order and determines the actual working price based on the current market and the specified price match mode. +3. **OrderAccepted event**: Nautilus generates an `OrderAccepted` event when the order is confirmed. +4. **OrderUpdated event**: If the Binance-accepted price differs from the original reference price, Nautilus immediately generates an `OrderUpdated` event with the actual working price. +5. **Price synchronization**: The order's limit price in the Nautilus cache is now synchronized with the actual price accepted by Binance. + +This ensures that the order price in your system accurately reflects what Binance has accepted, which is important for position management, risk calculations, and strategy logic. + +#### Example + +```python +order = strategy.order_factory.limit( + instrument_id=InstrumentId.from_str("BTCUSDT-PERP.BINANCE"), + order_side=OrderSide.BUY, + quantity=Quantity.from_int(1), + price=Price.from_str("65000"), # Reference price for local risk checks +) + +strategy.submit_order( + order, + params={"price_match": "QUEUE"}, +) +``` + +:::note +After submission, if Binance accepts the order at a different price (e.g., 64,995.50), you will receive both an `OrderAccepted` event followed by an `OrderUpdated` event with the new price. +::: + +### Trailing stops + +For trailing stop market orders on Binance: + +- Use `activation_price` (optional) to specify when the trailing mechanism activates +- When omitted, Binance uses the current market price at submission time +- Use `trailing_offset` for the callback rate (in basis points) + +:::warning +Do not use `trigger_price` for trailing stop orders - it will fail with an error. Use `activation_price` instead. +::: + +## Link & Trade + +The NautilusTrader integration ID is automatically prefixed to all +system-generated client order IDs for every order placed through the Binance +Rust adapter. This provides transparent order attribution through Binance's +[Link and Trade](https://developers.binance.com/docs/binance_link/link-and-trade) +program without requiring any user configuration. + +The adapter encodes outgoing `ClientOrderId` values into a compact format that +fits within Binance's 36-character `newClientOrderId` limit, and decodes +incoming order events back to the original ID before they reach strategies. +This transformation is fully transparent: strategies see only their original +`ClientOrderId` values at all times. + +:::note +The integration ID prefix applies to all order operations including +submissions, modifications, cancellations, and status queries. Orders placed +before this support was added are handled gracefully through passthrough +decoding. +::: + +:::info +This feature is currently available in the Rust adapter only. Users can opt out +by passing a custom `client_order_id` on their orders, or by removing the +encoding calls and recompiling. There is no technical limitation preventing +either approach. +::: + +## Order books + +Order books can be maintained at full or partial depths based on the subscription settings. +WebSocket stream update rates differ between Spot and Futures exchanges, with Nautilus using the +highest available streaming rate: + +- **Spot**: 100ms +- **Futures**: 0ms (unthrottled) + +There is a limitation of one order book per instrument per trader instance. +As stream subscriptions may vary, the latest order book data (deltas or snapshots) +subscription will be used by the Binance data client. + +Order book snapshot rebuilds will be triggered on: + +- Initial subscription of the order book data. +- Data websocket reconnects. + +The sequence of events is as follows: + +- Deltas will start buffered. +- Snapshot is requested and awaited. +- Snapshot response is parsed to `OrderBookDeltas`. +- Snapshot deltas are sent to the `DataEngine`. +- Buffered deltas are iterated, dropping those where the sequence number is not greater than the last delta in the snapshot. +- Deltas will stop buffering. +- Remaining deltas are sent to the `DataEngine`. + +## Binance data differences + +The `ts_event` field value for `QuoteTick` objects will differ between Spot and Futures exchanges, +where the former does not provide an event timestamp, so the `ts_init` is used (which means `ts_event` and `ts_init` are identical). + +## Binance specific data + +It's possible to subscribe to Binance specific data streams as they become available to the +adapter over time. + +:::note +Bars are not considered 'Binance specific' and can be subscribed to in the normal way. +As more adapters are built out which need for example mark price and funding rate updates, then these +methods may eventually become first-class (not requiring custom/generic subscriptions as below). +::: + +### `BinanceFuturesMarkPriceUpdate` + +You can subscribe to `BinanceFuturesMarkPriceUpdate` (including funding rate info) +data streams by subscribing in the following way from your actor or strategy: + +```python +from nautilus_trader.adapters.binance import BinanceFuturesMarkPriceUpdate +from nautilus_trader.model import DataType +from nautilus_trader.model import ClientId + +# In your `on_start` method +self.subscribe_data( + data_type=DataType(BinanceFuturesMarkPriceUpdate, metadata={"instrument_id": self.instrument.id}), + client_id=ClientId("BINANCE"), +) +``` + +This will result in your actor/strategy passing these received `BinanceFuturesMarkPriceUpdate` +objects to your `on_data` method. You will need to check the type, as this +method acts as a flexible handler for all custom/generic data. + +```python +from nautilus_trader.core import Data + +def on_data(self, data: Data): + # First check the type of data + if isinstance(data, BinanceFuturesMarkPriceUpdate): + # Do something with the data +``` + +## Funding rates + +The adapter subscribes to funding rate data through the +[Mark Price Stream](https://developers.binance.com/docs/derivatives/usds-margined-futures/websocket-market-streams/Mark-Price-Stream) +WebSocket endpoint, which provides the current funding rate and next funding time. + +The `interval` field on `FundingRateUpdate` is `None` for Binance because the Mark Price Stream +does not include a funding interval field. Binance exposes `fundingIntervalHours` through the +[Get Funding Rate Info](https://developers.binance.com/docs/derivatives/usds-margined-futures/market-data/rest-api/Get-Funding-Rate-Info) +REST endpoint, but this is not currently consumed by the adapter. + +## Rate limiting + +Binance uses an interval-based rate limiting system where request weight is tracked per fixed time window (every minute, resetting at :00 seconds). Each API endpoint has an assigned weight cost, and your total weight usage is tracked per IP address. + +### Global weight limits + +These are the primary limits shared across all endpoints: + +| Account Type | Weight Limit | Interval | +|--------------|--------------|----------| +| Spot/Margin | 6,000 | 1 minute | +| Futures | 2,400 | 1 minute | + +### Endpoint weight costs + +Some endpoints have higher weight costs per request: + +| Endpoint | Weight | Notes | +|---------------------------|--------|--------------------------------------------------------| +| `/api/v3/order` | 1 | Spot order placement. | +| `/api/v3/allOrders` | 20 | Spot historical orders (expensive). | +| `/api/v3/klines` | 2+ | Scales with `limit` parameter. | +| `/fapi/v1/order` | 1 | Futures order placement. | +| `/fapi/v1/allOrders` | 20 | Futures historical orders (expensive). | +| `/fapi/v1/commissionRate` | 20 | Futures commission rate query. | +| `/fapi/v1/klines` | 5+ | Scales with `limit` parameter. | + +### WebSocket API limits + +The WebSocket API (used for user data streams) shares the same weight quota as the REST API: + +| Limit Type | Value | Notes | +|------------------|--------|--------------------------------------------| +| Request weight | Shared | Counts against REST API weight quota. | +| Handshake | 5 | Weight cost per connection attempt. | +| Ping/pong frames | 5/sec | Maximum ping/pong rate. | + +### Adapter behavior + +The adapter uses token bucket rate limiters to approximate Binance's interval-based limits. This reduces the risk of quota violations while maintaining throughput for normal operations. + +For endpoints with dynamic weight (e.g., `/klines` scales with the `limit` parameter), the adapter draws a single token per call. Large history requests may need manual pacing. Monitor the `X-MBX-USED-WEIGHT-*` response headers to track actual usage. + +:::warning +Binance returns HTTP 429 when you exceed the allowed weight. Repeated violations trigger temporary IP bans (escalating from 2 minutes to 3 days for repeat offenders). +::: + +:::info +For the latest rate limits, query `/api/v3/exchangeInfo` (Spot) or `/fapi/v1/exchangeInfo` (Futures), or see: + +- [Spot API Limits](https://developers.binance.com/docs/binance-spot-api-docs/rest-api/limits) +- [Futures API Limits](https://developers.binance.com/docs/derivatives/usds-margined-futures/general-info) + +::: + +## Configuration + +### Data client configuration options + +| Option | Default | Description | +|------------------------------------|-----------|-------------| +| `venue` | `BINANCE` | Venue identifier used when registering the client. | +| `api_key` | `None` | Binance API key; loaded from environment variables when omitted. | +| `api_secret` | `None` | Binance API secret; loaded from environment variables when omitted. | +| `key_type` | `HMAC` | **Deprecated**: key type is now auto-detected from the API secret format. Only needed to force `RSA`. | +| `account_type` | `SPOT` | Account type for data endpoints (spot, margin, USDT futures, coin futures). | +| `base_url_http` | `None` | Override for the HTTP REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | +| `us` | `False` | Route requests to Binance US endpoints when `True`. | +| `environment` | `None` | Binance environment: `LIVE`, `TESTNET`, or `DEMO`. Defaults to `LIVE` when `None`. | +| `testnet` | `False` | **Deprecated**: use `environment=BinanceEnvironment.TESTNET` instead. | +| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalogue refreshes. | +| `use_agg_trade_ticks` | `False` | When `True`, subscribe to aggregated trade ticks instead of raw trades. | + +### Execution client configuration options + +| Option | Default | Description | +|--------------------------------------|-----------|-------------| +| `venue` | `BINANCE` | Venue identifier used when registering the client. | +| `api_key` | `None` | Binance API key; loaded from environment variables when omitted. | +| `api_secret` | `None` | Binance API secret; loaded from environment variables when omitted. | +| `key_type` | `HMAC` | **Deprecated**: key type is now auto-detected from the API secret format. Only needed to force `RSA` (data clients only, RSA is not supported for execution). | +| `account_type` | `SPOT` | Account type for order placement (spot, margin, USDT futures, coin futures). | +| `base_url_http` | `None` | Override for the HTTP REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket API base URL. | +| `base_url_ws_stream` | `None` | Override for the WebSocket stream URL (futures user data event delivery). | +| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | +| `us` | `False` | Route requests to Binance US endpoints when `True`. | +| `environment` | `None` | Binance environment: `LIVE`, `TESTNET`, or `DEMO`. Defaults to `LIVE` when `None`. | +| `testnet` | `False` | **Deprecated**: use `environment=BinanceEnvironment.TESTNET` instead. | +| `use_gtd` | `True` | When `False`, remaps GTD orders to GTC for local expiry management. | +| `use_reduce_only` | `True` | When `True`, passes through `reduce_only` instructions to Binance. | +| `use_position_ids` | `True` | Enable Binance hedging position IDs; set `False` for virtual hedging. | +| `use_trade_lite` | `False` | Use TRADE_LITE execution events that include derived fees. | +| `treat_expired_as_canceled` | `False` | Treat `EXPIRED` execution types as `CANCELED` when `True`. | +| `recv_window_ms` | `5,000` | Receive window (milliseconds) for signed REST requests. | +| `max_retries` | `None` | Maximum retry attempts for order submission/cancel/modify calls. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retry attempts. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retry attempts. | +| `futures_leverages` | `None` | Mapping of `BinanceSymbol` to initial leverage for futures accounts. | +| `futures_margin_types` | `None` | Mapping of `BinanceSymbol` to futures margin type (isolated/cross). | +| `log_rejected_due_post_only_as_warning` | `True` | Log post-only rejections as warnings when `True`; otherwise as errors. | + +The most common use case is to configure a live `TradingNode` to include Binance +data and execution clients. To achieve this, add a `BINANCE` section to your client +configuration(s): + +```python +from nautilus_trader.adapters.binance import BINANCE +from nautilus_trader.live.node import TradingNode + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + BINANCE: { + "api_key": "YOUR_BINANCE_API_KEY", + "api_secret": "YOUR_BINANCE_API_SECRET", + "account_type": "spot", # {spot, usdt_futures, coin_futures} + "base_url_http": None, # Override with custom endpoint + "base_url_ws": None, # Override with custom endpoint + "us": False, # If client is for Binance US + }, + }, + exec_clients={ + BINANCE: { + "api_key": "YOUR_BINANCE_API_KEY", + "api_secret": "YOUR_BINANCE_API_SECRET", + "account_type": "spot", # {spot, usdt_futures, coin_futures} + "base_url_http": None, # Override with custom endpoint + "base_url_ws": None, # Override with custom endpoint + "us": False, # If client is for Binance US + }, + }, +) +``` + +Then, create a `TradingNode` and add the client factories: + +```python +from nautilus_trader.adapters.binance import BINANCE +from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory +from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory +from nautilus_trader.live.node import TradingNode + +# Instantiate the live trading node with a configuration +node = TradingNode(config=config) + +# Register the client factories with the node +node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory) +node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory) + +# Finally build the node +node.build() +``` + +### Key types + +Binance supports three API key types: **Ed25519**, **HMAC-SHA256**, and **RSA**. +The adapter auto-detects the key type from your API secret format, so no configuration is needed. + +**Ed25519 is strongly recommended** for all API access. Binance recommends Ed25519 for +its superior performance and security, and a future version of NautilusTrader will +require Ed25519 exclusively. + +| Key Type | Data Clients | Execution Clients | Status | +|----------|--------------|-------------------|--------| +| Ed25519 | ✓ | ✓ | **Recommended** | +| HMAC | ✓ | ✓ | Deprecated, will be removed in a future version. | +| RSA | ✓ | - | Deprecated, not supported for execution. | + +:::tip +**We strongly recommend switching to Ed25519 keys now.** Generate an Ed25519 keypair and register +it with Binance. See [Generating Ed25519 keys](#generating-ed25519-keys) below for instructions. +::: + +:::note +Ed25519 keys must be provided in unencrypted PEM format (base64-encoded ASN.1/DER). +The implementation automatically extracts the 32-byte seed from the DER structure. +Encrypted (password-protected) PEM keys are not supported. If your key is encrypted, +decrypt it first: `openssl pkey -in encrypted.pem -out decrypted.pem` +::: + +#### Generating Ed25519 keys + +**Option 1: OpenSSL (recommended)** + +```bash +# Generate private key (PKCS#8 PEM format) +openssl genpkey -algorithm ed25519 -out binance_ed25519_private.pem + +# Extract public key +openssl pkey -in binance_ed25519_private.pem -pubout -out binance_ed25519_public.pem +``` + +**Option 2: Binance Key Generator** + +Download the [Binance Asymmetric Key Generator](https://github.com/binance/asymmetric-key-generator) from the releases page and run it to generate a keypair. + +**Registering with Binance** + +1. Log in to Binance and go to **Profile** → **API Management** +2. Click **Create API** and select **Self-generated** +3. Paste the contents of your public key file (including the `-----BEGIN PUBLIC KEY-----` header/footer) +4. Configure permissions (Enable Spot & Margin Trading, etc.) + +**Using with NautilusTrader** + +Set the private key as your API secret: + +```bash +export BINANCE_API_KEY="your-api-key-from-binance" +export BINANCE_API_SECRET="$(cat binance_ed25519_private.pem)" +``` + +Or pass the PEM content directly in your configuration. + +:::warning +Keep your private key secure. Never share it or commit it to version control. +::: + +### API credentials + +There are multiple options for supplying your credentials to the Binance clients. +Either pass the corresponding values to the configuration objects, or +set the appropriate environment variables (see [Environments](#environments) for per-environment variables). + +:::tip +We recommend using Ed25519 keys for all clients. HMAC keys still work for both data and +execution clients, but Ed25519 offers better performance and will become the only supported +key type in a future version. See [Key types](#key-types) for details. +::: + +:::warning +The `BINANCE_ED25519_*` and `BINANCE_*_ED25519_*` environment variables have been removed +for Spot/Margin. For Futures, they are deprecated with a warning and will be removed in a +future version. Rename them to the standard `BINANCE_API_KEY`/`BINANCE_API_SECRET` variables +(Ed25519 keys are now auto-detected). +::: + +When starting the trading node, you'll receive immediate confirmation of whether your +credentials are valid and have trading permissions. + +### Account type + +Set the `account_type` using the `BinanceAccountType` enum. The supported account types are: + +- `SPOT` +- `USDT_FUTURES` (USDT or BUSD stablecoins as collateral) +- `COIN_FUTURES` (other cryptocurrency as collateral) + +:::note +`MARGIN` and `ISOLATED_MARGIN` account types exist in the enum but margin trading +is not yet implemented. See [Product support](#product-support). +::: + +### Base URL overrides + +It's possible to override the default base URLs for both HTTP Rest and +WebSocket APIs. This is useful for configuring API clusters for performance reasons, +or when Binance has provided you with specialized endpoints. + +### Binance US + +There is support for Binance US accounts by setting the `us` option in the configs +to `True` (this is `False` by default). All functionality available to US accounts +should behave identically to standard Binance. + +### Environments + +Binance provides three trading environments, configured via the `environment` option: + +| Environment | Config | Description | +|--------------|--------------------------|---------------------------------------------------------------------| +| **Live** | `environment="LIVE"` | Production trading with real funds (default). | +| **Demo** | `environment="DEMO"` | Practice trading with simulated funds on production infrastructure. | +| **Testnet** | `environment="TESTNET"` | Separate test network for development and integration testing. | + +#### Live (production) + +The default environment for live trading with real funds. + +```python +config = BinanceExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + account_type=BinanceAccountType.SPOT, + # environment=BinanceEnvironment.LIVE (default) +) +``` + +Environment variables: `BINANCE_API_KEY`, `BINANCE_API_SECRET` + +#### Demo trading + +Practice trading with simulated funds. Spot demo uses `demo-api.binance.com`, USD-M Futures demo uses `demo-fapi.binance.com`, and COIN-M Futures demo shares testnet endpoints. Create demo API keys from the [Binance Demo Trading page](https://www.binance.com/en/demo-trading). + +```python +config = BinanceExecClientConfig( + api_key="YOUR_DEMO_API_KEY", + api_secret="YOUR_DEMO_API_SECRET", + account_type=BinanceAccountType.SPOT, + environment=BinanceEnvironment.DEMO, +) +``` + +Environment variables: `BINANCE_DEMO_API_KEY`, `BINANCE_DEMO_API_SECRET` (shared across Spot and Futures) + +:::warning +**Demo environment limitations:** + +- COIN-M Futures are **not supported** in demo mode. +- Futures demo shares testnet infrastructure, so market data and liquidity may differ from production. + +::: + +#### Testnet + +A separate test network for development and integration testing. Spot testnet is at `testnet.binance.vision`, Futures testnet at `testnet.binancefuture.com`. + +```python +config = BinanceExecClientConfig( + api_key="YOUR_TESTNET_API_KEY", + api_secret="YOUR_TESTNET_API_SECRET", + account_type=BinanceAccountType.SPOT, + environment=BinanceEnvironment.TESTNET, +) +``` + +Environment variables (Spot/Margin): `BINANCE_TESTNET_API_KEY`, `BINANCE_TESTNET_API_SECRET` +Environment variables (Futures): `BINANCE_FUTURES_TESTNET_API_KEY`, `BINANCE_FUTURES_TESTNET_API_SECRET` + +:::note +Testnet uses completely separate infrastructure from production. Market data and liquidity differ significantly from live. +::: + +:::warning +The `testnet` config option is deprecated and will be removed in a future version. Use `environment="TESTNET"` instead. +::: + +### Aggregated trades + +Binance provides aggregated trade data endpoints as an alternative source of trades. +In comparison to the default trade endpoints, aggregated trade data endpoints can return all +ticks between a `start_time` and `end_time`. + +To use aggregated trades and the endpoint features, set the `use_agg_trade_ticks` option +to `True` (this is `False` by default.) + +### Commission rate queries + +By default, Binance Futures instruments use fee tier tables based on your VIP level. +For market maker accounts with negative maker fees or when precise rates are required, +enable per-symbol commission rate queries: + +```python +from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig + +instrument_provider=BinanceInstrumentProviderConfig( + load_all=True, + query_commission_rates=True, # Query accurate rates per symbol +) +``` + +When enabled, the adapter queries Binance's `/fapi/v1/commissionRate` endpoint for +each symbol in parallel during instrument loading. This is particularly useful for: + +- Market maker accounts with negative maker fees. +- Accounts with custom fee arrangements. +- Ensuring exact commission rates for PnL calculations. + +The adapter uses parallel requests with proper rate limiting (120 requests/minute +accounting for the endpoint's weight of 20). If a query fails, it automatically +falls back to the fee tier table. + +### Parser warnings + +Some Binance instruments are unable to be parsed into Nautilus objects if they +contain enormous field values beyond what can be handled by the platform. +In these cases, a *warn and continue* approach is taken (the instrument will not +be available). + +These warnings may cause unnecessary log noise, and so it's possible to +configure the provider to not log the warnings, as per the client configuration +example below: + +```python +from nautilus_trader.config import InstrumentProviderConfig + +instrument_provider=InstrumentProviderConfig( + load_all=True, + log_warnings=False, +) +``` + +### Futures hedge mode + +Binance Futures Hedge mode is a position mode where a trader opens positions in both long and short +directions to mitigate risk and potentially profit from market volatility. + +To use Binance Future Hedge mode, you need to follow the three items below: + +- 1. Before starting the strategy, ensure that hedge mode is configured on Binance. +- 2. Set the `use_reduce_only` option to `False` in BinanceExecClientConfig (this is `True` by default). + + ```python + from nautilus_trader.adapters.binance import BINANCE + + config = TradingNodeConfig( + ..., # Omitted + data_clients={ + BINANCE: BinanceDataClientConfig( + api_key=None, # 'BINANCE_API_KEY' env var + api_secret=None, # 'BINANCE_API_SECRET' env var + account_type=BinanceAccountType.USDT_FUTURES, + base_url_http=None, # Override with custom endpoint + base_url_ws=None, # Override with custom endpoint + ), + }, + exec_clients={ + BINANCE: BinanceExecClientConfig( + api_key=None, # 'BINANCE_API_KEY' env var + api_secret=None, # 'BINANCE_API_SECRET' env var + account_type=BinanceAccountType.USDT_FUTURES, + base_url_http=None, # Override with custom endpoint + base_url_ws=None, # Override with custom endpoint + use_reduce_only=False, # Must be disabled for Hedge mode + ), + } + ) + ``` + +- 3. When submitting an order, use a suffix (`LONG` or `SHORT` ) in the `position_id` to indicate the position direction. + + ```python + class EMACrossHedgeMode(Strategy): + ..., # Omitted + def buy(self) -> None: + """ + Users simple buy method (example). + """ + order: MarketOrder = self.order_factory.market( + instrument_id=self.instrument_id, + order_side=OrderSide.BUY, + quantity=self.instrument.make_qty(self.trade_size), + # time_in_force=TimeInForce.FOK, + ) + + # LONG suffix is recognized as a long position by Binance adapter. + position_id = PositionId(f"{self.instrument_id}-LONG") + self.submit_order(order, position_id) + + def sell(self) -> None: + """ + Users simple sell method (example). + """ + order: MarketOrder = self.order_factory.market( + instrument_id=self.instrument_id, + order_side=OrderSide.SELL, + quantity=self.instrument.make_qty(self.trade_size), + # time_in_force=TimeInForce.FOK, + ) + # SHORT suffix is recognized as a short position by Binance adapter. + position_id = PositionId(f"{self.instrument_id}-SHORT") + self.submit_order(order, position_id) + ``` + +## Contributing + +:::info +For additional features or to contribute to the Binance adapter, see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/bitmex.md b/nautilus-docs/docs/integrations/bitmex.md new file mode 100644 index 0000000..4f3d2e7 --- /dev/null +++ b/nautilus-docs/docs/integrations/bitmex.md @@ -0,0 +1,871 @@ +# BitMEX + +Founded in 2014, BitMEX (Bitcoin Mercantile Exchange) is a cryptocurrency derivatives +trading platform offering spot, perpetual contracts, traditional futures, prediction +markets, and other advanced trading products. This integration supports live market data +ingest and order execution with BitMEX. + +## Overview + +This adapter is implemented in Rust, with optional Python bindings for ease of use in Python-based workflows. +It does not require external BitMEX client libraries; the core components are compiled as a static library and linked automatically during the build. + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/bitmex/). + +## Components + +This guide assumes a trader is setting up for both live market data feeds, and trade execution. +The BitMEX adapter includes multiple components, which can be used together or separately depending +on the use case. + +- `BitmexHttpClient`: Low-level HTTP API connectivity. +- `BitmexWebSocketClient`: Low-level WebSocket API connectivity. +- `BitmexInstrumentProvider`: Instrument parsing and loading functionality. +- `BitmexDataClient`: A market data feed manager. +- `BitmexExecutionClient`: An account management and trade execution gateway. +- `BitmexLiveDataClientFactory`: Factory for BitMEX data clients (used by the trading node builder). +- `BitmexLiveExecClientFactory`: Factory for BitMEX execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), +and won't need to necessarily work with these lower level components directly. +::: + +## BitMEX documentation + +BitMEX provides extensive documentation for users: + +- [BitMEX API Explorer](https://www.bitmex.com/app/restAPI) - Interactive API documentation. +- [BitMEX API Documentation](https://www.bitmex.com/app/apiOverview) - Complete API reference. +- [BitMEX Exchange Rules](https://www.bitmex.com/exchange-rules) - Official exchange rules and regulations. +- [Contract Guides](https://www.bitmex.com/app/contract) - Detailed contract specifications. +- [Spot Trading Guide](https://www.bitmex.com/app/spotGuide) - Spot trading overview. +- [Perpetual Contracts Guide](https://www.bitmex.com/app/perpetualContractsGuide) - Perpetual swaps explained. +- [Futures Contracts Guide](https://www.bitmex.com/app/futuresGuide) - Traditional futures information. + +It's recommended you refer to the BitMEX documentation in conjunction with this +NautilusTrader integration guide. + +## Product support + +| Product Type | Data Feed | Trading | Notes | +|-------------------|-----------|---------|-----------------------------------------------------| +| Spot | ✓ | ✓ | Limited pairs, unified wallet with derivatives. | +| Perpetual Swaps | ✓ | ✓ | Inverse and linear contracts available. | +| Stock Perpetuals | - | - | *Not yet supported*. Currently on testnet only. | +| Futures | ✓ | ✓ | Traditional fixed expiration contracts. | +| Quanto Futures | ✓ | ✓ | Settled in different currency than underlying. | +| Prediction Markets| ✓ | ✓ | Event-based contracts, 0-100 pricing, USDT settled. | +| Options | - | - | *Not provided by BitMEX*. | + +:::note +BitMEX has discontinued their options products to focus on their core derivatives and spot offerings. +::: + +### Spot trading + +- Direct token/coin trading with immediate settlement. +- Major pairs including XBT/USDT, ETH/USDT, ETH/XBT. +- Additional altcoin pairs (LINK, SOL, UNI, APE, AXS, BMEX against USDT). + +### Derivatives + +- **Perpetual contracts**: Inverse (e.g., XBTUSD) and linear (e.g., ETHUSDT). +- **Traditional futures**: Fixed expiration date contracts. +- **Quanto futures**: Contracts settled in a different currency than the underlying. +- **Prediction markets**: Event-based derivatives (e.g., P_FTXZ26, P_SBFJAILZ26) allowing traders to speculate on outcomes across crypto, finance, and other events. No leverage, priced 0-100, settled in USDT. +- **Stock perpetuals**: Equity-based perpetuals (e.g., SPYUSDT, CRCLUSDT). *Currently on testnet only; not yet supported by this adapter.* + +### Instrument type codes (CFI) + +BitMEX uses CFI (Classification of Financial Instruments) codes following the ISO 10962 standard. +The adapter recognizes the following instrument type codes: + +| Code | Type | Status | Description | +|----------|----------------------|-------------|-------------------------------------------------| +| `FFWCSX` | Perpetual Contract | Supported | Crypto-based perpetual swaps (e.g., XBTUSD). | +| `FFWCSF` | Perpetual FX | Supported | FX-based perpetual contracts. | +| `FFCCSX` | Futures | Supported | Calendar futures with fixed expiration. | +| `FFICSX` | Prediction Market | Supported | Event-based prediction contracts. | +| `IFXXXP` | Spot | Supported | Spot trading pairs. | +| `FFSCSX` | Stock Perpetual | Unsupported | Stock/equity-based perpetuals. Testnet only. | +| `SRMCSX` | Swap Rate | Unsupported | Yield-based swap products (historical). | +| `MR****` | Index | Reference | BitMEX indices (non-tradeable, for price ref). | + +See [BitMEX Typ Values](https://support.bitmex.com/hc/en-gb/articles/6299296145565-What-are-the-Typ-Values-for-Instrument-endpoint) for more details. + +## Symbology + +BitMEX uses a specific naming convention for its trading symbols. Understanding this +convention is crucial for correctly identifying and trading instruments. + +### Symbol format + +BitMEX symbols typically follow these patterns: + +- **Spot pairs**: Base currency + Quote currency (e.g., `XBT/USDT`, `ETH/USDT`). +- **Perpetual contracts**: Base currency + Quote currency (e.g., `XBTUSD`, `ETHUSD`). +- **Futures contracts**: Base currency + Expiry code (e.g., `XBTM24`, `ETHH25`). +- **Quanto contracts**: Special naming for non-USD settled contracts. +- **Prediction markets**: `P_` prefix + Event identifier + Expiry code (e.g., `P_POWELLK26`, `P_FTXZ26`). + +:::info +BitMEX uses `XBT` as the symbol for Bitcoin instead of `BTC`. This follows the ISO 4217 +currency code standard where "X" denotes non-national currencies. XBT and BTC refer to +the same asset - Bitcoin. +::: + +### Expiry codes + +Futures contracts use standard futures month codes: + +- `F` = January +- `G` = February +- `H` = March +- `J` = April +- `K` = May +- `M` = June +- `N` = July +- `Q` = August +- `U` = September +- `V` = October +- `X` = November +- `Z` = December + +Followed by the year (e.g., `24` for 2024, `25` for 2025). + +### NautilusTrader instrument IDs + +Within NautilusTrader, BitMEX instruments are identified using the native BitMEX symbol +directly, combined with the venue identifier: + +```python +from nautilus_trader.model.identifiers import InstrumentId + +# Spot pairs (note: no slash in the symbol) +spot_id = InstrumentId.from_str("XBTUSDT.BITMEX") # XBT/USDT spot +eth_spot_id = InstrumentId.from_str("ETHUSDT.BITMEX") # ETH/USDT spot + +# Perpetual contracts +perp_id = InstrumentId.from_str("XBTUSD.BITMEX") # Bitcoin perpetual (inverse) +linear_perp_id = InstrumentId.from_str("ETHUSDT.BITMEX") # Ethereum perpetual (linear) + +# Futures contract (June 2024) +futures_id = InstrumentId.from_str("XBTM24.BITMEX") # Bitcoin futures expiring June 2024 + +# Prediction market contracts +prediction_id = InstrumentId.from_str("P_XBTETFV23.BITMEX") # Bitcoin ETF SEC approval prediction expiring October 2023 +``` + +:::note +BitMEX spot symbols in NautilusTrader don't include the slash (/) that appears in the +BitMEX UI. Use `XBTUSDT` instead of `XBT/USDT`. +::: + +### Quantity scaling + +BitMEX reports spot and derivative quantities in *contract* units. The actual asset size per +contract is exchange-specific and published on the instrument definition: + +- `lotSize` – minimum number of contracts you can trade. +- `underlyingToPositionMultiplier` – number of contracts per unit of the underlying asset. + +For example, the SOL/USDT spot instrument (`SOLUSDT`) exposes `lotSize = 1000` and +`underlyingToPositionMultiplier = 10000`, meaning one contract represents `1 / 10000 = 0.0001` +SOL, and the minimum order (`lotSize * contract_size`) is `0.1` SOL. The adapter now derives the +contract size directly from these fields and scales both inbound market data and outbound orders +accordingly, so quantities in Nautilus are always expressed in base units (SOL, ETH, etc.). + +See the BitMEX API documentation for details on these fields: . + +## Orders capability + +The BitMEX integration supports the following order types and execution features. + +### Order types + +| Order Type | Supported | Notes | +|------------------------|-----------|-----------------------------------------------| +| `MARKET` | ✓ | Executed immediately at current market price. Quote quantity not supported. | +| `LIMIT` | ✓ | Executed only at specified price or better. | +| `STOP_MARKET` | ✓ | Supported (set `trigger_price`). | +| `STOP_LIMIT` | ✓ | Supported (set `price` and `trigger_price`). | +| `MARKET_IF_TOUCHED` | ✓ | Supported (set `trigger_price`). | +| `LIMIT_IF_TOUCHED` | ✓ | Supported (set `price` and `trigger_price`). | +| `TRAILING_STOP_MARKET` | ✓ | Supported (set `trailing_offset`). Price offset type only. | + +### Execution instructions + +| Instruction | Supported | Notes | +|---------------|-----------|-----------------------------------------------------------------------------------| +| `post_only` | ✓ | Supported via `ParticipateDoNotInitiate` execution instruction on `LIMIT` orders. | +| `reduce_only` | ✓ | Supported via `ReduceOnly` execution instruction. | + +:::note +Post-only orders that would cross the spread are canceled by BitMEX rather than rejected. The +integration surfaces these as rejections with `due_post_only=True` so strategies can handle them +consistently. +::: + +### Trigger types + +BitMEX supports multiple reference prices to evaluate stop/conditional order triggers for: + +- `STOP_MARKET` +- `STOP_LIMIT` +- `MARKET_IF_TOUCHED` +- `LIMIT_IF_TOUCHED` + +Choose the trigger type that matches your strategy and/or risk preferences. + +| Reference price | Nautilus `TriggerType` | BitMEX value | Notes | +|-----------------|------------------------|---------------|---------------------------------------------------------------------------------| +| Last trade | `LAST_PRICE` | `LastPrice` | BitMEX default; triggers on the last traded price. | +| Mark price | `MARK_PRICE` | `MarkPrice` | Recommended for many stop-loss use cases to reduce stop-outs from price spikes. | +| Index price | `INDEX_PRICE` | `IndexPrice` | Tracks the external index; useful for some contracts. | + +- If no `trigger_type` is provided, BitMEX uses its venue default (`LastPrice`). +- These trigger references are exchange-evaluated; the order remains resting at the venue until triggered. + +**Example**: + +```python +from nautilus_trader.model.enums import TriggerType + +order = self.order_factory.stop_market( + instrument_id=instrument_id, + order_side=order_side, + quantity=qty, + trigger_price=trigger, + trigger_type=TriggerType.MARK_PRICE, # Use BitMEX Mark Price as reference +) +``` + +`ExecTester` example configuration also demonstrates setting `stop_trigger_type=TriggerType.MARK_PRICE` +in `examples/live/bitmex/bitmex_exec_tester.py`. + +### Trailing stops + +BitMEX supports trailing stop orders that automatically adjust the stop price as the market moves +favorably. The adapter maps `TRAILING_STOP_MARKET` orders to BitMEX's pegged orders with +`TrailingStopPeg` price type. + +**Limitations:** + +- Only `PRICE` trailing offset type is supported (absolute price offset, not basis points or ticks). +- The offset sign is handled automatically: sell stops use negative offset, buy stops use positive. +- Trigger type can be combined with trailing stops for additional control. + +**Example**: + +```python +from nautilus_trader.model.enums import TrailingOffsetType + +order = self.order_factory.trailing_stop_market( + instrument_id=instrument_id, + order_side=OrderSide.SELL, + quantity=qty, + trailing_offset=Decimal("100"), # $100 trailing offset + trailing_offset_type=TrailingOffsetType.PRICE, + trigger_type=TriggerType.LAST_PRICE, # Optional +) +``` + +:::note +BitMEX updates trailing stop prices periodically as the market moves. +The stop price freezes when the market moves toward the trigger level. +See the [BitMEX API documentation](https://www.bitmex.com/app/perpetualContractsGuide) for current update cadence details. +::: + +### Pegged orders + +BitMEX supports pegged orders (BBO) that automatically track a reference price. The adapter +supports pegged orders via the `params` dict on `submit_order`, which overrides the order type +to `Pegged` on the exchange side. + +| Peg price type | Description | +|----------------|------------------------------------------------------------------| +| `PrimaryPeg` | Pegs to the best bid (buy) or best ask (sell). | +| `MarketPeg` | Pegs to the opposite side (best ask for buy, best bid for sell). | +| `MidPricePeg` | Pegs to the mid-price between bid and ask. | +| `LastPeg` | Pegs to the last traded price. | + +**Requirements**: + +- The underlying order must be a `LIMIT` order. Other order types are rejected. +- `peg_price_type` is required; `peg_offset_value` is optional (defaults to 0). +- `peg_offset_value` can be negative (e.g., sell-side offsets) or fractional. + +**Example**: + +```python +# Pegged to best bid with zero offset (BBO) +order = self.order_factory.limit( + instrument_id=instrument_id, + order_side=OrderSide.BUY, + quantity=qty, + price=price, # Required for LIMIT order, but overridden by peg +) +self.submit_order(order, params={"peg_price_type": "PrimaryPeg", "peg_offset_value": "0"}) + +# Pegged to mid-price with a -0.5 offset +self.submit_order(order, params={"peg_price_type": "MidPricePeg", "peg_offset_value": "-0.5"}) +``` + +:::note +The `price` field is still required when constructing the `LimitOrder`, but BitMEX ignores it +for pegged orders and instead continuously tracks the reference price plus offset. +::: + +### Time in force + +| Time in force | Supported | Notes | +|----------------|-----------|-----------------------------------------------------| +| `GTC` | ✓ | Good Till Canceled (default). | +| `GTD` | - | *Not supported by BitMEX*. | +| `FOK` | ✓ | Fill or Kill - fills entire order or cancels. | +| `IOC` | ✓ | Immediate or Cancel - partial fill allowed. | +| `DAY` | ✓ | Expires at 00:00 UTC (BitMEX trading day boundary). | + +:::note +`DAY` orders expire at 12:00am UTC, which marks the BitMEX trading day boundary (end of trading hours for that day). +See the [BitMEX Exchange Rules](https://www.bitmex.com/exchange-rules) and [API documentation](https://www.bitmex.com/api/explorer/) for complete details. +::: + +### Advanced order features + +| Feature | Supported | Notes | +|--------------------|-----------|--------------------------------------------------------------------------| +| Order Modification | ✓ | Modify price, quantity, and trigger price. | +| Bracket Orders | ✓ | Use `contingency_type` and `linked_order_ids`. | +| Iceberg Orders | ✓ | Use `display_qty`. | +| Trailing Stops | ✓ | Use `trailing_offset`. Price offset type only. | +| Pegged Orders | ✓ | Use `params` with `peg_price_type`. See [Pegged orders](#pegged-orders). | + +### Batch operations + +| Operation | Supported | Notes | +|--------------------|-----------|---------------------------------------------| +| Batch Submit | - | *Not supported by BitMEX*. | +| Batch Modify | - | *Not supported by BitMEX*. | +| Batch Cancel | ✓ | Cancel multiple orders in a single request. | + +### Position management + +| Feature | Supported | Notes | +|---------------------|-----------|----------------------------------------------------| +| Query positions | ✓ | REST and real-time position updates via WebSocket. | +| Cross margin | ✓ | Default margin mode. | +| Isolated margin | ✓ | | + +### Order querying + +| Feature | Supported | Notes | +|----------------------|-----------|----------------------------------------------| +| Query open orders | ✓ | List all active orders. | +| Query order history | ✓ | Historical order data. | +| Order status updates | ✓ | Real-time order state changes via WebSocket. | +| Trade history | ✓ | Execution and fill reports. | + +## Market data + +- Order book deltas: `L2_MBP` only; `depth` 0 (full book) or 25. +- Order book depth10 snapshots: fixed 10 levels via `orderBook10` channel. +- Quotes, trades, and instrument updates are supported via WebSocket. +- Funding rates, mark prices, and index prices are supported where applicable. +- Historical requests via REST: + - Trade ticks with optional `start`, `end`, and `limit` filters (up to 1,000 results per call). + - Time bars (`1m`, `5m`, `1h`, `1d`) for externally aggregated LAST prices, including optional partial bins. + +:::note +BitMEX caps each REST response at 1,000 rows and requires manual pagination via `start`/`startTime`. The current adapter returns only the +first page; wider pagination support is scheduled for a future update. +::: + +## Connection management + +### HTTP Keep-Alive + +The BitMEX adapter uses HTTP keep-alive for optimal performance: + +- **Connection pooling**: Connections are automatically pooled and reused. +- **Keep-alive timeout**: 90 seconds (matches BitMEX server-side timeout). +- **Automatic reconnection**: Failed connections are automatically re-established. +- **SSL session caching**: Reduces handshake overhead for subsequent requests. + +This configuration ensures low-latency communication with BitMEX servers by maintaining +persistent connections and avoiding the overhead of establishing new connections for each request. + +### Request authentication and expiration + +BitMEX uses an `api-expires` header for request authentication to prevent replay attacks: + +- Signed requests include an `api-expires` Unix timestamp set `recv_window_ms / 1000` seconds ahead (10 seconds by default). +- BitMEX rejects any request once that timestamp has passed, so keep latency within your configured window. + +## Funding rates + +The adapter receives funding rate data from the +[Funding](https://www.bitmex.com/app/wsAPI#Funding) +WebSocket stream. BitMEX returns a `fundingInterval` datetime field in each message, +and the adapter reads the hours and minutes to compute the `interval` field on +`FundingRateUpdate`. + +## Rate limiting + +BitMEX implements a dual-layer rate limiting system: + +### REST limits + +- **Burst limit**: 10 requests per second for authenticated users (applies to order placement, modification, and cancel endpoints). +- **Rolling minute limit**: 120 requests per minute for authenticated users (30 requests per minute for unauthenticated users). +- **Order caps**: 200 open orders and 10 stop orders per symbol; exceeding these caps triggers exchange-side rejections. + +The adapter enforces these quotas locally using the configured `max_requests_per_second` and `max_requests_per_minute` values. + +### WebSocket limits + +- Connection requests: follow the exchange guidance (currently 3 connections per second per IP). +- Private streams require authentication; the adapter reconnects automatically if a limit is exceeded. + +:::warning +Exceeding BitMEX rate limits returns HTTP 429 and may trigger temporary IP bans; persistent 4xx/5xx errors can extend the lockout period. +::: + +### Configurable rate limits + +The rate limits can be configured if your account has different limits than the defaults: + +| Parameter | Default (authenticated) | Default (unauthenticated) | Description | +|----------------------------|-------------------------|---------------------------|-----------------------------------------------------| +| `max_requests_per_second` | 10 | 10 | Maximum requests per second (burst limit). | +| `max_requests_per_minute` | 120 | 30 | Maximum requests per minute (rolling window). | + +:::info +For more details on rate limiting, see the [BitMEX API documentation on rate limits](https://www.bitmex.com/app/restAPI#Limits). +::: + +:::warning +**Cancel Broadcaster Rate Limit Considerations** + +The cancel broadcaster (when `canceller_pool_size > 1`) fans out each cancel request to multiple independent HTTP clients in parallel. Each client maintains its own rate limiter, which means the effective request rate is multiplied by the pool size. + +**Example**: With `canceller_pool_size=3` and `max_requests_per_second=10`, a single cancel operation consumes **3 requests** (one per client), potentially reaching **30 requests/second** if canceling rapidly. + +Since BitMEX enforces rate limits **at the account level** (not per connection), the broadcaster can push you over the exchange's default limits of 10 req/s burst and 120 req/min rolling window. + +**Mitigations**: Reduce `max_requests_per_second` and `max_requests_per_minute` proportionally (divide by `canceller_pool_size`), or adjust the pool size itself (see [Cancel broadcaster configuration](#cancel-broadcaster)). +Future versions may support shared rate limiters across the pool. +::: + +### Rate-limit headers + +BitMEX exposes the current allowance via response headers: + +- `x-ratelimit-limit`: total requests permitted in the current window. +- `x-ratelimit-remaining`: remaining requests before throttling occurs. +- `x-ratelimit-reset`: UNIX timestamp when the allowance resets. +- `retry-after`: seconds to wait after a 429 response. + +## Submit broadcaster + +The BitMEX execution client includes a submit broadcaster that provides higher assurance of market and limit orders being accepted at target prices through parallel request fanout, trading lower minimum latency for the risk of duplicate submissions. + +### Concepts + +Order submissions are time-critical operations - when a strategy decides to enter a position, any delay can result in missed opportunities or adverse pricing. The submit broadcaster addresses this by: + +- **Parallel fanout**: Submit requests are simultaneously broadcast to multiple independent HTTP client instances. +- **First-success short-circuiting**: The first successful response wins, minimizing latency to acceptance. +- **Shared client_order_id**: All transports use the same `client_order_id`. BitMEX rejects duplicate submissions with "duplicate clOrdID" (tracked as expected rejections). +- **Latency vs. duplicates tradeoff**: Accepts the risk of potential duplicate fills (if multiple transports succeed before rejection) in exchange for lower minimum latency and higher assurance of acceptance. + +This architecture reduces the minimum latency to order acceptance by parallelizing across multiple network paths. + +### Usage + +The submit broadcaster is opt-in and controlled via the `submit_tries` parameter when submitting orders. By default, orders are submitted through a single HTTP client. To enable broadcasting: + +```python +# Single submission (default behavior) +self.submit_order(order) + +# Broadcast to 2 parallel HTTP clients for redundancy +self.submit_order(order, params={"submit_tries": 2}) + +# Broadcast to 3 parallel HTTP clients (maximum recommended) +self.submit_order(order, params={"submit_tries": 3}) +``` + +**Key points**: + +- `submit_tries` must be a positive integer. +- Broadcasting only occurs when `submit_tries > 1`. Default submits go through a single HTTP client. +- If `submit_tries` exceeds `submitter_pool_size`, it will be capped at the pool size with a warning. +- All transports use the same `client_order_id`; BitMEX rejects duplicates as expected rejections. + +### Health monitoring + +Each HTTP client in the broadcaster pool maintains health metrics: + +- Successful submissions mark a client as healthy. +- Failed requests increment error counters. +- Background health checks periodically verify client connectivity. +- Degraded clients are tracked but remain in the pool to maintain fault tolerance. + +The broadcaster exposes metrics including total submits, successful submits, failed submits, and expected rejects for operational monitoring and debugging. + +#### Tracked metrics + +| Metric | Type | Description | +|--------------------------|--------|-----------------------------------------------------------------------------------------------------------------------| +| `total_submits` | `u64` | Total number of submit operations initiated. | +| `successful_submits` | `u64` | Number of submit operations that successfully received acknowledgement from BitMEX. | +| `failed_submits` | `u64` | Number of submit operations where all HTTP clients in the pool failed (no healthy clients or all requests failed). | +| `expected_rejects` | `u64` | Number of expected rejection patterns detected (e.g., duplicate clOrdID from parallel submissions). | +| `healthy_clients` | `usize`| Current number of healthy HTTP clients in the pool (clients that passed recent health checks). | +| `total_clients` | `usize`| Total number of HTTP clients configured in the pool (`submitter_pool_size`). | + +These metrics can be accessed programmatically via the `get_metrics()` method on the `SubmitBroadcaster` instance. + +### Configuration + +The submit broadcaster is configured via the execution client configuration: + +| Option | Default | Description | +|------------------------|---------|-------------------------------------------------------------------------------------------| +| `submitter_pool_size` | `None` | Size of the HTTP client pool. `None` resolves to 1 (single client, no redundancy). | +| `submitter_proxy_urls` | `None` | Optional list of proxy URLs for submit broadcaster path diversity. *Not yet wired through Python integration.* | + +**Example configuration**: + +```python +from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig + +exec_config = BitmexExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + submitter_pool_size=3, # Recommended pool size for redundancy +) +``` + +:::tip +For HFT strategies without higher rate limits, consider the advantages of using the submit broadcaster against potentially hitting rate limits, as each client has an independent rate limit budget. +The default `submitter_pool_size=None` disables the broadcaster. The recommended setting of `submitter_pool_size=3` broadcasts each submit request to 3 parallel HTTP clients for fault tolerance, which consumes 3× the rate limit quota per submit operation but provides higher assurance against network or exchange issues. +::: + +The broadcaster is automatically started when the execution client connects and stopped when it disconnects. Submit operations are routed through the broadcaster only when `submit_tries > 1`; default submits use a single HTTP client directly. + +## Cancel broadcaster + +The BitMEX execution client includes a cancel broadcaster that provides fault-tolerant order cancellation through parallel request fanout. + +### Concepts + +Order cancellations are time-critical operations - when a strategy decides to cancel an order, any delay or failure can result in unintended fills, slippage, or unwanted position exposure. The cancel broadcaster addresses this by: + +- **Parallel fanout**: Cancel requests are simultaneously broadcast to multiple independent HTTP client instances. +- **First-success short-circuiting**: The first successful response wins, and remaining in-flight requests are immediately aborted. +- **Fault tolerance**: If one HTTP client experiences network issues, DNS failures, or connection timeouts, other clients in the pool continue processing. +- **Idempotent success handling**: Responses indicating the order was already canceled (such as "orderID not found" or similar idempotent states) are treated as success rather than failure, preventing unnecessary error propagation. + +This architecture ensures that a single network path failure or slow connection doesn't block cancel operations, improving the reliability of risk management and position control in live trading. + +### Health monitoring + +Each HTTP client in the broadcaster pool maintains health metrics: + +- Successful cancellations mark a client as healthy. +- Failed requests increment error counters. +- Background health checks periodically verify client connectivity. +- Degraded clients are tracked but remain in the pool to maintain fault tolerance. + +The broadcaster exposes metrics including total cancels, successful cancels, failed cancels, expected rejects (already canceled orders), and idempotent successes for operational monitoring and debugging. + +#### Tracked metrics + +| Metric | Type | Description | +|--------------------------|--------|-----------------------------------------------------------------------------------------------------------------------| +| `total_cancels` | `u64` | Total number of cancel operations initiated (includes single, batch, and cancel-all requests). | +| `successful_cancels` | `u64` | Number of cancel operations that successfully received acknowledgement from BitMEX. | +| `failed_cancels` | `u64` | Number of cancel operations where all HTTP clients in the pool failed (no healthy clients or all requests failed). | +| `expected_rejects` | `u64` | Number of expected rejection patterns detected (e.g., post-only order rejections). | +| `idempotent_successes` | `u64` | Number of idempotent success responses (order already cancelled, order not found, unable to cancel due to state). | +| `healthy_clients` | `usize`| Current number of healthy HTTP clients in the pool (clients that passed recent health checks). | +| `total_clients` | `usize`| Total number of HTTP clients configured in the pool (`canceller_pool_size`). | + +These metrics can be accessed programmatically via the `get_metrics()` method on the `CancelBroadcaster` instance. + +### Configuration + +The cancel broadcaster is configured via the execution client configuration: + +| Option | Default | Description | +|------------------------|---------|-------------------------------------------------------------------------------------------| +| `canceller_pool_size` | `None` | Size of the HTTP client pool. `None` resolves to 1 (single client, no redundancy). | +| `canceller_proxy_urls` | `None` | Optional list of proxy URLs for cancel broadcaster path diversity. *Not yet wired through Python integration.* | + +**Example configuration**: + +```python +from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig + +exec_config = BitmexExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + canceller_pool_size=3, # Recommended pool size for redundancy +) +``` + +:::tip +For HFT strategies without higher rate limits, consider the advantages of using the cancel broadcaster against potentially hitting rate limits, as each client has an independent rate limit budget. +The default `canceller_pool_size=None` disables the broadcaster. The recommended setting of `canceller_pool_size=3` broadcasts each cancel request to 3 parallel HTTP clients for fault tolerance, which consumes 3× the rate limit quota per cancel operation but provides higher assurance against network or exchange issues. +::: + +The broadcaster is automatically started when the execution client connects and stopped when it disconnects. All cancel operations (`cancel_order`, `cancel_all_orders`, `batch_cancel_orders`) are automatically routed through the broadcaster without requiring any changes to strategy code. + +## Dead man's switch + +The adapter supports BitMEX's [dead man's switch](https://www.bitmex.com/app/restAPI#OrdercancelAllAfter) +(`cancelAllAfter`), which provides automatic order cancellation as a safety net against +connectivity failures. + +### How it works + +When enabled, a server-side timer is set on BitMEX. If the timer expires without being +refreshed, BitMEX cancels **all** open orders on the account. The adapter keeps the timer +alive by sending periodic heartbeat requests. If the adapter loses connectivity (network +failure, process crash, etc.), the heartbeats stop and BitMEX cancels the orders after the +configured timeout. + +The flow: + +1. On **connect**, the adapter calls `POST /api/v1/order/cancelAllAfter` with the configured + timeout (in milliseconds) to arm the server-side timer. +2. A background task sends the same request at a **refresh interval** of `timeout / 4` + (minimum 1 second) to keep resetting the timer before it expires. +3. On **disconnect**, the adapter waits for the background heartbeat task to fully shut + down, then calls `cancelAllAfter` with `timeout=0` to **disarm** the server-side timer. + +For example, with a 60-second timeout the adapter sends a heartbeat every 15 seconds. +If four consecutive heartbeats fail (60 seconds of lost connectivity), BitMEX cancels +all open orders. + +### Disconnect ordering + +Disarming the dead man's switch during disconnect requires careful ordering. The disarm +request (`timeout=0`) should be the last `cancelAllAfter` call to reach BitMEX. If an +in-flight heartbeat were processed after the disarm, it would re-arm the server-side timer +and orders could be unexpectedly cancelled after the timeout expires, even though the +adapter disconnected gracefully. + +The adapter mitigates this in both implementations: + +- **Rust**: The heartbeat task is immediately stopped (abort + await) so disconnect does + not stall waiting for a sleep or HTTP timeout to elapse. The disarm request is then sent + after the task has exited. +- **Python**: The heartbeat task is cancelled and awaited, ensuring the coroutine fully + unwinds before the disarm request is sent. + +In a force-stop scenario (e.g., process shutdown via `stop()`), the heartbeat task is +aborted without disarming. This is intentional, as the server-side timer provides the +desired safety behavior when the process exits unexpectedly. + +:::note +Each heartbeat consumes one REST rate limit token. A 60-second timeout uses approximately +4 requests per minute from the 120/min budget. +::: + +### Configuration + +Enable the dead man's switch by setting `deadmans_switch_timeout_secs` on the execution +client config: + +```python +from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig + +exec_config = BitmexExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + deadmans_switch_timeout_secs=60, # Cancel all orders after 60s of lost connectivity +) +``` + +When enabled, the adapter logs: + +``` +Starting dead man's switch: timeout=60s, refresh_interval=15s +``` + +on connect, and: + +``` +Disarming dead man's switch +``` + +on disconnect. + +:::tip +A timeout of **60 seconds** is the recommended starting point. Shorter timeouts provide +faster protection but are more sensitive to transient network blips. Longer timeouts are +more tolerant of brief outages but leave orders exposed longer during a real failure. +::: + +:::warning +The dead man's switch applies to **all** open orders on the account, not just orders placed +by the adapter. If other systems place orders on the same account, enabling the dead man's +switch will affect those orders too. +::: + +## Configuration + +### API credentials + +BitMEX API credentials can be provided either directly in the configuration or via environment variables: + +- `BITMEX_API_KEY`: Your BitMEX API key for production. +- `BITMEX_API_SECRET`: Your BitMEX API secret for production. +- `BITMEX_TESTNET_API_KEY`: Your BitMEX API key for testnet (when `testnet=True`). +- `BITMEX_TESTNET_API_SECRET`: Your BitMEX API secret for testnet (when `testnet=True`). + +To generate API keys: + +1. Log in to your BitMEX account. +2. Navigate to Account & Security → API Keys. +3. Create a new API key with appropriate permissions. +4. For testnet, use [testnet.bitmex.com](https://testnet.bitmex.com). + +:::note +**Testnet API endpoints**: + +- REST API: `https://testnet.bitmex.com/api/v1` +- WebSocket: `wss://ws.testnet.bitmex.com/realtime` + +The adapter automatically routes requests to the correct endpoints when `testnet=True` is configured. +::: + +### Data client configuration options + +The BitMEX data client provides the following configuration options: + +| Option | Default | Description | +|-----------------------------------|----------|-------------| +| `api_key` | `None` | Optional API key; if `None`, loaded from `BITMEX_API_KEY` or `BITMEX_TESTNET_API_KEY` (when `testnet=True`). | +| `api_secret` | `None` | Optional API secret; if `None`, loaded from `BITMEX_API_SECRET` or `BITMEX_TESTNET_API_SECRET` (when `testnet=True`). | +| `base_url_http` | `None` | Override for the REST base URL (defaults to production). | +| `base_url_ws` | `None` | Override for the WebSocket base URL (defaults to production). | +| `testnet` | `False` | Route requests to the BitMEX testnet when `True`. | +| `http_timeout_secs` | `60` | Request timeout applied to HTTP calls. | +| `max_retries` | `None` | Maximum retry attempts for HTTP calls (disabled when `None`). | +| `retry_delay_initial_ms` | `1,000` | Initial backoff delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `5,000` | Maximum backoff delay (milliseconds) between retries. | +| `recv_window_ms` | `10,000` | Expiration window (milliseconds) for signed requests. See [Request authentication](#request-authentication-and-expiration). | +| `update_instruments_interval_mins`| `60` | Interval (minutes) between instrument catalogue refreshes. | +| `max_requests_per_second` | `10` | Burst rate limit enforced by the adapter for REST calls. | +| `max_requests_per_minute` | `120` | Rolling minute rate limit enforced by the adapter for REST calls. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. *Not yet implemented; reserved for future use.* | + +### Execution client configuration options + +The BitMEX execution client provides the following configuration options: + +| Option | Default | Description | +|--------------------------|----------|-------------| +| `api_key` | `None` | Optional API key; if `None`, loaded from `BITMEX_API_KEY` or `BITMEX_TESTNET_API_KEY` (when `testnet=True`). | +| `api_secret` | `None` | Optional API secret; if `None`, loaded from `BITMEX_API_SECRET` or `BITMEX_TESTNET_API_SECRET` (when `testnet=True`). | +| `base_url_http` | `None` | Override for the REST base URL (defaults to production). | +| `base_url_ws` | `None` | Override for the WebSocket base URL (defaults to production). | +| `testnet` | `False` | Route orders to the BitMEX testnet when `True`. | +| `http_timeout_secs` | `60` | Request timeout applied to HTTP calls. | +| `max_retries` | `None` | Maximum retry attempts for HTTP calls (disabled when `None`). | +| `retry_delay_initial_ms` | `1,000` | Initial backoff delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `5,000` | Maximum backoff delay (milliseconds) between retries. | +| `recv_window_ms` | `10,000` | Expiration window (milliseconds) for signed requests. See [Request authentication](#request-authentication-and-expiration). | +| `max_requests_per_second`| `10` | Burst rate limit enforced by the adapter for REST calls. | +| `max_requests_per_minute`| `120` | Rolling minute rate limit enforced by the adapter for REST calls. | +| `deadmans_switch_timeout_secs` | `None` | Timeout in seconds for the dead man's switch. `None` disables. See [Dead man's switch](#dead-mans-switch). | +| `canceller_pool_size` | `None` | Number of HTTP clients in the cancel broadcaster pool. `None` resolves to 1. See [Cancel broadcaster](#cancel-broadcaster). | +| `submitter_pool_size` | `None` | Number of HTTP clients in the submit broadcaster pool. `None` resolves to 1. See [Submit broadcaster](#submit-broadcaster). | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. *Not yet implemented; reserved for future use.* | +| `submitter_proxy_urls` | `None` | Optional list of proxy URLs for submit broadcaster path diversity. *Not yet wired through Python integration.* | +| `canceller_proxy_urls` | `None` | Optional list of proxy URLs for cancel broadcaster path diversity. *Not yet wired through Python integration.* | + +### Configuration examples + +A typical BitMEX configuration for live trading includes both testnet and mainnet options: + +```python +from nautilus_trader.adapters.bitmex.config import BitmexDataClientConfig +from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig + +# Using environment variables (recommended) +testnet_data_config = BitmexDataClientConfig( + testnet=True, # API credentials loaded from BITMEX_TESTNET_API_KEY and BITMEX_TESTNET_API_SECRET +) + +# Using explicit credentials +mainnet_data_config = BitmexDataClientConfig( + api_key="YOUR_API_KEY", # Or use os.getenv("BITMEX_API_KEY") + api_secret="YOUR_API_SECRET", # Or use os.getenv("BITMEX_API_SECRET") + testnet=False, +) + +mainnet_exec_config = BitmexExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + testnet=False, +) +``` + +## Trading considerations + +### Contingent orders + +The BitMEX execution adapter now maps Nautilus contingent order lists to the exchange's +native `clOrdLinkID`/`contingencyType` mechanics. When the engine submits +`ContingencyType::Oco` or `ContingencyType::Oto` orders the adapter will: + +- Create/maintain the linked order group on BitMEX so child stops and targets inherit the + parent order status. +- Propagate order list updates and cancellations so that contingent peers stay aligned with + the current position state. +- Surface execution reports with the appropriate contingency metadata, enabling strategy-level + tracking without additional manual wiring. + +This means common bracket flows (entry + stop + take-profit) and multi-leg stop structures can +now be managed directly by BitMEX instead of being emulated client-side. When defining +strategies, continue to use Nautilus `OrderList`/`ContingencyType` abstractions. The adapter +handles the required BitMEX wiring automatically. + +### Contract specifications + +- **Inverse contracts**: Settled in cryptocurrency (e.g., XBTUSD settled in XBT). +- **Linear contracts**: Settled in stablecoin (e.g., ETHUSDT settled in USDT). +- **Contract size**: Varies by instrument, check specifications carefully. +- **Tick size**: Minimum price increment varies by contract. + +### Margin requirements + +- Initial margin requirements vary by contract and market conditions. +- Maintenance margin is typically lower than initial margin. +- Liquidation occurs when maintenance margin requirement is not satisfied. +- BitMEX supports both isolated margin and cross margin modes. +- Risk limits can be adjusted based on position size per the [Exchange Rules](https://www.bitmex.com/exchange-rules). + +### Fees + +- **Maker fees**: Typically negative (rebate) for providing liquidity. +- **Taker fees**: Positive fee for taking liquidity. +- **Funding rates**: Apply to perpetual contracts every 8 hours. +- **Prediction market fees**: Maker 0.00%, Taker 0.25% (no leverage allowed). + +## Contributing + +:::info +For additional features or to contribute to the BitMEX adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/blockchain.md b/nautilus-docs/docs/integrations/blockchain.md new file mode 100644 index 0000000..c116184 --- /dev/null +++ b/nautilus-docs/docs/integrations/blockchain.md @@ -0,0 +1,92 @@ +# Blockchain + +## Core Primitives + +Nautilus Trader's blockchain integration is built on foundational primitives defined in the DeFi domain model (`nautilus_model::defi`). These building blocks provide type-safe abstractions for working with EVM-based blockchains. + +### Chain + +The `Chain` struct represents a blockchain network with its connection endpoints and metadata. Each chain instance contains: + +**Fields:** + +- `name` (`Blockchain`): The blockchain network type (enum of 80+ supported chains) +- `chain_id` (`u32`): Unique EVM chain identifier (e.g., 1 for Ethereum, 42161 for Arbitrum) +- `hypersync_url` (`String`): Endpoint for high-performance Hypersync data streaming +- `rpc_url` (`Option`): Optional HTTP/WSS RPC endpoint for direct node communication +- `native_currency_decimals` (`u8`): Decimal precision for the chain's native gas token (typically 18) + +**Chain Retrieval:** + +Chains can be retrieved by numeric ID or string name (case-insensitive): + +- **By Chain ID:** Lookup using EVM chain identifier with `from_chain_id` +- **By Name:** Lookup using blockchain name (case-insensitive: "ethereum", "Ethereum", "ETHEREUM" all work) with `from_chain_name` +- **Static Instances:** Pre-configured chains available as constants + +Each chain has a native currency used for gas fees. The `native_currency()` method returns a properly configured Currency instance: + +| Chain Family | Code | Name | Decimals | +|-------------------------------------------------|------|--------------|----------| +| Ethereum & L2s (Arbitrum, Base, Optimism, etc.) | ETH | Ethereum | 18 | +| Polygon | POL | Polygon | 18 | +| Avalanche | AVAX | Avalanche | 18 | +| BSC | BNB | Binance Coin | 18 | + +## Contracts + +High-performance interface for querying EVM smart contracts with type-safe Rust abstractions. Supports token metadata, DEX pools, and DeFi protocols through efficient batch operations. + +### Base (Multicall3) + +Batches multiple contract calls into a single RPC request using Multicall3 (`0xcA11bde05977b3631167028862bE2a173976CA11`). + +- Always uses `allow_failure: true` for partial success and detailed errors +- Executes atomically in the same block +- Errors: `RpcError` (network issues), `AbiDecodingError` (decode failures) + +### ERC20 + +Inherits from `BaseContract` to use Multicall3 for efficient batch operations. Fetches token metadata, handling non-standard implementations. + +**Methods:** + +- `fetch_token_info`: Single token metadata (uses multicall internally for name, symbol, decimals) +- `batch_fetch_token_info`: Multiple tokens in one multicall (3 calls per token) +- `enforce_token_fields`: Validate non-empty name/symbol + +**Error Types:** + +1. **`CallFailed`** - Contract missing or function not implemented → Skip token +2. **`DecodingError`** - Raw bytes instead of ABI encoding (e.g., `0x5269636f...`) → Skip token +3. **`EmptyTokenField`** - Function returns empty string → Skip if enforced + +**Best Practices:** + +- Skip pools with any token errors +- `raw_data` field preserves original response for debugging +- Non-standard tokens often have other issues (transfer fees, rebasing) + +## Configuration + +| Option | Default | Description | +|---------------------------------|---------|-------------| +| `chain` | Required | `nautilus_trader.model.Chain` to synchronize (e.g., `Chain.ETHEREUM`). | +| `dex_ids` | Required | Sequence of `DexType` identifiers describing which DEX integrations to enable. | +| `http_rpc_url` | Required | HTTPS RPC endpoint used for EVM calls and Multicall requests. | +| `wss_rpc_url` | `None` | Optional WSS endpoint for streaming live updates. | +| `rpc_requests_per_second` | `None` | Optional throttle for outbound RPC calls (requests per second). | +| `multicall_calls_per_rpc_request` | `200` | Maximum number of Multicall targets batched per RPC request. | +| `use_hypersync_for_live_data` | `True` | When `True`, bootstrap and stream using Hypersync for lower-latency diffs. | +| `from_block` | `None` | Optional starting block height for historical backfill. | +| `pool_filters` | `DexPoolFilters()` | Filtering rules applied when selecting DEX pools to monitor. | +| `postgres_cache_database_config`| `None` | Optional `PostgresConnectOptions` enabling on-disk caching of decoded pool state. | +| `http_proxy_url` | `None` | Reserved for future use; not yet configurable via the constructor. | +| `ws_proxy_url` | `None` | Reserved for future use; not yet configurable via the constructor. | + +## Contributing + +:::info +For additional features or to contribute to the Blockchain adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/bybit.md b/nautilus-docs/docs/integrations/bybit.md new file mode 100644 index 0000000..7fecb58 --- /dev/null +++ b/nautilus-docs/docs/integrations/bybit.md @@ -0,0 +1,673 @@ +# Bybit + +Founded in 2018, Bybit is one of the largest cryptocurrency exchanges in terms +of daily trading volume, and open interest of crypto assets and crypto +derivative products. This integration supports live market data ingest and order +execution with Bybit. + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/bybit/). + +## Overview + +This guide assumes a trader is setting up for both live market data feeds, and trade execution. +The Bybit adapter includes multiple components, which can be used together or separately depending +on the use case. + +- `BybitHttpClient`: Low-level HTTP API connectivity. +- `BybitWebSocketClient`: Low-level WebSocket API connectivity. +- `BybitInstrumentProvider`: Instrument parsing and loading functionality. +- `BybitDataClient`: A market data feed manager. +- `BybitExecutionClient`: An account management and trade execution gateway. +- `BybitLiveDataClientFactory`: Factory for Bybit data clients (used by the trading node builder). +- `BybitLiveExecClientFactory`: Factory for Bybit execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), +and won't need to necessarily work with these lower level components directly. +::: + +## Bybit documentation + +Bybit provides extensive documentation for users which can be found in the [Bybit help center](https://www.bybit.com/en/help-center). +It’s recommended you also refer to the Bybit documentation in conjunction with this NautilusTrader integration guide. + +## Products + +A product is an umbrella term for a group of related instrument types. + +:::note +Product is also referred to as `category` in the Bybit v5 API. +::: + +The following product types are supported on Bybit: + +| Product Type | Supported | Notes | +|-----------------------------|-----------|------------------------------------------| +| Spot cryptocurrencies | ✓ | Native spot markets with margin support. | +| Linear perpetual contracts | ✓ | USDT/USDC margined perpetual swaps. | +| Linear futures contracts | ✓ | Delivery-settled linear futures. | +| Inverse perpetual contracts | ✓ | Coin-margined perpetual swaps. | +| Inverse futures contracts | ✓ | Coin-margined delivery futures. | +| Option contracts | ✓ | USDC-settled options. | + +## Symbology + +To distinguish between different product types on Bybit, Nautilus uses specific product category suffixes for symbols: + +- `-SPOT`: Spot cryptocurrencies +- `-LINEAR`: Perpetual and futures contracts +- `-INVERSE`: Inverse perpetual and inverse futures contracts +- `-OPTION`: Option contracts + +These suffixes must be appended to the Bybit raw symbol string to identify the specific product type +for the instrument ID. For example: + +- The Ether/Tether spot currency pair is identified with `-SPOT`, such as `ETHUSDT-SPOT`. +- The BTCUSDT perpetual futures contract is identified with `-LINEAR`, such as `BTCUSDT-LINEAR`. +- The BTCUSD inverse perpetual futures contract is identified with `-INVERSE`, such as `BTCUSD-INVERSE`. + +## Environments + +Bybit provides three trading environments. Configure the appropriate environment using the `demo` and `testnet` flags in your client configuration. + +| Environment | Config | Description | +|--------------|-------------------------------|------------------------------------------------------------------| +| **Mainnet** | `demo=False, testnet=False` | Production trading with real funds. | +| **Demo** | `demo=True` | Practice trading with simulated funds on mainnet infrastructure. | +| **Testnet** | `testnet=True` | Separate test network for development and integration testing. | + +### Mainnet (Production) + +The default environment for live trading with real funds. + +```python +config = BybitExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + # demo=False (default) + # testnet=False (default) +) +``` + +Environment variables: `BYBIT_API_KEY`, `BYBIT_API_SECRET` + +### Demo trading + +Demo trading uses Bybit's mainnet infrastructure with simulated funds. Create demo API keys from the [Bybit demo trading page](https://www.bybit.com/en/demo-trading). + +```python +config = BybitExecClientConfig( + api_key="YOUR_DEMO_API_KEY", + api_secret="YOUR_DEMO_API_SECRET", + demo=True, +) +``` + +Environment variables: `BYBIT_DEMO_API_KEY`, `BYBIT_DEMO_API_SECRET` + +:::warning +**Demo environment limitations:** + +- The WebSocket Trade API is **not supported** for demo trading. NautilusTrader automatically uses the HTTP REST API for order operations in demo mode. +- Some advanced order features available via WebSocket (trigger orders, post-only with is_quote_quantity) are not available in demo mode. + +::: + +### Testnet + +A separate test network for development. Create testnet API keys from [testnet.bybit.com](https://testnet.bybit.com). + +```python +config = BybitExecClientConfig( + api_key="YOUR_TESTNET_API_KEY", + api_secret="YOUR_TESTNET_API_SECRET", + testnet=True, +) +``` + +Environment variables: `BYBIT_TESTNET_API_KEY`, `BYBIT_TESTNET_API_SECRET` + +:::note +Testnet supports all trading features including the WebSocket Trade API. It uses completely separate infrastructure from mainnet, so market data and liquidity differ significantly from production. +::: + +### Environment priority + +If both `demo` and `testnet` are set to `True`, demo takes priority. + +## Orders capability + +Bybit offers a flexible combination of trigger types, enabling a broader range of Nautilus orders. +All the order types listed below can be used as *either* entries or exits, except for trailing stops +(which use a position-related API). + +### Order types + +| Order Type | Spot | Linear | Inverse | Notes | +|------------------------|------|--------|---------|---------------------------| +| `MARKET` | ✓ | ✓ | ✓ | Supports quote quantity. | +| `LIMIT` | ✓ | ✓ | ✓ | | +| `STOP_MARKET` | ✓ | ✓ | ✓ | | +| `STOP_LIMIT` | ✓ | ✓ | ✓ | | +| `MARKET_IF_TOUCHED` | ✓ | ✓ | ✓ | | +| `LIMIT_IF_TOUCHED` | ✓ | ✓ | ✓ | | +| `TRAILING_STOP_MARKET` | - | ✓ | ✓ | *Not supported for Spot*. | + +### Execution instructions + +| Instruction | Spot | Linear | Inverse | Notes | +|---------------|------|--------|---------|------------------------------------| +| `post_only` | ✓ | ✓ | ✓ | Only supported on `LIMIT` orders. | +| `reduce_only` | - | ✓ | ✓ | *Not supported for Spot*. | + +### Time in force + +| Time in force | Spot | Linear | Inverse | Notes | +|---------------|------|--------|---------|------------------------------| +| `GTC` | ✓ | ✓ | ✓ | Good Till Canceled. | +| `GTD` | - | - | - | *Not supported*. | +| `FOK` | ✓ | ✓ | ✓ | Fill or Kill. | +| `IOC` | ✓ | ✓ | ✓ | Immediate or Cancel. | + +### Advanced order features + +| Feature | Spot | Linear | Inverse | Notes | +|--------------------|------|--------|---------|----------------------------------------| +| Order Modification | ✓ | ✓ | ✓ | Price and quantity modification. | +| Bracket/OCO Orders | ✓ | ✓ | ✓ | UI only; API users implement manually. | +| Iceberg Orders | ✓ | ✓ | ✓ | Max 10 per account, 1 per symbol. | + +### Batch operations + +| Operation | Spot | Linear | Inverse | Notes | +|--------------------|------|--------|---------|-------------------------------------------| +| Batch Submit | ✓ | ✓ | ✓ | Submit multiple orders in single request. | +| Batch Modify | ✓ | ✓ | ✓ | Modify multiple orders in single request. | +| Batch Cancel | ✓ | ✓ | ✓ | Cancel multiple orders in single request. | + +### Position management + +| Feature | Spot | Linear | Inverse | Notes | +|---------------------|------|--------|---------|------------------------------------------| +| Query positions | - | ✓ | ✓ | Real-time position updates. | +| Position mode | - | ✓ | ✓ | One-Way vs Hedge mode. | +| Leverage control | - | ✓ | ✓ | Dynamic leverage adjustment per symbol. | +| Margin mode | - | ✓ | ✓ | Cross vs Isolated margin. | + +### Order querying + +| Feature | Spot | Linear | Inverse | Notes | +|---------------------|------|--------|---------|-----------------------------------------| +| Query open orders | ✓ | ✓ | ✓ | List all active orders. | +| Query order history | ✓ | ✓ | ✓ | Historical order data. | +| Order status updates| ✓ | ✓ | ✓ | Real-time order state changes. | +| Trade history | ✓ | ✓ | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Spot | Linear | Inverse | Notes | +|---------------------|------|--------|---------|-----------------------------------------| +| Order lists | - | - | - | *Not supported*. | +| OCO orders | ✓ | ✓ | ✓ | UI only; API users implement manually. | +| Bracket orders | ✓ | ✓ | ✓ | UI only; API users implement manually. | +| Conditional orders | ✓ | ✓ | ✓ | Stop and limit-if-touched orders. | + +### Order parameters + +Individual orders can be customized using the `params` dictionary when submitting orders: + +| Parameter | Type | Description | +|---------------|--------|--------------------------------------------------------------------------------| +| `is_leverage` | `bool` | For Spot products only. If `True`, enables margin trading (borrowing) for the order. Default: `False`. See [Bybit's isLeverage documentation](https://bybit-exchange.github.io/docs/v5/order/create-order#request-parameters). | + +#### Example: Spot margin trading + +```python +# Submit a Spot order with margin enabled +order = strategy.order_factory.market( + instrument_id=InstrumentId.from_str("BTCUSDT-SPOT.BYBIT"), + order_side=OrderSide.BUY, + quantity=Quantity.from_str("0.1"), + params={"is_leverage": True} # Enable margin for this order +) +strategy.submit_order(order) +``` + +:::note +Without `is_leverage=True` in the params, Spot orders will only use your available balance +and won't borrow funds, even if you have auto-borrow enabled on your Bybit account. +::: + +For a complete example of using order parameters including `is_leverage`, see the +[bybit_exec_tester.py](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/bybit/bybit_exec_tester.py) example. + +## Spot margin borrowing and repayment + +NautilusTrader provides automated spot margin borrow repayment functionality to prevent interest accrual after closing short positions on Bybit. + +### Background + +When trading Spot with margin enabled (`is_leverage=True`), Bybit automatically borrows coins when you execute short positions. +However, after you close the short position (BUY order fills), the borrowed coins are **NOT automatically repaid** - they continue accruing hourly interest charges until manually repaid. +This can result in significant interest costs if left unattended. + +### Automatic repayment (recommended) + +NautilusTrader automatically repays spot margin borrows immediately after BUY orders fill on Spot instruments. +This feature is **enabled by default** via the `auto_repay_spot_borrows` configuration flag. + +**How it works:** + +1. When a Spot BUY order fills, the execution client automatically attempts to repay any outstanding borrows for that coin. +2. The repayment uses Bybit's `no-convert-repay` endpoint, which repays the full outstanding borrow amount. +3. If the repayment fails (e.g., API error), it logs the error but does not crash the execution client. +4. Repayments are automatically skipped during Bybit's UTC blackout window (see below). + +**Example:** + +```python +from nautilus_trader.adapters.bybit import BybitExecClientConfig + +config = BybitExecClientConfig( + api_key="YOUR_API_KEY", + api_secret="YOUR_API_SECRET", + product_types=[BybitProductType.SPOT], + auto_repay_spot_borrows=True, # Default is True +) +``` + +### Manual margin operations + +Strategies can control margin borrowing and repayment directly via `query_account` with the +`BybitMarginAction` enum: + +| Action | Description | +|---------------------------------------|--------------------------------------| +| `BybitMarginAction.BORROW` | Borrow funds for margin trading. | +| `BybitMarginAction.REPAY` | Repay borrowed funds. | +| `BybitMarginAction.GET_BORROW_AMOUNT` | Query current borrowed amount. | + +#### Borrow + +```python +self.query_account( + account_id=self.account_id, + params={"action": BybitMarginAction.BORROW, "coin": "USDT", "amount": 1000}, +) +``` + +#### Repay + +```python +# Repay specific amount +self.query_account( + account_id=self.account_id, + params={"action": BybitMarginAction.REPAY, "coin": "USDT", "amount": 500}, +) + +# Repay all (omit amount) +self.query_account( + account_id=self.account_id, + params={"action": BybitMarginAction.REPAY, "coin": "USDT"}, +) +``` + +#### Query borrow amount + +```python +self.query_account( + account_id=self.account_id, + params={"action": BybitMarginAction.GET_BORROW_AMOUNT, "coin": "USDT"}, +) +``` + +:::note +The `account_id` can be obtained from `self.portfolio.account(BYBIT_VENUE).id` or stored +during strategy initialization via the config. +::: + +#### Receiving results + +Results are published as custom data on the message bus. Subscribe in your strategy to receive them: + +```python +from nautilus_trader.adapters.bybit import BybitMarginAction +from nautilus_trader.adapters.bybit import BybitMarginBorrowResult +from nautilus_trader.adapters.bybit import BybitMarginRepayResult +from nautilus_trader.adapters.bybit import BybitMarginStatusResult +from nautilus_trader.model.data import DataType + + +class MyStrategy(Strategy): + def on_start(self): + self.subscribe_data(DataType(BybitMarginBorrowResult)) + self.subscribe_data(DataType(BybitMarginRepayResult)) + self.subscribe_data(DataType(BybitMarginStatusResult)) + + def on_data(self, data): + if isinstance(data, BybitMarginBorrowResult): + if data.success: + self.log.info(f"Borrowed {data.amount} {data.coin}") + else: + self.log.error(f"Borrow failed: {data.message}") + elif isinstance(data, BybitMarginRepayResult): + if data.success: + self.log.info(f"Repaid {data.amount or 'all'} {data.coin}") + else: + self.log.error(f"Repay failed: {data.message}") + elif isinstance(data, BybitMarginStatusResult): + self.log.info(f"Borrow amount for {data.coin}: {data.borrow_amount}") +``` + +### UTC blackout window + +Bybit blocks `no-convert-repay` operations daily during **04:00-05:30 UTC** for interest calculation processing. NautilusTrader automatically detects this window and skips repayment attempts, logging a warning instead. + +During the blackout window, any BUY order fills will trigger a warning like: + +``` +Skipping borrow repayment for BTC due to Bybit blackout window (04:00-05:30 UTC daily). Will need manual repayment. +``` + +**Important:** If your BUY orders fill during the blackout window, you'll need to manually repay the borrows after 05:30 UTC to stop interest accrual, or wait for the next BUY order fill outside the blackout window. + +### Configuration options + +| Option | Type | Default | Description | +|---------------------------|--------|---------|-----------------------------------------------------------------------------| +| `auto_repay_spot_borrows` | `bool` | `True` | If `True`, automatically repay spot margin borrows after BUY orders fill. Prevents interest accrual on borrowed coins. Repayment is skipped during blackout window. | + +### Important notes + +- Auto-repayment only triggers on **Spot BUY orders**, not derivatives. +- Repayment uses the `no-convert-repay` endpoint which repays the full outstanding borrow by default. +- The feature gracefully handles API errors and logs failures without crashing. +- Bybit is planning to release an auto-repay mode at the venue level (end of month), which may make this feature redundant in the future. +- Manual borrowing is still required before opening short positions unless auto-borrow is enabled on your Bybit account. + +### Spot trading limitations + +The following limitations apply to Spot products, as positions are not tracked on the venue side: + +- `reduce_only` orders are *not supported*. +- Trailing stop orders are *not supported*. + +### Trailing stops + +Trailing stops on Bybit do not have a client order ID on the venue side (though there is a `venue_order_id`). +This is because trailing stops are associated with a netted position for an instrument. +Consider the following points when using trailing stops on Bybit: + +- `reduce_only` instruction is available +- When the position associated with a trailing stop is closed, the trailing stop is automatically "deactivated" (closed) on the venue side. +- You cannot query trailing stop orders that are not already open (the `venue_order_id` is unknown until then). +- You can manually adjust the trigger price in the GUI, which will update the Nautilus order. + +## Funding rates + +The adapter receives funding rate data from the +[Linear Ticker](https://bybit-exchange.github.io/docs/v5/websocket/public/ticker#linear-inverse-perpetual-response) +WebSocket stream. Bybit provides the `fundingIntervalHour` field in ticker updates, +which the adapter uses to populate the `interval` field on `FundingRateUpdate`. + +The adapter caches the last known `fundingIntervalHour` per symbol so that partial +ticker updates (which may omit the field) still carry the correct interval. + +For historical funding rate requests, the adapter computes the interval from consecutive +funding timestamps. + +## Rate limiting + +Every HTTP call consumes the global token bucket as well as any keyed quota(s). When usage exceeds a bucket, requests are queued automatically, so manual throttling is rarely required. + +| Key / Endpoint | Limit (requests/sec) | Notes | +|---------------------------|----------------------|------------------------------------------------------| +| `bybit:global` | 120 | Exchange-wide 600 req / 5 s ceiling. | +| `/v5/market/kline` | 20 | Historical sweeps throttled slightly below global. | +| `/v5/market/trades` | 24 | Matches the global quota. | +| `/v5/order/create` | 10 | Standard order placement. | +| `/v5/order/cancel` | 10 | Single-order cancellation. | +| `/v5/order/create-batch` | 5 | Batch placement endpoints. | +| `/v5/order/cancel-batch` | 5 | Batch cancellation endpoints. | +| `/v5/order/cancel-all` | 2 | Full book cancel to mirror Bybit guidance. | + +:::warning +Bybit responds with error code `10016` when the rate limit is exceeded and may temporarily block the IP if requests continue without back-off. +::: + +:::info +For more details on rate limiting, see the official documentation: . +::: + +### Data clients + +If no product types are specified then all product types will be loaded and available. + +### Execution clients + +The adapter automatically determines the account type based on configured product types: + +- **Spot only**: Uses `CASH` account type with borrowing support enabled +- **Derivatives or mixed products**: Uses `MARGIN` account type (UTA - Unified Trading Account) + +This allows you to trade Spot alongside derivatives in a single Unified Trading Account, which is the standard account type for most Bybit users. + +:::info +**Unified Trading Accounts (UTA) and Spot margin trading** + +Most Bybit users now have Unified Trading Accounts (UTA) as Bybit steers new users to this account type. +Classic accounts are considered legacy. + +For Spot margin trading on UTA accounts: + +- Borrowing is **NOT automatically enabled** - it requires explicit API configuration +- To use Spot margin via API, you must submit orders with `is_leverage=True` in the parameters (see [Bybit docs](https://bybit-exchange.github.io/docs/v5/order/create-order#request-parameters)) +- If auto-borrow/auto-repay is enabled on your Bybit account, the venue will automatically borrow/repay funds for those margin orders +- Without auto-borrow enabled, you'll need to manually manage borrowing through Bybit's interface + +**Important**: The Nautilus Bybit adapter defaults to `is_leverage=False` for Spot orders, +meaning they won't use margin unless you explicitly enable it. +::: + +## Fee currency logic + +Understanding how Bybit determines the currency for trading fees is important for accurate accounting and position tracking. The fee currency rules vary between Spot and derivatives products. + +### Spot trading fees + +For Spot trading, the fee currency depends on the order side and whether the fee is a rebate (negative fee for maker orders): + +#### Normal fees (positive) + +- **BUY orders**: Fee is charged in the **base currency** (e.g., BTC for BTCUSDT) +- **SELL orders**: Fee is charged in the **quote currency** (e.g., USDT for BTCUSDT) + +#### Maker rebates (negative fees) + +When maker fees are negative (rebates), the currency logic is **inverted**: + +- **BUY orders with maker rebate**: Rebate is paid in the **quote currency** (e.g., USDT for BTCUSDT) +- **SELL orders with maker rebate**: Rebate is paid in the **base currency** (e.g., BTC for BTCUSDT) + +:::note +**Taker orders never have inverted logic**, even if the maker fee rate is negative. Taker fees always follow the normal fee currency rules. +::: + +#### Example: BTCUSDT Spot + +- **Buy 1 BTC as taker (0.1% fee)**: Pay 0.001 BTC in fees +- **Sell 1 BTC as taker (0.1% fee)**: Pay equivalent USDT in fees +- **Buy 1 BTC as maker (-0.01% rebate)**: Receive USDT rebate (inverted) +- **Sell 1 BTC as maker (-0.01% rebate)**: Receive BTC rebate (inverted) + +### Derivatives trading fees + +For all derivatives products (LINEAR, INVERSE, OPTION), fees are always charged in the **settlement currency**: + +| Product Type | Settlement Currency | Fee Currency | +|--------------|---------------------------------------|--------------| +| LINEAR | USDT (typically) | USDT | +| INVERSE | Base coin (e.g., BTC for BTCUSD) | Base coin | +| OPTION | USDC (legacy) or USDT (post Feb 2025) | USDC/USDT | + +### Fee calculation + +When the WebSocket execution message doesn't provide the exact fee amount (`execFee`), the adapter calculates fees as follows: + +#### Spot products + +- **BUY orders**: `fee = base_quantity × fee_rate` +- **SELL orders**: `fee = notional_value × fee_rate` (where `notional_value = quantity × price`) + +#### Derivatives + +- All derivatives: `fee = notional_value × fee_rate` + +### Official documentation + +For complete details on Bybit's fee structure and currency rules, refer to: + +- [Bybit WebSocket Private Execution](https://bybit-exchange.github.io/docs/v5/websocket/private/execution) +- [Bybit Spot Fee Currency Instruction](https://bybit-exchange.github.io/docs/v5/enum#spot-fee-currency-instruction) + +## Configuration + +The product types for each client must be specified in the configurations. + +### Data client configuration options + +| Option | Default | Description | +|----------------------------------|---------|-------------| +| `api_key` | `None` | API key; loaded from `BYBIT_API_KEY`/`BYBIT_TESTNET_API_KEY` when omitted. | +| `api_secret` | `None` | API secret; loaded from `BYBIT_API_SECRET`/`BYBIT_TESTNET_API_SECRET` when omitted. | +| `product_types` | `None` | Sequence of `BybitProductType` values to enable; loads all products when `None`. | +| `base_url_http` | `None` | Override for the REST base URL. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Optional WebSocket proxy URL (not yet implemented). | +| `demo` | `False` | Connect to the Bybit demo environment when `True`. | +| `testnet` | `False` | Connect to the Bybit testnet when `True`. | +| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalogue refreshes. | +| `recv_window_ms` | `5,000`| Receive window (milliseconds) for signed REST requests. | +| `bars_timestamp_on_close` | `True` | Timestamp bars on the close (`True`) or open (`False`) of the interval. | +| `max_retries` | `None` | Maximum retry attempts for REST/WebSocket recovery. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | + +### Execution client configuration options + +| Option | Default | Description | +|----------------------------------|---------|-------------| +| `api_key` | `None` | API key; loaded from `BYBIT_API_KEY`/`BYBIT_TESTNET_API_KEY` when omitted. | +| `api_secret` | `None` | API secret; loaded from `BYBIT_API_SECRET`/`BYBIT_TESTNET_API_SECRET` when omitted. | +| `product_types` | `None` | Sequence of `BybitProductType` values to enable (Spot cannot be mixed with derivatives for execution). | +| `base_url_http` | `None` | Override for the REST base URL. | +| `base_url_ws_private` | `None` | Override for the private WebSocket base URL. | +| `base_url_ws_trade` | `None` | Override for the trade WebSocket base URL. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Optional WebSocket proxy URL (not yet implemented). | +| `demo` | `False` | Connect to the Bybit demo environment when `True`. | +| `testnet` | `False` | Connect to the Bybit testnet when `True`. | +| `use_gtd` | `False` | Remap GTD orders to GTC when `True` (Bybit lacks native GTD support). | +| `use_ws_execution_fast` | `False` | Subscribe to the low-latency execution stream. | +| `use_http_batch_api` | `False` | Use Bybit's HTTP batch trading API (deprecated). | +| `use_spot_position_reports` | `False` | Report Spot wallet balances as positions when `True`. | +| `auto_repay_spot_borrows` | `True` | Automatically repay Spot margin borrows after BUY orders fully fill (Spot only). | +| `repay_queue_interval_secs` | `1.0` | Interval (seconds) between processing repayment queues for spot borrows. | +| `ignore_uncached_instrument_executions` | `False` | Ignore execution messages for instruments not yet cached. | +| `max_retries` | `None` | Maximum retry attempts for order submission/cancel/modify calls. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | +| `recv_window_ms` | `5,000`| Receive window (milliseconds) for signed REST requests. | +| `ws_trade_timeout_secs` | `5.0` | Timeout (seconds) waiting for trade WebSocket acknowledgements. | +| `ws_auth_timeout_secs` | `5.0` | Timeout (seconds) waiting for auth WebSocket acknowledgements. | +| `futures_leverages` | `None` | Mapping of `BybitSymbol` to leverage settings. | +| `position_mode` | `None` | Mapping of `BybitSymbol` to position mode (one-way vs hedge). | +| `margin_mode` | `None` | Margin mode setting for the account. | + +The most common use case is to configure a live `TradingNode` to include Bybit +data and execution clients. To achieve this, add a `BYBIT` section to your client +configuration(s): + +```python +from nautilus_trader.adapters.bybit import BYBIT +from nautilus_trader.adapters.bybit import BybitProductType +from nautilus_trader.live.node import TradingNode + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + BYBIT: { + "api_key": "YOUR_BYBIT_API_KEY", + "api_secret": "YOUR_BYBIT_API_SECRET", + "base_url_http": None, # Override with custom endpoint + "product_types": [BybitProductType.LINEAR], + "testnet": False, + }, + }, + exec_clients={ + BYBIT: { + "api_key": "YOUR_BYBIT_API_KEY", + "api_secret": "YOUR_BYBIT_API_SECRET", + "base_url_http": None, # Override with custom endpoint + "product_types": [BybitProductType.LINEAR], + "testnet": False, + }, + }, +) +``` + +Then, create a `TradingNode` and add the client factories: + +```python +from nautilus_trader.adapters.bybit import BYBIT +from nautilus_trader.adapters.bybit import BybitLiveDataClientFactory +from nautilus_trader.adapters.bybit import BybitLiveExecClientFactory +from nautilus_trader.live.node import TradingNode + +# Instantiate the live trading node with a configuration +node = TradingNode(config=config) + +# Register the client factories with the node +node.add_data_client_factory(BYBIT, BybitLiveDataClientFactory) +node.add_exec_client_factory(BYBIT, BybitLiveExecClientFactory) + +# Finally build the node +node.build() +``` + +### API credentials + +There are two options for supplying your credentials to the Bybit clients. +Either pass the corresponding `api_key` and `api_secret` values to the configuration objects, or +set the following environment variables: + +For Bybit live clients, you can set: + +- `BYBIT_API_KEY` +- `BYBIT_API_SECRET` + +For Bybit demo clients, you can set: + +- `BYBIT_DEMO_API_KEY` +- `BYBIT_DEMO_API_SECRET` + +For Bybit testnet clients, you can set: + +- `BYBIT_TESTNET_API_KEY` +- `BYBIT_TESTNET_API_SECRET` + +:::tip +We recommend using environment variables to manage your credentials. +::: + +When starting the trading node, you'll receive immediate confirmation of whether your +credentials are valid and have trading permissions. + +## Contributing + +:::info +For additional features or to contribute to the Bybit adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/databento.md b/nautilus-docs/docs/integrations/databento.md new file mode 100644 index 0000000..09172d2 --- /dev/null +++ b/nautilus-docs/docs/integrations/databento.md @@ -0,0 +1,881 @@ +# Databento + +NautilusTrader includes an adapter for the [Databento](https://databento.com/) API and +[Databento Binary Encoding (DBN)](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) format data. +Databento is a market data provider only. The adapter does not include an execution client, +but you can pair it with a sandbox for simulated execution. +You can also match Databento data with Interactive Brokers execution, +or calculate traditional asset class signals for crypto trading. + +The adapter supports: + +- Loading historical data from DBN files and decoding to Nautilus objects for backtesting or catalog storage. +- Requesting historical data decoded to Nautilus objects for live trading and backtesting. +- Subscribing to real-time data feeds decoded to Nautilus objects for live trading and sandbox environments. + +:::tip +[Databento](https://databento.com/signup) offers 125 USD in free data credits (historical only) for new sign-ups. + +With careful requests, this covers testing and evaluation. +Check the [/metadata.get_cost](https://databento.com/docs/api-reference-historical/metadata/metadata-get-cost) +endpoint before requesting data. +::: + +## Overview + +The adapter uses the [databento-rs](https://crates.io/crates/databento) crate, +Databento's official Rust client library. + +:::info +No separate `databento` installation is needed. The adapter compiles as a static +library and links automatically during the build. +::: + +The following adapter classes are available: + +- `DatabentoDataLoader`: Loads DBN data from files. +- `DatabentoInstrumentProvider`: Fetches latest or historical instrument definitions via the Databento HTTP API. +- `DatabentoHistoricalClient`: Fetches historical market data via the Databento HTTP API. +- `DatabentoLiveClient`: Subscribes to real-time data feeds via Databento's raw TCP API. +- `DatabentoDataClient`: `LiveMarketDataClient` implementation for live trading nodes. + +:::info +Most users configure a live trading node (covered below) and do not work with +these components directly. +::: + +## Examples + +Live example scripts are available [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/databento/). + +## Databento documentation + +See the [Databento new users guide](https://databento.com/docs/quickstart/new-user-guides). +Refer to it alongside this integration guide. + +## Databento Binary Encoding (DBN) + +Databento Binary Encoding (DBN) is a fast message encoding and storage format for +normalized market data. The [DBN specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) +includes a self-describing metadata header and a fixed set of struct definitions +that standardize how market data is normalized. + +The adapter decodes DBN data to Nautilus objects. The same Rust decoder handles: + +- Loading and decoding DBN files from disk. +- Decoding historical and live data in real time. + +## Supported schemas + +The following Databento schemas are supported by NautilusTrader: + +| Databento schema | Nautilus data type | Description | +|:------------------------------------------------------------------------------|:----------------------------------|:--------------------------------| +| [MBO](https://databento.com/docs/schemas-and-data-formats/mbo) | `OrderBookDelta` | Market by order (L3). | +| [MBP_1](https://databento.com/docs/schemas-and-data-formats/mbp-1) | `(QuoteTick, TradeTick \| None)` | Market by price (L1). | +| [MBP_10](https://databento.com/docs/schemas-and-data-formats/mbp-10) | `OrderBookDepth10` | Market depth (L2). | +| [BBO_1S](https://databento.com/docs/schemas-and-data-formats/bbo-1s) | `QuoteTick` | 1-second best bid/offer. | +| [BBO_1M](https://databento.com/docs/schemas-and-data-formats/bbo-1m) | `QuoteTick` | 1-minute best bid/offer. | +| [CMBP_1](https://databento.com/docs/schemas-and-data-formats/cmbp-1) | `(QuoteTick, TradeTick \| None)` | Consolidated MBP across venues. | +| [CBBO_1S](https://databento.com/docs/schemas-and-data-formats/cbbo-1s) | `QuoteTick` | Consolidated 1-second BBO. | +| [CBBO_1M](https://databento.com/docs/schemas-and-data-formats/cbbo-1m) | `QuoteTick` | Consolidated 1-minute BBO. | +| [TCBBO](https://databento.com/docs/schemas-and-data-formats/tcbbo) | `(QuoteTick, TradeTick)` | Trade-sampled consolidated BBO. | +| [TBBO](https://databento.com/docs/schemas-and-data-formats/tbbo) | `(QuoteTick, TradeTick)` | Trade-sampled best bid/offer. | +| [TRADES](https://databento.com/docs/schemas-and-data-formats/trades) | `TradeTick` | Trade ticks. | +| [OHLCV_1S](https://databento.com/docs/schemas-and-data-formats/ohlcv-1s) | `Bar` | 1-second bars. | +| [OHLCV_1M](https://databento.com/docs/schemas-and-data-formats/ohlcv-1m) | `Bar` | 1-minute bars. | +| [OHLCV_1H](https://databento.com/docs/schemas-and-data-formats/ohlcv-1h) | `Bar` | 1-hour bars. | +| [OHLCV_1D](https://databento.com/docs/schemas-and-data-formats/ohlcv-1d) | `Bar` | Daily bars. | +| [OHLCV_EOD](https://databento.com/docs/schemas-and-data-formats/ohlcv-eod) | `Bar` | End-of-day bars. | +| [DEFINITION](https://databento.com/docs/schemas-and-data-formats/definition) | `Instrument` (various types) | Instrument definitions. | +| [IMBALANCE](https://databento.com/docs/schemas-and-data-formats/imbalance) | `DatabentoImbalance` | Auction imbalance data. | +| [STATISTICS](https://databento.com/docs/schemas-and-data-formats/statistics) | `DatabentoStatistics` | Market statistics. | +| [STATUS](https://databento.com/docs/schemas-and-data-formats/status) | `InstrumentStatus` | Market status updates. | + +### Schema considerations + +- **TBBO and TCBBO**: Trade-sampled feeds that pair every trade with the BBO immediately *before* the trade's effect (TBBO per-venue, TCBBO consolidated). Use when you need trades aligned with contemporaneous quotes without managing two streams. +- **MBP-1 and CMBP-1 (L1)**: Event-level updates; emit trades only on trade events. Choose for a complete top-of-book event tape. For quote+trade alignment, prefer TBBO/TCBBO; otherwise use TRADES. +- **MBP-10 (L2)**: Top 10 levels with trades. Lighter than MBO for depth-aware strategies. Includes orders per level. +- **MBO (L3)**: Per-order events for queue position modeling and exact book reconstruction. Highest volume/cost; start at node initialization for proper replay context. +- **BBO_1S/BBO_1M and CBBO_1S/CBBO_1M**: Sampled top-of-book quotes at fixed intervals (1s/1m), no trades. Good for monitoring, spreads, and low-cost signals. Not suited for microstructure work. +- **TRADES**: Trades only. Pair with MBP-1 (`include_trades=True`) or use TBBO/TCBBO for quote context with trades. +- **OHLCV_ (incl. OHLCV_EOD)**: Aggregated bars from trades. Use for higher-timeframe analytics. Set `bars_timestamp_on_close=True` for close timestamps. +- **Imbalance / Statistics / Status**: Venue operational data; subscribe via `subscribe_data` with a `DataType` carrying `instrument_id` metadata. + +:::tip +Consolidated schemas (CMBP_1, CBBO_1S, CBBO_1M, TCBBO) aggregate data across +multiple venues. Useful for cross-venue analysis. +::: + +:::info +See also the Databento [Schemas and data formats](https://databento.com/docs/schemas-and-data-formats) guide. +::: + +## Schema selection for live subscriptions + +Nautilus subscription methods map to Databento schemas as follows: + +| Nautilus Subscription Method | Default Schema | Available Databento Schemas | Nautilus Data Type | +|:--------------------------------|:---------------|:-----------------------------------------------------------------------------|:-------------------| +| `subscribe_quote_ticks()` | `mbp-1` | `mbp-1`, `bbo-1s`, `bbo-1m`, `cmbp-1`, `cbbo-1s`, `cbbo-1m`, `tbbo`, `tcbbo` | `QuoteTick` | +| `subscribe_trade_ticks()` | `trades` | `trades`, `tbbo`, `tcbbo`, `mbp-1`, `cmbp-1` | `TradeTick` | +| `subscribe_order_book_depth()` | `mbp-10` | `mbp-10` | `OrderBookDepth10` | +| `subscribe_order_book_deltas()` | `mbo` | `mbo` | `OrderBookDeltas` | +| `subscribe_bars()` | varies | `ohlcv-1s`, `ohlcv-1m`, `ohlcv-1h`, `ohlcv-1d` | `Bar` | + +:::note +The examples below assume a `Strategy` or `Actor` context where `self` has +subscription methods. Import the required types: + +```python +from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID +from nautilus_trader.model import BarType +from nautilus_trader.model.enums import BookType +from nautilus_trader.model.identifiers import InstrumentId +``` + +::: + +### Quote subscriptions (MBP / L1) + +```python +# Default MBP-1 quotes (may include trades) +self.subscribe_quote_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) + +# Explicit MBP-1 schema +self.subscribe_quote_ticks( + instrument_id=instrument_id, + params={"schema": "mbp-1"}, + client_id=DATABENTO_CLIENT_ID, +) + +# 1-second BBO snapshots (quotes only, no trades) +self.subscribe_quote_ticks( + instrument_id=instrument_id, + params={"schema": "bbo-1s"}, + client_id=DATABENTO_CLIENT_ID, +) + +# Consolidated quotes across venues +self.subscribe_quote_ticks( + instrument_id=instrument_id, + params={"schema": "cbbo-1s"}, # or "cmbp-1" for consolidated MBP + client_id=DATABENTO_CLIENT_ID, +) + +# Trade-sampled BBO (includes both quotes AND trades) +self.subscribe_quote_ticks( + instrument_id=instrument_id, + params={"schema": "tbbo"}, # Will receive both QuoteTick and TradeTick onto the message bus + client_id=DATABENTO_CLIENT_ID, +) +``` + +### Trade subscriptions + +```python +# Trade ticks only +self.subscribe_trade_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) + +# Trades from MBP-1 feed (only when trade events occur) +self.subscribe_trade_ticks( + instrument_id=instrument_id, + params={"schema": "mbp-1"}, + client_id=DATABENTO_CLIENT_ID, +) + +# Trade-sampled data (includes quotes at trade time) +self.subscribe_trade_ticks( + instrument_id=instrument_id, + params={"schema": "tbbo"}, # Also provides quotes at trade events + client_id=DATABENTO_CLIENT_ID, +) +``` + +### Order book depth subscriptions (MBP / L2) + +```python +# Subscribe to top 10 levels of market depth +self.subscribe_order_book_depth( + instrument_id=instrument_id, + depth=10 # MBP-10 schema is automatically selected +) + +# The depth parameter must be 10 for Databento +# This will receive OrderBookDepth10 updates +``` + +### Order book deltas subscriptions (MBO / L3) + +```python +# Subscribe to full order book updates (market by order) +self.subscribe_order_book_deltas( + instrument_id=instrument_id, + book_type=BookType.L3_MBO # Uses MBO schema +) + +# Note: MBO subscriptions must be made at node startup for Databento +# to ensure proper replay from session start +``` + +### Bar subscriptions + +```python +# Subscribe to 1-minute bars (automatically uses ohlcv-1m schema) +self.subscribe_bars( + bar_type=BarType.from_str(f"{instrument_id}-1-MINUTE-LAST-EXTERNAL") +) + +# Subscribe to 1-second bars (automatically uses ohlcv-1s schema) +self.subscribe_bars( + bar_type=BarType.from_str(f"{instrument_id}-1-SECOND-LAST-EXTERNAL") +) + +# Subscribe to hourly bars (automatically uses ohlcv-1h schema) +self.subscribe_bars( + bar_type=BarType.from_str(f"{instrument_id}-1-HOUR-LAST-EXTERNAL") +) + +# Subscribe to daily bars (automatically uses ohlcv-1d schema) +self.subscribe_bars( + bar_type=BarType.from_str(f"{instrument_id}-1-DAY-LAST-EXTERNAL") +) + +# Subscribe to daily bars with end-of-day schema (only valid for DAY aggregation) +self.subscribe_bars( + bar_type=BarType.from_str(f"{instrument_id}-1-DAY-LAST-EXTERNAL"), + params={"schema": "ohlcv-eod"}, # Override to use end-of-day bars +) +``` + +### Custom data type subscriptions + +Imbalance, statistics, and status data require the generic `subscribe_data` method: + +```python +from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID +from nautilus_trader.adapters.databento import DatabentoImbalance +from nautilus_trader.adapters.databento import DatabentoStatistics +from nautilus_trader.model import DataType + +# Subscribe to imbalance data +self.subscribe_data( + data_type=DataType(DatabentoImbalance, metadata={"instrument_id": instrument_id}), + client_id=DATABENTO_CLIENT_ID, +) + +# Subscribe to statistics data +self.subscribe_data( + data_type=DataType(DatabentoStatistics, metadata={"instrument_id": instrument_id}), + client_id=DATABENTO_CLIENT_ID, +) + +# Subscribe to instrument status updates +from nautilus_trader.model.data import InstrumentStatus +self.subscribe_data( + data_type=DataType(InstrumentStatus, metadata={"instrument_id": instrument_id}), + client_id=DATABENTO_CLIENT_ID, +) +``` + +## Instrument IDs and symbology + +Databento market data includes an `instrument_id` field: an integer assigned by +the source venue or by Databento during normalization. This differs from the +Nautilus `InstrumentId`, a string of symbol + venue separated by a period: +`"{symbol}.{venue}"`. + +The decoder maps the Databento `raw_symbol` to the Nautilus `symbol` and uses an +[ISO 10383 MIC](https://www.iso20022.org/market-identifier-codes) (Market Identifier Code) from the +definition message for the Nautilus `venue`. + +Databento identifies datasets with a *dataset ID*, separate from venue identifiers. +See [Databento dataset naming conventions](https://databento.com/docs/api-reference-historical/basics/datasets) +for details. + +For CME Globex MDP 3.0 (`GLBX.MDP3`), these exchanges group under the `GLBX` venue. +The instrument's `exchange` field determines the mapping: + +- `CBCM`: XCME-XCBT inter-exchange spread +- `NYUM`: XNYM-DUMX inter-exchange spread +- `XCBT`: Chicago Board of Trade (CBOT) +- `XCEC`: Commodities Exchange Center (COMEX) +- `XCME`: Chicago Mercantile Exchange (CME) +- `XFXS`: CME FX Link spread +- `XNYM`: New York Mercantile Exchange (NYMEX) + +:::info +Other venue MICs are in the `venue` field of responses from +the [metadata.list_publishers](https://databento.com/docs/api-reference-historical/metadata/metadata-list-publishers) endpoint. +::: + +## Timestamps + +Databento data includes these timestamp fields: + +- `ts_event`: Matching-engine-received timestamp in nanoseconds since the UNIX epoch. +- `ts_in_delta`: Matching-engine-sending timestamp in nanoseconds before `ts_recv`. +- `ts_recv`: Capture-server-received timestamp in nanoseconds since the UNIX epoch. +- `ts_out`: Databento sending timestamp. + +Nautilus data requires at least two timestamps (per the `Data` contract): + +- `ts_event`: UNIX timestamp (nanoseconds) when the data event occurred. +- `ts_init`: UNIX timestamp (nanoseconds) when the data instance was created. + +The decoder maps Databento `ts_recv` to Nautilus `ts_event`. This timestamp is +more reliable and monotonically increases per instrument. The exceptions are +`DatabentoImbalance` and `DatabentoStatistics`, which carry all timestamp fields +since they are adapter-specific types. + +:::info +See the following Databento docs for further information: + +- [Databento standards and conventions - timestamps](https://databento.com/docs/standards-and-conventions/common-fields-enums-types#timestamps) +- [Databento timestamping guide](https://databento.com/docs/architecture/timestamping-guide) + +::: + +## Data types + +This section covers Databento schema to Nautilus data type mapping. + +:::info +See Databento [schemas and data formats](https://databento.com/docs/schemas-and-data-formats). +::: + +### Instrument definitions + +Databento uses a single schema for all instrument classes. The decoder maps each +to the appropriate Nautilus `Instrument` type. + +| Databento instrument class | Code | Nautilus instrument type | +|----------------------------|------|--------------------------| +| Stock | `K` | `Equity` | +| Future | `F` | `FuturesContract` | +| Call | `C` | `OptionContract` | +| Put | `P` | `OptionContract` | +| Future spread | `S` | `FuturesSpread` | +| Option spread | `T` | `OptionSpread` | +| Mixed spread | `M` | `OptionSpread` | +| FX spot | `X` | `CurrencyPair` | +| Bond | `B` | Not yet available | + +### Price precision + +Databento raw prices are fixed-point integers scaled by 1e-9. The adapter derives +price precision from the instrument's tick size in the definition message. + +For live feeds, the feed handler maintains a per-instrument precision map populated +from `InstrumentDefMsg` records as they arrive. Market data handlers look up +precision from this map. Without a prior definition, precision falls back to 2 +(USD default). + +**Instrument definitions must arrive before market data** for correct precision on +instruments with non-standard tick sizes (e.g., treasury futures with fractional +ticks like 1/256). Subscribe to `DEFINITION` schema for your instruments before +or alongside market data subscriptions. + +For historical and file-based loading, pass an explicit `price_precision` parameter +to override the default. + +:::tip +The Python adapter automatically subscribes to instrument definitions before +market data, so the precision map populates without extra configuration. For +direct Rust client usage, subscribe to `DEFINITION` schema before market data. +::: + +### MBO (market by order) + +MBO is the highest granularity data from Databento, representing full order book +depth. Some messages include trade data. The decoder produces an `OrderBookDelta` +and optionally a `TradeTick`. + +The live client buffers MBO messages until it sees an `F_LAST` flag, then passes +an `OrderBookDeltas` container to the handler. + +The client also buffers order book snapshots into `OrderBookDeltas` during the +replay startup sequence. + +### MBP-1 (market by price, top-of-book) + +MBP-1 represents top-of-book quotes and trades. Some messages carry trade data. +The decoder produces a `QuoteTick` and also a `TradeTick` when the message is +a trade. + +### TBBO and TCBBO (top-of-book with trades) + +TBBO and TCBBO provide both quote and trade data in each message. Both schemas +emit `QuoteTick` and `TradeTick` per message, more efficient than separate quote +and trade subscriptions. TCBBO provides consolidated data across venues. + +### OHLCV (bar aggregates) + +Databento timestamps bar messages at the **open** of the interval. The decoder +normalizes `ts_event` to the bar **close** (original `ts_event` + interval). + +### Imbalance & Statistics + +The `imbalance` and `statistics` schemas have no built-in Nautilus equivalents. +The adapter defines `DatabentoImbalance` and `DatabentoStatistics` in Rust. + +PyO3 bindings expose these types in Python. Their attributes are PyO3 objects +and may not be compatible with methods expecting Cython types. See the API +reference for PyO3 to Cython conversion methods. + +Convert a PyO3 `Price` to a Cython `Price`: + +```python +price = Price.from_raw(pyo3_price.raw, pyo3_price.precision) +``` + +Requesting and subscribing to these types requires the generic `subscribe_data` +method. Subscribe to `imbalance` for `AAPL.XNAS`: + +```python +from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID +from nautilus_trader.adapters.databento import DatabentoImbalance +from nautilus_trader.model import DataType + +instrument_id = InstrumentId.from_str("AAPL.XNAS") +self.subscribe_data( + data_type=DataType(DatabentoImbalance, metadata={"instrument_id": instrument_id}), + client_id=DATABENTO_CLIENT_ID, +) +``` + +Request the previous day's `statistics` for the `ES.FUT` parent symbol +(all active E-mini S&P 500 futures): + +```python +from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID +from nautilus_trader.adapters.databento import DatabentoStatistics +from nautilus_trader.model import DataType + +instrument_id = InstrumentId.from_str("ES.FUT.GLBX") +metadata = { + "instrument_id": instrument_id, + "start": "2024-03-06", +} +self.request_data( + data_type=DataType(DatabentoStatistics, metadata=metadata), + client_id=DATABENTO_CLIENT_ID, +) +``` + +### Catalog persistence + +Both types support Arrow serialization for catalog storage. The Arrow serializers +register automatically when you import the adapter package. + +#### Writing to the catalog + +```python +from nautilus_trader.adapters.databento import DatabentoDataLoader +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.persistence.catalog import ParquetDataCatalog + +catalog = ParquetDataCatalog.from_env() +loader = DatabentoDataLoader() + +imbalances = loader.from_dbn_file( + path="aapl-imbalance.dbn.zst", + instrument_id=InstrumentId.from_str("AAPL.XNAS"), + as_legacy_cython=False, # Required for Databento-specific types +) + +catalog.write_data(imbalances) +``` + +#### Reading from the catalog + +```python +from nautilus_trader.adapters.databento import DatabentoImbalance + +results = catalog.query(DatabentoImbalance, identifiers=["AAPL.XNAS"]) + +for imbalance in results: + print(imbalance.ref_price) # DatabentoImbalance fields +``` + +:::warning +Catalog persistence supports writing and querying these types, but streaming +them through `BacktestNode` or `BacktestEngine` is not yet supported. For +backtesting with imbalance or statistics data, query the catalog directly and +process the results in your strategy or analysis code. +::: + +#### Encoding and decoding in Rust + +The `nautilus_databento::arrow` module provides Arrow record batch encoding and +decoding. Requires the `arrow` feature flag. + +```rust +use nautilus_databento::arrow::imbalance::{ + decode_imbalance_batch, + imbalance_to_arrow_record_batch, +}; + +let batch = imbalance_to_arrow_record_batch(imbalances)?; + +let metadata = batch.schema().metadata().clone(); +let decoded = decode_imbalance_batch(&metadata, batch)?; +``` + +The `statistics` module follows the same pattern with +`decode_statistics_batch` and `statistics_to_arrow_record_batch`. + +## Performance considerations + +Two options for backtesting with DBN data: + +- Store data as DBN (`.dbn.zst`) files and decode to Nautilus objects every run. +- Convert DBN files to Nautilus objects once and write to the data catalog (Nautilus Parquet format). + +The DBN decoder is optimized Rust, but writing to the catalog once gives the +best backtest performance. + +[DataFusion](https://arrow.apache.org/datafusion/) streams Nautilus Parquet data +from disk at high throughput, at least an order of magnitude faster than +decoding DBN per run. + +:::note +Performance benchmarks are under development. +::: + +## Loading DBN data + +The `DatabentoDataLoader` class loads DBN files and converts records to Nautilus +objects. Two primary uses: + +- Pass data to `BacktestEngine.add_data` for backtesting. +- Write data to `ParquetDataCatalog` for streaming with a `BacktestNode`. + +### DBN data to a BacktestEngine + +Load DBN data and pass to a `BacktestEngine`. The engine requires an instrument. +This example uses `TestInstrumentProvider` (an instrument parsed from a DBN +file also works). The data covers one month of TSLA trades on Nasdaq: + +```python +# Add instrument +TSLA_NASDAQ = TestInstrumentProvider.equity(symbol="TSLA") +engine.add_instrument(TSLA_NASDAQ) + +# Decode data to legacy Cython objects +loader = DatabentoDataLoader() +trades = loader.from_dbn_file( + path=TEST_DATA_DIR / "databento" / "temp" / "tsla-xnas-20240107-20240206.trades.dbn.zst", + instrument_id=TSLA_NASDAQ.id, +) + +# Add data +engine.add_data(trades) +``` + +### DBN data to a ParquetDataCatalog + +Load DBN data and write to a `ParquetDataCatalog`. Set `as_legacy_cython=False` +to decode as PyO3 objects. Cython objects also work with `write_data` but require +conversion under the hood, so PyO3 objects are faster. + +### Loading instruments + +**Important**: Load instrument definitions from DEFINITION schema files before +loading market data into a catalog. The catalog requires instruments before it +can store market data. Market data files do not contain instrument definitions. + +```python +# Initialize the catalog interface +# (will use the `NAUTILUS_PATH` env var as the path) +catalog = ParquetDataCatalog.from_env() + +loader = DatabentoDataLoader() + +# Step 1: Load instrument definitions FIRST +# You must obtain DEFINITION schema files from Databento for your instruments +instruments = loader.from_dbn_file( + path=TEST_DATA_DIR / "databento" / "temp" / "tsla-xnas-definition.dbn.zst", + as_legacy_cython=False, # Use PyO3 for optimal performance +) + +# Write instruments to catalog +catalog.write_data(instruments) + +# Step 2: Now load and write market data +instrument_id = InstrumentId.from_str("TSLA.XNAS") + +# Decode trades to pyo3 objects +trades = loader.from_dbn_file( + path=TEST_DATA_DIR / "databento" / "temp" / "tsla-xnas-20240107-20240206.trades.dbn.zst", + instrument_id=instrument_id, + as_legacy_cython=False, # This is an optimization for writing to the catalog +) + +# Write market data +catalog.write_data(trades) +``` + +#### Loading multiple data types for backtesting + +Always load instruments before market data: + +```python +from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.persistence.catalog import ParquetDataCatalog + +catalog = ParquetDataCatalog.from_env() +loader = DatabentoDataLoader() + +# Step 1: Load instrument definitions from DEFINITION files +instruments = loader.from_dbn_file( + path="equity-definitions.dbn.zst", + as_legacy_cython=False, +) +catalog.write_data(instruments) + +# Step 2: Load market data (MBO, trades, quotes, etc.) +instrument_id = InstrumentId.from_str("AAPL.XNAS") + +# Load MBO order book deltas +deltas = loader.from_dbn_file( + path="aapl-mbo.dbn.zst", + instrument_id=instrument_id, # Optional but improves performance + as_legacy_cython=False, +) +catalog.write_data(deltas) + +# Load trades +trades = loader.from_dbn_file( + path="aapl-trades.dbn.zst", + instrument_id=instrument_id, + as_legacy_cython=False, +) +catalog.write_data(trades) + +# Verify instruments are in the catalog +print(catalog.instruments()) # Should show your loaded instruments +``` + +:::tip +Call `catalog.instruments()` to verify. An empty list means you need to load +DEFINITION files first. +::: + +:::info +Download DEFINITION schema files through the Databento API or CLI for your +symbols and date ranges. See the +[Databento documentation](https://databento.com/docs/api-reference-historical/timeseries/timeseries-get-range) +for details. +::: + +:::info +See also the [Data concepts guide](../concepts/data.md). +::: + +### Historical loader options + +Parameters for `from_dbn_file`: + +- `instrument_id`: Speeds up decoding by skipping symbology lookup. +- `price_precision`: Overrides the default price precision. +- `include_trades`: For MBP-1/CMBP-1 schemas, `True` emits both `QuoteTick` and `TradeTick` when trade data is present. +- `as_legacy_cython`: Set to `False` for IMBALANCE/STATISTICS schemas (required) or for better catalog write performance. + +:::warning +IMBALANCE and STATISTICS schemas require `as_legacy_cython=False` (PyO3-only +types). `True` raises a `ValueError`. +::: + +### Loading consolidated data + +Consolidated schemas aggregate data across multiple venues: + +```python +# Load consolidated MBP-1 quotes +loader = DatabentoDataLoader() +cmbp_quotes = loader.from_dbn_file( + path="consolidated.cmbp-1.dbn.zst", + instrument_id=InstrumentId.from_str("AAPL.XNAS"), + include_trades=True, # Get both quotes and trades if available + as_legacy_cython=True, +) + +# Load consolidated BBO quotes +cbbo_quotes = loader.from_dbn_file( + path="consolidated.cbbo-1s.dbn.zst", + instrument_id=InstrumentId.from_str("AAPL.XNAS"), + as_legacy_cython=False, # Use PyO3 for better performance +) + +# Load TCBBO (trade-sampled consolidated BBO) - provides both quotes and trades +# Note: include_trades=True loads quotes, include_trades=False loads trades +tcbbo_quotes = loader.from_dbn_file( + path="consolidated.tcbbo.dbn.zst", + instrument_id=InstrumentId.from_str("AAPL.XNAS"), + include_trades=True, # Loads quotes + as_legacy_cython=True, +) + +tcbbo_trades = loader.from_dbn_file( + path="consolidated.tcbbo.dbn.zst", + instrument_id=InstrumentId.from_str("AAPL.XNAS"), + include_trades=False, # Loads trades + as_legacy_cython=True, +) +``` + +:::tip +Avoid subscribing to both TBBO/TCBBO and separate trade feeds for the same +instrument. These schemas already include trades. Duplicating wastes cost and +creates duplicate data. +::: + +## Real-time client architecture + +The `DatabentoDataClient` wraps the other Databento adapter classes. Each +dataset uses two `DatabentoLiveClient` instances: + +- One for MBO (order book deltas) real-time feeds +- One for all other real-time feeds + +:::warning +All MBO subscriptions for a dataset must be made at node startup to replay from +session start. Subscriptions after start are logged as errors and ignored. + +This limitation does not apply to other schemas. +::: + +A single `DatabentoHistoricalClient` serves both `DatabentoInstrumentProvider` +and `DatabentoDataClient` for historical requests. + +## Configuration + +Add a `DATABENTO` section to your `TradingNode` client configuration: + +```python +from nautilus_trader.adapters.databento import DATABENTO +from nautilus_trader.live.node import TradingNode + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + DATABENTO: { + "api_key": None, # 'DATABENTO_API_KEY' env var + "http_gateway": None, # Override for the default HTTP historical gateway + "live_gateway": None, # Override for the default raw TCP real-time gateway + "instrument_provider": InstrumentProviderConfig(load_all=True), + "instrument_ids": None, # Nautilus instrument IDs to load on start + "parent_symbols": None, # Databento parent symbols to load on start + }, + }, + ..., # Omitted +) +``` + +Create the `TradingNode` and register the factory: + +```python +from nautilus_trader.adapters.databento.factories import DatabentoLiveDataClientFactory +from nautilus_trader.live.node import TradingNode + +# Instantiate the live trading node with a configuration +node = TradingNode(config=config) + +# Register the client factory with the node +node.add_data_client_factory(DATABENTO, DatabentoLiveDataClientFactory) + +# Finally build the node +node.build() +``` + +### Configuration parameters + +| Option | Default | Description | +|---------------------------|---------|----------------------------------------------------------------------------------------------------------------------| +| `api_key` | `None` | Databento API secret. Falls back to the `DATABENTO_API_KEY` environment variable when `None`. | +| `http_gateway` | `None` | Historical HTTP gateway override for testing custom endpoints. | +| `live_gateway` | `None` | Raw TCP real-time gateway override, typically for testing only. | +| `use_exchange_as_venue` | `True` | Use the exchange MIC for Nautilus venues (e.g., `XCME`). `False` retains the default GLBX mapping. | +| `timeout_initial_load` | `15.0` | Seconds to wait for instrument definitions per dataset before proceeding. | +| `mbo_subscriptions_delay` | `3.0` | Seconds to buffer before enabling MBO/L3 streams so initial snapshots replay in order. | +| `bars_timestamp_on_close` | `True` | Timestamp bars on the close (`ts_event`/`ts_init`). `False` timestamps on the open. | +| `reconnect_timeout_mins` | `10` | Minutes to attempt reconnection before giving up. `None` retries indefinitely. See [Connection stability](#connection-stability). | +| `venue_dataset_map` | `None` | Optional Nautilus venue to Databento dataset code mapping. | +| `parent_symbols` | `None` | Optional `{dataset: {parent symbols}}` to preload definition trees (e.g., `{"GLBX.MDP3": {"ES.FUT", "ES.OPT"}}`). | +| `instrument_ids` | `None` | Nautilus `InstrumentId` values to preload definitions for at startup. | + +:::tip +Use environment variables for credentials. +::: + +### Connection stability + +The live client reconnects automatically on: + +- **Network interruptions**: Temporary connectivity issues. +- **Gateway restarts**: Databento Sunday maintenance (see [Maintenance Schedule](https://databento.com/docs/api-reference-live/basics#maintenance-schedule)). +- **Market closures**: Sessions ending during off-hours. + +#### Reconnection strategy + +Backoff strategy depends on the timeout configuration: + +**With timeout** (default 10 minutes): + +- Exponential backoff capped at **60 seconds**. +- Pattern: 1s, 2s, 4s, 8s, 16s, 32s, 60s, 60s... (with jitter). +- Reconnects quickly within the timeout window. + +**Without timeout** (`reconnect_timeout_mins=None`): + +- Exponential backoff capped at **10 minutes**. +- Pattern: 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 600s, 600s... (with jitter). +- Suited for unattended systems through overnight closures and scheduled maintenance. + +All reconnections include: + +- **Jitter**: Random delay (up to 1 second) to prevent simultaneous reconnection storms. +- **Automatic resubscription**: Restores all active subscriptions after reconnecting. +- **Cycle reset**: Each successful session (>60s) resets the timeout clock. + +#### Timeout configuration + +The `reconnect_timeout_mins` parameter controls how long the client attempts reconnection: + +**Default (10 minutes)**: Suitable for most use cases. + +- Handles transient network issues. +- Survives scheduled gateway restarts. +- Stops retrying overnight when markets close. +- Requires manual intervention for longer outages. + +:::warning +Setting `reconnect_timeout_mins=None` retries indefinitely. Use only for +unattended systems that must survive overnight market closures. This can mask +persistent configuration or authentication issues. +::: + +#### Scheduled maintenance + +Databento restarts live gateways every Sunday (all clients disconnect): + +| Dataset | Maintenance Time (UTC) | +|--------------------|------------------------| +| CME Globex | 09:30 | +| All ICE venues | 09:45 | +| All other datasets | 10:30 | + +The default 10-minute timeout covers typical restarts. For unattended systems, +use `reconnect_timeout_mins=None` or a longer value. See the +[Databento Maintenance Schedule](https://databento.com/docs/api-reference-live/basics/maintenance-schedule) +for details. + +## Contributing + +:::info +To contribute, see the +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/deribit.md b/nautilus-docs/docs/integrations/deribit.md new file mode 100644 index 0000000..b1e4109 --- /dev/null +++ b/nautilus-docs/docs/integrations/deribit.md @@ -0,0 +1,648 @@ +# Deribit + +Founded in 2016, Deribit is a cryptocurrency derivatives exchange specializing in Bitcoin and +Ethereum options and futures. It is one of the largest crypto options exchanges by volume, +and a leading platform for crypto derivatives trading. + +This integration supports live market data ingest and order execution with Deribit. + +## Overview + +This adapter is implemented in Rust, with optional Python bindings for use in Python-based workflows. +Deribit uses JSON-RPC 2.0 over both HTTP and WebSocket transports. +WebSocket is preferred for subscriptions and real-time data. + +The official Deribit API reference can be found at [docs.deribit.com](https://docs.deribit.com/). + +The Deribit adapter includes multiple components, which can be used together or separately depending +on your use case: + +- `DeribitHttpClient`: Low-level HTTP API connectivity (JSON-RPC over HTTP). +- `DeribitWebSocketClient`: Low-level WebSocket API connectivity (JSON-RPC over WebSocket). +- `DeribitInstrumentProvider`: Instrument parsing and loading functionality. +- `DeribitDataClient`: Market data feed manager. +- `DeribitExecutionClient`: Account management and trade execution gateway. +- `DeribitLiveDataClientFactory`: Factory for Deribit data clients (used by the trading node builder). +- `DeribitLiveExecClientFactory`: Factory for Deribit execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as shown below), +and won't need to work directly with these lower-level components. +::: + +### Product support + +| Product Type | Data Feed | Trading | Notes | +|-------------------|-----------|---------|----------------------------------| +| Perpetual Futures | ✓ | ✓ | BTC-PERPETUAL, ETH-PERPETUAL. | +| Dated Futures | ✓ | ✓ | Futures with fixed expiry dates. | +| Options | ✓ | ✓ | BTC and ETH options. | +| Spot | ✓ | ✓ | BTC_USDC, ETH_USDC pairs. | +| Future Combos | ✓ | ✓ | Calendar spreads for futures. | +| Option Combos | ✓ | ✓ | Option spread strategies. | + +## Symbology + +Deribit uses specific symbol conventions for different instrument types. +All instrument IDs should include the `.DERIBIT` suffix when referencing them +(e.g., `BTC-PERPETUAL.DERIBIT` for BTC perpetual). + +### Perpetual Futures + +Format: `{Currency}-PERPETUAL` + +Examples: + +- `BTC-PERPETUAL` - Bitcoin perpetual swap +- `ETH-PERPETUAL` - Ethereum perpetual swap + +To subscribe to BTC perpetual in your strategy: + +```python +InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") +``` + +### Dated Futures + +Format: `{Currency}-{DDMMMYY}` + +Examples: + +- `BTC-28MAR25` - Bitcoin futures expiring March 28, 2025 +- `ETH-27JUN25` - Ethereum futures expiring June 27, 2025 + +```python +InstrumentId.from_str("BTC-28MAR25.DERIBIT") +``` + +### Options + +Format: `{Currency}-{DDMMMYY}-{Strike}-{Type}` + +Examples: + +- `BTC-28MAR25-100000-C` - Bitcoin call option, $100,000 strike, expiring March 28, 2025 +- `BTC-28MAR25-80000-P` - Bitcoin put option, $80,000 strike, expiring March 28, 2025 +- `ETH-28MAR25-4000-C` - Ethereum call option, $4,000 strike + +Where: + +- `C` = Call option +- `P` = Put option + +```python +InstrumentId.from_str("BTC-28MAR25-100000-C.DERIBIT") +``` + +### Spot + +Format: `{BaseCurrency}_{QuoteCurrency}` + +Examples: + +- `BTC_USDC` - Bitcoin against USDC +- `ETH_USDC` - Ethereum against USDC + +```python +InstrumentId.from_str("BTC_USDC.DERIBIT") +``` + +## Order book subscriptions + +Deribit provides two types of order book feeds, each suited for different use cases. + +### Raw feeds (tick-by-tick) + +Raw channels deliver every single update as an individual message. Subscribing to a raw order book +gives you a notification for every order insertion, update, or deletion in the book. + +- Requires authenticated connection (safeguard against abuse). +- Use when you need every price level change for HFT or market making. +- Higher message volume. + +### Aggregated feeds (batched) + +Aggregated channels deliver updates in batches at a fixed interval (e.g., every 100ms). +This groups multiple order book changes into single messages. + +- Available without authentication. +- Recommended for most use cases. +- Lower message volume, easier to process. +- Default interval: 100ms. + +### Subscription parameters + +The Nautilus adapter supports both feed types via subscription parameters: + +| Parameter | Values | Notes | +|-----------|--------|-------| +| `interval` | `raw`, `100ms`, `agg2` | Default: `100ms`. `agg2` batches at ~1 second intervals. `raw` requires auth. | +| `depth` | `1`, `10`, `20` | Default: `10`. Number of price levels per side. | + +```python +from nautilus_trader.model.identifiers import InstrumentId + +instrument_id = InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") + +# Default: 100ms aggregated feed (no authentication required) +strategy.subscribe_order_book_deltas(instrument_id) + +# Raw feed (requires API credentials) +strategy.subscribe_order_book_deltas( + instrument_id, + params={"interval": "raw"}, +) +``` + +:::note +Raw order book feeds require an authenticated WebSocket connection. Ensure API credentials are +configured before subscribing to raw feeds. +::: + +:::tip +For most strategies, the default 100ms aggregated feed provides sufficient granularity with +significantly lower message overhead. Only use raw feeds when tick-by-tick precision is essential. +::: + +### Sequence gap recovery + +The adapter tracks `change_id` / `prev_change_id` sequence numbers on every book update. +When a gap is detected (missed message), the adapter automatically: + +1. Drops all incoming deltas for the affected instrument. +2. Unsubscribes from the book channel. +3. Resubscribes to obtain a fresh snapshot. +4. Resumes normal processing once the snapshot arrives. + +During resync, the strategy will not receive stale or incomplete book updates. + +## Orders capability + +Below are the order types, execution instructions, and time-in-force options supported on Deribit. + +### Order types + +| Order Type | Supported | Notes | +|---------------|-----------|------------------------------------------| +| `MARKET` | ✓ | Immediate execution at market price. | +| `LIMIT` | ✓ | Execution at specified price or better. | +| `STOP_MARKET` | ✓ | Conditional market order on trigger. | +| `STOP_LIMIT` | ✓ | Conditional limit order on trigger. | + +### Execution instructions + +| Instruction | Supported | Notes | +|----------------|-----------|------------------------------------------------| +| `post_only` | ✓ | Order will be rejected if it would take liquidity. Uses `reject_post_only=true`. | +| `reduce_only` | ✓ | Order can only reduce an existing position. | + +### Time in force + +| Time in force | Supported | Notes | +|---------------|-----------|-----------------------------------------------------| +| `GTC` | ✓ | Good Till Canceled (`good_til_cancelled`). | +| `GTD` | ✓ | Good Till Day - expires at 8:00 UTC (`good_til_day`). | +| `IOC` | ✓ | Immediate or Cancel (`immediate_or_cancel`). | +| `FOK` | ✓ | Fill or Kill (`fill_or_kill`). | + +:::note +**GTD on Deribit**: Unlike other exchanges where GTD accepts an arbitrary expiry time, +Deribit's `good_til_day` always expires at 8:00 UTC the same or next day. Custom expiry times +will be logged as warnings and the order will use the exchange's fixed expiry behavior. +::: + +### Trigger types + +Conditional orders (stop orders) support different trigger price sources: + +| Trigger Type | Supported | Notes | +|---------------|-----------|------------------------------------------| +| `last_price` | ✓ | Uses the last traded price (default). | +| `mark_price` | ✓ | Uses the mark price. | +| `index_price` | ✓ | Uses the underlying index price. | + +```python +# Example: Stop loss using mark price trigger +stop_order = order_factory.stop_market( + instrument_id=instrument_id, + order_side=OrderSide.SELL, + quantity=Quantity.from_str("0.1"), + trigger_price=Price.from_str("45000.0"), + trigger_type=TriggerType.MARK_PRICE, # Use mark price for trigger +) +strategy.submit_order(stop_order) +``` + +### Batch operations + +| Operation | Supported | Notes | +|---------------|-----------|--------------------------------------------| +| Batch Submit | - | *Not yet implemented*. | +| Batch Modify | - | *Not yet implemented*. | +| Batch Cancel | - | *Not yet implemented*. | + +### Post-only behavior + +Deribit offers two post-only modes: + +1. **Price adjustment (Deribit default)**: If a post-only order would cross the spread and execute, + Deribit automatically adjusts the price to one tick inside the spread. +2. **Reject mode**: Order is immediately rejected if it would cross the spread. + +The Nautilus adapter uses **reject mode** (`reject_post_only=true`) for deterministic behavior. +If a post-only order would take liquidity, it is rejected with error code `11054`, and an `OrderRejected` +event is emitted with the `due_post_only` flag set to `true`. + +This allows strategies to differentiate between: + +- Orders rejected due to post-only violation (attempted to take liquidity). +- Orders rejected for other reasons (insufficient margin, invalid price, etc.). + +### Order modification + +The adapter uses Deribit's native `private/edit` endpoint rather than cancel-and-replace. +This provides several advantages: + +| Benefit | Description | +|----------------------------|--------------------------------------------------------------------| +| Single request | Faster execution, lower latency than cancel + new order. | +| Queue priority preservation | Keeps position when only reducing quantity or keeping same price. | +| Fill history maintained | Partial fills remain linked to the same order ID. | + +**Queue priority rules:** + +- **Decreasing quantity only**: Keeps queue position. +- **Same price**: Keeps queue position. +- **Increasing quantity or changing price**: Loses queue position (treated as new order). + +### Position management + +| Feature | Supported | Notes | +|-------------------|-----------|-------------------------------------------| +| Query positions | ✓ | Real-time position updates. | +| Position mode | - | Deribit uses net position mode only. | +| Leverage control | - | Leverage set at account level via UI. | +| Margin mode | - | Portfolio margin via Deribit UI settings. | + +### Order querying + +| Feature | Supported | Notes | +|----------------------|-----------|------------------------------------| +| Query open orders | ✓ | List all active orders. | +| Query order history | ✓ | Historical order data. | +| Order status updates | ✓ | Real-time order state changes. | +| Trade history | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Supported | Notes | +|---------------------|-----------|------------------------------------| +| Order lists | - | *Not supported*. | +| OCO orders | - | *Not supported*. | +| Bracket orders | - | *Not supported*. | +| Conditional orders | ✓ | Stop market and stop limit orders. | + +## Funding rates + +Deribit exchanges funding continuously (every few seconds) rather than at fixed intervals +like most other exchanges. The `interval` field on `FundingRateUpdate` is `None` for +Deribit because this continuous model does not map to a discrete period. + +## Rate limiting + +Deribit uses a credit-based rate limiting system. Each API request consumes credits, which are replenished +over time. The adapter enforces these quotas to prevent rate limit violations. + +### REST limits + +| Bucket / Key | Limit | Notes | +|---------------------|------------------|---------------------------------------------| +| `deribit:global` | 20 req/sec (100 burst) | Default bucket for all REST requests. | +| `deribit:orders` | 5 req/sec (20 burst) | Matching engine operations (buy, sell, edit, cancel). | +| `deribit:account` | 5 req/sec | Account information endpoints. | + +### WebSocket limits + +| Operation | Limit | Notes | +|---------------------|------------------|---------------------------------------------| +| Subscribe/unsubscribe | 3 req/sec (10 burst) | Subscription operations. | +| Order operations | 5 req/sec (20 burst) | Buy, sell, edit, cancel via WebSocket. | + +:::note +The Nautilus adapter uses WebSocket for order submission (not REST) for lower latency. +Order operations are rate-limited by `DERIBIT_WS_ORDER_QUOTA` (5 req/sec, 20 burst). +::: + +### Credit-based system details + +Deribit uses a sophisticated credit-based rate limiting system where credits are replenished +continuously at a fixed rate. Each second, credits "drip" back into your sub-account's credit pool. + +**Non-matching engine requests:** + +| Parameter | Value | Notes | +|------------------|--------------------|---------------------------------| +| Cost per request | 500 credits | Each API call consumes credits. | +| Maximum pool | 50,000 credits | Allows 100 request burst. | +| Refill rate | 10,000 credits/sec | ~20 sustained requests/second. | + +**Matching engine requests (default tier):** + +| Parameter | Value | Notes | +|----------------|----------------|------------------------------------| +| Sustained rate | 5 requests/sec | Continuous rate limit. | +| Burst capacity | 20 requests | Maximum burst before throttling. | + +Higher matching engine limits are available for market makers and high-volume traders based on +7-day trading volume tiers. + +The Nautilus adapter implements this using token bucket rate limiters configured as: + +- `DERIBIT_HTTP_REST_QUOTA`: 20 req/sec with 100 burst (non-matching REST) +- `DERIBIT_HTTP_ORDER_QUOTA`: 5 req/sec with 20 burst (matching engine REST) +- `DERIBIT_WS_ORDER_QUOTA`: 5 req/sec with 20 burst (matching engine WebSocket) +- `DERIBIT_WS_SUBSCRIPTION_QUOTA`: 3 req/sec with 10 burst (subscribe/unsubscribe) + +For more details, see the [Rate Limits article](https://support.deribit.com/hc/en-us/articles/25944617523357-Rate-Limits). + +:::warning +Deribit returns error code `10028` (too_many_requests) when you exceed the allowed quota. +Repeated violations may result in temporary throttling. +::: + +## Connection management + +### Platform limits + +| Limit | Value | +|-----------------------------------|-------| +| Maximum connections per IP | 32 | +| Maximum sessions per API key | 16 | +| Maximum API keys per (sub)account | 8 | + +### Session-based authentication + +The adapter uses **separate WebSocket sessions** for data and execution clients, each with its own +authentication scope: + +| Client | Session Name | Purpose | +|------------------|----------------------|------------------------------------------------------| +| Data client | `nautilus-data` | Market data subscriptions (raw feeds require auth). | +| Execution client | `nautilus-execution` | Order operations (buy, sell, edit, cancel). | + +**Authentication flow:** + +1. WebSocket connects to Deribit. +2. Client authenticates using `client_signature` grant type with session scope. +3. Tokens are automatically refreshed at 80% of expiry time (continuous refresh cycle). +4. On reconnection, re-authentication is retried with exponential backoff (up to 3 attempts). + If all attempts fail, only public channel subscriptions are restored. + +This session-based approach allows: + +- Independent token management per client type. +- Isolated failure domains (data auth failure does not affect execution). +- Clear audit trail in Deribit's session logs. + +### Best practices + +The adapter follows Deribit's +[recommended connection practices](https://support.deribit.com/hc/en-us/articles/25944603459613): + +1. **Uses WebSocket subscriptions** for real-time data instead of REST polling, resulting in fewer requests, + lower latency, and reduced rate limit consumption. +2. **Authenticates all connections** when credentials are provided. Authenticated users benefit + from higher rate limits and are less likely to be IP rate-limited. +3. **Implements heartbeats** (30 second interval) to maintain connection health and detect + disconnections early. +4. **Handles reconnection** automatically with re-authentication and subscription recovery. + +:::tip +Always provide API credentials even for public data access. Authenticated connections have higher +rate limits, and Deribit contacts authenticated clients before applying restrictions during +high-load periods. +::: + +:::note +The adapter uses a 30 second heartbeat interval, which is the lower bound of Deribit's recommended +30-60 second range. More frequent heartbeats may trigger stricter rate limits. +::: + +## Authentication + +Deribit uses API key authentication with HMAC-SHA256 signatures for private endpoints. + +To create API credentials: + +1. Log into your Deribit account at [deribit.com](https://www.deribit.com) (or [test.deribit.com](https://test.deribit.com) for testnet). +2. Navigate to **Account** → **API**. +3. Click **Add new key** and configure permissions: + - Enable **read** for market data access + - Enable **trade** for order execution + - Enable **wallet** if you need account balance access +4. Note down your **Client ID** (API key) and **Client Secret** (API secret). + +:::warning +Keep your API secret secure. Never share it or commit it to version control. +::: + +### API key scopes + +Each API key on Deribit is assigned a default access scope, which defines the maximum permissions. +Configure appropriate permissions when +[creating your API key](https://support.deribit.com/hc/en-us/articles/26268257333661): + +| Scope | Required For | +|--------------------|----------------------------------------| +| `account:read` | Account information, portfolio data. | +| `trade:read` | View orders and positions. | +| `trade:read_write` | Place, modify, and cancel orders. | +| `wallet:read` | View balances and transaction history. | + +**Recommended minimum for trading:** `account:read`, `trade:read_write`, `wallet:read` + +:::tip +Follow the principle of least privilege. For data-only access (market data, no trading), +create a read-only key without `trade:read_write`. +::: + +## Testnet + +Deribit provides a testnet environment for testing strategies without real funds. +To use the testnet, set `is_testnet=True` in your client configuration: + +```python +config = TradingNodeConfig( + data_clients={ + DERIBIT: DeribitDataClientConfig( + is_testnet=True, # Enable testnet mode + # ... other config + ), + }, + exec_clients={ + DERIBIT: DeribitExecClientConfig( + is_testnet=True, # Enable testnet mode + # ... other config + ), + }, +) +``` + +When testnet mode is enabled: + +- HTTP requests use `https://test.deribit.com`. +- WebSocket connections use `wss://test.deribit.com/ws/api/v2`. +- Loads credentials from `DERIBIT_TESTNET_API_KEY` and `DERIBIT_TESTNET_API_SECRET` environment variables. + +:::note +Testnet API keys are separate from production keys. Create API keys specifically +for the testnet through the testnet interface at [test.deribit.com](https://test.deribit.com). +::: + +## Configuration + +### Data client configuration options + +| Option | Default | Description | +|------------------------------------|------------|-------------| +| `api_key` | `None` | Deribit API key; loads from environment variables when omitted. | +| `api_secret` | `None` | Deribit API secret; loads from environment variables when omitted. | +| `product_types` | `None` | Product types to load (Future, Option, Spot, etc.). If `None`, defaults to Future. | +| `base_url_http` | `None` | Override for the HTTP REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `is_testnet` | `False` | Use Deribit testnet endpoints when `True`. | +| `http_timeout_secs` | `60` | Request timeout (seconds) for REST calls. | +| `max_retries` | `3` | Maximum retry attempts for recoverable errors. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) before retrying. | +| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | +| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument refreshes. | + +### Execution client configuration options + +| Option | Default | Description | +|--------------------------|------------|-------------| +| `api_key` | `None` | Deribit API key; loads from environment variables when omitted. | +| `api_secret` | `None` | Deribit API secret; loads from environment variables when omitted. | +| `product_types` | `None` | Product types to load (Future, Option, Spot, etc.). If `None`, defaults to Future. | +| `base_url_http` | `None` | Override for the HTTP REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `is_testnet` | `False` | Use Deribit testnet endpoints when `True`. | +| `http_timeout_secs` | `60` | Request timeout (seconds) for REST calls. | +| `max_retries` | `3` | Maximum retry attempts for recoverable errors. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) before retrying. | +| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | + +### Production configuration + +Below is an example configuration for a live trading node using Deribit data and execution clients: + +```python +from nautilus_trader.adapters.deribit import DERIBIT +from nautilus_trader.adapters.deribit import DeribitDataClientConfig +from nautilus_trader.adapters.deribit import DeribitExecClientConfig +from nautilus_trader.adapters.deribit import DeribitLiveDataClientFactory +from nautilus_trader.adapters.deribit import DeribitLiveExecClientFactory +from nautilus_trader.config import InstrumentProviderConfig +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.core.nautilus_pyo3 import DeribitProductType +from nautilus_trader.live.node import TradingNode + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + DERIBIT: DeribitDataClientConfig( + api_key=None, # Uses DERIBIT_API_KEY env var + api_secret=None, # Uses DERIBIT_API_SECRET env var + product_types=(DeribitProductType.Future,), + instrument_provider=InstrumentProviderConfig(load_all=True), + is_testnet=False, + ), + }, + exec_clients={ + DERIBIT: DeribitExecClientConfig( + api_key=None, + api_secret=None, + product_types=(DeribitProductType.Future,), + instrument_provider=InstrumentProviderConfig(load_all=True), + is_testnet=False, + ), + }, +) + +node = TradingNode(config=config) +node.add_data_client_factory(DERIBIT, DeribitLiveDataClientFactory) +node.add_exec_client_factory(DERIBIT, DeribitLiveExecClientFactory) +node.build() +``` + +### API credentials + +There are multiple options for supplying your credentials to the Deribit clients. +Either pass the corresponding values to the configuration objects, or +set the following environment variables: + +For Deribit live (production) clients: + +- `DERIBIT_API_KEY` +- `DERIBIT_API_SECRET` + +For Deribit testnet clients: + +- `DERIBIT_TESTNET_API_KEY` +- `DERIBIT_TESTNET_API_SECRET` + +:::tip +We recommend using environment variables to manage your credentials. +::: + +### Product types + +The `product_types` configuration option controls which Deribit product families are loaded. +Available options via the `DeribitProductType` enum: + +- `DeribitProductType.Future` - Perpetual and dated futures. +- `DeribitProductType.Option` - Call and put options. +- `DeribitProductType.Spot` - Spot trading pairs. +- `DeribitProductType.FutureCombo` - Future spread instruments. +- `DeribitProductType.OptionCombo` - Option spread instruments. + +Example loading multiple product types: + +```python +from nautilus_trader.core.nautilus_pyo3 import DeribitProductType + +config = DeribitDataClientConfig( + product_types=( + DeribitProductType.Future, + DeribitProductType.Option, + ), + # ... other config +) +``` + +### Base URL overrides + +It's possible to override the default base URLs for both HTTP and WebSocket APIs: + +| Environment | HTTP URL | WebSocket URL | +|-------------|----------------------------|------------------------------------| +| Production | `https://www.deribit.com` | `wss://www.deribit.com/ws/api/v2` | +| Testnet | `https://test.deribit.com` | `wss://test.deribit.com/ws/api/v2` | + +## Server infrastructure + +Deribit's matching engine is located in **Equinix LD4, Slough, UK**. For latency-sensitive strategies, +consider hosting in or near London. Colocation and cross-connect options are available directly +from Deribit for institutional clients. + +For most users connecting via internet, the adapter's built-in retry logic, heartbeat monitoring, +and automatic reconnection handling provide reliable connectivity. + +For more details, see the [Server Infrastructure article](https://support.deribit.com/hc/en-us/articles/25944617582877). + +## Contributing + +:::info +For additional features or to contribute to the Deribit adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/dydx.md b/nautilus-docs/docs/integrations/dydx.md new file mode 100644 index 0000000..ea0d4e4 --- /dev/null +++ b/nautilus-docs/docs/integrations/dydx.md @@ -0,0 +1,803 @@ +# dYdX v4 + +dYdX is one of the largest decentralized cryptocurrency exchanges for crypto derivative products. +This integration supports live market data ingestion and order execution with dYdX v4, running on +its own Cosmos SDK application-specific blockchain (dYdX Chain) with CometBFT consensus. The order +book and matching engine run on-chain as part of the validator process. Orders are submitted as +Cosmos transactions via gRPC and settled each block. An Indexer service exposes REST and WebSocket +APIs for market data and account state. + +This is the Rust-backed adapter with Python bindings. + +## Installation + +:::note +No additional installation extras are required. The adapter is implemented in Rust and +compiled into the core `nautilus_trader` package automatically during the build. +::: + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/dydx/). + +## Overview + +This adapter is implemented in Rust with Python bindings via PyO3. It provides direct integration +with dYdX's Indexer API (REST/WebSocket) for market data and gRPC for Cosmos SDK transaction +submission, without requiring external client libraries. + +### Product support + +| Product Type | Data Feed | Trading | Notes | +|-------------------|-----------|---------|----------------------------------------| +| Perpetual Futures | ✓ | ✓ | All perpetuals are USDC-settled. | +| Spot | - | - | dYdX offers spot on Solana; not supported by this adapter. | +| Options | - | - | *Not available on dYdX*. | + +:::note +This adapter supports perpetual futures only. All markets are quoted in USD and settled in USDC. +::: + +## Chain architecture + +Unlike centralized exchanges (CEXs) that expose a single REST/WebSocket API, dYdX v4 runs on its +own **Cosmos SDK application-specific blockchain**. This means every trade is a Cosmos transaction +that goes through consensus, and the adapter must manage sequences, gas, and block-height-based +expiration. + +### Transport layers + +The adapter communicates through three independent transport layers: + +``` + ┌─────────────────────────────────────────────┐ + │ dYdX v4 Chain │ + │ │ + ┌──────────┐ HTTP │ ┌──────────────────────┐ │ + │ │───────────►│ │ Indexer (read-only) │ │ + │ │ WebSocket │ │ - REST API │ │ + │ Nautilus │───────────►│ │ - Streaming API │ │ + │ Adapter │ │ └──────────────────────┘ │ + │ │ gRPC │ ┌──────────────────────┐ │ + │ │───────────►│ │ Validator (write) │ │ + └──────────┘ │ │ - Cosmos Tx submit │ │ + │ │ - Sequence mgmt │ │ + │ └──────────────────────┘ │ + └─────────────────────────────────────────────┘ +``` + +| Layer | Target | Direction | Purpose | +|-----------|-----------|------------|------------------------------------------------------| +| HTTP | Indexer | Read-only | Instrument metadata, historical data, account state. | +| WebSocket | Indexer | Read-only | Real-time market data, order/fill/position updates. | +| gRPC | Validator | Write | Order placement, cancellation, and batch operations. | + +### Block-based settlement + +dYdX blocks are produced approximately every **~0.5 seconds** (actual times vary). The adapter +includes a `BlockTimeMonitor` that tracks observed block times from the WebSocket feed to +dynamically estimate `seconds_per_block`. This estimate is used to convert time-based order expiry +into block-height offsets for short-term orders. + +## Architecture + +The dYdX v4 adapter includes multiple components which can be used together or separately: + +- `DydxHttpClient`: Rust-backed HTTP client for Indexer REST API queries. +- `DydxWebSocketClient`: Rust-backed WebSocket client for real-time market data and account updates. +- `DydxGrpcClient`: Rust-backed gRPC client for Cosmos SDK transaction submission. +- `DydxInstrumentProvider`: Instrument parsing and loading functionality. +- `DydxDataClient`: Market data feed manager. +- `DydxExecutionClient`: Account management and trade execution gateway. +- `DydxLiveDataClientFactory`: Factory for dYdX v4 data clients (used by the trading node builder). +- `DydxLiveExecClientFactory`: Factory for dYdX v4 execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), +and won't need to work with these lower level components directly. +::: + +:::warning[First-time account activation] +A dYdX v4 trading account (sub-account 0) is created only after the wallet's first deposit or trade. +Until then, every gRPC/Indexer query returns `NOT_FOUND`, so `DydxExecutionClient.connect()` fails. + +Before starting a live `TradingNode`, send any positive amount of USDC or other supported collateral +from the same wallet on the same network (mainnet/testnet). Once the transaction has finalised +(a few blocks), restart the node and the client will connect cleanly. +::: + +## Troubleshooting + +### `StatusCode.NOT_FOUND` account not found + +**Cause:** The wallet/sub-account has never been funded and therefore does not yet exist on-chain. + +**Fix:** + +1. Deposit any positive amount of USDC to sub-account 0 on the correct network. +2. Wait for finality (roughly 30 seconds on mainnet, longer on testnet). +3. Restart the `TradingNode`; the connection should now succeed. + +:::tip +In unattended deployments, wrap the `connect()` call in an exponential-backoff loop so the +client retries until the deposit appears. +::: + +## Symbology + +dYdX uses specific symbol conventions for perpetual futures contracts. + +### Symbol format + +Format: `{Base}-USD-PERP` + +All perpetuals on dYdX are: + +- Quoted in USD +- Settled in USDC +- Use the `.DYDX` venue suffix in Nautilus + +Examples: + +- `BTC-USD-PERP.DYDX` - Bitcoin perpetual futures +- `ETH-USD-PERP.DYDX` - Ethereum perpetual futures +- `SOL-USD-PERP.DYDX` - Solana perpetual futures + +To subscribe in your strategy: + +```python +InstrumentId.from_str("BTC-USD-PERP.DYDX") +InstrumentId.from_str("ETH-USD-PERP.DYDX") +``` + +:::info +The `-PERP` suffix is appended for consistency with other adapters and future-proofing. While dYdX +currently only supports perpetuals, this naming convention allows for potential expansion to other +product types. +::: + +## Orders capability + +dYdX supports perpetual futures trading with a full set of order types and execution +features. The Rust adapter automatically classifies orders as short-term or long-term based on +time-in-force and expiry, so no manual tagging is needed (unlike the legacy Python adapter). + +### Order types + +| Order Type | Perpetuals | Notes | +|------------------------|------------|----------------------------------------------------| +| `MARKET` | ✓ | Immediate execution at best available price. | +| `LIMIT` | ✓ | | +| `STOP_MARKET` | ✓ | Stop-loss conditional order, always long-term. | +| `STOP_LIMIT` | ✓ | Conditional order, always long-term. | +| `MARKET_IF_TOUCHED` | ✓ | Take-profit market order, triggers on price touch. | +| `LIMIT_IF_TOUCHED` | ✓ | Take-profit limit order, triggers on price touch. | +| `TRAILING_STOP_MARKET` | - | *Not supported*. | + +### Execution instructions + +| Instruction | Perpetuals | Notes | +|---------------|------------|--------------------------------------------------------------| +| `post_only` | ✓ | Supported on LIMIT, STOP_LIMIT, and LIMIT_IF_TOUCHED orders. | +| `reduce_only` | ✓ | Passed for all order types. Venue support may vary by order type. | + +### Time in force options + +| Time in force | Perpetuals | Notes | +|---------------|------------|----------------------| +| `GTC` | ✓ | Good Till Canceled. | +| `GTD` | ✓ | Good Till Date. | +| `FOK` | ✓ | Fill or Kill. | +| `IOC` | ✓ | Immediate or Cancel. | + +### Advanced order features + +| Feature | Perpetuals | Notes | +|--------------------|------------|------------------| +| Order modification | - | Not supported. dYdX supports short-term order [replacement](https://docs.dydx.xyz/concepts/trading/limit-orderbook#replacements) (same ID, higher GTB); not yet exposed as `ModifyOrder`. | +| Bracket/OCO orders | - | *Not supported*. | +| Iceberg orders | - | *Not supported*. | + +### Batch operations + +| Operation | Perpetuals | Notes | +|--------------|------------|------------------------------------------------------------------------------------------------------------------------| +| Batch submit | - | *Not supported*. | +| Batch modify | - | *Not supported*. | +| Batch cancel | ✓ | Partitioned: short-term orders use `MsgBatchCancel` (single gRPC call), long-term orders use batched `MsgCancelOrder`. | + +### Position management + +| Feature | Perpetuals | Notes | +|------------------|------------|-------------------------------| +| Query positions | ✓ | Real-time position updates. | +| Position mode | - | Netting only (see below). | +| Leverage control | ✓ | Per-market leverage settings. | +| Margin mode | - | Cross margin only. | + +:::note +dYdX supports netting (one position per instrument) at the venue level. The adapter currently +operates in `NETTING` mode only. Hedging support is planned for a future version. +::: + +### Order querying + +| Feature | Perpetuals | Notes | +|----------------------|------------|--------------------------------| +| Query open orders | ✓ | List all active orders. | +| Query order history | ✓ | Historical order data. | +| Order status updates | ✓ | Real-time order state changes. | +| Trade history | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Perpetuals | Notes | +|--------------------|------------|--------------------------------------------------| +| Order lists | - | *Not supported*. | +| OCO orders | - | *Not supported*. | +| Bracket orders | - | *Not supported*. | +| Conditional orders | ✓ | Stop, take-profit market, and take-profit limit. | + +### Order classification + +dYdX classifies every order into one of three on-chain categories. The Rust adapter +automatically determines the category based on time-in-force and expiry, so no manual +configuration is required. + +| Category | Placement | Expiry | Typical use | +|-----------------|-------------|-------------------|-----------------------------------------------| +| Short-term | In-memory | Block height | IOC/FOK, or orders expiring within 40 blocks. | +| Long-term | On-chain | Timestamp (UTC) | GTC/GTD with expiry beyond the short-term window (~20s at ~0.5s/block). | +| Conditional | On-chain | Timestamp (UTC) | Stop-loss and take-profit triggers. | + +At the protocol level, **all dYdX orders are limit orders**. The `MARKET` order type +is a Nautilus convenience that the adapter implements as an aggressive IOC limit order +priced well through the book. This means market orders follow the same +`Submitted > Accepted > Filled` lifecycle as limit orders (an `OrderAccepted` event is +expected before the fill). + +See the [dYdX order documentation](https://docs.dydx.xyz/concepts/trading/orders) +for full protocol-level details on short-term vs stateful order mechanics. + +#### Short-term orders + +Short-term orders live **in validator memory only** and expire by block height (max 40 blocks, +roughly ~20 seconds at ~0.5s/block). They are the fastest order type on dYdX because they skip +on-chain storage. + +Key properties: + +- **IOC and FOK are always short-term**, regardless of other parameters +- **GTD orders** are automatically classified as short-term when the expiry falls within the + dynamic short-term window (`40 blocks × seconds_per_block`) +- Use Good-Til-Block (GTB) for replay protection instead of Cosmos SDK sequences +- Can be broadcast **concurrently** (no semaphore, cached sequence) +- Expire silently without generating cancel events +- Cannot be batched in a single transaction (one `MsgPlaceOrder` per tx) + +#### Long-term orders + +Long-term (stateful) orders are **stored on-chain** and expire by UTC timestamp. They generate +explicit cancel events when they expire or are cancelled. + +Key properties: + +- **GTC** orders default to 90-day expiration (protocol limit is 95 days) +- **GTD** orders use the user-provided expiry timestamp +- Require proper Cosmos SDK sequence management (serialized via semaphore) +- Must be broadcast **serially** with incrementing sequence numbers +- Can be batched in a single transaction + +#### Conditional orders + +Conditional orders (stop-loss, take-profit) are **always stored on-chain** and triggered by +price conditions on the validator. + +Key properties: + +- Always use timestamp-based expiry (default 90 days for GTC, protocol limit 95 days) +- Always use the long-term broadcast path (serialized with semaphore) +- Include `StopMarket`, `StopLimit`, `TakeProfitMarket`, and `TakeProfitLimit` + +#### Automatic routing + +The adapter determines order lifetime automatically using the `BlockTimeMonitor`: + +``` +max_short_term_secs = SHORT_TERM_ORDER_MAXIMUM_LIFETIME (40) × seconds_per_block +``` + +If the order's time until expiry is within `max_short_term_secs`, it is routed as short-term. +Otherwise, it is routed as long-term. No manual configuration is needed. + +#### MARKET order implementation + +dYdX has no native market order type. The adapter implements `MARKET` orders as aggressive +**IOC limit orders** priced at: + +- **Buy**: `oracle_price × (1 + 0.05)` (5% above oracle) +- **Sell**: `oracle_price × (1 - 0.05)` (5% below oracle) + +This 5% slippage buffer (`DEFAULT_MARKET_ORDER_SLIPPAGE = 0.05`) sets the worst-case price +(the "pay-through price"). Because the order is IOC, unfilled slippage is not consumed. The +buffer is intentionally wide to maximize fill probability across volatile conditions. + +### Client order ID encoding + +dYdX requires `u32` client IDs on-chain, but Nautilus uses string-based `ClientOrderId` values +(e.g., `O-20260220-031943-001-000-51`). The adapter encodes these bidirectionally so that orders +can be reconciled across restarts without persisted state. + +For the standard O-format (`O-YYYYMMDD-HHMMSS-TTT-SSS-CCC`), the encoding is deterministic: + +| dYdX field | Bits | Contents | +|-------------------|------|----------------------------------------------------| +| `client_id` | 32 | `[trader:10][strategy:10][count:12]` (unique key). | +| `client_metadata` | 32 | Seconds since 2020-01-01 UTC (timestamp). | + +Because the encoding is deterministic, the adapter can decode any reconciled order back to its +original `ClientOrderId` string without needing a database or mapping file. + +Non-standard `ClientOrderId` formats (custom strings, plain numbers) fall back to sequential +allocation with an in-memory reverse map. These IDs can only be decoded within the same session. + +#### Restart collision prevention + +On restart, Nautilus resets the internal order counter based on the number of reconciled orders, +which may be lower than the highest counter value used in the previous session (e.g., if some +orders have expired from the API response). This can cause a new order to produce the same +`client_id` as a previous session's order, resulting in a duplicate venue order UUID. + +The adapter prevents this by registering every `client_id` seen during reconciliation. If a new +O-format encoding produces a `client_id` that was already used, the encoder logs a warning and +falls back to sequential allocation. Sequential allocation also skips any registered values. + +:::note +This protection is automatic and requires no user configuration. The warning log +`[ENCODER] client_id ... collides with reconciled order` is informational. The order will +still be submitted successfully with an alternative ID. +::: + +## Broadcasting and retry strategy + +### Short-term broadcast + +Short-term orders use Good-Til-Block (GTB) for replay protection. The chain's `ClobDecorator` +ante handler skips Cosmos SDK sequence checking for short-term messages, so: + +- **No semaphore**: broadcasts are fully concurrent +- **Cached sequence**: no increment or allocation needed +- **No retry**: if the broadcast fails, it fails immediately +- Benign cancel errors are treated as success (see below) + +### Long-term broadcast + +Long-term and conditional orders require proper Cosmos SDK sequence management: + +- **Semaphore** with 1 permit serializes all long-term broadcasts +- **Exponential backoff**: 500ms → 1s → 2s → 4s (max 5 retries) +- **10-second total budget** prevents indefinite retry loops +- On sequence mismatch, the sequence is **resynced from chain** before retry + +### Sequence mismatch detection + +| Error code | Source | Meaning | +|------------|----------------------|--------------------------------------------------| +| `code=32` | Cosmos SDK | Account sequence mismatch | +| `code=104` | dYdX authenticator | Signature verification failed (sequence-related) | + +Both trigger automatic resync + retry via the `RetryManager`. + +### Benign cancel errors + +These errors during short-term cancel operations are treated as **success**: + +| Error code | Meaning | +|-------------|----------------------------------------------------------------| +| `code=19` | Transaction already in mempool cache (duplicate tx) | +| `code=9` | Cancel already exists in memclob with >= GoodTilBlock | +| `code=3006` | Order to cancel does not exist (already filled/expired/cancelled) | + +### Batch cancel partitioning + +When cancelling multiple orders, the adapter partitions them by lifetime: + +1. **Short-term orders** → Single `MsgBatchCancel` via `broadcast_short_term()` +2. **Long-term orders** → Batched `MsgCancelOrder` messages via `broadcast_with_retry()` + +This ensures each group uses the appropriate broadcast strategy. + +## Funding rates + +dYdX perpetual futures use a fixed 1-hour funding interval. The adapter sets `interval` +to `60` (minutes) on all `FundingRateUpdate` objects for both WebSocket and historical +funding data. + +## Rate limiting + +### gRPC rate limiting + +The adapter rate-limits gRPC `broadcast_tx` calls to prevent `ResourceExhausted` (429) errors +from validator nodes. + +| Setting | Default | Description | +|-------------------------------|---------|-------------------------------------------| +| `grpc_rate_limit_per_second` | `4` | Maximum gRPC broadcast requests per second. Set to `None` to disable. | + +### Provider limits + +Known rate limits for public gRPC providers: + +| Provider | Limit | Notes | +|------------|--------------------|-----------------| +| Polkachu | 300 req/min (~5/s) | | +| KingNodes | 250 req/min (~4.2/s) | | +| AutoStake | 4 req/s | | + +The default of 4 req/s is conservative and works across all public providers. + +### Multiple gRPC URL fallback + +The adapter supports configuring multiple gRPC URLs for failover: + +```python +exec_config = DydxExecClientConfig( + base_url_grpc="https://primary-grpc.example.com:443,https://fallback-grpc.example.com:443", + # ... +) +``` + +## Price and size quantization + +dYdX uses integer-based quantization for prices and sizes. The adapter handles all conversions +automatically via `OrderMessageBuilder`, but understanding the parameters helps with debugging. + +### Market parameters + +| Parameter | Description | +|--------------------------------|----------------------------------------------------------| +| `atomic_resolution` | Exponent for converting human-readable size to quantums | +| `quantum_conversion_exponent` | Exponent for converting quantums to tokens | +| `step_base_quantums` | Minimum order size step in quantums | +| `subticks_per_tick` | Price granularity within each tick | + +### Market order pricing + +Market orders use the oracle price with a 5% slippage buffer (the "pay-through price"): + +- **Buy**: `oracle_price × 1.05` +- **Sell**: `oracle_price × 0.95` + +The oracle price is cached from the Indexer and refreshed periodically. + +### Automatic handling + +All price and size quantization is handled automatically by `OrderMessageBuilder`. +No manual conversion is needed when submitting orders through Nautilus. + +## Data subscriptions + +The v4 adapter supports the following data subscriptions: + +| Data type | Subscription | Historical request | Notes | +|---------------------|--------------|--------------------|-------------------------------------------| +| Trade ticks | ✓ | ✓ | | +| Quote ticks | ✓ | - | Synthesized from order book top-of-book. | +| Order book deltas | ✓ | ✓ | L2 depth only. Snapshot via HTTP request. | +| Bars | ✓ | ✓ | See supported resolutions below. | +| Mark prices | ✓ | - | Via markets channel. | +| Index prices | ✓ | - | Via markets channel. | +| Funding rates | ✓ | - | Via markets channel. | +| Instrument status | ✓ | - | Via markets channel. | + +### Supported bar resolutions + +| Resolution | dYdX candle | +|------------|-------------| +| 1-MINUTE | `1MIN` | +| 5-MINUTE | `5MINS` | +| 15-MINUTE | `15MINS` | +| 30-MINUTE | `30MINS` | +| 1-HOUR | `1HOUR` | +| 4-HOUR | `4HOURS` | +| 1-DAY | `1DAY` | + +## Subaccounts + +dYdX supports multiple subaccounts per wallet address, allowing segregation of trading strategies +and risk management within a single wallet. + +### Key concepts + +- Each wallet address can have multiple numbered subaccounts (0, 1, 2, ..., 127). +- Subaccount 0 is the **default** and is automatically created on first deposit. +- Each subaccount maintains its own: + - Positions + - Open orders + - Collateral balance + - Margin requirements + +### Configuration + +Specify the subaccount number in the execution client config: + +```python +config = TradingNodeConfig( + exec_clients={ + "DYDX": DydxExecClientConfig( + subaccount=0, # Default subaccount + ), + }, +) +``` + +:::note +Most users will use subaccount `0` (the default). Advanced users can configure multiple execution +clients for different subaccounts to implement strategy segregation or risk isolation. +::: + +## Testnet setup + +The dYdX testnet (`dydx-testnet-4`) is a full replica of mainnet for testing strategies +without risking real funds. All default testnet endpoints are resolved automatically when +`is_testnet=True`. + +### 1. Create a testnet wallet + +**Option A: Via the dYdX testnet web app (easiest)** + +1. Go to [v4.testnet.dydx.exchange](https://v4.testnet.dydx.exchange) +2. Connect with MetaMask, Keplr, Phantom, or WalletConnect +3. A dYdX account is generated automatically +4. Export your secret phrase: click your address (top-right) and select "Export secret phrase" + +**Option B: Use an existing secp256k1 private key** + +Any 32-byte hex-encoded secp256k1 private key will work. The adapter derives the `dydx1...` +address from the key automatically using Cosmos bech32 encoding. + +### 2. Fund the testnet account + +A subaccount must be funded before the adapter can connect (see [First-time account activation](#architecture)). + +**Via the testnet web app:** + +Click the deposit/recharge button on [v4.testnet.dydx.exchange](https://v4.testnet.dydx.exchange) +to receive testnet USDC automatically. + +**Via the faucet API directly:** + +```bash +# Fund subaccount 0 with 2000 USDC +curl -X POST https://faucet.v4testnet.dydx.exchange/faucet/tokens \ + -H "Content-Type: application/json" \ + -d '{"address": "dydx1...", "subaccountNumber": 0, "amount": 2000}' + +# Fund native tokens (for gas fees) +curl -X POST https://faucet.v4testnet.dydx.exchange/faucet/native-token \ + -H "Content-Type: application/json" \ + -d '{"address": "dydx1..."}' +``` + +### 3. Set environment variables + +```bash +export DYDX_TESTNET_WALLET_ADDRESS="dydx1..." +export DYDX_TESTNET_PRIVATE_KEY="0x..." # hex-encoded, 0x prefix optional +``` + +### 4. Configure the trading node + +Set `is_testnet=True` on both data and execution clients: + +```python +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + DYDX: DydxDataClientConfig( + wallet_address=None, # Falls back to DYDX_TESTNET_WALLET_ADDRESS env var + instrument_provider=InstrumentProviderConfig(load_all=True), + is_testnet=True, + ), + }, + exec_clients={ + DYDX: DydxExecClientConfig( + wallet_address=None, # Falls back to DYDX_TESTNET_WALLET_ADDRESS env var + private_key=None, # Falls back to DYDX_TESTNET_PRIVATE_KEY env var + subaccount=0, + instrument_provider=InstrumentProviderConfig(load_all=True), + is_testnet=True, + ), + }, +) +``` + +### Testnet endpoints + +Default testnet endpoints are used automatically. Override with `base_url_*` config options if needed. + +| Service | Default URL | +|-----------|------------------------------------------------------| +| HTTP | `https://indexer.v4testnet.dydx.exchange` | +| WebSocket | `wss://indexer.v4testnet.dydx.exchange/v4/ws` | +| gRPC | `https://test-dydx-grpc.kingnodes.com:443` (primary) | +| Faucet | `https://faucet.v4testnet.dydx.exchange` | +| Web app | `https://v4.testnet.dydx.exchange` | + +### Mainnet endpoints + +Default mainnet endpoints are used automatically. Override with `base_url_*` +config options if needed. + +| Service | Default URL | +|-----------|-----------------------------------------------------| +| HTTP | `https://indexer.dydx.trade` | +| WebSocket | `wss://indexer.dydx.trade/v4/ws` | +| gRPC | `https://dydx-ops-grpc.kingnodes.com:443` (primary) | + +## Configuration + +Configure the dYdX adapter through the trading node configuration. Both data and execution +clients support environment variable fallbacks for credentials and network-specific settings. + +### Data client configuration options + +| Option | Default | Description | +|---------------------------|---------|------------------------------------------------------------------------------------------| +| `wallet_address` | `None` | dYdX wallet address. Falls back to `DYDX_WALLET_ADDRESS` / `DYDX_TESTNET_WALLET_ADDRESS` env var. | +| `is_testnet` | `False` | Connect to dYdX testnet when `True`. | +| `bars_timestamp_on_close` | `True` | Use bar close time for `ts_event` timestamps. Set `False` to use venue-native open time. | +| `base_url_http` | `None` | HTTP API endpoint override. | +| `base_url_ws` | `None` | WebSocket endpoint override. | +| `max_retries` | `3` | Maximum retry attempts for REST/WebSocket recovery. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | + +### Execution client configuration options + +| Option | Default | Description | +|--------------------------------|---------|----------------------------------------------------------------------------------------------------| +| `wallet_address` | `None` | dYdX wallet address. Falls back to `DYDX_WALLET_ADDRESS` / `DYDX_TESTNET_WALLET_ADDRESS` env var. | +| `subaccount` | `0` | Subaccount number (0-127). Subaccount 0 is the default. | +| `private_key` | `None` | Hex-encoded private key for signing. Falls back to `DYDX_PRIVATE_KEY` / `DYDX_TESTNET_PRIVATE_KEY` env var. | +| `authenticator_ids` | `None` | List of authenticator IDs for permissioned key trading (institutional setups). | +| `is_testnet` | `False` | Connect to dYdX testnet when `True`. | +| `base_url_http` | `None` | HTTP client custom endpoint override. | +| `base_url_ws` | `None` | WebSocket client custom endpoint override. | +| `base_url_grpc` | `None` | gRPC client custom endpoint override. Supports fallback with multiple URLs. | +| `max_retries` | `3` | Maximum retry attempts for order operations. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | +| `grpc_rate_limit_per_second` | `4` | Maximum gRPC requests per second. Set to `None` to disable. | + +### Basic setup + +Configure a live `TradingNode` to include dYdX data and execution clients: + +```python +from nautilus_trader.adapters.dydx import DydxDataClientConfig +from nautilus_trader.adapters.dydx import DydxExecClientConfig +from nautilus_trader.adapters.dydx.constants import DYDX +from nautilus_trader.config import InstrumentProviderConfig +from nautilus_trader.config import TradingNodeConfig + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + DYDX: DydxDataClientConfig( + wallet_address=None, # Falls back to env var + instrument_provider=InstrumentProviderConfig(load_all=True), + is_testnet=False, + ), + }, + exec_clients={ + DYDX: DydxExecClientConfig( + wallet_address=None, # Falls back to env var + private_key=None, # Falls back to env var + subaccount=0, + instrument_provider=InstrumentProviderConfig(load_all=True), + is_testnet=False, + ), + }, +) +``` + +Then, create a `TradingNode` and register the client factories: + +```python +from nautilus_trader.adapters.dydx import DydxLiveDataClientFactory +from nautilus_trader.adapters.dydx import DydxLiveExecClientFactory +from nautilus_trader.adapters.dydx.constants import DYDX +from nautilus_trader.live.node import TradingNode + +node = TradingNode(config=config) + +node.add_data_client_factory(DYDX, DydxLiveDataClientFactory) +node.add_exec_client_factory(DYDX, DydxLiveExecClientFactory) + +node.build() +``` + +### API credentials + +Credentials can be passed directly via the Python config (`wallet_address`, `private_key`) or +resolved automatically from environment variables based on the `is_testnet` setting. + +#### Environment variables + +| Variable | Network | Description | +|---------------------------------|----------|------------------------------------------------| +| `DYDX_WALLET_ADDRESS` | Mainnet | Bech32-encoded wallet address (`dydx1...`). | +| `DYDX_PRIVATE_KEY` | Mainnet | Hex-encoded secp256k1 private key for signing. | +| `DYDX_TESTNET_WALLET_ADDRESS` | Testnet | Testnet wallet address (`dydx1...`). | +| `DYDX_TESTNET_PRIVATE_KEY` | Testnet | Testnet private key. | + +#### Resolution priority + +1. Value passed in the Python config (if non-empty) +2. Environment variable (selected by `is_testnet` flag) + +### Permissioned key trading + +#### What are API Trading Keys + +API Trading Keys let you delegate trading to a separate signing key without sharing your main +wallet's seed phrase. The API key can place trades using all available margin in the owner's +cross-margin account, but cannot withdraw funds or transfer assets. + +#### Creating an API key + +1. In the dYdX web app, navigate to **More → API Trading Keys** +2. Click **Generate New API Key** +3. Save the **API Wallet Address** and **Private Key** (shown once, not stored by dYdX) +4. Click **Authorize API Key** (this registers the key on-chain as an authenticator) +5. The key is now active and can be used for trading + +See the [dYdX API Trading Keys guide](https://docs.dydx.xyz/concepts/trading/api-trading-keys) for full details on creating and managing API keys. + +#### Adapter configuration + +There are two ways to configure the adapter for API Trading Key usage: + +**Auto-resolution (recommended):** Set the API key's private key as `DYDX_PRIVATE_KEY` and the +owner's wallet address as `DYDX_WALLET_ADDRESS`. The adapter detects the mismatch during connect +and automatically queries the chain for matching authenticator IDs. No manual ID configuration +needed. + +```python +config = DydxExecClientConfig( + wallet_address="dydx1owner...", # Owner account (holds margin) + private_key="0xapikey...", # API Trading Key private key + # authenticator_ids resolved automatically +) +``` + +**Manual override:** If you know the authenticator IDs (e.g., from the dYdX TypeScript client), +pass them directly to skip auto-resolution: + +```python +config = DydxExecClientConfig( + wallet_address="dydx1owner...", + private_key="0xapikey...", + authenticator_ids=[1, 2], # Skip auto-resolution +) +``` + +:::note +API Trading Keys only work with **cross-margin** accounts and cross markets. Isolated margin +is not supported. +::: + +## Order books + +Order books can be maintained at full depth or top-of-book quotes depending on the subscription. +The venue does not provide quotes directly. Instead, the adapter subscribes to order book deltas +and synthesizes quotes for the `DataEngine` when there is a top-of-book price or size change. +Only L2 (MBP) book type is supported. + +## Contributing + +:::info +For additional features or to contribute to the dYdX adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/hyperliquid.md b/nautilus-docs/docs/integrations/hyperliquid.md new file mode 100644 index 0000000..eebaa95 --- /dev/null +++ b/nautilus-docs/docs/integrations/hyperliquid.md @@ -0,0 +1,494 @@ +# Hyperliquid + +[Hyperliquid](https://hyperliquid.gitbook.io/hyperliquid-docs) is a decentralized perpetual futures +and spot exchange built on the Hyperliquid L1, a purpose-built blockchain optimized for trading. +HyperCore provides a fully on-chain order book and matching engine. This integration supports +live market data ingest and order execution on Hyperliquid. + +## Overview + +This adapter is implemented in Rust with Python bindings. It provides direct integration +with Hyperliquid's REST and WebSocket APIs without requiring external client libraries. + +The Hyperliquid adapter includes multiple components: + +- `HyperliquidHttpClient`: Low-level HTTP API connectivity. +- `HyperliquidWebSocketClient`: Low-level WebSocket API connectivity. +- `HyperliquidInstrumentProvider`: Instrument parsing and loading functionality. +- `HyperliquidDataClient`: Market data feed manager. +- `HyperliquidExecutionClient`: Account management and trade execution gateway. +- `HyperliquidLiveDataClientFactory`: Factory for Hyperliquid data clients (used by the trading node builder). +- `HyperliquidLiveExecClientFactory`: Factory for Hyperliquid execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as shown below) +and won't need to work directly with these lower-level components. +::: + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/hyperliquid/). + +## Revoking builder code approval + +Previous versions of NautilusTrader required users to approve a builder code fee before trading. +**This is no longer required.** If you previously approved the builder fee and wish to revoke it, +you can run the revoke script. + +The script reads your private key from environment variables (`HYPERLIQUID_PK` or `HYPERLIQUID_TESTNET_PK`). + +```bash +python nautilus_trader/adapters/hyperliquid/scripts/builder_fee_revoke.py +``` + +Testnet: + +```bash +HYPERLIQUID_TESTNET=true python nautilus_trader/adapters/hyperliquid/scripts/builder_fee_revoke.py +``` + +Alternatively, from Rust: + +```bash +cargo run --bin hyperliquid-builder-fee-revoke +``` + +## Testnet setup + +Hyperliquid provides a testnet environment for testing strategies with mock funds. + +:::info +**Mainnet account required.** Hyperliquid's testnet faucet only works for wallets that have +previously deposited on mainnet. You must fund a mainnet account first before you can obtain +testnet USDC. +::: + +### Getting testnet funds + +To receive testnet USDC, you must first have deposited on **mainnet** using the same wallet address: + +1. Visit the [Hyperliquid mainnet portal](https://app.hyperliquid.xyz/) and make a deposit with your wallet. +2. Visit the [testnet faucet](https://app.hyperliquid-testnet.xyz/drip) using the same wallet. +3. Claim 1,000 mock USDC from the faucet. + +:::note +**Email wallet users**: Email login generates different addresses for mainnet vs testnet. +To use the faucet, export your email wallet from mainnet, import it into MetaMask or Rabby, +then connect the extension to testnet. +::: + +### Creating a testnet account + +1. Visit the [Hyperliquid testnet portal](https://app.hyperliquid-testnet.xyz/). +2. Connect your wallet (MetaMask, WalletConnect, or email). +3. The testnet automatically creates an account for your wallet address. + +### Exporting your private key + +To use your testnet account with NautilusTrader, you need to export your wallet's private key: + +**MetaMask:** + +1. Click the three dots menu next to your account. +2. Select "Account details". +3. Click "Show private key". +4. Enter your password and copy the private key. + +:::warning +**Never share your private keys.** +Store private keys securely using environment variables; never commit them to version control. +::: + +### Setting environment variables + +Set your testnet credentials as environment variables: + +```bash +export HYPERLIQUID_TESTNET_PK="your_private_key_here" +# Optional: for vault trading +export HYPERLIQUID_TESTNET_VAULT="vault_address_here" +``` + +The adapter automatically loads these when `testnet=True` in the configuration. + +## Product support + +Hyperliquid offers linear perpetual futures and native spot markets. + +| Product Type | Data Feed | Trading | Notes | +|-------------------|-----------|---------|----------------------------| +| Perpetual Futures | ✓ | ✓ | USDC-settled linear perps. | +| Spot | ✓ | ✓ | Native spot markets. | + +:::note +Perpetual futures on Hyperliquid are settled in USDC. Spot markets are standard currency pairs. +::: + +## Symbology + +Hyperliquid uses a specific symbol format for instruments: + +### Perpetual futures + +Format: `{Base}-USD-PERP` + +Examples: + +- `BTC-USD-PERP` - Bitcoin perpetual futures +- `ETH-USD-PERP` - Ethereum perpetual futures +- `SOL-USD-PERP` - Solana perpetual futures + +To subscribe in your strategy: + +```python +InstrumentId.from_str("BTC-USD-PERP.HYPERLIQUID") +InstrumentId.from_str("ETH-USD-PERP.HYPERLIQUID") +``` + +### Spot markets + +Format: `{Base}-{Quote}-SPOT` + +Examples: + +- `PURR-USDC-SPOT` - PURR/USDC spot pair +- `HYPE-USDC-SPOT` - HYPE/USDC spot pair + +To subscribe in your strategy: + +```python +InstrumentId.from_str("PURR-USDC-SPOT.HYPERLIQUID") +``` + +:::note +Spot instruments may include vault tokens (prefixed with `vntls:`). These are automatically +handled by the instrument provider. +::: + +## Instrument provider + +The instrument provider supports filtering when loading instruments via +`InstrumentProviderConfig(filters=...)`: + +| Filter key | Type | Description | +|-----------------------------|-------------|---------------------------------------------| +| `market_types` (or `kinds`) | `list[str]` | `"perp"` or `"spot"`. | +| `bases` | `list[str]` | Base currency codes, e.g. `["BTC", "ETH"]`. | +| `quotes` | `list[str]` | Quote currency codes, e.g. `["USDC"]`. | +| `symbols` | `list[str]` | Full symbols, e.g. `["BTC-USD-PERP"]`. | + +Example loading only perpetual instruments: + +```python +instrument_provider=InstrumentProviderConfig( + load_all=True, + filters={"market_types": ["perp"]}, +) +``` + +## Data subscriptions + +The adapter supports the following data subscriptions: + +| Data type | Subscription | Historical | Nautilus type | Notes | +|-------------------|--------------|------------|--------------------|--------------------------------------------| +| Trade ticks | ✓ | - | `TradeTick` | Via WebSocket trades channel. | +| Quote ticks | ✓ | - | `QuoteTick` | Best bid/offer from WebSocket. | +| Order book deltas | ✓ | - | `OrderBookDelta` | L2 depth. Each message is a full snapshot. | +| Bars | ✓ | ✓ | `Bar` | See supported intervals below. | +| Mark prices | ✓ | - | `MarkPriceUpdate` | Perpetual mark price ticks. | +| Index prices | ✓ | - | `IndexPriceUpdate` | Underlying index reference prices. | +| Funding rates | ✓ | - | `FundingRate` | Perpetual funding rate updates. | + +:::note +Historical quote tick and trade tick requests are not yet supported by this adapter. +::: + +### Supported bar intervals + +| Resolution | Hyperliquid candle | +|------------|--------------------| +| 1-MINUTE | `1m` | +| 3-MINUTE | `3m` | +| 5-MINUTE | `5m` | +| 15-MINUTE | `15m` | +| 30-MINUTE | `30m` | +| 1-HOUR | `1h` | +| 2-HOUR | `2h` | +| 4-HOUR | `4h` | +| 8-HOUR | `8h` | +| 12-HOUR | `12h` | +| 1-DAY | `1d` | +| 3-DAY | `3d` | +| 1-WEEK | `1w` | +| 1-MONTH | `1M` | + +## Orders capability + +Hyperliquid supports a full set of order types and execution options. + +### Order types + +| Order Type | Perpetuals | Spot | Notes | +|---------------------|------------|------|-------------------------------------------| +| `MARKET` | ✓ | ✓ | IOC limit at 0.5% slippage from best BBO. | +| `LIMIT` | ✓ | ✓ | | +| `STOP_MARKET` | ✓ | ✓ | Stop loss orders. | +| `STOP_LIMIT` | ✓ | ✓ | Stop loss with limit execution. | +| `MARKET_IF_TOUCHED` | ✓ | ✓ | Take profit at market. | +| `LIMIT_IF_TOUCHED` | ✓ | ✓ | Take profit with limit execution. | + +:::info +Conditional orders (stop and if-touched) are implemented using Hyperliquid's native trigger +order functionality with automatic TP/SL mode detection. All trigger orders are evaluated +against the [mark price](https://hyperliquid.gitbook.io/hyperliquid-docs/trading/robust-price-indices). +::: + +:::note +Market orders require cached quote data. The adapter uses the best ask (for buys) or best bid +(for sells) with 0.5% slippage. Prices are rounded to 5 significant figures, which is a +Hyperliquid API requirement for all limit prices. Ensure you subscribe to quotes for any +instrument you intend to trade with market orders. +::: + +:::note +`STOP_MARKET` and `MARKET_IF_TOUCHED` orders do not carry a limit price. The adapter derives +one from the trigger price with 0.5% slippage, rounds to 5 significant figures, and clamps to +the instrument's price precision (ceiling for buys, floor for sells). This guarantees +Hyperliquid's `limit_px >= trigger_px` (buys) / `limit_px <= trigger_px` (sells) constraint. +::: + +:::warning +**Price normalization is enabled by default.** Hyperliquid enforces a maximum of 5 significant +figures on all order prices. This is a dynamic constraint that depends on the price magnitude +and cannot be fully encoded in the static instrument price precision. For example, if ETH is +trading at $2,600 (4 integer digits), only 1 decimal place is allowed despite the instrument +having `price_precision=2`. + +By default, the adapter normalizes all outgoing limit and trigger prices to 5 significant +figures to prevent order rejections. This means your submitted prices may shift slightly. +To disable this and take full control of price formatting, set `normalize_prices=False` +in your `HyperliquidExecClientConfig`. + +If you disable normalization, you can apply the same rounding in your strategy: + +```python +from decimal import Decimal, ROUND_DOWN + +def round_to_sig_figs(price: Decimal, sig_figs: int = 5) -> Decimal: + if price == 0: + return Decimal(0) + shift = sig_figs - int(price.adjusted()) - 1 + if shift <= 0: + factor = Decimal(10) ** (-shift) + return (price / factor).to_integral_value() * factor + return round(price, shift) +``` + +::: + +### Time in force + +| Time in force | Perpetuals | Spot | Notes | +|---------------|------------|------|----------------------| +| `GTC` | ✓ | ✓ | Good Till Canceled. | +| `IOC` | ✓ | ✓ | Immediate or Cancel. | +| `FOK` | - | - | *Not supported*. | +| `GTD` | - | - | *Not supported*. | + +### Execution instructions + +| Instruction | Perpetuals | Spot | Notes | +|---------------|------------|------|----------------------------------| +| `post_only` | ✓ | ✓ | Equivalent to ALO time in force. | +| `reduce_only` | ✓ | ✓ | Close-only orders. | + +:::info +Post-only orders that would immediately match are rejected by Hyperliquid. The adapter detects +this and generates an `OrderRejected` event. Post-only orders are routed through Hyperliquid's +ALO (Add-Liquidity-Only) lane. +::: + +### Order operations + +| Operation | Perpetuals | Spot | Notes | +|-------------------|------------|------|-------------------------------------------------| +| Submit order | ✓ | ✓ | Single order submission. | +| Submit order list | ✓ | ✓ | Batch order submission (single API call). | +| Modify order | ✓ | ✓ | Requires venue order ID. | +| Cancel order | ✓ | ✓ | Cancel by client order ID. | +| Cancel all orders | ✓ | ✓ | Iterates cached open orders by instrument/side. | +| Batch cancel | ✓ | ✓ | Iterates provided cancel list. | + +:::warning +Cancel all and batch cancel issue individual cancel requests per order. +::: + +:::info +Orders placed outside NautilusTrader (e.g. via the Hyperliquid web UI or another client) +are detected and tracked as external orders. They appear in order status reports and position +reconciliation. +::: + +## Order books + +Order books are maintained via L2 WebSocket subscription. Each message delivers a full-depth +snapshot (clear + rebuild), not incremental deltas. + +:::note +There is a limitation of one order book per instrument per trader instance. +::: + +## Account and position management + +The adapter uses cross-margin mode and reports account state with USDC balances and margin +usage. On connect, the execution client performs a full reconciliation of orders, fills, and +positions against Hyperliquid's clearinghouse state. This ensures the local cache is +consistent even after restarts or disconnections. + +:::note +Leverage is managed directly through the Hyperliquid web UI or API, not through the adapter. +Set your desired leverage per instrument on Hyperliquid before trading. +::: + +## Connection management + +The adapter automatically reconnects on WebSocket disconnection using exponential backoff +(starting at 250ms, up to 5s). On reconnect, all active subscriptions are resubscribed +automatically, and order book snapshots are rebuilt. No manual intervention is required. + +A heartbeat ping is sent every 30 seconds to keep the connection alive (Hyperliquid closes +idle connections after 60 seconds). + +## API credentials + +There are two options for supplying your credentials to the Hyperliquid clients. +Either pass the corresponding values to the configuration objects, or +set the following environment variables: + +For Hyperliquid mainnet clients, you can set: + +- `HYPERLIQUID_PK` +- `HYPERLIQUID_VAULT` (optional, for vault trading) + +For Hyperliquid testnet clients, you can set: + +- `HYPERLIQUID_TESTNET_PK` +- `HYPERLIQUID_TESTNET_VAULT` (optional, for vault trading) + +:::tip +We recommend using environment variables to manage your credentials. +::: + +## Vault trading + +Hyperliquid supports [vault trading](https://hyperliquid.gitbook.io/hyperliquid-docs/trading/vaults), +where a wallet operates on behalf of a vault (sub-account). Orders are signed with the +wallet's private key but include the vault address in the signature payload. + +To trade via a vault, set the `vault_address` in your execution client config (or set the +`HYPERLIQUID_VAULT` / `HYPERLIQUID_TESTNET_VAULT` environment variable). + +:::warning +When vault trading is enabled, WebSocket subscriptions for order and fill updates automatically +use the vault address instead of the wallet address. This is required to receive the vault's +order and fill events. +::: + +## Funding rates + +Hyperliquid perpetual futures use a fixed 1-hour funding interval. The adapter sets +`interval` to `60` (minutes) on all `FundingRateUpdate` objects. + +## Rate limiting + +The adapter implements a token bucket rate limiter for Hyperliquid's REST API with a capacity +of 1200 weight per minute. HTTP info requests are automatically retried with exponential +backoff (full jitter) on rate limit (429) and server error (5xx) responses. + +## Configuration + +### Data client configuration options + +| Option | Default | Description | +|---------------------|---------|-------------------------------------------------| +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `testnet` | `False` | Connect to the Hyperliquid testnet when `True`. | +| `http_timeout_secs` | `10` | Timeout (seconds) applied to REST calls. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Reserved; WebSocket proxy not yet implemented. | + +### Execution client configuration options + +| Option | Default | Description | +|----------------------------|---------|-------------------------------------------------------------------------------------------| +| `private_key` | `None` | EVM private key; loaded from `HYPERLIQUID_PK` or `HYPERLIQUID_TESTNET_PK` when omitted. | +| `vault_address` | `None` | Vault address; loaded from `HYPERLIQUID_VAULT` or `HYPERLIQUID_TESTNET_VAULT` if omitted. | +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `testnet` | `False` | Connect to the Hyperliquid testnet when `True`. | +| `max_retries` | `None` | Maximum retry attempts for submit, cancel, or modify order requests. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | +| `http_timeout_secs` | `10` | Timeout (seconds) applied to REST calls. | +| `normalize_prices` | `True` | Normalize order prices to 5 significant figures before submission. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Reserved; WebSocket proxy not yet implemented. | + +### Configuration example + +```python +from nautilus_trader.adapters.hyperliquid import HYPERLIQUID +from nautilus_trader.adapters.hyperliquid import HyperliquidDataClientConfig +from nautilus_trader.adapters.hyperliquid import HyperliquidExecClientConfig +from nautilus_trader.config import InstrumentProviderConfig +from nautilus_trader.config import TradingNodeConfig + +config = TradingNodeConfig( + data_clients={ + HYPERLIQUID: HyperliquidDataClientConfig( + instrument_provider=InstrumentProviderConfig(load_all=True), + testnet=True, # Use testnet + ), + }, + exec_clients={ + HYPERLIQUID: HyperliquidExecClientConfig( + private_key=None, # Loads from HYPERLIQUID_TESTNET_PK env var + vault_address=None, # Optional: loads from HYPERLIQUID_TESTNET_VAULT + instrument_provider=InstrumentProviderConfig(load_all=True), + testnet=True, # Use testnet + normalize_prices=True, # Rounds prices to 5 significant figures + ), + }, +) +``` + +:::note +When `testnet=True`, the adapter automatically uses testnet environment variables +(`HYPERLIQUID_TESTNET_PK` and `HYPERLIQUID_TESTNET_VAULT`) instead of mainnet variables. +::: + +Then, create a `TradingNode` and add the client factories: + +```python +from nautilus_trader.adapters.hyperliquid import HYPERLIQUID +from nautilus_trader.adapters.hyperliquid import HyperliquidLiveDataClientFactory +from nautilus_trader.adapters.hyperliquid import HyperliquidLiveExecClientFactory +from nautilus_trader.live.node import TradingNode + +# Instantiate the live trading node with a configuration +node = TradingNode(config=config) + +# Register the client factories with the node +node.add_data_client_factory(HYPERLIQUID, HyperliquidLiveDataClientFactory) +node.add_exec_client_factory(HYPERLIQUID, HyperliquidLiveExecClientFactory) + +# Finally build the node +node.build() +``` + +## Contributing + +:::info +For additional features or to contribute to the Hyperliquid adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/ib.md b/nautilus-docs/docs/integrations/ib.md new file mode 100644 index 0000000..c745ea7 --- /dev/null +++ b/nautilus-docs/docs/integrations/ib.md @@ -0,0 +1,2058 @@ +# Interactive Brokers + +Interactive Brokers (IB) is a trading platform providing market access across a wide range of financial instruments, including stocks, options, futures, currencies, bonds, funds, and cryptocurrencies. NautilusTrader offers an adapter to integrate with IB using their [Trader Workstation (TWS) API](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/) through their Python library, [ibapi](https://github.com/nautechsystems/ibapi). + +The TWS API is an interface to IB's standalone trading applications: TWS and IB Gateway. Both can be downloaded from the IB website. If you haven't installed TWS or IB Gateway yet, refer to the [Initial Setup](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/#tws-download) guide. In NautilusTrader, you'll establish a connection to one of these applications via the `InteractiveBrokersClient`. + +Alternatively, you can start with a [dockerized version](https://github.com/gnzsnz/ib-gateway-docker) of the IB Gateway, which is particularly useful when deploying trading strategies on a hosted cloud platform. This requires having [Docker](https://www.docker.com/) installed on your machine, along with the [docker](https://pypi.org/project/docker/) Python package, which NautilusTrader conveniently includes as an extra package. + +:::note +The standalone TWS and IB Gateway applications require manually inputting username, password, and trading mode (live or paper) at startup. The dockerized version of the IB Gateway handles these steps programmatically. +::: + +## Installation + +To install NautilusTrader with Interactive Brokers (and Docker) support: + +```bash +uv pip install "nautilus_trader[ib,docker]" +``` + +To build from source with all extras (including IB and Docker): + +```bash +uv sync --all-extras +``` + +:::note +Because IB does not provide wheels for `ibapi`, NautilusTrader [repackages](https://pypi.org/project/nautilus-ibapi/) it for release on PyPI. +::: + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/interactive_brokers/). + +## Getting started + +Before implementing your trading strategies, make sure that either TWS (Trader Workstation) or IB Gateway is running. You can log in to one of these standalone applications with your credentials, or connect programmatically via `DockerizedIBGateway`. + +### Connection methods + +There are two primary ways to connect to Interactive Brokers: + +1. **Connect to an existing TWS or IB Gateway instance** +2. **Use the dockerized IB Gateway (recommended for automated deployments)** + +### Default ports + +Interactive Brokers uses different default ports depending on the application and trading mode: + +| Application | Paper Trading | Live Trading | +|-------------|---------------|--------------| +| TWS | 7497 | 7496 | +| IB Gateway | 4002 | 4001 | + +### Establish connection to an existing gateway or TWS + +When connecting to a pre-existing gateway or TWS, specify the `ibg_host` and `ibg_port` parameters in both the `InteractiveBrokersDataClientConfig` and `InteractiveBrokersExecClientConfig`: + +```python +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig + +# Example for TWS paper trading (default port 7497) +data_config = InteractiveBrokersDataClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=1, +) + +exec_config = InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=1, + account_id="DU123456", # Your paper trading account ID +) +``` + +### Establish connection to Dockerized IB Gateway + +For automated deployments, the dockerized gateway is recommended. Supply `dockerized_gateway` with an instance of `DockerizedIBGatewayConfig` in both client configurations. The `ibg_host` and `ibg_port` parameters are not needed as they're managed automatically. + +```python +from nautilus_trader.adapters.interactive_brokers.config import DockerizedIBGatewayConfig +from nautilus_trader.adapters.interactive_brokers.gateway import DockerizedIBGateway + +gateway_config = DockerizedIBGatewayConfig( + username="your_username", # Or set TWS_USERNAME env var + password="your_password", # Or set TWS_PASSWORD env var + trading_mode="paper", # "paper" or "live" + read_only_api=True, # Set to False to allow order execution + timeout=300, # Startup timeout in seconds +) + +# This may take a short while to start up, especially the first time +gateway = DockerizedIBGateway(config=gateway_config) +gateway.start() + +# Confirm you are logged in +print(gateway.is_logged_in(gateway.container)) + +# Inspect the logs +print(gateway.container.logs()) +``` + +### Environment variables + +To supply credentials to the Interactive Brokers Gateway, either pass the `username` and `password` to the `DockerizedIBGatewayConfig`, or set the following environment variables: + +- `TWS_USERNAME`: Your IB account username. +- `TWS_PASSWORD`: Your IB account password. +- `TWS_ACCOUNT`: Your IB account ID (used as the fallback for `account_id`). + +### Connection management + +The adapter includes connection management features: + +- **Automatic reconnection**: Configure retries with the `IB_MAX_CONNECTION_ATTEMPTS` environment variable. +- **Connection timeout**: Adjust the timeout with the `connection_timeout` parameter (default: 300 seconds). +- **Connection watchdog**: Monitor connection health and trigger reconnection automatically when required. +- **Graceful error handling**: Handle diverse connection scenarios with error classification. + +## Overview + +The Interactive Brokers adapter provides an integration with IB's TWS API. The adapter includes several major components: + +### Core components + +- **`InteractiveBrokersClient`**: The central client that executes TWS API requests using `ibapi`. Manages connections, handles errors, and coordinates all API interactions. +- **`InteractiveBrokersDataClient`**: Connects to the Gateway for streaming market data including quotes, trades, and bars. +- **`InteractiveBrokersExecutionClient`**: Handles account information, order management, and trade execution. +- **`InteractiveBrokersInstrumentProvider`**: Retrieves and manages instrument definitions, including support for options and futures chains. +- **`HistoricInteractiveBrokersClient`**: Provides methods for retrieving instruments and historical data, useful for backtesting and research. + +### Supporting components + +- **`DockerizedIBGateway`**: Manages dockerized IB Gateway instances for automated deployments. +- **Configuration classes**: Provide configuration options for all components. +- **Factory classes**: Create and configure client instances with the necessary dependencies. + +### Supported asset classes + +The adapter supports trading across all major asset classes available through Interactive Brokers: + +- **Equities**: Stocks, ETFs, and equity options. +- **Fixed income**: Bonds and bond funds. +- **Derivatives**: Futures, options, and warrants. +- **Foreign exchange**: Spot FX and FX forwards. +- **Cryptocurrencies**: Bitcoin, Ethereum, and other digital assets. +- **Commodities**: Physical commodities and commodity futures. +- **Indices**: Index products and index options. + +## The Interactive Brokers client + +The `InteractiveBrokersClient` is the central component of the IB adapter, overseeing a range of functions. These include establishing and maintaining connections, handling API errors, executing trades, and gathering various types of data such as market data, contract/instrument data, and account details. + +The `InteractiveBrokersClient` is divided into specialized mixin classes, each handling a specific responsibility. + +### Client architecture + +The client uses a mixin-based architecture where each mixin handles a specific aspect of the IB API: + +#### Connection management (`InteractiveBrokersClientConnectionMixin`) + +- Establishes and maintains socket connections to TWS/Gateway. +- Handles connection timeouts and reconnection logic. +- Manages connection state and health monitoring. +- Supports configurable reconnection attempts via `IB_MAX_CONNECTION_ATTEMPTS` environment variable. + +#### Error handling (`InteractiveBrokersClientErrorMixin`) + +- Processes all API errors and warnings. +- Categorizes errors by type (client errors, connectivity issues, request errors). +- Handles subscription and request-specific error scenarios. +- Provides error logging and debugging information. + +#### Account management (`InteractiveBrokersClientAccountMixin`) + +- Retrieves account information and balances. +- Manages position data and portfolio updates. +- Handles multi-account scenarios. +- Processes account-related notifications. + +#### Contract/instrument management (`InteractiveBrokersClientContractMixin`) + +- Retrieves contract details and specifications. +- Handles instrument searches and lookups. +- Manages contract validation and verification. +- Supports complex instrument types (options chains, futures chains). + +#### Market data management (`InteractiveBrokersClientMarketDataMixin`) + +- Handles real-time and historical market data subscriptions. +- Processes quotes, trades, and bar data. +- Manages market data type settings (real-time, delayed, frozen). +- Handles tick-by-tick data and market depth. + +#### Order management (`InteractiveBrokersClientOrderMixin`) + +- Processes order placement, modification, and cancellation. +- Handles order status updates and execution reports. +- Manages order validation and error handling. +- Supports complex order types and conditions. + +### Key features + +- **Asynchronous operation**: All operations are fully asynchronous using Python's asyncio. +- **Error handling**: Error categorization and handling. +- **Connection resilience**: Automatic reconnection with configurable retry logic. +- **Message processing**: Efficient message queue processing for high-throughput scenarios. +- **State management**: Proper state tracking for connections, subscriptions, and requests. + +:::tip +To troubleshoot TWS API incoming message issues, consider starting at the `InteractiveBrokersClient._process_message` method, which acts as the primary gateway for processing all messages received from the API. +::: + +## Symbology + +The `InteractiveBrokersInstrumentProvider` supports three methods for constructing `InstrumentId` instances, which can be configured via the `symbology_method` enum in `InteractiveBrokersInstrumentProviderConfig`. + +### Symbology methods + +#### 1. Simplified symbology (`IB_SIMPLIFIED`) - default + +When `symbology_method` is set to `IB_SIMPLIFIED` (the default setting), the system uses intuitive, human-readable symbology rules: + +**Format Rules by Asset Class:** + +- **Forex**: `{symbol}/{currency}.{exchange}` + - Example: `EUR/USD.IDEALPRO` +- **Stocks**: `{localSymbol}.{primaryExchange}` + - Spaces in localSymbol are replaced with hyphens + - Example: `BF-B.NYSE`, `SPY.ARCA` +- **Futures**: `{localSymbol}.{exchange}` + - Individual contracts use single digit years + - Example: `ESM4.CME`, `CLZ7.NYMEX` +- **Continuous Futures**: `{symbol}.{exchange}` + - Represents front month, automatically rolling + - Example: `ES.CME`, `CL.NYMEX` +- **Options on Futures (FOP)**: `{localSymbol}.{exchange}` + - Format: `{symbol}{month}{year} {right}{strike}` + - Example: `ESM4 C4200.CME` +- **Options**: `{localSymbol}.{exchange}` + - All spaces removed from localSymbol + - Example: `AAPL230217P00155000.SMART` +- **Indices**: `^{localSymbol}.{exchange}` + - Example: `^SPX.CBOE`, `^NDX.NASDAQ` +- **Bonds**: `{localSymbol}.{exchange}` + - Example: `912828XE8.SMART` +- **Cryptocurrencies**: `{symbol}/{currency}.{exchange}` + - Example: `BTC/USD.PAXOS`, `ETH/USD.PAXOS` + +#### 2. Raw symbology (`IB_RAW`) + +Setting `symbology_method` to `IB_RAW` enforces stricter parsing rules that align directly with the fields defined in the IB API. This method provides maximum compatibility across all regions and instrument types: + +**Format Rules:** + +- **CFDs**: `{localSymbol}={secType}.IBCFD` +- **Commodities**: `{localSymbol}={secType}.IBCMDTY` +- **Default for Other Types**: `{localSymbol}={secType}.{exchange}` + +**Examples:** + +- `IBUS30=CFD.IBCFD` +- `XAUUSD=CMDTY.IBCMDTY` +- `AAPL=STK.SMART` + +This configuration ensures explicit instrument identification and supports instruments from any region, especially those with non-standard symbology where simplified parsing may fail. + +### MIC venue conversion + +The adapter supports converting Interactive Brokers exchange codes to Market Identifier Codes (MIC) for standardized venue identification: + +#### `convert_exchange_to_mic_venue` + +When set to `True`, the adapter automatically converts IB exchange codes to their corresponding MIC codes: + +```python +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + convert_exchange_to_mic_venue=True, # Enable MIC conversion + symbology_method=SymbologyMethod.IB_SIMPLIFIED, +) +``` + +**Examples of MIC Conversion:** + +- `CME` → `XCME` (Chicago Mercantile Exchange) +- `NASDAQ` → `XNAS` (Nasdaq Stock Market) +- `NYSE` → `XNYS` (New York Stock Exchange) +- `LSE` → `XLON` (London Stock Exchange) + +#### `symbol_to_mic_venue` + +For custom venue mapping, use the `symbol_to_mic_venue` dictionary to override default conversions: + +```python +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + convert_exchange_to_mic_venue=True, + symbol_to_mic_venue={ + "ES": "XCME", # All ES futures/options use CME MIC + "SPY": "ARCX", # SPY specifically uses ARCA + }, +) +``` + +### Supported instrument formats + +The adapter supports various instrument formats based on Interactive Brokers' contract specifications: + +#### Futures month codes + +- **F** = January, **G** = February, **H** = March, **J** = April +- **K** = May, **M** = June, **N** = July, **Q** = August +- **U** = September, **V** = October, **X** = November, **Z** = December + +#### Supported exchanges by asset class + +**Futures Exchanges:** + +- `CME`, `CBOT`, `NYMEX`, `COMEX`, `KCBT`, `MGE`, `NYBOT`, `SNFE` + +**Options Exchanges:** + +- `SMART` (IB's smart routing) + +**Forex Exchanges:** + +- `IDEALPRO` (IB's forex platform) + +**Cryptocurrency Exchanges:** + +- `PAXOS` (IB's crypto platform) + +**CFD/Commodity Exchanges:** + +- `IBCFD`, `IBCMDTY` (IB's internal routing) + +### Choosing the right symbology method + +- **Use `IB_SIMPLIFIED`** (default) for most use cases - provides clean, readable instrument IDs +- **Use `IB_RAW`** when dealing with complex international instruments or when simplified parsing fails +- **Enable `convert_exchange_to_mic_venue`** when you need standardized MIC venue codes for compliance or data consistency + +## Instruments and contracts + +In Interactive Brokers, a NautilusTrader `Instrument` corresponds to an IB [Contract](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/#contracts). The adapter handles two types of contract representations: + +### Contract types + +#### Basic contract (`IBContract`) + +- Contains essential contract identification fields +- Used for contract searches and basic operations +- Cannot be directly converted to a NautilusTrader `Instrument` + +#### Contract details (`IBContractDetails`) + +- Contains contract information including: + - Order types supported + - Trading hours and calendar + - Margin requirements + - Price increments and multipliers + - Market data permissions +- Can be converted to a NautilusTrader `Instrument` +- Required for trading operations + +### Contract discovery + +To search for contract information, use the [IB Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/). + +### Loading instruments + +There are two primary methods for loading instruments: + +#### 1. Using `load_ids` (recommended) + +Use `symbology_method=SymbologyMethod.IB_SIMPLIFIED` (default) with `load_ids` for clean, intuitive instrument identification: + +```python +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig +from nautilus_trader.adapters.interactive_brokers.config import SymbologyMethod + +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + symbology_method=SymbologyMethod.IB_SIMPLIFIED, + load_ids=frozenset([ + "EUR/USD.IDEALPRO", # Forex + "SPY.ARCA", # Stock + "ESM24.CME", # Future + "BTC/USD.PAXOS", # Crypto + "^SPX.CBOE", # Index + ]), +) +``` + +#### 2. Using `load_contracts` (for complex instruments) + +Use `load_contracts` with `IBContract` instances for complex scenarios like options/futures chains: + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBContract + +# Load options chain for specific expiry +options_chain_expiry = IBContract( + secType="IND", + symbol="SPX", + exchange="CBOE", + build_options_chain=True, + lastTradeDateOrContractMonth='20240718', +) + +# Load options chain for date range +options_chain_range = IBContract( + secType="IND", + symbol="SPX", + exchange="CBOE", + build_options_chain=True, + min_expiry_days=0, + max_expiry_days=30, +) + +# Load futures chain +futures_chain = IBContract( + secType="CONTFUT", + exchange="CME", + symbol="ES", + build_futures_chain=True, +) + +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + load_contracts=frozenset([ + options_chain_expiry, + options_chain_range, + futures_chain, + ]), +) +``` + +### IBContract examples by asset class + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBContract + +# Stocks +IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY') +IBContract(secType='STK', exchange='SMART', primaryExchange='NASDAQ', symbol='AAPL') + +# Bonds +IBContract(secType='BOND', secIdType='ISIN', secId='US03076KAA60') +IBContract(secType='BOND', secIdType='CUSIP', secId='912828XE8') + +# Individual Options +IBContract(secType='OPT', exchange='SMART', symbol='SPY', + lastTradeDateOrContractMonth='20251219', strike=500, right='C') + +# Options Chain (loads all strikes/expirations) +IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY', + build_options_chain=True, min_expiry_days=10, max_expiry_days=60) + +# CFDs +IBContract(secType='CFD', symbol='IBUS30') +IBContract(secType='CFD', symbol='DE40EUR', exchange='SMART') + +# Individual Futures +IBContract(secType='FUT', exchange='CME', symbol='ES', + lastTradeDateOrContractMonth='20240315') + +# Futures Chain (loads all expirations) +IBContract(secType='CONTFUT', exchange='CME', symbol='ES', build_futures_chain=True) + +# Options on Futures (FOP) - Individual +IBContract(secType='FOP', exchange='CME', symbol='ES', + lastTradeDateOrContractMonth='20240315', strike=4200, right='C') + +# Options on Futures Chain (loads all strikes/expirations) +IBContract(secType='CONTFUT', exchange='CME', symbol='ES', + build_options_chain=True, min_expiry_days=7, max_expiry_days=60) + +# Forex +IBContract(secType='CASH', exchange='IDEALPRO', symbol='EUR', currency='USD') +IBContract(secType='CASH', exchange='IDEALPRO', symbol='GBP', currency='JPY') + +# Cryptocurrencies +IBContract(secType='CRYPTO', symbol='BTC', exchange='PAXOS', currency='USD') +IBContract(secType='CRYPTO', symbol='ETH', exchange='PAXOS', currency='USD') + +# Indices +IBContract(secType='IND', symbol='SPX', exchange='CBOE') +IBContract(secType='IND', symbol='NDX', exchange='NASDAQ') + +# Commodities +IBContract(secType='CMDTY', symbol='XAUUSD', exchange='SMART') +``` + +### Advanced configuration options + +```python +# Options chain with custom exchange +IBContract( + secType="STK", + symbol="AAPL", + exchange="SMART", + primaryExchange="NASDAQ", + build_options_chain=True, + options_chain_exchange="CBOE", # Use CBOE for options instead of SMART + min_expiry_days=7, + max_expiry_days=45, +) + +# Futures chain with specific months +IBContract( + secType="CONTFUT", + exchange="NYMEX", + symbol="CL", # Crude Oil + build_futures_chain=True, + min_expiry_days=30, + max_expiry_days=180, +) +``` + +### Continuous futures + +For continuous futures contracts (using `secType='CONTFUT'`), the adapter creates instrument IDs using just the symbol and venue: + +```python +# Continuous futures examples +IBContract(secType='CONTFUT', exchange='CME', symbol='ES') # → ES.CME +IBContract(secType='CONTFUT', exchange='NYMEX', symbol='CL') # → CL.NYMEX + +# With MIC venue conversion enabled +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + convert_exchange_to_mic_venue=True, +) +# Results in: +# ES.XCME (instead of ES.CME) +# CL.XNYM (instead of CL.NYMEX) +``` + +**Continuous Futures vs Individual Futures:** + +- **Continuous**: `ES.CME` - Represents the front month contract, automatically rolls +- **Individual**: `ESM4.CME` - Specific March 2024 contract + +:::note +When using `build_options_chain=True` or `build_futures_chain=True`, the `secType` and `symbol` should be specified for the underlying contract. The adapter will automatically discover and load all related derivative contracts within the specified expiry range. +::: + +## Option spreads + +Interactive Brokers supports option spreads through BAG contracts, which combine multiple option legs into a single tradeable instrument. NautilusTrader provides support for creating, loading, and trading option spreads. + +### Creating option spread instrument IDs + +Option spreads are created using the `new_generic_spread_id()` function, which combines individual option legs with their respective ratios: + +```python +from nautilus_trader.model.identifiers import InstrumentId, new_generic_spread_id + +# Create individual option instrument IDs +call_leg = InstrumentId.from_str("SPY C400.SMART") +put_leg = InstrumentId.from_str("SPY P390.SMART") + +# Create a 1:1 call spread (long call, short call) +call_spread_id = new_generic_spread_id([ + (call_leg, 1), # Long 1 contract + (put_leg, -1), # Short 1 contract +]) + +# Create a 1:2 ratio spread +ratio_spread_id = new_generic_spread_id([ + (call_leg, 1), # Long 1 contract + (put_leg, 2), # Long 2 contracts +]) +``` + +### Dynamic spread loading + +Option spreads must be requested before they can be traded or subscribed to for market data. Use the `request_instrument()` method to dynamically load spread instruments: + +```python +# In your strategy's on_start method +def on_start(self): + # Request the spread instrument + self.request_instrument(spread_id) + +def on_instrument(self, instrument): + # Handle the loaded spread instrument + self.log.info(f"Loaded spread: {instrument.id}") + + # Now you can subscribe to market data + self.subscribe_quote_ticks(instrument.id) + + # And place orders + order = self.order_factory.market( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(1), + time_in_force=TimeInForce.DAY, + ) + self.submit_order(order) +``` + +### Spread trading requirements + +1. **Load individual legs first**: Ensure the individual option legs are available before creating spreads. +2. **Request the spread instrument**: Use `request_instrument()` to load the spread before trading. +3. **Subscribe to market data**: Request quote ticks after the spread is loaded. +4. **Place orders**: Any order type can be used once the spread is available. + +## Historical data and backtesting + +The `HistoricInteractiveBrokersClient` provides methods for retrieving historical data from Interactive Brokers for backtesting and research purposes. + +### Supported data types + +- **Bar data**: OHLCV bars with time, tick, and volume aggregations. +- **Tick data**: Trade ticks and quote ticks with microsecond precision. +- **Instrument data**: Complete contract specifications and trading rules. + +### Historical data client + +```python +from nautilus_trader.adapters.interactive_brokers.historical.client import HistoricInteractiveBrokersClient +from ibapi.common import MarketDataTypeEnum + +# Initialize the client +client = HistoricInteractiveBrokersClient( + host="127.0.0.1", + port=7497, + client_id=1, + market_data_type=MarketDataTypeEnum.DELAYED_FROZEN, # Use delayed data if no subscription + log_level="INFO" +) + +# Connect to TWS/Gateway +await client.connect() +``` + +### Retrieving instruments + +#### Basic instrument retrieval + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBContract + +# Define contracts +contracts = [ + IBContract(secType="STK", symbol="AAPL", exchange="SMART", primaryExchange="NASDAQ"), + IBContract(secType="STK", symbol="MSFT", exchange="SMART", primaryExchange="NASDAQ"), + IBContract(secType="CASH", symbol="EUR", currency="USD", exchange="IDEALPRO"), +] + +# Request instrument definitions +instruments = await client.request_instruments(contracts=contracts) +``` + +#### Option chain retrieval with catalog storage + +You can download entire option chains using `request_instruments` in your strategy, with the added benefit of saving the data to the catalog using `update_catalog=True`: + +```python +# In your strategy's on_start method +def on_start(self): + self.request_instruments( + venue=IB_VENUE, + update_catalog=True, + params={ + "ib_contracts": ( + # SPY options + { + "secType": "STK", + "symbol": "SPY", + "exchange": "SMART", + "primaryExchange": "ARCA", + "build_options_chain": True, + "min_expiry_days": 7, + "max_expiry_days": 30, + }, + # QQQ options + { + "secType": "STK", + "symbol": "QQQ", + "exchange": "SMART", + "primaryExchange": "NASDAQ", + "build_options_chain": True, + "min_expiry_days": 7, + "max_expiry_days": 30, + }, + # ES futures options + { + "secType": "CONTFUT", + "exchange": "CME", + "symbol": "ES", + "build_options_chain": True, + "min_expiry_days": 0, + "max_expiry_days": 60, + }, + # SPX index options + { + "secType": "IND", + "symbol": "SPX", + "exchange": "CBOE", + "build_options_chain": True, + "min_expiry_days": 0, + "max_expiry_days": 5, + }, + # ES futures chain and futures options + { + "secType": "CONTFUT", + "exchange": "CME", + "symbol": "ES", + "build_futures_chain": True, + "build_options_chain": True, + "min_expiry_days": 0, + "max_expiry_days": 2, + }, + # ESTX50 index options (Eurex) + { + "secType": "IND", + "exchange": "EUREX", + "symbol": "ESTX50", + "build_options_chain": True, + "min_expiry_days": 0, + "max_expiry_days": 2, + }, + ), + }, + ) +``` + +### Retrieving historical bars + +```python +import datetime + +# Request historical bars +bars = await client.request_bars( + bar_specifications=[ + "1-MINUTE-LAST", # 1-minute bars using last price + "5-MINUTE-MID", # 5-minute bars using midpoint + "1-HOUR-LAST", # 1-hour bars using last price + "1-DAY-LAST", # Daily bars using last price + ], + start_date_time=datetime.datetime(2023, 11, 1, 9, 30), + end_date_time=datetime.datetime(2023, 11, 6, 16, 30), + tz_name="America/New_York", + contracts=contracts, + use_rth=True, # Regular Trading Hours only + timeout=120, # Request timeout in seconds +) +``` + +### Retrieving historical ticks + +```python +# Request historical tick data (use tick_type="TRADES" or "BID_ASK" for quote ticks) +ticks = await client.request_ticks( + tick_type="TRADES", + start_date_time=datetime.datetime(2023, 11, 6, 9, 30), + end_date_time=datetime.datetime(2023, 11, 6, 16, 30), + tz_name="America/New_York", + contracts=contracts, + use_rth=True, + timeout=120, +) +``` + +### Bar specifications + +The adapter supports various bar specifications: + +#### Time-based bars + +- `"1-SECOND-LAST"`, `"5-SECOND-LAST"`, `"10-SECOND-LAST"`, `"15-SECOND-LAST"`, `"30-SECOND-LAST"` +- `"1-MINUTE-LAST"`, `"2-MINUTE-LAST"`, `"3-MINUTE-LAST"`, `"5-MINUTE-LAST"`, `"10-MINUTE-LAST"`, `"15-MINUTE-LAST"`, `"20-MINUTE-LAST"`, `"30-MINUTE-LAST"` +- `"1-HOUR-LAST"`, `"2-HOUR-LAST"`, `"3-HOUR-LAST"`, `"4-HOUR-LAST"`, `"8-HOUR-LAST"` +- `"1-DAY-LAST"`, `"1-WEEK-LAST"`, `"1-MONTH-LAST"` + +#### Price types + +- `LAST` - Last traded price +- `MID` - Midpoint of bid/ask +- `BID` - Bid price +- `ASK` - Ask price + +### Complete example + +```python +import asyncio +import datetime +from nautilus_trader.adapters.interactive_brokers.common import IBContract +from nautilus_trader.adapters.interactive_brokers.historical.client import HistoricInteractiveBrokersClient +from nautilus_trader.persistence.catalog import ParquetDataCatalog + + +async def download_historical_data(): + # Initialize client + client = HistoricInteractiveBrokersClient( + host="127.0.0.1", + port=7497, + client_id=5, + ) + + # Connect + await client.connect() + await asyncio.sleep(2) # Allow connection to stabilize + + # Define contracts + contracts = [ + IBContract(secType="STK", symbol="AAPL", exchange="SMART", primaryExchange="NASDAQ"), + IBContract(secType="CASH", symbol="EUR", currency="USD", exchange="IDEALPRO"), + ] + + # Request instruments + instruments = await client.request_instruments(contracts=contracts) + + # Request historical bars + bars = await client.request_bars( + bar_specifications=["1-HOUR-LAST", "1-DAY-LAST"], + start_date_time=datetime.datetime(2023, 11, 1, 9, 30), + end_date_time=datetime.datetime(2023, 11, 6, 16, 30), + tz_name="America/New_York", + contracts=contracts, + use_rth=True, + ) + + # Request tick data + ticks = await client.request_ticks( + tick_type="TRADES", + start_date_time=datetime.datetime(2023, 11, 6, 14, 0), + end_date_time=datetime.datetime(2023, 11, 6, 15, 0), + tz_name="America/New_York", + contracts=contracts, + ) + + # Save to catalog + catalog = ParquetDataCatalog("./catalog") + catalog.write_data(instruments) + catalog.write_data(bars) + catalog.write_data(ticks) + + print(f"Downloaded {len(instruments)} instruments") + print(f"Downloaded {len(bars)} bars") + print(f"Downloaded {len(ticks)} ticks") + + # Disconnect + await client.disconnect() + +# Run the example +if __name__ == "__main__": + asyncio.run(download_historical_data()) +``` + +### Data limitations + +Be aware of Interactive Brokers' historical data limitations: + +- **Rate Limits**: IB enforces rate limits on historical data requests +- **Data Availability**: Historical data availability varies by instrument and subscription level +- **Market Data Permissions**: Some data requires specific market data subscriptions +- **Time Ranges**: Maximum lookback periods vary by bar size and instrument type + +### Best practices + +1. **Use Delayed Data**: For backtesting, `MarketDataTypeEnum.DELAYED_FROZEN` is often sufficient +2. **Batch Requests**: Group multiple instruments in single requests when possible +3. **Handle Timeouts**: Set appropriate timeout values for large data requests +4. **Respect Rate Limits**: Add delays between requests to avoid hitting rate limits +5. **Validate Data**: Always check data quality and completeness before backtesting + +:::warning +Interactive Brokers enforces pacing limits; excessive historical-data or order requests trigger pacing violations and IB can disable the API session for several minutes. +::: + +## Live trading + +Live trading with Interactive Brokers requires setting up a `TradingNode` that incorporates both `InteractiveBrokersDataClient` and `InteractiveBrokersExecutionClient`. These clients depend on the `InteractiveBrokersInstrumentProvider` for instrument management. + +### Architecture overview + +The live trading setup consists of three main components: + +1. **InstrumentProvider**: Manages instrument definitions and contract details +2. **DataClient**: Handles real-time market data subscriptions +3. **ExecutionClient**: Manages orders, positions, and account information + +### InstrumentProvider configuration + +The `InteractiveBrokersInstrumentProvider` provides access to financial instrument data from IB. It supports loading individual instruments, options chains, and futures chains. + +#### Basic configuration + +```python +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig +from nautilus_trader.adapters.interactive_brokers.config import SymbologyMethod +from nautilus_trader.adapters.interactive_brokers.common import IBContract + +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + symbology_method=SymbologyMethod.IB_SIMPLIFIED, + build_futures_chain=False, # Set to True if fetching futures chains + build_options_chain=False, # Set to True if fetching options chains + min_expiry_days=10, # Minimum days to expiry for derivatives + max_expiry_days=60, # Maximum days to expiry for derivatives + convert_exchange_to_mic_venue=False, # Use MIC codes for venue mapping + cache_validity_days=1, # Cache instrument data for 1 day + load_ids=frozenset([ + # Individual instruments using simplified symbology + "EUR/USD.IDEALPRO", # Forex + "BTC/USD.PAXOS", # Cryptocurrency + "SPY.ARCA", # Stock ETF + "V.NYSE", # Individual stock + "ESM4.CME", # Future contract (single digit year) + "^SPX.CBOE", # Index + ]), + load_contracts=frozenset([ + # Complex instruments using IBContract + IBContract(secType='STK', symbol='AAPL', exchange='SMART', primaryExchange='NASDAQ'), + IBContract(secType='CASH', symbol='GBP', currency='USD', exchange='IDEALPRO'), + ]), +) +``` + +#### Advanced configuration for derivatives + +```python +# Configuration for options and futures chains +advanced_config = InteractiveBrokersInstrumentProviderConfig( + symbology_method=SymbologyMethod.IB_SIMPLIFIED, + build_futures_chain=True, # Enable futures chain loading + build_options_chain=True, # Enable options chain loading + min_expiry_days=7, # Load contracts expiring in 7+ days + max_expiry_days=90, # Load contracts expiring within 90 days + load_contracts=frozenset([ + # Load SPY options chain + IBContract( + secType='STK', + symbol='SPY', + exchange='SMART', + primaryExchange='ARCA', + build_options_chain=True, + ), + # Load ES futures chain + IBContract( + secType='CONTFUT', + exchange='CME', + symbol='ES', + build_futures_chain=True, + ), + ]), +) +``` + +#### Filtering security types + +Use `filter_sec_types` to ignore specific IB `secType` values. Any contract whose `secType` matches an entry in this frozenset is skipped with a warning (for example unsupported types such as `WAR` or `IOPT`): + +```python +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + load_ids=frozenset(["SPY.ARCA"]), + filter_sec_types=frozenset({"WAR", "IOPT"}), # Opt out from unsupported asset types +) +``` + +### Integration with external data providers + +The Interactive Brokers adapter can be used alongside other data providers for enhanced market data coverage. When using multiple data sources: + +- Use consistent symbology methods across providers +- Consider using `convert_exchange_to_mic_venue=True` for standardized venue identification +- Ensure instrument cache management is handled properly to avoid conflicts + +### Data client configuration + +The `InteractiveBrokersDataClient` interfaces with IB for streaming and retrieving real-time market data. Upon connection, it configures the [market data type](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/#delayed-market-data) and loads instruments based on the `InteractiveBrokersInstrumentProviderConfig` settings. + +#### Supported data types + +- **Quote Ticks**: Real-time bid/ask prices and sizes +- **Trade Ticks**: Real-time trade prices and volumes +- **Bar Data**: Real-time OHLCV bars (1-second to 1-day intervals) +- **Market Depth**: Level 2 order book data (where available) + +#### Market data types + +Interactive Brokers supports several market data types: + +- `REALTIME`: Live market data (requires market data subscriptions) +- `DELAYED`: 15-20 minute delayed data (free for most markets) +- `DELAYED_FROZEN`: Delayed data that doesn't update (useful for testing) +- `FROZEN`: Last known real-time data (when market is closed) + +#### Basic data client configuration + +```python +from nautilus_trader.adapters.interactive_brokers.config import IBMarketDataTypeEnum +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig + +data_client_config = InteractiveBrokersDataClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, # TWS paper trading port + ibg_client_id=1, + use_regular_trading_hours=True, # RTH only for stocks + market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN, # Use delayed data + ignore_quote_tick_size_updates=False, # Include size-only updates + instrument_provider=instrument_provider_config, + connection_timeout=300, # 5 minutes + request_timeout_secs=60, # 1 minute +) +``` + +#### Advanced data client configuration + +```python +# Configuration for production with real-time data +production_data_config = InteractiveBrokersDataClientConfig( + ibg_host="127.0.0.1", + ibg_port=4001, # IB Gateway live trading port + ibg_client_id=1, + use_regular_trading_hours=False, # Include extended hours + market_data_type=IBMarketDataTypeEnum.REALTIME, # Real-time data + ignore_quote_tick_size_updates=True, # Reduce tick volume + handle_revised_bars=True, # Handle bar revisions + instrument_provider=instrument_provider_config, + dockerized_gateway=dockerized_gateway_config, # If using Docker + connection_timeout=300, + request_timeout_secs=60, +) +``` + +### Data client configuration options + +| Option | Default | Description | +|---------------------------------|-------------------------------------------------|-------------| +| `instrument_provider` | `InteractiveBrokersInstrumentProviderConfig()` | Instrument provider settings controlling which contracts load at startup. | +| `ibg_host` | `127.0.0.1` | Hostname or IP for TWS/IB Gateway. | +| `ibg_port` | `None` | Port for TWS/IB Gateway (`7497`/`7496` for TWS, `4002`/`4001` for IBG). | +| `ibg_client_id` | `1` | Unique client identifier used when connecting to TWS/IB Gateway. | +| `use_regular_trading_hours` | `True` | Request bars limited to regular trading hours when `True`. | +| `market_data_type` | `REALTIME` | Market data feed type (`REALTIME`, `DELAYED`, `DELAYED_FROZEN`, etc.). | +| `ignore_quote_tick_size_updates`| `False` | Suppress quote ticks where only size changes when `True`. | +| `handle_revised_bars` | `False` | When `True`, processes bar revisions from IB (bars can be updated after initial publication). | +| `dockerized_gateway` | `None` | Optional `DockerizedIBGatewayConfig` for containerized setups. | +| `connection_timeout` | `300` | Seconds to wait for the initial API connection. | +| `request_timeout_secs` | `60` | Seconds to wait for historical data requests before timing out. | + +#### Notes + +- **`use_regular_trading_hours`**: When `True`, only requests data during regular trading hours. Primarily affects bar data for stocks. +- **`ignore_quote_tick_size_updates`**: When `True`, filters out quote ticks where only the size changed (not price), reducing data volume. +- **`handle_revised_bars`**: When `True`, processes bar revisions from IB (bars can be updated after initial publication). +- **`connection_timeout`**: Maximum time to wait for initial connection establishment. +- **`request_timeout_secs`**: Maximum time to wait for historical data requests. + +### Execution client configuration options + +| Option | Default | Description | +|-----------------------------------------|-------------------------------------------------|-------------| +| `instrument_provider` | `InteractiveBrokersInstrumentProviderConfig()` | Instrument provider settings controlling which contracts load at startup. | +| `ibg_host` | `127.0.0.1` | Hostname or IP for TWS/IB Gateway. | +| `ibg_port` | `None` | Port for TWS/IB Gateway (`7497`/`7496` for TWS, `4002`/`4001` for IBG). | +| `ibg_client_id` | `1` | Unique client identifier used when connecting to TWS/IB Gateway. | +| `account_id` | `None` | Interactive Brokers account identifier (falls back to `TWS_ACCOUNT` env var). | +| `dockerized_gateway` | `None` | Optional `DockerizedIBGatewayConfig` for containerized setups. | +| `connection_timeout` | `300` | Seconds to wait for the initial API connection. | +| `request_timeout_secs` | `60` | Seconds to wait for request responses (contract details, etc.). | +| `fetch_all_open_orders` | `False` | When `True`, pulls open orders for every API client ID (not just this session). | +| `track_option_exercise_from_position_update` | `False` | Subscribe to real-time position updates to detect option exercises when `True`. | + +### Execution client configuration + +The `InteractiveBrokersExecutionClient` handles trade execution, order management, account information, and position tracking. It provides order lifecycle management and real-time account updates. + +#### Supported functionality + +- **Order Management**: Place, modify, and cancel orders +- **Order Types**: Market, limit, stop, stop-limit, trailing stop, and more +- **Account Information**: Real-time balance and margin updates +- **Position Tracking**: Real-time position updates and P&L +- **Trade Reporting**: Execution reports and fill notifications +- **Risk Management**: Pre-trade risk checks and position limits + +#### Supported order types + +The adapter supports most Interactive Brokers order types: + +- **Market Orders**: `OrderType.MARKET` +- **Limit Orders**: `OrderType.LIMIT` +- **Stop Orders**: `OrderType.STOP_MARKET` +- **Stop-Limit Orders**: `OrderType.STOP_LIMIT` +- **Market-If-Touched**: `OrderType.MARKET_IF_TOUCHED` +- **Limit-If-Touched**: `OrderType.LIMIT_IF_TOUCHED` +- **Trailing Stop Market**: `OrderType.TRAILING_STOP_MARKET` +- **Trailing Stop Limit**: `OrderType.TRAILING_STOP_LIMIT` +- **Market-on-Close**: `OrderType.MARKET` with `TimeInForce.AT_THE_CLOSE` +- **Limit-on-Close**: `OrderType.LIMIT` with `TimeInForce.AT_THE_CLOSE` + +#### Time in force options + +- **Day Orders**: `TimeInForce.DAY` +- **Good-Till-Canceled**: `TimeInForce.GTC` +- **Immediate-or-Cancel**: `TimeInForce.IOC` +- **Fill-or-Kill**: `TimeInForce.FOK` +- **Good-Till-Date**: `TimeInForce.GTD` +- **At-the-Open**: `TimeInForce.AT_THE_OPEN` +- **At-the-Close**: `TimeInForce.AT_THE_CLOSE` + +#### Batch operations + +| Operation | Supported | Notes | +|--------------------|-----------|----------------------------------------------| +| Batch Submit | ✓ | Submit multiple orders in single request. | +| Batch Modify | ✓ | Modify multiple orders in single request. | +| Batch Cancel | ✓ | Cancel multiple orders in single request. | + +#### Position management + +| Feature | Supported | Notes | +|--------------------|-----------|----------------------------------------------| +| Query positions | ✓ | Real-time position updates. | +| Position mode | ✓ | Net vs separate long/short positions. | +| Leverage control | ✓ | Account-level margin requirements. | +| Margin mode | ✓ | Portfolio vs individual margin. | + +#### Order querying + +| Feature | Supported | Notes | +|--------------------|-----------|----------------------------------------------| +| Query open orders | ✓ | List all active orders. | +| Query order history | ✓ | Historical order data. | +| Order status updates| ✓ | Real-time order state changes. | +| Trade history | ✓ | Execution and fill reports. | + +#### Contingent orders + +| Feature | Supported | Notes | +|--------------------|-----------|----------------------------------------------| +| Order lists | ✓ | Atomic multi-order submission. | +| OCO orders | ✓ | One-Cancels-Other with customizable OCA types (1, 2, 3). | +| Bracket orders | ✓ | Parent-child order relationships. | +| Conditional orders | ✓ | Advanced order conditions and triggers. | + +#### Basic execution client configuration + +```python +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig +from nautilus_trader.config import RoutingConfig + +exec_client_config = InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, # TWS paper trading port + ibg_client_id=1, + account_id="DU123456", # Your IB account ID (paper or live) + instrument_provider=instrument_provider_config, + connection_timeout=300, + routing=RoutingConfig(default=True), # Route all orders through this client +) +``` + +#### Advanced execution client configuration + +```python +# Production configuration with dockerized gateway +production_exec_config = InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=4001, # IB Gateway live trading port + ibg_client_id=1, + account_id=None, # Will use TWS_ACCOUNT environment variable + instrument_provider=instrument_provider_config, + dockerized_gateway=dockerized_gateway_config, + connection_timeout=300, + routing=RoutingConfig(default=True), +) +``` + +#### Account ID configuration + +The `account_id` parameter is crucial and must match the account logged into TWS/Gateway: + +```python +# Option 1: Specify directly in config +exec_config = InteractiveBrokersExecClientConfig( + account_id="DU123456", # Paper trading account + # ... other parameters +) + +# Option 2: Use environment variable +import os +os.environ["TWS_ACCOUNT"] = "DU123456" +exec_config = InteractiveBrokersExecClientConfig( + account_id=None, # Will use TWS_ACCOUNT env var + # ... other parameters +) +``` + +#### Order tags and advanced features + +The adapter supports IB-specific order parameters through order tags: + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags + +# Create order with IB-specific parameters +order_tags = IBOrderTags( + allOrNone=True, # All-or-none order + ocaGroup="MyGroup1", # One-cancels-all group + ocaType=1, # Cancel with block + activeStartTime="20240315 09:30:00 EST", # GTC activation time + activeStopTime="20240315 16:00:00 EST", # GTC deactivation time + goodAfterTime="20240315 09:35:00 EST", # Good after time +) + +# Apply tags to an order +order = order_factory.limit( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(100), + price=instrument.make_price(100.0), + tags=[order_tags.value], +) +``` + +#### OCA (one-cancels-all) orders + +The adapter provides support for OCA orders through explicit configuration using `IBOrderTags`: + +### Basic OCA configuration + +All OCA functionality must be explicitly configured using `IBOrderTags`: + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags + +# Create OCA configuration +oca_tags = IBOrderTags( + ocaGroup="MY_OCA_GROUP", + ocaType=1, # Type 1: Cancel All with Block (recommended) +) + +# Apply to bracket orders +bracket_order = order_factory.bracket( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(100), + tp_price=instrument.make_price(110.0), + sl_trigger_price=instrument.make_price(90.0), + tp_tags=[oca_tags.value], # Must explicitly add OCA tags + sl_tags=[oca_tags.value], # Must explicitly add OCA tags +) +``` + +### Advanced OCA configuration + +You can specify different OCA types and behaviors using `IBOrderTags`: + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags + +# Create custom OCA configuration +custom_oca_tags = IBOrderTags( + ocaGroup="MY_CUSTOM_GROUP", + ocaType=2, # Use Type 2: Reduce with Block +) + +# Apply to individual orders +order = order_factory.limit( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(100), + price=instrument.make_price(100.0), + tags=[custom_oca_tags.value], +) +``` + +### OCA types + +Interactive Brokers supports three OCA types: + +| Type | Name | Behavior | Use Case | +|------|------|----------|----------| +| **1** | Cancel All with Block | Cancel all remaining orders with block protection | **Default** - Safest option, prevents overfills | +| **2** | Reduce with Block | Proportionally reduce remaining orders with block protection | Partial fills with overfill protection | +| **3** | Reduce without Block | Proportionally reduce remaining orders without block protection | Fastest execution, higher overfill risk | + +#### Multiple orders in same OCA group + +```python +# Create multiple orders with the same OCA group +oca_tags = IBOrderTags( + ocaGroup="MULTI_ORDER_GROUP", + ocaType=3, # Use Type 3: Reduce without Block +) + +order1 = order_factory.limit( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(50), + price=instrument.make_price(99.0), + tags=[oca_tags.value], +) + +order2 = order_factory.limit( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(50), + price=instrument.make_price(101.0), + tags=[oca_tags.value], +) +``` + +### OCA configuration requirements + +OCA functionality is **only** available through explicit configuration: + +1. **IBOrderTags Required** - OCA settings must be explicitly specified in order tags +2. **No Automatic Detection** - `ContingencyType.OCO` and `ContingencyType.OUO` do not automatically create OCA groups +3. **Manual Configuration** - All OCA groups and types must be manually specified + +### Conditional orders + +The adapter supports Interactive Brokers conditional orders through the `conditions` parameter in `IBOrderTags`. Conditional orders allow you to specify criteria that must be met before an order is transmitted or cancelled. + +#### Supported condition types + +- **Price Conditions**: Trigger based on price movements of a specific instrument +- **Time Conditions**: Trigger at a specific date and time +- **Volume Conditions**: Trigger based on trading volume thresholds +- **Execution Conditions**: Trigger when trades occur for a specific instrument +- **Margin Conditions**: Trigger based on account margin levels +- **Percent Change Conditions**: Trigger based on percentage price changes + +#### Basic conditional order example + +```python +from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags + +# Create a price condition: trigger when SPY goes above $250 +price_condition = { + "type": "price", + "conId": 265598, # SPY contract ID + "exchange": "SMART", + "isMore": True, # Trigger when price is greater than threshold + "price": 250.00, + "triggerMethod": 0, # Default trigger method + "conjunction": "and", +} + +# Create order tags with condition +order_tags = IBOrderTags( + conditions=[price_condition], + conditionsCancelOrder=False, # Transmit order when condition is met +) + +# Apply to order +order = order_factory.limit( + instrument_id=instrument.id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(100), + price=instrument.make_price(251.00), + tags=[order_tags.value], +) +``` + +#### Multiple conditions with logic + +```python +# Create multiple conditions with AND/OR logic +conditions = [ + { + "type": "price", + "conId": 265598, + "exchange": "SMART", + "isMore": True, + "price": 250.00, + "triggerMethod": 0, + "conjunction": "and", # AND with next condition + }, + { + "type": "time", + "time": "20250315-09:30:00", + "isMore": True, + "conjunction": "or", # OR with next condition + }, + { + "type": "volume", + "conId": 265598, + "exchange": "SMART", + "isMore": True, + "volume": 10000000, + "conjunction": "and", + }, +] + +order_tags = IBOrderTags( + conditions=conditions, + conditionsCancelOrder=False, +) +``` + +#### Condition parameters + +**Price Condition:** + +- `conId`: Contract ID of the instrument to monitor +- `exchange`: Exchange to monitor (e.g., "SMART", "NASDAQ") +- `isMore`: True for >=, False for <= +- `price`: Price threshold +- `triggerMethod`: 0=Default, 1=DoubleBidAsk, 2=Last, 3=DoubleLast, 4=BidAsk, 7=LastBidAsk, 8=MidPoint + +**Time Condition:** + +- `time`: Time string in UTC format "YYYYMMDD-HH:MM:SS" (e.g., "20250315-09:30:00") +- `isMore`: True for after time, False for before time + +**Volume Condition:** + +- `conId`: Contract ID of the instrument to monitor +- `exchange`: Exchange to monitor +- `isMore`: True for >=, False for <= +- `volume`: Volume threshold + +**Execution Condition:** + +- `symbol`: Symbol to monitor for trades +- `secType`: Security type (e.g., "STK", "OPT", "FUT") +- `exchange`: Exchange to monitor + +**Margin Condition:** + +- `percent`: Margin cushion percentage threshold +- `isMore`: True for >=, False for <= + +**Percent Change Condition:** + +- `conId`: Contract ID of the instrument to monitor +- `exchange`: Exchange to monitor +- `isMore`: True for >=, False for <= +- `changePercent`: Percentage change threshold + +#### Complete example: all condition types + +```python +# Example showing all 6 supported condition types +from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags + +# 1. Price Condition - trigger when ES futures > 6000 +price_condition = { + "type": "price", + "conId": 495512563, # ES futures contract ID + "exchange": "CME", + "isMore": True, + "price": 6000.0, + "triggerMethod": 0, + "conjunction": "and", +} + +# 2. Time Condition - trigger at specific time +time_condition = { + "type": "time", + "time": "20250315-09:30:00", # UTC format + "isMore": True, + "conjunction": "and", +} + +# 3. Volume Condition - trigger when volume > 100,000 +volume_condition = { + "type": "volume", + "conId": 495512563, + "exchange": "CME", + "isMore": True, + "volume": 100000, + "conjunction": "and", +} + +# 4. Execution Condition - trigger when SPY trades +execution_condition = { + "type": "execution", + "symbol": "SPY", + "secType": "STK", + "exchange": "SMART", + "conjunction": "and", +} + +# 5. Margin Condition - trigger when margin cushion > 75% +margin_condition = { + "type": "margin", + "percent": 75, + "isMore": True, + "conjunction": "and", +} + +# 6. Percent Change Condition - trigger when price changes > 5% +percent_change_condition = { + "type": "percent_change", + "conId": 495512563, + "exchange": "CME", + "changePercent": 5.0, + "isMore": True, + "conjunction": "and", +} + +# Use any combination of conditions +order_tags = IBOrderTags( + conditions=[price_condition, time_condition], # Multiple conditions + conditionsCancelOrder=False, # Transmit when conditions met +) +``` + +#### Order behavior + +Set `conditionsCancelOrder` to control what happens when conditions are met: + +- `False`: Transmit the order when conditions are satisfied +- `True`: Cancel the order when conditions are satisfied + +#### Implementation notes + +- **All 6 condition types are fully supported** and tested with live Interactive Brokers orders +- **Price conditions** work correctly despite a known bug in the ibapi library where `PriceCondition.__str__` is incorrectly decorated as a property +- **Time conditions** use UTC format with dash separator (`YYYYMMDD-HH:MM:SS`) for reliable parsing +- **Conjunction logic** allows complex condition combinations using "and"/"or" operators + +### Complete trading node configuration + +Setting up a complete trading environment involves configuring a `TradingNodeConfig` with all necessary components. Here are examples for different scenarios. + +#### Paper trading configuration + +```python +import os +from nautilus_trader.adapters.interactive_brokers.common import IB +from nautilus_trader.adapters.interactive_brokers.common import IB_VENUE +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig +from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig +from nautilus_trader.adapters.interactive_brokers.config import IBMarketDataTypeEnum +from nautilus_trader.adapters.interactive_brokers.config import SymbologyMethod +from nautilus_trader.adapters.interactive_brokers.factories import InteractiveBrokersLiveDataClientFactory +from nautilus_trader.adapters.interactive_brokers.factories import InteractiveBrokersLiveExecClientFactory +from nautilus_trader.config import LiveDataEngineConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import RoutingConfig +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.live.node import TradingNode + +# Instrument provider configuration +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + symbology_method=SymbologyMethod.IB_SIMPLIFIED, + load_ids=frozenset([ + "EUR/USD.IDEALPRO", + "GBP/USD.IDEALPRO", + "SPY.ARCA", + "QQQ.NASDAQ", + "AAPL.NASDAQ", + "MSFT.NASDAQ", + ]), +) + +# Data client configuration +data_client_config = InteractiveBrokersDataClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, # TWS paper trading + ibg_client_id=1, + use_regular_trading_hours=True, + market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN, + instrument_provider=instrument_provider_config, +) + +# Execution client configuration +exec_client_config = InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, # TWS paper trading + ibg_client_id=1, + account_id="DU123456", # Your paper trading account + instrument_provider=instrument_provider_config, + routing=RoutingConfig(default=True), +) + +# Trading node configuration +config_node = TradingNodeConfig( + trader_id="PAPER-TRADER-001", + logging=LoggingConfig(log_level="INFO"), + data_clients={IB: data_client_config}, + exec_clients={IB: exec_client_config}, + data_engine=LiveDataEngineConfig( + time_bars_timestamp_on_close=False, # IB standard: use bar open time + validate_data_sequence=True, # Discard out-of-sequence bars + ), + timeout_connection=90.0, + timeout_reconciliation=5.0, + timeout_portfolio=5.0, + timeout_disconnection=5.0, + timeout_post_stop=2.0, +) + +# Create and configure the trading node +node = TradingNode(config=config_node) +node.add_data_client_factory(IB, InteractiveBrokersLiveDataClientFactory) +node.add_exec_client_factory(IB, InteractiveBrokersLiveExecClientFactory) +node.build() + +if __name__ == "__main__": + try: + node.run() + finally: + node.dispose() +``` + +## Live trading with Dockerized gateway + +```python +from nautilus_trader.adapters.interactive_brokers.config import DockerizedIBGatewayConfig + +# Dockerized gateway configuration +dockerized_gateway_config = DockerizedIBGatewayConfig( + username=os.environ.get("TWS_USERNAME"), + password=os.environ.get("TWS_PASSWORD"), + trading_mode="live", # "paper" or "live" + read_only_api=False, # Allow order execution + timeout=300, +) + +# Data client with dockerized gateway +data_client_config = InteractiveBrokersDataClientConfig( + ibg_client_id=1, + use_regular_trading_hours=False, # Include extended hours + market_data_type=IBMarketDataTypeEnum.REALTIME, + instrument_provider=instrument_provider_config, + dockerized_gateway=dockerized_gateway_config, +) + +# Execution client with dockerized gateway +exec_client_config = InteractiveBrokersExecClientConfig( + ibg_client_id=1, + account_id=os.environ.get("TWS_ACCOUNT"), # Live account ID + instrument_provider=instrument_provider_config, + dockerized_gateway=dockerized_gateway_config, + routing=RoutingConfig(default=True), +) + +# Live trading node configuration +config_node = TradingNodeConfig( + trader_id="LIVE-TRADER-001", + logging=LoggingConfig(log_level="INFO"), + data_clients={IB: data_client_config}, + exec_clients={IB: exec_client_config}, + data_engine=LiveDataEngineConfig( + time_bars_timestamp_on_close=False, + validate_data_sequence=True, + ), +) +``` + +### Multi-client configuration + +For advanced setups, you can configure multiple clients with different purposes: + +```python +# Separate data and execution clients with different client IDs +data_client_config = InteractiveBrokersDataClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=1, # Data client uses ID 1 + market_data_type=IBMarketDataTypeEnum.REALTIME, + instrument_provider=instrument_provider_config, +) + +exec_client_config = InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=2, # Execution client uses ID 2 + account_id="DU123456", + instrument_provider=instrument_provider_config, + routing=RoutingConfig(default=True), +) +``` + +### Multiple IB execution clients for different accounts + +NautilusTrader supports using multiple Interactive Brokers execution clients simultaneously, each connected to a different IB account. This is useful when you need to trade with multiple accounts, such as: + +- Separate accounts for different strategies +- Paper trading and live trading accounts running simultaneously +- Multiple managed accounts under the same IB login + +To configure multiple IB execution clients, provide multiple entries in the `exec_clients` dictionary with unique keys. Each entry specifies a different `account_id`: + +```python +from nautilus_trader.adapters.interactive_brokers.config import ( + InteractiveBrokersDataClientConfig, + InteractiveBrokersExecClientConfig, + InteractiveBrokersInstrumentProviderConfig, + SymbologyMethod, + IBMarketDataTypeEnum, +) +from nautilus_trader.live.config import TradingNodeConfig, RoutingConfig, LoggingConfig +from nautilus_trader.model.identifiers import AccountId, Venue, ClientId + +# Shared instrument provider configuration +instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( + symbology_method=SymbologyMethod.IB_SIMPLIFIED, +) + +# Data client (shared across all accounts) +data_client_config = InteractiveBrokersDataClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=1, + market_data_type=IBMarketDataTypeEnum.REALTIME, + instrument_provider=instrument_provider_config, +) + +# Configuration for multiple IB execution clients +config_node = TradingNodeConfig( + trader_id="MULTI-ACCOUNT-001", + logging=LoggingConfig(log_level="INFO"), + + # Single data client shared across accounts + data_clients={ + "IB": data_client_config, + }, + + # Multiple execution clients, one per account + exec_clients={ + # First account: Paper trading account + "IB-PAPER": InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=2, # Unique IB API client ID + account_id="DU123456", # Paper trading account ID + instrument_provider=instrument_provider_config, + routing=RoutingConfig(default=False), # Not default + ), + + # Second account: Live trading account + "IB-LIVE": InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=3, # Unique IB API client ID + account_id="U987654", # Live account ID + instrument_provider=instrument_provider_config, + routing=RoutingConfig(default=True), # Set as default + ), + + # Third account: Another managed account + "IB-ACCOUNT3": InteractiveBrokersExecClientConfig( + ibg_host="127.0.0.1", + ibg_port=7497, + ibg_client_id=4, # Unique IB API client ID + account_id="U456789", # Another account ID + instrument_provider=instrument_provider_config, + routing=RoutingConfig(default=False), + ), + }, +) +``` + +**Key points for multiple IB execution clients:** + +1. **Unique keys**: Each entry in `exec_clients` must have a unique key (e.g., `"IB-PAPER"`, `"IB-LIVE"`). This key becomes the `account_issuer` for that client. + +2. **Unique client IDs**: Each execution client must use a different `ibg_client_id` (2, 3, 4, etc.). IB Gateway/TWS requires each API connection to use a unique client ID. + +3. **Account ID**: Each execution client must specify a different `account_id` matching the account logged into IB Gateway/TWS. + +4. **Account identifiers**: The system creates `AccountId` instances like: + - `AccountId("IB-PAPER-DU123456")` + - `AccountId("IB-LIVE-U987654")` + - `AccountId("IB-ACCOUNT3-U456789")` + +5. **Routing**: Orders and queries are automatically routed to the correct execution client based on: + - Explicit `client_id` in the command + - `account_id` issuer (for `QueryAccount` commands or orders with account_id set) + - Default client (if one is marked with `routing=RoutingConfig(default=True)`) + +6. **Portfolio queries**: When querying portfolio properties, you can specify either: + - `account_id` for account-specific queries: `portfolio.realized_pnls(account_id=AccountId("IB-PAPER-DU123456"))` + - `venue` for aggregated queries across all accounts with that venue: `portfolio.realized_pnls(venue=Venue("IB-PAPER"))` + +**Example: Using multiple IB execution clients in a strategy:** + +```python +from nautilus_trader.model.identifiers import AccountId, ClientId +from nautilus_trader.trading.strategy import Strategy + +class MultiAccountStrategy(Strategy): + """Example strategy using multiple IB accounts.""" + + def on_start(self): + # Define account IDs for easy reference + self.paper_account = AccountId("IB-PAPER-DU123456") + self.live_account = AccountId("IB-LIVE-U987654") + + # Query paper account balance + paper_account_state = self.cache.account(self.paper_account) + if paper_account_state: + self.log.info(f"Paper account balance: {paper_account_state.balance_total()}") + + # Query live account balance + live_account_state = self.cache.account(self.live_account) + if live_account_state: + self.log.info(f"Live account balance: {live_account_state.balance_total()}") + + def submit_order_to_paper(self, order): + """Submit order to paper trading account.""" + self.submit_order(order, client_id=ClientId("IB-PAPER")) + + def submit_order_to_live(self, order): + """Submit order to live trading account.""" + self.submit_order(order, client_id=ClientId("IB-LIVE")) + + def check_paper_pnl(self, instrument_id): + """Check realized PnL for paper account.""" + pnl = self.portfolio.realized_pnl( + instrument_id=instrument_id, + account_id=self.paper_account + ) + return pnl + + def check_live_pnl(self, instrument_id): + """Check realized PnL for live account.""" + pnl = self.portfolio.realized_pnl( + instrument_id=instrument_id, + account_id=self.live_account + ) + return pnl +``` + +**Example: Querying account information with multiple IB clients:** + +```python +from nautilus_trader.model.identifiers import AccountId + +# Query specific account +paper_account = cache.account(AccountId("IB-PAPER-DU123456")) +live_account = cache.account(AccountId("IB-LIVE-U987654")) + +# Query account using account_id (preferred method) +paper_account_by_id = cache.account(AccountId("IB-PAPER-DU123456")) + +# Alternative: Query account using account_id parameter (also works) +paper_account_via_account_id = cache.account_for_venue( + account_id=AccountId("IB-PAPER-DU123456") +) + +# Query portfolio properties by account +paper_realized_pnl = portfolio.realized_pnl( + instrument_id=instrument_id, + account_id=AccountId("IB-PAPER-DU123456") +) + +# Query portfolio properties aggregated across all IB accounts +# Note: This aggregates across all accounts with the same venue +all_ib_realized_pnl = portfolio.realized_pnls(venue=Venue("IB")) +``` + +### Running the trading node + +```python +def run_trading_node(): + """Run the trading node with proper error handling.""" + node = None + try: + # Create and build node + node = TradingNode(config=config_node) + node.add_data_client_factory(IB, InteractiveBrokersLiveDataClientFactory) + node.add_exec_client_factory(IB, InteractiveBrokersLiveExecClientFactory) + node.build() + + # Add your strategies here + # node.trader.add_strategy(YourStrategy()) + + # Run the node + node.run() + + except KeyboardInterrupt: + print("Shutting down...") + except Exception as e: + print(f"Error: {e}") + finally: + if node: + node.dispose() + +if __name__ == "__main__": + run_trading_node() +``` + +### Additional configuration options + +#### Environment variables + +Set these environment variables for easier configuration: + +```bash +export TWS_USERNAME="your_ib_username" +export TWS_PASSWORD="your_ib_password" +export TWS_ACCOUNT="your_account_id" +export IB_MAX_CONNECTION_ATTEMPTS="5" # Optional: limit reconnection attempts +``` + +#### Logging configuration + +```python +# Enhanced logging configuration +logging_config = LoggingConfig( + log_level="INFO", + log_level_file="DEBUG", + log_file_format="json", # JSON format for structured logging + log_component_levels={ + "InteractiveBrokersClient": "DEBUG", + "InteractiveBrokersDataClient": "INFO", + "InteractiveBrokersExecutionClient": "INFO", + }, +) +``` + +You can find additional examples here: + +## Troubleshooting + +### Common connection issues + +#### Connection refused + +- **Cause**: TWS/Gateway not running or wrong port +- **Solution**: Verify TWS/Gateway is running and check port configuration +- **Default Ports**: TWS (7497/7496), IB Gateway (4002/4001) + +#### Authentication errors + +- **Cause**: Incorrect credentials or account not logged in +- **Solution**: Verify username/password and ensure account is logged into TWS/Gateway + +#### Client ID conflicts + +- **Cause**: Multiple clients using the same client ID +- **Solution**: Use unique client IDs for each connection + +#### Market data permissions + +- **Cause**: Insufficient market data subscriptions +- **Solution**: Use `IBMarketDataTypeEnum.DELAYED_FROZEN` for testing or subscribe to required data feeds + +### Error codes + +Interactive Brokers uses specific error codes. Common ones include: + +- **200**: No security definition found +- **201**: Order rejected - reason follows +- **202**: Order cancelled +- **300**: Can't find EId with ticker ID +- **354**: Requested market data is not subscribed +- **2104**: Market data farm connection is OK +- **2106**: HMDS data farm connection is OK + +### Performance optimization + +#### Reduce data volume + +```python +# Reduce quote tick volume by ignoring size-only updates +data_config = InteractiveBrokersDataClientConfig( + ignore_quote_tick_size_updates=True, + # ... other config +) +``` + +#### Connection management + +```python +# Set reasonable timeouts +config = InteractiveBrokersDataClientConfig( + connection_timeout=300, # 5 minutes + request_timeout_secs=60, # 1 minute + # ... other config +) +``` + +#### Memory management + +- Use appropriate bar sizes for your strategy +- Limit the number of simultaneous subscriptions +- Consider using historical data for backtesting instead of live data + +### Best practices + +#### Security + +- Never hardcode credentials in source code +- Use environment variables for sensitive information +- Use paper trading for development and testing +- Set `read_only_api=True` for data-only applications + +#### Development workflow + +1. **Start with Paper Trading**: Always test with paper trading first +2. **Use Delayed Data**: Use `DELAYED_FROZEN` market data for development +3. **Implement Proper Error Handling**: Handle connection losses and API errors gracefully +4. **Monitor Logs**: Enable appropriate logging levels for debugging +5. **Test Reconnection**: Test your strategy's behavior during connection interruptions + +#### Production deployment + +- Use dockerized gateway for automated deployments +- Implement proper monitoring and alerting +- Set up log aggregation and analysis +- Use real-time data subscriptions only when necessary +- Implement circuit breakers and position limits + +#### Order management + +- Always validate orders before submission +- Implement proper position sizing +- Use appropriate order types for your strategy +- Monitor order status and handle rejections +- Implement timeout handling for order operations + +### Debugging tips + +#### Enable debug logging + +```python +logging_config = LoggingConfig( + log_level="DEBUG", + log_component_levels={ + "InteractiveBrokersClient": "DEBUG", + }, +) +``` + +#### Monitor connection status + +```python +# Check connection status in your strategy +if not self.data_client.is_connected: + self.log.warning("Data client disconnected") +``` + +#### Validate instruments + +```python +# Ensure instruments are loaded before trading +instruments = self.cache.instruments() +if not instruments: + self.log.error("No instruments loaded") +``` + +### Support and resources + +- **IB API Documentation**: [TWS API Guide](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/) +- **NautilusTrader Examples**: [GitHub Examples](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/interactive_brokers) +- **IB Contract Search**: [Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/) +- **Market Data Subscriptions**: [IB Market Data](https://www.interactivebrokers.com/en/pricing/market-data-pricing.php) + +## Contributing + +:::info +For additional features or to contribute to the Interactive Brokers adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/index.md b/nautilus-docs/docs/integrations/index.md new file mode 100644 index 0000000..6f89cb8 --- /dev/null +++ b/nautilus-docs/docs/integrations/index.md @@ -0,0 +1,60 @@ +# Integrations + +NautilusTrader uses modular *adapters* to connect to trading venues and data providers, translating raw APIs into a unified interface and normalized domain model. + +The following integrations are currently supported: + +| Name | ID | Type | Status | Docs | +| :--------------------------------------------------------------------------- | :-------------------- | :---------------------- | :------------------------------------------------------ | :------------------------ | +| [AX Exchange](https://architect.exchange) | `AX` | Perpetuals Exchange | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](architect_ax.md) | +| [Architect](https://architect.co) | `ARCHITECT` | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/planned-gray) | - | +| [Betfair](https://betfair.com) | `BETFAIR` | Sports Betting Exchange | ![status](https://img.shields.io/badge/stable-green) | [Guide](betfair.md) | +| [Binance](https://binance.com) | `BINANCE` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](binance.md) | +| [BitMEX](https://www.bitmex.com) | `BITMEX` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](bitmex.md) | +| [Bybit](https://www.bybit.com) | `BYBIT` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](bybit.md) | +| [Databento](https://databento.com) | `DATABENTO` | Data Provider | ![status](https://img.shields.io/badge/stable-green) | [Guide](databento.md) | +| [Deribit](https://www.deribit.com) | `DERIBIT` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](deribit.md) | +| [dYdX](https://dydx.exchange/) | `DYDX` | Crypto Exchange (DEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](dydx.md) | +| [Hyperliquid](https://hyperliquid.xyz) | `HYPERLIQUID` | Crypto Exchange (DEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](hyperliquid.md) | +| [Interactive Brokers](https://www.interactivebrokers.com) | `INTERACTIVE_BROKERS` | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/stable-green) | [Guide](ib.md) | +| [Kraken](https://kraken.com) | `KRAKEN` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](kraken.md) | +| [OKX](https://okx.com) | `OKX` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](okx.md) | +| [Polymarket](https://polymarket.com) | `POLYMARKET` | Prediction Market (DEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](polymarket.md) | +| [Tardis](https://tardis.dev) | `TARDIS` | Crypto Data Provider | ![status](https://img.shields.io/badge/stable-green) | [Guide](tardis.md) | + +- **ID**: The default client ID for the integrations adapter clients. +- **Type**: The type of integration (often the venue type). + +## Status + +- `planned`: Planned for future development. +- `building`: Under construction and likely not in a usable state. +- `beta`: Completed to a minimally working state and in a 'beta' testing phase. +- `stable`: Stabilized feature set and API, the integration has been tested by both developers and users to a reasonable level (some bugs may still remain). + +## Implementation goals + +The primary goal of NautilusTrader is to provide a unified trading system for +use with a variety of integrations. To support the widest range of trading +strategies, priority will be given to *standard* functionality: + +- Requesting historical market data. +- Streaming live market data. +- Reconciling execution state. +- Submitting standard order types with standard execution instructions. +- Modifying existing orders (if possible on an exchange). +- Canceling orders. + +The implementation of each integration aims to meet the following criteria: + +- Low-level client components should match the exchange API as closely as possible. +- The full range of an exchange's functionality (where applicable to NautilusTrader) should *eventually* be supported. +- Exchange specific data types will be added to support the functionality and return types which are reasonably expected by a user. +- Actions unsupported by an exchange or NautilusTrader will be logged as a warning or error when invoked. + +## API unification + +All integrations must conform to NautilusTrader’s system API, requiring normalization and standardization: + +- Symbols should use the venue’s native symbol format unless disambiguation is required (e.g., Binance Spot vs. Binance Futures). +- Timestamps must use UNIX epoch nanoseconds. If milliseconds are used, field/property names should explicitly end with `_ms`. diff --git a/nautilus-docs/docs/integrations/kraken.md b/nautilus-docs/docs/integrations/kraken.md new file mode 100644 index 0000000..8d55c10 --- /dev/null +++ b/nautilus-docs/docs/integrations/kraken.md @@ -0,0 +1,555 @@ +# Kraken + +Founded in 2011, Kraken is one of the most established cryptocurrency exchanges +globally and the largest exchange in Europe by euro trading volume. The platform +offers spot and derivatives trading across a wide range of digital assets. This +integration connects to Kraken Pro and supports live market data ingest and order +execution for both Kraken Spot and Kraken Derivatives (Futures) markets. + +## Overview + +This adapter is implemented in Rust with Python bindings for ease of use in +Python-based workflows. It does not require external Kraken client libraries; the +core components are compiled as a static library and linked automatically during +the build. + +This guide assumes a trader is setting up for both live market data feeds and +trade execution. The Kraken adapter includes multiple components, which can be +used together or separately depending on the use case. + +- `KrakenRawHttpClient`: Low-level HTTP API connectivity for Spot and Futures. +- `KrakenHttpClient`: Higher-level HTTP client with instrument caching and reconciliation support. +- `KrakenInstrumentProvider`: Instrument parsing and loading functionality. +- `KrakenDataClient`: Market data feed manager. +- `KrakenExecutionClient`: Account management and trade execution gateway. +- `KrakenLiveDataClientFactory`: Factory for Kraken data clients (used by the + trading node builder). +- `KrakenLiveExecClientFactory`: Factory for Kraken execution clients (used by + the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), and +won't need to work directly with these lower-level components. +::: + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/kraken/). + +## Kraken documentation + +Kraken provides extensive documentation for users: + +- [Kraken API Documentation](https://docs.kraken.com/api/) +- [Kraken Spot REST API](https://docs.kraken.com/api/docs/guides/spot-rest-intro) +- [Kraken Futures REST API](https://docs.kraken.com/api/docs/futures-api) + +Refer to the Kraken documentation in conjunction with this NautilusTrader +integration guide. + +## Products + +Kraken supports two primary product categories: + +| Product Type | Supported | Notes | +|--------------------------|-----------|-----------------------------------------------------------| +| Spot | ✓ | Standard cryptocurrency pairs with margin support. | +| Futures (Perpetual) | ✓ | Inverse (`PI_`) and USD-margined (`PF_`) perpetual swaps. | +| Futures (Dated/Flex) | ✓ | Fixed maturity (`FI_`) and flex (`FF_`) contracts. | + +:::note +**Dual-product deployments**: When both `SPOT` and `FUTURES` product types are +configured, the adapter queries both APIs and merges the account states. This +ensures the execution engine has visibility into collateral across both markets. +::: + +## Bar streaming + +### Supported intervals + +The Kraken adapter supports real-time bar (OHLC) streaming for Spot markets via +WebSocket. The following intervals are available: + +| Interval | BarType specification | +|------------|-----------------------| +| 1 minute | `1-MINUTE-LAST` | +| 5 minutes | `5-MINUTE-LAST` | +| 15 minutes | `15-MINUTE-LAST` | +| 30 minutes | `30-MINUTE-LAST` | +| 1 hour | `1-HOUR-LAST` | +| 4 hours | `4-HOUR-LAST` | +| 1 day | `1-DAY-LAST` | +| 1 week | `1-WEEK-LAST` | +| 15 days | `15-DAY-LAST` | + +:::note +**Futures limitation**: Kraken Futures does not support bar streaming via +WebSocket. Use `request_bars()` for historical bar data instead. +::: + +### Bar emission latency + +Kraken's WebSocket OHLC channel pushes updates for the *current* (incomplete) +bar on every trade. Unlike some exchanges (e.g., Binance), Kraken does not +provide an "is_closed" indicator to signal when a bar is complete. + +To avoid emitting partial/incomplete bars, the adapter buffers the current bar +and only emits it when the next bar period begins (i.e., when a message with a +new `interval_begin` timestamp arrives). This means: + +- Bars are emitted with a delay of up to one bar period. +- For 1-minute bars, the maximum delay is ~1 minute. +- The emitted bar data is complete and final. + +We chose this approach over timer-based emission because: + +- Timer-based emission could miss the final update before the bar closes. +- Kraken's updates are not guaranteed to arrive at exact interval boundaries. +- Buffering ensures data integrity at the cost of latency. + +:::warning +If bar latency matters for your strategy, consider using trade tick data +and aggregating bars locally with `BarAggregator`. +::: + +:::tip +For most use cases, we recommend using `INTERNAL` bar aggregation (subscribing to +trades and aggregating bars locally) rather than `EXTERNAL` exchange-provided bars: + +- Bars are emitted immediately when complete, with no buffering delay. +- Consistent behavior across all exchanges, simplifying multi-venue strategies. + +::: + +## Symbology + +### Bitcoin symbol format (BTC vs XBT) + +Kraken uses different Bitcoin symbol conventions across their APIs: + +| Market | Symbol Format | Example | Notes | +|---------|---------------|--------------------|---------------------------------------------| +| Spot | `BTC` | `BTC/USD.KRAKEN` | Adapter normalizes XBT → BTC at load time. | +| Futures | `XBT` | `PI_XBTUSD.KRAKEN` | Uses Kraken's native XBT format. | + +:::note +Kraken's REST API returns `XBT` for Bitcoin (following ISO 4217 conventions for +supranational currencies), but their WebSocket v2 API requires the `BTC` format. +The adapter automatically normalizes spot symbols to `BTC` when loading instruments, +whether XBT appears as the base currency (e.g., `XBT/USD` → `BTC/USD`) or quote +currency (e.g., `ETH/XBT` → `ETH/BTC`). Futures retain Kraken's native `XBT` format. +::: + +### Spot markets + +NautilusTrader uses ISO 4217-A3 format for Kraken Spot instrument symbols, +which provides a standardized representation across exchanges. The adapter +handles translation to Kraken's native format internally. + +**Instrument ID format:** + +```python +InstrumentId.from_str("BTC/USD.KRAKEN") # Spot BTC/USD +InstrumentId.from_str("ETH/USD.KRAKEN") # Spot ETH/USD +InstrumentId.from_str("SOL/USD.KRAKEN") # Spot SOL/USD +InstrumentId.from_str("BTC/USDT.KRAKEN") # Spot BTC/USDT +InstrumentId.from_str("ETH/BTC.KRAKEN") # Spot ETH/BTC (normalized from ETH/XBT) +``` + +### Futures markets + +Kraken Futures instruments use a specific naming convention with prefixes: + +- `PI_` - Perpetual Inverse contracts (e.g., `PI_XBTUSD`) +- `PF_` - Perpetual Fixed-margin contracts (e.g., `PF_XBTUSD`) +- `FI_` - Fixed maturity Inverse contracts (e.g., `FI_XBTUSD_230929`) +- `FF_` - Flex futures contracts + +**Instrument ID format:** + +```python +InstrumentId.from_str("PI_XBTUSD.KRAKEN") # Perpetual inverse BTC +InstrumentId.from_str("PI_ETHUSD.KRAKEN") # Perpetual inverse ETH +InstrumentId.from_str("PF_XBTUSD.KRAKEN") # Perpetual fixed-margin BTC +``` + +## Orders capability + +### Order types + +| Order Type | Spot | Futures | Notes | +|------------------------|------|---------|--------------------------------------------| +| `MARKET` | ✓ | ✓ | Immediate execution at market price. | +| `LIMIT` | ✓ | ✓ | Execution at specified price or better. | +| `STOP_MARKET` | ✓ | ✓ | Conditional market order (stop-loss). | +| `MARKET_IF_TOUCHED` | ✓ | ✓ | Conditional market order (take-profit). | +| `STOP_LIMIT` | ✓ | ✓ | Conditional limit order (stop-loss-limit). | +| `LIMIT_IF_TOUCHED` | ✓ | - | *Futures: not yet implemented*. | + +### Time in force + +| Time in Force | Spot | Futures | Notes | +|---------------|------|---------|-----------------------------------------------------| +| `GTC` | ✓ | ✓ | Good Till Canceled. | +| `GTD` | ✓ | - | Good Till Date (Spot only, requires `expire_time`). | +| `IOC` | ✓ | ✓ | Immediate or Cancel. | +| `FOK` | - | - | *Not supported by Kraken*. | + +:::note +**Market orders** are inherently immediate and do not support time-in-force. +`IOC` only applies to limit-type orders. +::: + +### Execution instructions + +| Instruction | Spot | Futures | Notes | +|---------------|------|---------|---------------------------------------------| +| `post_only` | ✓ | ✓ | Available for limit orders. | +| `reduce_only` | - | ✓ | Futures only. Reduces position, no reversal.| + +### Batch operations + +| Operation | Spot | Futures | Notes | +|--------------------|------|---------|----------------------------------------------| +| Batch Submit | - | - | *Not yet implemented*. | +| Batch Modify | - | - | *Not yet implemented* (Futures only). | +| Batch Cancel | ✓ | ✓ | Auto-chunks into batches of 50. | + +:::note +**Cancel all orders**: + +- Order side filtering is not supported; all orders are canceled regardless of side. +- Spot: Cancels all open orders across all symbols. +- Futures: Requires an `instrument_id`; cancels orders for that symbol only. + +::: + +### Position management + +| Feature | Spot | Futures | Notes | +|-------------------|------|---------|-----------------------------------------------------------| +| Query positions | ✓* | ✓ | *Spot: opt-in via `use_spot_position_reports`. See below. | +| Position mode | - | - | Single position per instrument. | +| Leverage control | - | ✓ | Configured per account tier. | +| Margin mode | - | ✓ | Cross margin for Futures. | + +### Order querying + +| Feature | Spot | Futures | Notes | +|----------------------|------|---------|----------------------------------------------| +| Query open orders | ✓ | ✓ | List all active orders. | +| Query order history | ✓ | ✓ | Historical order data with pagination. | +| Order status updates | ✓ | ✓ | Real-time order state changes via WebSocket. | +| Trade history | ✓ | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Spot | Futures | Notes | +|---------------------|------|---------|------------------------------------------| +| Order lists | - | - | *Not supported*. | +| OCO orders | - | - | *Not supported*. | +| Bracket orders | - | - | *Not supported*. | +| Conditional orders | ✓ | ✓ | Stop and take-profit orders. | + +## Reconciliation + +The Kraken adapter provides reconciliation capabilities for both +Spot and Futures markets, allowing traders to synchronize their local state with +the exchange state at startup or during operation. + +### Spot reconciliation + +**Order status reports:** + +- Open orders: Fetches all currently active orders. +- Closed orders: Fetches historical orders with pagination support. +- Time-bounded queries: Supports filtering by start/end timestamps. + +**Fill reports:** + +- Trade history: Fetches execution history with pagination. +- Time-bounded queries: Supports filtering by start/end timestamps. +- All fill types: Market, limit, and conditional order fills. + +### Futures reconciliation + +**Order status reports:** + +- Open orders: Fetches all currently active futures orders. +- Historical orders: Fetches closed and filled orders when `open_only=False`. +- Order events: Full order lifecycle history via `/api/history/v2/orders` + endpoint. + +**Fill reports:** + +- Fill history: Fetches all execution reports. +- Time filtering: Client-side filtering by start/end timestamps (parses + RFC3339 timestamps). +- All fill types: Maker and taker fills with fee information. + +**Position status reports:** + +- Open positions: Fetches all active futures positions. +- Real-time data: Includes unrealized funding, average price, and position size. + +:::note +**Futures time filtering**: The Kraken Futures fills endpoint does not support +server-side time range filtering. The adapter implements client-side filtering +by parsing `fillTime` fields and comparing against requested start/end +timestamps. +::: + +### Spot position reports + +The Kraken adapter can optionally report wallet balances as position status +reports for spot instruments. This feature is disabled by default and must be +explicitly enabled via configuration. + +**How it works:** + +- When enabled, wallet balances are converted to `PositionStatusReport` objects. +- Positive balances are reported as `LONG` positions. +- Only instruments matching the configured quote currency are reported (default: `USDT`). +- This prevents duplicate reports when the same asset is available with multiple quote currencies (e.g., BTC/USD, BTC/USDT, BTC/EUR). + +**Configuration:** + +```python +exec_clients={ + KRAKEN: { + "use_spot_position_reports": True, + "spot_positions_quote_currency": "USDT", # Default + }, +} +``` + +:::warning +**Use with caution**: Enabling spot position reports may lead to unintended +behavior if your strategy is not designed to handle spot positions. For example, +a strategy that expects to close positions may attempt to sell your wallet +holdings. +::: + +## Funding rates + +The adapter receives funding rate data from the +[Ticker](https://docs.kraken.com/api/docs/futures-api/websocket/ticker) +WebSocket feed, which provides `relative_funding_rate` and `next_funding_rate_time` for +perpetual futures. + +The `interval` field on `FundingRateUpdate` is `None` for Kraken because the ticker feed +does not include a funding interval field and the Kraken API documentation does not +specify a fixed funding period. + +## Rate limiting + +The adapter implements automatic rate limiting to comply with Kraken's API requirements. + +| Endpoint Type | Limit (requests/sec) | Notes | +|-----------------------|----------------------|--------------------------------------| +| Spot REST (global) | 5 | Global rate limit for Spot API. | +| Futures REST (global) | 5 | Global rate limit for Futures API. | + +:::info +Kraken uses a counter-based rate limiting system with tier-dependent limits: + +- **Starter tier**: 15 max counter, -0.33/sec decay +- **Intermediate tier**: 20 max counter, -0.5/sec decay +- **Pro tier**: 20 max counter, -1/sec decay + +Ledger/trade history calls add +2 to the counter; other calls add +1. +::: + +:::warning +Kraken may temporarily block IP addresses that exceed rate limits. The adapter +automatically queues requests when limits are approached. +::: + +## Configuration + +The product types for each client must be specified in the configurations. + +### Data client configuration options + +| Option | Default | Description | +|---------------------------------|-----------|-------------------------------------------------------------------------| +| `api_key` | `None` | API key; loaded from environment variables (see below) when omitted. | +| `api_secret` | `None` | API secret; loaded from environment variables (see below) when omitted. | +| `environment` | `mainnet` | Trading environment (`mainnet` or `demo`); demo only for Futures. | +| `product_types` | `(SPOT,)` | Product types tuple (e.g., `(KrakenProductType.SPOT,)`). | +| `base_url_http_spot` | `None` | Override for Kraken Spot REST base URL. | +| `base_url_http_futures` | `None` | Override for Kraken Futures REST base URL. | +| `base_url_ws_spot` | `None` | Override for Kraken Spot WebSocket URL. | +| `base_url_ws_futures` | `None` | Override for Kraken Futures WebSocket URL. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | WebSocket proxy URL (*not yet implemented*). | +| `update_instruments_interval_mins` | `60` | Interval (minutes) to reload instruments; `None` to disable. | +| `max_retries` | `None` | Maximum retry attempts for REST requests. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | +| `http_timeout_secs` | `None` | HTTP request timeout in seconds. | +| `ws_heartbeat_secs` | `30` | WebSocket heartbeat interval in seconds. | +| `max_requests_per_second` | `None` | Override rate limit (default 5 req/s); for higher tier accounts. | + +### Execution client configuration options + +| Option | Default | Description | +|---------------------------------|-----------|-------------------------------------------------------------------------| +| `api_key` | `None` | API key; loaded from environment variables (see below) when omitted. | +| `api_secret` | `None` | API secret; loaded from environment variables (see below) when omitted. | +| `environment` | `mainnet` | Trading environment (`mainnet` or `demo`); demo only for Futures. | +| `product_types` | `(SPOT,)` | Product types tuple; `SPOT` uses CASH, `FUTURES` uses MARGIN account. | +| `base_url_http_spot` | `None` | Override for Kraken Spot REST base URL. | +| `base_url_http_futures` | `None` | Override for Kraken Futures REST base URL. | +| `base_url_ws_spot` | `None` | Override for Kraken Spot WebSocket URL. | +| `base_url_ws_futures` | `None` | Override for Kraken Futures WebSocket URL. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | WebSocket proxy URL (*not yet implemented*). | +| `max_retries` | `None` | Maximum retry attempts for order submission/cancel calls. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | +| `http_timeout_secs` | `None` | HTTP request timeout in seconds. | +| `ws_heartbeat_secs` | `30` | WebSocket heartbeat interval in seconds. | +| `max_requests_per_second` | `None` | Override rate limit (default 5 req/s); for higher tier accounts. | +| `use_spot_position_reports` | `False` | Report wallet balances as positions (see below). | +| `spot_positions_quote_currency` | `"USDT"` | Quote currency filter for spot position reports. | + +### Demo environment setup + +To test with Kraken Futures demo (paper trading): + +1. Sign up at [https://demo-futures.kraken.com](https://demo-futures.kraken.com) + and generate API credentials. +2. Set environment variables with your demo credentials: + - `KRAKEN_FUTURES_DEMO_API_KEY` + - `KRAKEN_FUTURES_DEMO_API_SECRET` +3. Configure the adapter with `environment=KrakenEnvironment.DEMO` and + `product_types=(KrakenProductType.FUTURES,)`. + +```python +from nautilus_trader.adapters.kraken import KRAKEN +from nautilus_trader.adapters.kraken import KrakenEnvironment +from nautilus_trader.adapters.kraken import KrakenProductType + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + KRAKEN: { + "environment": KrakenEnvironment.DEMO, + "product_types": (KrakenProductType.FUTURES,), + }, + }, + exec_clients={ + KRAKEN: { + "environment": KrakenEnvironment.DEMO, + "product_types": (KrakenProductType.FUTURES,), + }, + }, +) +``` + +### Production configuration + +The most common use case is to configure a live `TradingNode` to include Kraken +data and execution clients. Add a `KRAKEN` section to your client +configuration(s): + +```python +from nautilus_trader.adapters.kraken import KRAKEN +from nautilus_trader.adapters.kraken import KrakenEnvironment +from nautilus_trader.adapters.kraken import KrakenProductType +from nautilus_trader.live.node import TradingNode + +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + KRAKEN: { + "environment": KrakenEnvironment.MAINNET, + "product_types": (KrakenProductType.SPOT,), + }, + }, + exec_clients={ + KRAKEN: { + "environment": KrakenEnvironment.MAINNET, + "product_types": (KrakenProductType.SPOT,), + }, + }, +) +``` + +### Dual-product configuration (Spot + Futures) + +When trading both Spot and Futures markets, include both product types: + +```python +config = TradingNodeConfig( + ..., # Omitted + data_clients={ + KRAKEN: { + "environment": KrakenEnvironment.MAINNET, + "product_types": (KrakenProductType.SPOT, KrakenProductType.FUTURES), + }, + }, + exec_clients={ + KRAKEN: { + "environment": KrakenEnvironment.MAINNET, + "product_types": (KrakenProductType.SPOT, KrakenProductType.FUTURES), + }, + }, +) +``` + +Then, create a `TradingNode` and add the client factories: + +```python +from nautilus_trader.adapters.kraken import KRAKEN +from nautilus_trader.adapters.kraken import KrakenLiveDataClientFactory +from nautilus_trader.adapters.kraken import KrakenLiveExecClientFactory +from nautilus_trader.live.node import TradingNode + +# Instantiate the live trading node with a configuration +node = TradingNode(config=config) + +# Register the client factories with the node +node.add_data_client_factory(KRAKEN, KrakenLiveDataClientFactory) +node.add_exec_client_factory(KRAKEN, KrakenLiveExecClientFactory) + +# Finally build the node +node.build() +``` + +### API credentials + +There are two options for supplying your credentials to the Kraken clients. +Either pass the corresponding `api_key` and `api_secret` values to the +configuration objects, or set the following environment variables: + +| Environment Variable | Description | +|----------------------------------|------------------------------------------| +| `KRAKEN_SPOT_API_KEY` | API key for Kraken Spot (mainnet). | +| `KRAKEN_SPOT_API_SECRET` | API secret for Kraken Spot (mainnet). | +| `KRAKEN_FUTURES_API_KEY` | API key for Kraken Futures (mainnet). | +| `KRAKEN_FUTURES_API_SECRET` | API secret for Kraken Futures (mainnet). | +| `KRAKEN_FUTURES_DEMO_API_KEY` | API key for Kraken Futures (demo). | +| `KRAKEN_FUTURES_DEMO_API_SECRET` | API secret for Kraken Futures (demo). | + +:::note +**Demo environment**: Only Kraken Futures offers a demo environment +(`https://demo-futures.kraken.com`) for testing without real funds. Kraken Spot +does not have a testnet - the `environment` setting only affects Futures +connections. +::: + +:::tip +We recommend using environment variables to manage your credentials. +::: + +When starting the trading node, you'll receive immediate confirmation of whether +your credentials are valid and have trading permissions. + +## Contributing + +:::info +For additional features or to contribute to the Kraken adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/okx.md b/nautilus-docs/docs/integrations/okx.md new file mode 100644 index 0000000..3782bc1 --- /dev/null +++ b/nautilus-docs/docs/integrations/okx.md @@ -0,0 +1,675 @@ +# OKX + +Founded in 2017, OKX is a leading cryptocurrency exchange offering spot, perpetual swap, +futures, and options trading. This integration supports live market data ingest and order +execution on OKX. + +## Overview + +This adapter is implemented in Rust, with optional Python bindings for ease of use in Python-based workflows. +It does not require external OKX client libraries; the core components are compiled as a static library and linked automatically during the build. + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/okx/). + +### Product support + +| Product Type | Data Feed | Trading | Notes | +|-------------------|-----------|---------|--------------------------------------------------| +| Spot | ✓ | ✓ | Use for index prices. | +| Perpetual Swaps | ✓ | ✓ | Linear and inverse contracts. | +| Futures | ✓ | ✓ | Specific expiration dates. | +| Margin | ✓ | ✓ | Spot trading with margin/leverage (spot margin). | +| Options | ✓ | - | *Data feed supported, trading coming soon*. | + +:::note +**Options support**: While you can subscribe to options market data and receive price updates, order execution for options is not yet implemented. You can use the symbology format shown above to subscribe to options data feeds. +::: + +:::info +**Instrument multipliers**: For derivatives (SWAP, FUTURES, OPTIONS), instrument multipliers are calculated as the product of OKX's `ctMult` (contract multiplier) and `ctVal` (contract value) fields. This ensures position sizing accurately reflects both the contract size and value. +::: + +The OKX adapter includes multiple components, which can be used separately or together depending on your use case. + +- `OKXHttpClient`: Low-level HTTP API connectivity. +- `OKXWebSocketClient`: Low-level WebSocket API connectivity. +- `OKXInstrumentProvider`: Instrument parsing and loading functionality. +- `OKXDataClient`: Market data feed manager. +- `OKXExecutionClient`: Account management and trade execution gateway. +- `OKXLiveDataClientFactory`: Factory for OKX data clients (used by the trading node builder). +- `OKXLiveExecClientFactory`: Factory for OKX execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as shown below), +and won’t need to work directly with these lower-level components. +::: + +## Symbology + +OKX uses specific symbol conventions for different instrument types. All instrument IDs should include the `.OKX` suffix when referencing them (e.g., `BTC-USDT.OKX` for spot Bitcoin). + +### Symbol format by instrument type + +#### SPOT + +Format: `{BaseCurrency}-{QuoteCurrency}` + +Examples: + +- `BTC-USDT` - Bitcoin against USDT (Tether) +- `BTC-USDC` - Bitcoin against USDC +- `ETH-USDT` - Ethereum against USDT +- `SOL-USDT` - Solana against USDT + +To subscribe to spot Bitcoin USD in your strategy: + +```python +InstrumentId.from_str("BTC-USDT.OKX") # For USDT-quoted spot +InstrumentId.from_str("BTC-USDC.OKX") # For USDC-quoted spot +``` + +#### SWAP (Perpetual Futures) + +Format: `{BaseCurrency}-{QuoteCurrency}-SWAP` + +Examples: + +- `BTC-USDT-SWAP` - Bitcoin perpetual swap (linear, USDT-margined) +- `BTC-USD-SWAP` - Bitcoin perpetual swap (inverse, coin-margined) +- `ETH-USDT-SWAP` - Ethereum perpetual swap (linear) +- `ETH-USD-SWAP` - Ethereum perpetual swap (inverse) + +Linear vs Inverse contracts: + +- **Linear** (USDT-margined): Uses stablecoins like USDT as margin. +- **Inverse** (coin-margined): Uses the base cryptocurrency as margin. + +#### FUTURES (Dated Futures) + +Format: `{BaseCurrency}-{QuoteCurrency}-{YYMMDD}` + +Examples: + +- `BTC-USD-251226` - Bitcoin futures expiring December 26, 2025 +- `ETH-USD-251226` - Ethereum futures expiring December 26, 2025 +- `BTC-USD-250328` - Bitcoin futures expiring March 28, 2025 + +Note: Futures are typically inverse contracts (coin-margined). + +#### OPTIONS + +Format: `{BaseCurrency}-{QuoteCurrency}-{YYMMDD}-{Strike}-{Type}` + +Examples: + +- `BTC-USD-251226-100000-C` - Bitcoin call option, $100,000 strike, expiring December 26, 2025 +- `BTC-USD-251226-100000-P` - Bitcoin put option, $100,000 strike, expiring December 26, 2025 +- `ETH-USD-251226-4000-C` - Ethereum call option, $4,000 strike, expiring December 26, 2025 + +Where: + +- `C` = Call option +- `P` = Put option + +### Common questions + +**Q: How do I subscribe to spot Bitcoin USD?** +A: Use `BTC-USDT.OKX` for USDT-margined spot or `BTC-USDC.OKX` for USDC-margined spot. + +**Q: What's the difference between BTC-USDT-SWAP and BTC-USD-SWAP?** +A: `BTC-USDT-SWAP` is a linear perpetual (USDT-margined), while `BTC-USD-SWAP` is an inverse perpetual (BTC-margined). + +**Q: How do I know which contract type to use?** +A: Check the `contract_types` parameter in the configuration: + +- For linear contracts: `OKXContractType.LINEAR`. +- For inverse contracts: `OKXContractType.INVERSE`. + +## Orders capability + +Below are the order types, execution instructions, and time-in-force options supported +for linear perpetual swap products on OKX. + +### Client order ID requirements + +:::note +OKX has specific requirements for client order IDs: + +- **No hyphens allowed**: OKX does not accept hyphens (`-`) in client order IDs. +- Maximum length: 32 characters. +- Allowed characters: alphanumeric characters and underscores only. + +When configuring your strategy, ensure you set: + +```python +use_hyphens_in_client_order_ids=False +``` + +::: + +### Order types + +| Order Type | Linear Perpetual Swap | Notes | +|---------------------|-----------------------|---------------------------------------------------------------| +| `MARKET` | ✓ | Immediate execution at market price. Supports quote quantity. | +| `MARKET_TO_LIMIT` | ✓ | Market order converted to IOC limit. | +| `LIMIT` | ✓ | Execution at specified price or better. | +| `STOP_MARKET` | ✓ | Conditional market order (OKX algo order). | +| `STOP_LIMIT` | ✓ | Conditional limit order (OKX algo order). | +| `MARKET_IF_TOUCHED` | ✓ | Conditional market order (OKX algo order). | +| `LIMIT_IF_TOUCHED` | ✓ | Conditional limit order (OKX algo order). | +| `TRAILING_STOP_MARKET` | ✓ | Trailing stop market order (OKX advance algo order). | + +:::info +**Conditional orders**: `STOP_MARKET`, `STOP_LIMIT`, `MARKET_IF_TOUCHED`, `LIMIT_IF_TOUCHED`, and `TRAILING_STOP_MARKET` are implemented as OKX algo orders, providing advanced trigger capabilities with multiple price sources. `TRAILING_STOP_MARKET` uses OKX's advance algo order API (`move_order_stop`) and requires the separate `cancel-advance-algos` endpoint for cancellation. +::: + +### Quantity semantics for spot margin trading + +When using spot margin trading (`use_spot_margin=True`), OKX interprets order quantities differently depending on the order side: + +- **Limit** orders interpret `quantity` as the number of base currency units. +- **Market SELL** orders also use base-unit quantities. +- **Market BUY** orders interpret `quantity` as quote notional (e.g., USDT). + +:::warning +**When submitting spot margin market BUY orders, you must**: + +1. Set `quote_quantity=True` on the order (or pre-compute the quote-denominated amount). +2. Configure the execution engine with `convert_quote_qty_to_base=False` so the quote amount reaches the adapter unchanged. + +The OKX execution client will deny base-denominated market buy orders for spot margin to prevent unintended fills. + +**On the first fill**, the order quantity will be automatically updated from the quote quantity to the actual base quantity received, +reflecting the executed trade. +::: + +```python +from nautilus_trader.execution.config import ExecEngineConfig +from nautilus_trader.execution.engine import ExecutionEngine + +# Disable automatic conversion for quote quantities +config = ExecEngineConfig(convert_quote_qty_to_base=False) +engine = ExecutionEngine(msgbus=msgbus, cache=cache, clock=clock, config=config) + +# Correct: Spot margin market BUY with quote quantity (spend $100 USDT) +order = strategy.order_factory.market( + instrument_id=instrument_id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(100.0), + quote_quantity=True, # Interpret as USDT notional +) +strategy.submit_order(order) +``` + +### Execution instructions + +| Instruction | Linear Perpetual Swap | Notes | +|----------------|-----------------------|------------------------| +| `post_only` | ✓ | Only for LIMIT orders. | +| `reduce_only` | ✓ | Only for derivatives. | + +### Time in force + +| Time in force | Linear Perpetual Swap | Notes | +|---------------|-----------------------|---------------------------------------------------| +| `GTC` | ✓ | Good Till Canceled. | +| `FOK` | ✓ | Fill or Kill. | +| `IOC` | ✓ | Immediate or Cancel. | +| `GTD` | ✗ | *Not supported by OKX API.* | + +:::note +**GTD (Good Till Date) time in force**: OKX does not support native GTD functionality through their API. + +If you need GTD functionality, you must use Nautilus's strategy-managed GTD feature, which will handle the order expiration by canceling the order at the specified expiry time. +::: + +### Batch operations + +| Operation | Linear Perpetual Swap | Notes | +|--------------------|-----------------------|-------------------------------------------| +| Batch Submit | ✓ | Submit multiple orders in single request. | +| Batch Modify | ✓ | Modify multiple orders in single request. | +| Batch Cancel | ✓ | Cancel multiple orders in single request. | + +### Position management + +| Feature | Linear Perpetual Swap | Notes | +|-------------------|-----------------------|------------------------------------------------------| +| Query positions | ✓ | Real-time position updates. | +| Position mode | ✓ | Net vs Long/Short mode (see below). | +| Leverage control | ✓ | Dynamic leverage adjustment per instrument. | +| Margin mode | ✓ | Supports cash, isolated, and cross modes. | + +#### Position modes + +OKX supports two position modes for derivatives trading: + +- **Net mode** (Netting): Single position per instrument that can be positive (LONG) or negative (SHORT). Buy and sell orders net against each other. This is the default and recommended for most traders. +- **Long/Short mode** (Hedging): Separate long and short positions for the same instrument. Allows simultaneous long and short positions, useful for hedging strategies. + +:::note +Position mode must be configured via the OKX Web/App interface and applies account-wide. The adapter automatically detects the current position mode and handles position reporting accordingly. +::: + +### Trade modes and margin configuration + +OKX's unified account system supports different trade modes for spot and derivatives trading. The adapter automatically determines the correct trade mode based on your configuration and instrument type. + +:::note +**Important**: Account modes must be initially configured via the OKX Web/App interface. The API cannot set the account mode for the first time. +::: + +For more details on OKX's account modes and margin system, see the [OKX Account Mode documentation](https://www.okx.com/docs-v5/en/#overview-account-mode). + +#### Trade modes overview + +OKX supports four trade modes, which the adapter selects automatically based on your configuration: + +| Mode | Used For | Leverage | Borrowing | Configuration | +|---------------------|--------------------------------------------|----------|-----------|---------------| +| **`cash`** | Simple spot trading | - | - | `use_spot_margin=False` (default for SPOT) | +| **`isolated`** | Spot margin or derivatives (default) | ✓ | ✓ | `use_spot_margin=True` with `margin_mode=ISOLATED` (or unset) for SPOT; default for derivatives | +| **`cross`** | Spot margin or derivatives, shared pool | ✓ | ✓ | `use_spot_margin=True` with `margin_mode=CROSS` for SPOT; `margin_mode=CROSS` for derivatives | + +#### Configuration-based trade mode selection + +**The adapter automatically selects the correct trade mode** based on: + +1. **Instrument type** (SPOT vs derivatives) +2. **Configuration settings** (`use_spot_margin` for SPOT, `margin_mode` for derivatives) + +##### For SPOT trading + +```python +# Simple SPOT trading without leverage (uses 'cash' mode) +exec_clients={ + OKX: OKXExecClientConfig( + instrument_types=(OKXInstrumentType.SPOT,), + use_spot_margin=False, # Default - simple SPOT + # ... other config + ), +} + +# SPOT trading WITH margin/leverage (uses 'isolated' or 'cross' mode) +exec_clients={ + OKX: OKXExecClientConfig( + instrument_types=(OKXInstrumentType.SPOT,), + use_spot_margin=True, # Enable margin trading for SPOT + margin_mode=OKXMarginMode.ISOLATED, # Or CROSS for shared margin + # ... other config + ), +} +``` + +##### For derivatives trading (SWAP/FUTURES/OPTIONS) + +```python +# Derivatives with isolated margin (default - uses 'isolated' mode) +exec_clients={ + OKX: OKXExecClientConfig( + instrument_types=(OKXInstrumentType.SWAP,), + margin_mode=OKXMarginMode.ISOLATED, # Or omit - ISOLATED is default + # ... other config + ), +} + +# Derivatives with cross margin (uses 'cross' mode) +exec_clients={ + OKX: OKXExecClientConfig( + instrument_types=(OKXInstrumentType.SWAP,), + margin_mode=OKXMarginMode.CROSS, # Share margin across all positions + # ... other config + ), +} +``` + +##### For mixed SPOT and derivatives trading + +When trading both SPOT and derivatives instruments simultaneously, the adapter automatically determines the correct trade mode **per-order** based on the instrument being traded: + +```python +# Mixed SPOT + SWAP configuration +exec_clients={ + OKX: OKXExecClientConfig( + instrument_types=(OKXInstrumentType.SPOT, OKXInstrumentType.SWAP), + use_spot_margin=True, # Applies to SPOT orders only + margin_mode=OKXMarginMode.CROSS, # Applies to SWAP orders only + # ... other config + ), +} +``` + +**How it works:** + +- **SPOT orders** → Uses `cross` mode (because `use_spot_margin=True` and `margin_mode=CROSS`) +- **SWAP orders** → Uses `cross` mode (because `margin_mode=CROSS`) +- Each order automatically gets the correct `tdMode` based on its instrument type +- No manual intervention required + +This enables strategies that trade across multiple instrument types with different margin configurations, such as: + +- Spot-futures arbitrage strategies +- Delta-neutral strategies combining spot and perpetual swaps +- Market making across spot and derivatives markets + +:::warning +**Manual trade mode override**: While you can still manually override the trade mode per order using `params={"td_mode": "..."}`, this is **not recommended** as it bypasses automatic mode selection and can lead to order rejection if the wrong mode is specified for the instrument type (e.g., using `isolated` for SPOT instruments). + +Only use manual override if you have specific requirements that cannot be met through configuration. +::: + +#### Benefits of configuration-based approach + +- **Type-safe**: Configuration is validated at startup before placing any orders. +- **Automatic**: System chooses correct mode based on instrument type and intent. +- **Clear**: Field names explain purpose (`use_spot_margin` vs obscure `td_mode` parameter). +- **Safe**: Impossible to use incompatible combinations (e.g., `isolated` mode for SPOT). +- **Backwards compatible**: Default values maintain existing behavior. + +### Order querying + +| Feature | Linear Perpetual Swap | Notes | +|----------------------|-----------------------|-------------------------------------------| +| Query open orders | ✓ | List all active orders. | +| Query order history | ✓ | Historical order data. | +| Order status updates | ✓ | Real-time order state changes. | +| Trade history | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Linear Perpetual Swap | Notes | +|---------------------|-----------------------|--------------------------------------------| +| Order lists | - | *Not supported*. | +| OCO orders | ✓ | One-Cancels-Other orders. | +| Bracket orders | ✓ | Stop loss + take profit combinations. | +| Conditional orders | ✓ | Stop and limit-if-touched orders. | + +#### Conditional order architecture + +Conditional orders (OKX algo orders) use a hybrid architecture for optimal performance and reliability: + +- **Submission**: Via HTTP REST API (`/api/v5/trade/order-algo`) +- **Status updates**: Via WebSocket business endpoint (`/ws/v5/business`) on the `orders-algo` channel +- **Cancellation**: Via HTTP REST API using algo order ID tracking + +This design ensures: + +- Immediate submission acknowledgment through HTTP. +- Real-time status updates through WebSocket. +- Proper order lifecycle management with algo order ID mapping. + +#### Supported conditional order types + +| Order Type | Trigger Types | Notes | +|---------------------|------------------------|-------------------------------------------| +| `STOP_MARKET` | Last, Mark, Index | Market execution when triggered. | +| `STOP_LIMIT` | Last, Mark, Index | Limit order placement when triggered. | +| `MARKET_IF_TOUCHED` | Last, Mark, Index | Market execution when price touched. | +| `LIMIT_IF_TOUCHED` | Last, Mark, Index | Limit order placement when price touched. | +| `TRAILING_STOP_MARKET` | Last, Mark, Index | Trailing stop with callback ratio. | + +#### Trigger price types + +Conditional orders support different trigger price sources: + +- **Last Price** (`TriggerType.LAST_PRICE`): Uses the last traded price (default). +- **Mark Price** (`TriggerType.MARK_PRICE`): Uses the mark price (recommended for derivatives). +- **Index Price** (`TriggerType.INDEX_PRICE`): Uses the underlying index price. + +```python +# Example: Stop loss using mark price trigger +stop_order = order_factory.stop_market( + instrument_id=instrument_id, + order_side=OrderSide.SELL, + quantity=Quantity.from_str("0.1"), + trigger_price=Price.from_str("45000.0"), + trigger_type=TriggerType.MARK_PRICE, # Use mark price for trigger +) +strategy.submit_order(stop_order) +``` + +## Risk management + +### Liquidation and ADL event handling + +The OKX adapter automatically detects and handles exchange-initiated risk management events: + +- **Liquidation orders**: When a position is liquidated by the exchange (full or partial), the adapter detects the liquidation category and logs warnings with order details. These orders are processed normally through the order and fill pipeline. +- **Auto-Deleveraging (ADL)**: When your position is closed by the exchange to offset a counterparty's liquidation, the adapter detects and logs the ADL event with position details. + +:::info +**Liquidation and ADL events are logged at WARNING level** with details including order ID, instrument, and state. Monitor your logs for these events as part of your risk management process. + +The adapter handles these exchange-generated orders, generating appropriate `OrderFilled` events and updating positions accordingly. No special handling is required in your strategy code. +::: + +## Authentication + +To use the OKX adapter, you'll need to create API credentials in your OKX account: + +1. Log into your OKX account and navigate to the API management page. +2. Create a new API key with the required permissions for trading and data access. +3. Note down your API key, secret key, and passphrase. + +You can provide these credentials through environment variables: + +```bash +export OKX_API_KEY="your_api_key" +export OKX_API_SECRET="your_api_secret" +export OKX_API_PASSPHRASE="your_passphrase" +``` + +Or pass them directly in the configuration (not recommended for production). + +## Demo trading + +OKX provides a demo trading environment for testing strategies without real funds. + +### Setting up a demo account + +1. Log into your OKX account at [okx.com](https://www.okx.com). +2. Navigate to **Trade** → **Demo Trading**. +3. Go to **Personal Center** within Demo Trading. +4. Select **Demo Trading API** and create a new API key. +5. Note down your demo API key, secret, and passphrase. + +You can provide demo credentials through environment variables: + +```bash +export OKX_API_KEY="your_demo_api_key" +export OKX_API_SECRET="your_demo_api_secret" +export OKX_API_PASSPHRASE="your_demo_passphrase" +``` + +### Configuration + +Set `is_demo=True` in your client configuration: + +```python +config = TradingNodeConfig( + data_clients={ + OKX: OKXDataClientConfig( + is_demo=True, # Enable demo mode + # ... other config + ), + }, + exec_clients={ + OKX: OKXExecClientConfig( + is_demo=True, # Enable demo mode + # ... other config + ), + }, +) +``` + +When demo mode is enabled: + +- REST API requests include the `x-simulated-trading: 1` header. +- WebSocket connections use demo endpoints (`wspap.okx.com`). + +:::note +Demo API keys are separate from production keys. You must create API keys specifically for demo trading through the Demo Trading interface. Production API keys will not work in demo mode. +::: + +## Funding rates + +The adapter receives funding rate data from the +[Funding Rate Channel](https://www.okx.com/docs-v5/en/#public-data-websocket-funding-rate-channel) +WebSocket stream. OKX provides both `fundingTime` and `nextFundingTime` in each message, +and the adapter computes `interval` as the difference between these two values. + +For historical funding rate requests, the adapter computes the interval from consecutive +funding timestamps returned by the +[Get Funding Rate History](https://www.okx.com/docs-v5/en/#public-data-rest-api-get-funding-rate-history) +endpoint. + +## Rate limiting + +The adapter enforces OKX’s per-endpoint quotas while keeping sensible defaults for both REST and WebSocket calls. + +### REST limits + +- Global cap: 250 requests per second (matches 500 requests / 2 seconds IP allowance). +- Endpoint-specific quotas appear in the table below and mirror OKX’s published limits where available. + +### WebSocket limits + +- Connection establishment: 3 requests per second (per IP). +- Subscription operations (subscribe/unsubscribe/login): 480 requests per hour per connection. +- Order actions (place/cancel/amend): 250 requests per second. + +:::warning +OKX enforces per-endpoint and per-account quotas; exceeding them leads to HTTP 429 responses and temporary throttling on that key. +::: + +| Key / Endpoint | Limit (req/sec) | Notes | +|----------------------------------|-----------------|---------------------------------------------------------| +| `okx:global` | 250 | Matches 500 req / 2 s IP allowance. | +| `/api/v5/public/instruments` | 10 | Matches OKX 20 req / 2 s docs. | +| `/api/v5/market/candles` | 50 | Higher allowance for streaming candles. | +| `/api/v5/market/history-candles` | 20 | Conservative quota for large historical pulls. | +| `/api/v5/market/history-trades` | 30 | Trade history pulls. | +| `/api/v5/account/balance` | 5 | OKX guidance: 10 req / 2 s. | +| `/api/v5/trade/order` | 30 | 60 requests / 2 seconds per-instrument limit. | +| `/api/v5/trade/orders-pending` | 20 | Open order fetch. | +| `/api/v5/trade/orders-history` | 20 | Historical orders. | +| `/api/v5/trade/fills` | 30 | Execution reports. | +| `/api/v5/trade/order-algo` | 10 | Algo placements (conditional orders). | +| `/api/v5/trade/cancel-algos` | 10 | Algo cancellation. | + +All keys automatically include the `okx:global` bucket. URLs are normalised (query strings removed) before rate limiting, so requests with different filters share the same quota. + +:::info +For more details on rate limiting, see the official documentation: . +::: + +## Configuration + +### Configuration options + +The OKX data client provides the following configuration options: + +#### Data client + +| Option | Default | Description | +|--------------------------------------|---------------------------------|-------------| +| `instrument_types` | `(OKXInstrumentType.SPOT,)` | Controls which OKX instrument families are loaded (spot, swap, futures, options). | +| `contract_types` | `None` | Restricts loading to specific contract styles when combined with `instrument_types`. | +| `instrument_families` | `None` | Instrument families to load (e.g., "BTC-USD", "ETH-USD"). Required for OPTIONS. Optional for FUTURES/SWAP. Not applicable for SPOT/MARGIN. | +| `base_url_http` | `None` | Override for the OKX REST endpoint; defaults to the production URL resolved at runtime. | +| `base_url_ws` | `None` | Override for the market data WebSocket endpoint. | +| `api_key` | `None` | Falls back to `OKX_API_KEY` environment variable when unset. | +| `api_secret` | `None` | Falls back to `OKX_API_SECRET` environment variable when unset. | +| `api_passphrase` | `None` | Falls back to `OKX_API_PASSPHRASE` environment variable when unset. | +| `is_demo` | `False` | Connects to the OKX demo environment when `True`. | +| `http_timeout_secs` | `60` | Request timeout (seconds) for REST market data calls. | +| `max_retries` | `3` | Maximum retry attempts for recoverable REST errors. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) before retrying a failed request. | +| `retry_delay_max_ms` | `10,000` | Upper bound for exponential backoff delay between retries. | +| `update_instruments_interval_mins` | `60` | Interval, in minutes, between background instrument refreshes. | +| `vip_level` | `None` | Enables higher-depth order book channels when set to the matching OKX VIP tier. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. | + +The OKX execution client provides the following configuration options: + +#### Execution client + +| Option | Default | Description | +|----------------------------|-------------|-------------| +| `instrument_types` | `(OKXInstrumentType.SPOT,)` | Instrument families that should be tradable for this client. | +| `contract_types` | `None` | Restricts tradable contracts (linear, inverse, options) when paired with `instrument_types`. | +| `instrument_families` | `None` | Instrument families to load (e.g., "BTC-USD", "ETH-USD"). Required for OPTIONS. Optional for FUTURES/SWAP. Not applicable for SPOT/MARGIN. | +| `base_url_http` | `None` | Override for the OKX trading REST endpoint. | +| `base_url_ws` | `None` | Override for the private WebSocket endpoint. | +| `api_key` | `None` | Falls back to `OKX_API_KEY` environment variable when unset. | +| `api_secret` | `None` | Falls back to `OKX_API_SECRET` environment variable when unset. | +| `api_passphrase` | `None` | Falls back to `OKX_API_PASSPHRASE` environment variable when unset. | +| `margin_mode` | `None` | Margin mode for derivatives trading (`ISOLATED` or `CROSS`). Only applies to SWAP/FUTURES/OPTIONS. Defaults to `ISOLATED` if not specified. | +| `use_spot_margin` | `False` | Enables margin/leverage for SPOT trading. When `True`, uses `isolated` or `cross` trade mode (determined by `margin_mode`). When `False`, uses `cash` trade mode (no leverage). Only applies to SPOT instruments. | +| `is_demo` | `False` | Connects to the OKX demo trading environment. | +| `http_timeout_secs` | `60` | Request timeout (seconds) for REST trading calls. | +| `use_fills_channel` | `False` | Subscribes to the dedicated fills channel (VIP5+ required) for lower-latency fill reports. | +| `use_mm_mass_cancel` | `False` | Uses the market-maker bulk cancel endpoint when available; otherwise falls back to per-order cancels. | +| `max_retries` | `3` | Maximum retry attempts for recoverable REST errors. | +| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) applied before retrying a failed request. | +| `retry_delay_max_ms` | `10,000` | Upper bound for the exponential backoff delay between retries. | +| `use_spot_cash_position_reports` | `False` | Generate position reports for SPOT CASH instruments based on wallet balances. | +| `http_proxy_url` | `None` | Optional HTTP proxy URL. | +| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. | + +Below is an example configuration for a live trading node using OKX data and execution clients: + +```python +from nautilus_trader.adapters.okx import OKX +from nautilus_trader.adapters.okx import OKXDataClientConfig, OKXExecClientConfig +from nautilus_trader.adapters.okx.factories import OKXLiveDataClientFactory, OKXLiveExecClientFactory +from nautilus_trader.config import InstrumentProviderConfig, TradingNodeConfig +from nautilus_trader.core.nautilus_pyo3 import OKXContractType +from nautilus_trader.core.nautilus_pyo3 import OKXInstrumentType +from nautilus_trader.core.nautilus_pyo3 import OKXMarginMode +from nautilus_trader.live.node import TradingNode + +config = TradingNodeConfig( + ..., + data_clients={ + OKX: OKXDataClientConfig( + api_key=None, # Will use OKX_API_KEY env var + api_secret=None, # Will use OKX_API_SECRET env var + api_passphrase=None, # Will use OKX_API_PASSPHRASE env var + base_url_http=None, + instrument_provider=InstrumentProviderConfig(load_all=True), + instrument_types=(OKXInstrumentType.SWAP,), + contract_types=(OKXContractType.LINEAR,), + is_demo=False, + ), + }, + exec_clients={ + OKX: OKXExecClientConfig( + api_key=None, + api_secret=None, + api_passphrase=None, + base_url_http=None, + base_url_ws=None, + instrument_provider=InstrumentProviderConfig(load_all=True), + instrument_types=(OKXInstrumentType.SWAP,), + contract_types=(OKXContractType.LINEAR,), + is_demo=False, + ), + }, +) +node = TradingNode(config=config) +node.add_data_client_factory(OKX, OKXLiveDataClientFactory) +node.add_exec_client_factory(OKX, OKXLiveExecClientFactory) +node.build() +``` + +## Contributing + +:::info +For additional features or to contribute to the OKX adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/polymarket.md b/nautilus-docs/docs/integrations/polymarket.md new file mode 100644 index 0000000..cca5294 --- /dev/null +++ b/nautilus-docs/docs/integrations/polymarket.md @@ -0,0 +1,801 @@ +# Polymarket + +Founded in 2020, Polymarket is the world’s largest decentralized prediction market platform, +enabling traders to speculate on the outcomes of world events by buying and selling binary option contracts using cryptocurrency. + +NautilusTrader provides a venue integration for data and execution via Polymarket's Central Limit Order Book (CLOB) API. +The integration uses the [official Python CLOB client library](https://github.com/Polymarket/py-clob-client) +to interact with the Polymarket platform. + +NautilusTrader supports multiple Polymarket signature types for order signing, providing flexibility for different wallet configurations. +This integration ensures that traders can execute orders securely and efficiently across various wallet types, +while NautilusTrader abstracts the complexity of signing and preparing orders for execution. + +## Installation + +To install NautilusTrader with Polymarket support: + +```bash +uv pip install "nautilus_trader[polymarket]" +``` + +To build from source with all extras (including Polymarket): + +```bash +uv sync --all-extras +``` + +## Examples + +You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/polymarket/). + +## Binary options + +A [binary option](https://en.wikipedia.org/wiki/Binary_option) is a type of financial exotic option contract in which traders bet on the outcome of a yes-or-no proposition. +If the prediction is correct, the trader receives a fixed payout; otherwise, they receive nothing. + +All assets traded on Polymarket are quoted and settled in **USDC.e (PoS)**, [see below](#usdce-pos) for more information. + +## Polymarket documentation + +Polymarket offers resources for different audiences: + +- [Polymarket Learn](https://learn.polymarket.com/): Educational content and guides for users to understand the platform and how to engage with it. +- [Polymarket CLOB API](https://docs.polymarket.com/#introduction): Technical documentation for developers interacting with the Polymarket CLOB API. + +## Overview + +This guide assumes a trader is setting up for both live market data feeds and trade execution. +The Polymarket integration adapter includes multiple components, which can be used together or +separately depending on the use case. + +- `PolymarketWebSocketClient`: Low-level WebSocket API connectivity (built on top of the Nautilus `WebSocketClient` written in Rust). +- `PolymarketInstrumentProvider`: Instrument parsing and loading functionality for `BinaryOption` instruments. +- `PolymarketDataClient`: A market data feed manager. +- `PolymarketExecutionClient`: A trade execution gateway. +- `PolymarketLiveDataClientFactory`: Factory for Polymarket data clients (used by the trading node builder). +- `PolymarketLiveExecClientFactory`: Factory for Polymarket execution clients (used by the trading node builder). + +:::note +Most users will define a configuration for a live trading node (as below), +and won't need to work with these lower-level components directly. +::: + +## USDC.e (PoS) + +**USDC.e** is a bridged version of USDC from Ethereum to the Polygon network, operating on Polygon's **Proof of Stake (PoS)** chain. +This enables faster, more cost-efficient transactions on Polygon while maintaining backing by USDC on Ethereum. + +The contract address is [0x2791bca1f2de4661ed88a30c99a7a9449aa84174](https://polygonscan.com/address/0x2791bca1f2de4661ed88a30c99a7a9449aa84174) on the Polygon blockchain. +More information can be found in this [blog](https://polygon.technology/blog/phase-one-of-native-usdc-migration-on-polygon-pos-is-underway). + +## Wallets and accounts + +To interact with Polymarket via NautilusTrader, you'll need a **Polygon**-compatible wallet (such as MetaMask). + +### Signature types + +Polymarket supports multiple signature types for order signing and verification: + +| Signature Type | Wallet Type | Description | Use Case | +|----------------|--------------------------------|-------------|----------| +| `0` | EOA (Externally Owned Account) | Standard EIP712 signatures from wallets with direct private key control. | **Default.** Direct wallet connections (MetaMask, hardware wallets, etc.). | +| `1` | Email/Magic Wallet Proxy | Smart contract wallet for email-based accounts (Magic Link). Only the email-associated address can execute functions. | Polymarket Proxy associated with Email/Magic accounts. Requires `funder` address. | +| `2` | Browser Wallet Proxy | Modified Gnosis Safe (1-of-1 multisig) for browser wallets. | Polymarket Proxy associated with browser wallets. Enables UI verification. Requires `funder` address. | + +:::note +See also: [Proxy wallet](https://docs.polymarket.com/developers/proxy-wallet) in the Polymarket documentation for more details about signature types and proxy wallet infrastructure. +::: + +NautilusTrader defaults to signature type 0 (EOA) but can be configured to use any of the supported signature types via the `signature_type` configuration parameter. + +A single wallet address is supported per trader instance when using environment variables, +or multiple wallets could be configured with multiple `PolymarketExecutionClient` instances. + +:::note +Ensure your wallet is funded with **USDC.e**, otherwise you will encounter the "not enough balance / allowance" API error when submitting orders. +::: + +### Setting allowances for Polymarket contracts + +Before you can start trading, you need to ensure that your wallet has allowances set for Polymarket's smart contracts. +You can do this by running the provided script located at `nautilus_trader/adapters/polymarket/scripts/set_allowances.py`. + +This script is adapted from a [gist](https://gist.github.com/poly-rodr/44313920481de58d5a3f6d1f8226bd5e) created by @poly-rodr. + +:::note +You only need to run this script **once** per EOA wallet that you intend to use for trading on Polymarket. +::: + +This script automates the process of approving the necessary allowances for the Polymarket contracts. +It sets approvals for the USDC token and Conditional Token Framework (CTF) contract to allow the +Polymarket CLOB Exchange to interact with your funds. + +Before running the script, ensure the following prerequisites are met: + +- Install the web3 Python package: `uv pip install "web3==7.12.1"`. +- Have a **Polygon**-compatible wallet funded with some POL (used for gas fees). +- Set the following environment variables in your shell: + - `POLYGON_PRIVATE_KEY`: Your private key for the **Polygon**-compatible wallet. + - `POLYGON_PUBLIC_KEY`: Your public key for the **Polygon**-compatible wallet. + +Once you have these in place, the script will: + +- Approve the maximum possible amount of USDC (using the `MAX_INT` value) for the Polymarket USDC token contract. +- Set the approval for the CTF contract, allowing it to interact with your account for trading purposes. + +:::note +You can also adjust the approval amount in the script instead of using `MAX_INT`, +with the amount specified in *fractional units* of **USDC.e**, though this has not been tested. +::: + +Ensure that your private key and public key are correctly stored in the environment variables before running the script. +Here's an example of how to set the variables in your terminal session: + +```bash +export POLYGON_PRIVATE_KEY="YOUR_PRIVATE_KEY" +export POLYGON_PUBLIC_KEY="YOUR_PUBLIC_KEY" +``` + +Run the script using: + +```bash +python nautilus_trader/adapters/polymarket/scripts/set_allowances.py +``` + +### Script breakdown + +The script performs the following actions: + +- Connects to the Polygon network via an RPC URL (). +- Signs and sends a transaction to approve the maximum USDC allowance for Polymarket contracts. +- Sets approval for the CTF contract to manage Conditional Tokens on your behalf. +- Repeats the approval process for specific addresses like the Polymarket CLOB Exchange and Neg Risk adapter. + +This allows Polymarket to interact with your funds when executing trades and ensures smooth integration with the CLOB Exchange. + +## API keys + +To trade with Polymarket, you'll need to generate API credentials. Follow these steps: + +1. Ensure the following environment variables are set: + - `POLYMARKET_PK`: Your private key for signing transactions. + - `POLYMARKET_FUNDER`: The wallet address (public key) on the **Polygon** network used for funding trades on Polymarket. + +2. Run the script using: + + ```bash + python nautilus_trader/adapters/polymarket/scripts/create_api_key.py + ``` + +The script will generate and print API credentials, which you should save to the following environment variables: + +- `POLYMARKET_API_KEY` +- `POLYMARKET_API_SECRET` +- `POLYMARKET_PASSPHRASE` + +These can then be used for Polymarket client configurations: + +- `PolymarketDataClientConfig` +- `PolymarketExecClientConfig` + +## Configuration + +When setting up NautilusTrader to work with Polymarket, it’s crucial to properly configure the necessary parameters, particularly the private key. + +**Key parameters**: + +- `private_key`: The private key for your wallet used to sign orders. The interpretation depends on your `signature_type` configuration. If not explicitly provided in the configuration, it will automatically source the `POLYMARKET_PK` environment variable. +- `funder`: The **USDC.e** funding wallet address used for funding trades. If not provided, will source the `POLYMARKET_FUNDER` environment variable. +- API credentials: You will need to provide the following API credentials to interact with the Polymarket CLOB: + - `api_key`: If not provided, will source the `POLYMARKET_API_KEY` environment variable. + - `api_secret`: If not provided, will source the `POLYMARKET_API_SECRET` environment variable. + - `passphrase`: If not provided, will source the `POLYMARKET_PASSPHRASE` environment variable. + +:::tip +We recommend using environment variables to manage your credentials. +::: + +## Orders capability + +Polymarket operates as a prediction market with a more limited set of order types and instructions compared to traditional exchanges. + +### Order types + +| Order Type | Binary Options | Notes | +|------------------------|----------------|-------------------------------------| +| `MARKET` | ✓ | **BUY orders require quote quantity**, SELL orders require base quantity. | +| `LIMIT` | ✓ | | +| `STOP_MARKET` | - | *Not supported by Polymarket*. | +| `STOP_LIMIT` | - | *Not supported by Polymarket*. | +| `MARKET_IF_TOUCHED` | - | *Not supported by Polymarket*. | +| `LIMIT_IF_TOUCHED` | - | *Not supported by Polymarket*. | +| `TRAILING_STOP_MARKET` | - | *Not supported by Polymarket*. | + +### Quantity semantics + +Polymarket interprets order quantities differently depending on the order type *and* side: + +- **Limit** orders interpret `quantity` as the number of conditional tokens (base units). +- **Market SELL** orders also use base-unit quantities. +- **Market BUY** orders interpret `quantity` as quote notional in **USDC.e**. + +As a result, a market buy order submitted with a base-denominated quantity will execute far more size than intended. + +:::warning +When submitting market BUY orders, set `quote_quantity=True` (or pre-compute the quote-denominated amount) +and configure the execution engine with `convert_quote_qty_to_base=False` so the quote amount reaches the adapter unchanged. +The Polymarket execution client denies base-denominated market buys to prevent unintended fills. + +**NautilusTrader now forwards market orders to Polymarket's native market-order endpoint, so the +quote amount you specify for a BUY is executed directly (no more synthetic max-price limits).** +::: + +```python +from nautilus_trader.execution.config import ExecEngineConfig +from nautilus_trader.execution.engine import ExecutionEngine + +# Temporary: disable automatic conversion until the behaviour is fully removed in a future release +config = ExecEngineConfig(convert_quote_qty_to_base=False) +engine = ExecutionEngine(msgbus=msgbus, cache=cache, clock=clock, config=config) + +# Correct: Market BUY with quote quantity (spend $10 USDC) +order = strategy.order_factory.market( + instrument_id=instrument_id, + order_side=OrderSide.BUY, + quantity=instrument.make_qty(10.0), + quote_quantity=True, # Interpret as USDC.e notional +) +strategy.submit_order(order) +``` + +### Execution instructions + +| Instruction | Binary Options | Notes | +|---------------|----------------|------------------------------------------| +| `post_only` | - | *Not supported by Polymarket*. | +| `reduce_only` | - | *Not supported by Polymarket*. | + +### Time-in-force options + +| Time in force | Binary Options | Notes | +|---------------|----------------|------------------------------------------| +| `GTC` | ✓ | Good Till Canceled. | +| `GTD` | ✓ | Good Till Date. | +| `FOK` | ✓ | Fill or Kill. | +| `IOC` | ✓ | Immediate or Cancel (maps to FAK). | + +:::note +FAK (Fill and Kill) is Polymarket's terminology for Immediate or Cancel (IOC) semantics. +::: + +### Advanced order features + +| Feature | Binary Options | Notes | +|--------------------|----------------|------------------------------------| +| Order modification | - | Cancellation functionality only. | +| Bracket/OCO orders | - | *Not supported by Polymarket.* | +| Iceberg orders | - | *Not supported by Polymarket.* | + +### Batch operations + +| Operation | Binary Options | Notes | +|--------------------|----------------|-------------------------------------| +| Batch Submit | - | *Not supported by Polymarket*. | +| Batch Modify | - | *Not supported by Polymarket*. | +| Batch Cancel | - | *Not supported by Polymarket*. | + +### Position management + +| Feature | Binary Options | Notes | +|------------------|----------------|-----------------------------------| +| Query positions | ✓ | Contract balance-based positions. | +| Position mode | - | Binary outcome positions only. | +| Leverage control | - | No leverage available. | +| Margin mode | - | No margin trading. | + +### Order querying + +| Feature | Binary Options | Notes | +|----------------------|----------------|--------------------------------| +| Query open orders | ✓ | Active orders only. | +| Query order history | ✓ | Limited historical data. | +| Order status updates | ✓ | Real-time order state changes. | +| Trade history | ✓ | Execution and fill reports. | + +### Contingent orders + +| Feature | Binary Options | Notes | +|--------------------|----------------|-------------------------------------| +| Order lists | - | *Not supported by Polymarket*. | +| OCO orders | - | *Not supported by Polymarket*. | +| Bracket orders | - | *Not supported by Polymarket*. | +| Conditional orders | - | *Not supported by Polymarket*. | + +### Precision limits + +Polymarket enforces different precision constraints based on tick size and order type. + +**Binary Option instruments** typically support up to 6 decimal places for amounts (with 0.0001 tick size), but **market orders have stricter precision requirements**: + +- **FOK (Fill-or-Kill) market orders:** + - Sell orders: maker amount limited to **2 decimal places**. + - Taker amount: limited to **4 decimal places**. + - The product `size × price` must not exceed **2 decimal places**. + +- **Regular GTC orders:** More flexible precision based on market tick size. + +### Tick size precision hierarchy + +| Tick Size | Price Decimals | Size Decimals | Amount Decimals | +|-----------|----------------|---------------|-----------------| +| 0.1 | 1 | 2 | 3 | +| 0.01 | 2 | 2 | 4 | +| 0.001 | 3 | 2 | 5 | +| 0.0001 | 4 | 2 | 6 | + +:::note + +- The tick size precision hierarchy is defined in the [`py-clob-client` `ROUNDING_CONFIG`](https://github.com/Polymarket/py-clob-client/blob/main/py_clob_client/order_builder/builder.py). +- FOK market order precision limits (2 decimals for maker amount) are based on API error responses documented in [issue #121](https://github.com/Polymarket/py-clob-client/issues/121). +- Tick sizes can change dynamically during market conditions, particularly when markets become one-sided. + +::: + +## Trades + +Trades on Polymarket can have the following statuses: + +- `MATCHED`: Trade has been matched and sent to the executor service by the operator. The executor service submits the trade as a transaction to the Exchange contract. +- `MINED`: Trade is observed to be mined into the chain, and no finality threshold is established. +- `CONFIRMED`: Trade has achieved strong probabilistic finality and was successful. +- `RETRYING`: Trade transaction has failed (revert or reorg) and is being retried/resubmitted by the operator. +- `FAILED`: Trade has failed and is not being retried. + +Once a trade is initially matched, subsequent trade status updates will be received via the WebSocket. +NautilusTrader records the initial trade details in the `info` field of the `OrderFilled` event, +with additional trade events stored in the cache as JSON under a custom key to retain this information. + +## Fees + +Polymarket charges **zero fees** on most markets. The exception is **15-minute crypto markets**: + +| Trade Type | Fee Deducted In | Rate Range | +|------------|-----------------|-------------| +| Buy | Tokens | 0.2% - 1.6% | +| Sell | USDC | 0.8% - 3.7% | + +Fees are rounded to 4 decimal places (0.0001 USDC minimum). Market makers receive daily rebates from collected taker fees. + +:::note +For the latest rates, see Polymarket's [Fees](https://docs.polymarket.com/polymarket-learn/trading/fees) and [Maker Rebates](https://docs.polymarket.com/polymarket-learn/trading/maker-rebates-program) documentation. +::: + +## Reconciliation + +The Polymarket API returns either all **active** (open) orders or specific orders when queried by the +Polymarket order ID (`venue_order_id`). The execution reconciliation procedure for Polymarket is as follows: + +- Generate order reports for all instruments with active (open) orders, as reported by Polymarket. +- Generate position reports from contract balances reported by Polymarket, for instruments available in the cache. +- Compare these reports with Nautilus execution state. +- Generate missing orders to bring Nautilus execution state in line with positions reported by Polymarket. + +**Note**: Polymarket does not directly provide data for orders which are no longer active. + +:::warning +An optional execution client configuration, `generate_order_history_from_trades`, is currently under development. +It is not recommended for production use at this time. +::: + +## WebSockets + +The `PolymarketWebSocketClient` is built on top of the high-performance Nautilus `WebSocketClient` base class, written in Rust. + +### Data + +The main data WebSocket handles all `market` channel subscriptions received during the initial +connection sequence, up to `ws_connection_delay_secs`. For any additional subscriptions, a new `PolymarketWebSocketClient` is +created for each new instrument (asset). + +### Execution + +The main execution WebSocket manages all `user` channel subscriptions based on the Polymarket instruments +available in the cache during the initial connection sequence. When trading commands are issued for additional instruments, +a separate `PolymarketWebSocketClient` is created for each new instrument (asset). + +:::note +Polymarket does not support unsubscribing from channel streams once subscribed. +::: + +### Subscription limits + +Polymarket enforces a **maximum of 500 instruments per WebSocket connection** (undocumented limitation). + +When you attempt to subscribe to 501 or more instruments on a single WebSocket connection: + +- You will **not** receive the initial order book snapshot for each instrument. +- You will only receive subsequent order book updates. + +NautilusTrader automatically manages WebSocket connections to handle this limitation: + +- The adapter defaults to **200 instrument subscriptions per connection** (configurable via `ws_max_subscriptions_per_connection`). +- When the subscription count exceeds this limit, additional WebSocket connections are created automatically. +- This ensures you receive complete order book data (including initial snapshots) for all subscribed instruments. + +:::tip +If you need to subscribe to a large number of instruments (e.g., 5000+), the adapter will automatically distribute these subscriptions across multiple WebSocket connections. You can tune the per-connection limit up to 500 via `ws_max_subscriptions_per_connection`. +::: + +## Rate limiting + +Polymarket enforces rate limits via Cloudflare throttling. When limits are exceeded, the API returns +HTTP 429 responses and requests are throttled rather than immediately rejected. + +### REST limits + +| Endpoint Type | Limit | Notes | +|----------------------|--------------------|---------------------------------------------| +| Public endpoints | 100 req/min per IP | Unauthenticated market data requests. | +| Authenticated reads | 300 req/min per key| Authenticated data queries. | +| Trading endpoints | 60 orders/min | Order placement and cancellation. | +| Order endpoint | 3,000 req/10 min | Rolling or fixed window (unconfirmed). | + +### WebSocket limits + +- **Subscriptions per connection**: 500 instruments maximum (adapter defaults to 200, configurable via `ws_max_subscriptions_per_connection`). +- **Connections**: 20 subscriptions per connection for some endpoints. + +:::warning +Exceeding Polymarket rate limits returns HTTP 429 and may result in temporary blocking. +::: + +### Data loader rate limiting + +The `PolymarketDataLoader` includes built-in rate limiting when using the default HTTP client. +Requests are automatically throttled to 100 requests per minute, matching Polymarket's +public endpoint limit. + +When fetching large date ranges across multiple markets: + +- Multiple loaders sharing the same `http_client` instance will coordinate rate limiting automatically. +- For higher throughput with authenticated requests, pass a custom `http_client` with adjusted quotas. +- The loader does not implement automatic retry on 429 errors, so implement backoff if needed. + +:::info +For the latest rate limit details, see the official Polymarket documentation: + +::: + +## Limitations and considerations + +The following limitations are currently known: + +- Order signing via the Polymarket Python client is slow, taking around one second. +- Post-only orders are not supported. +- Reduce-only orders are not supported. + +## Configuration + +### Data client configuration options + +| Option | Default | Description | +|------------------------------------|--------------|-------------| +| `venue` | `POLYMARKET` | Venue identifier registered for the data client. | +| `private_key` | `None` | Wallet private key; sourced from `POLYMARKET_PK` when omitted. | +| `signature_type` | `0` | Signature scheme (0 = EOA, 1 = email proxy, 2 = browser wallet proxy). | +| `funder` | `None` | USDC.e funding wallet; sourced from `POLYMARKET_FUNDER` when omitted. | +| `api_key` | `None` | API key; sourced from `POLYMARKET_API_KEY` when omitted. | +| `api_secret` | `None` | API secret; sourced from `POLYMARKET_API_SECRET` when omitted. | +| `passphrase` | `None` | API passphrase; sourced from `POLYMARKET_PASSPHRASE` when omitted. | +| `base_url_http` | `None` | Override for the REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `ws_connection_initial_delay_secs` | `5` | Delay (seconds) before the first WebSocket connection to buffer subscriptions. | +| `ws_connection_delay_secs` | `0.1` | Delay (seconds) between subsequent WebSocket connection attempts. | +| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalogue refreshes. | +| `compute_effective_deltas` | `False` | Compute effective order book deltas for bandwidth savings. | +| `drop_quotes_missing_side` | `True` | Drop quotes with missing bid/ask prices instead of substituting boundary values. | +| `instrument_config` | `None` | Optional `PolymarketInstrumentProviderConfig` for instrument loading. | +| `ws_max_subscriptions_per_connection` | `200` | Maximum instrument subscriptions per WebSocket connection (Polymarket limit is 500). | + +### Execution client configuration options + +| Option | Default | Description | +|--------------------------------------|--------------|-------------| +| `venue` | `POLYMARKET` | Venue identifier registered for the execution client. | +| `private_key` | `None` | Wallet private key; sourced from `POLYMARKET_PK` when omitted. | +| `signature_type` | `0` | Signature scheme (0 = EOA, 1 = email proxy, 2 = browser wallet proxy). | +| `funder` | `None` | USDC.e funding wallet; sourced from `POLYMARKET_FUNDER` when omitted. | +| `api_key` | `None` | API key; sourced from `POLYMARKET_API_KEY` when omitted. | +| `api_secret` | `None` | API secret; sourced from `POLYMARKET_API_SECRET` when omitted. | +| `passphrase` | `None` | API passphrase; sourced from `POLYMARKET_PASSPHRASE` when omitted. | +| `base_url_http` | `None` | Override for the REST base URL. | +| `base_url_ws` | `None` | Override for the WebSocket base URL. | +| `max_retries` | `None` | Maximum retry attempts for submit/cancel requests. | +| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | +| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | +| `ack_timeout_secs` | `5.0` | Timeout (seconds) to wait for order/trade acknowledgment from cache. | +| `generate_order_history_from_trades` | `False` | Generate synthetic order history from trade reports when `True` (experimental). | +| `log_raw_ws_messages` | `False` | Log raw WebSocket payloads at INFO level when `True`. | +| `instrument_config` | `None` | Optional `PolymarketInstrumentProviderConfig` for instrument loading. | +| `ws_max_subscriptions_per_connection` | `200` | Maximum instrument subscriptions per WebSocket connection (Polymarket limit is 500). | +| `use_data_api` | `False` | Use the Data API instead of CLOB API for fetching user positions (experimental). | + +### Instrument provider configuration options + +The instrument provider config is passed via the `instrument_config` parameter on the data client config. + +| Option | Default | Description | +|----------------------|---------|---------------------------------------------------------------------------------------------------| +| `load_all` | `False` | Load all venue instruments on start. Auto-set to `True` when `event_slug_builder` is provided. | +| `event_slug_builder` | `None` | Fully qualified path to a callable returning event slugs (e.g., `"mymodule:build_slugs"`). | + +#### Event slug builder + +The `event_slug_builder` feature enables efficient loading of niche markets without downloading +all 151k+ instruments. Instead of loading everything, you provide a function that returns +event slugs for the specific markets you need. + +```python +from nautilus_trader.adapters.polymarket.providers import PolymarketInstrumentProviderConfig + +# Configure with a slug builder function +instrument_config = PolymarketInstrumentProviderConfig( + event_slug_builder="myproject.slugs:build_temperature_slugs", +) +``` + +The callable must have signature `() -> list[str]` and return a list of event slugs: + +```python +# myproject/slugs.py +from datetime import UTC, datetime, timedelta + +def build_temperature_slugs() -> list[str]: + """Build slugs for NYC temperature markets.""" + slugs = [] + today = datetime.now(tz=UTC).date() + + for i in range(7): + date = today + timedelta(days=i) + slug = f"highest-temperature-in-nyc-on-{date.strftime('%B-%d').lower()}" + slugs.append(slug) + + return slugs +``` + +See `examples/live/polymarket/slug_builders.py` for more examples including crypto UpDown markets. + +## Historical data loading + +The `PolymarketDataLoader` provides methods for fetching and parsing historical market data +for research and backtesting purposes. The loader integrates with multiple Polymarket APIs to provide the required data. + +:::note +All data fetching methods are **asynchronous** and must be called with `await`. The loader can optionally accept an `http_client` parameter for dependency injection (useful for testing). +::: + +### Data sources + +The loader fetches data from two primary sources: + +1. **Polymarket Gamma API** - Market metadata, instrument details, and active market listings. +2. **Polymarket CLOB API** - Price/trade history timeseries and order book history snapshots. + +### Method naming conventions + +The loader provides two ways to access the Polymarket APIs: + +| Prefix | Type | Use case | +|-----------|------------------|------------------------------------------------------------------------| +| `query_*` | Static methods | API exploration without an instrument. No loader instance needed. | +| `fetch_*` | Instance methods | Data fetching with a configured loader. Uses the loader's HTTP client. | + +**Use `query_*` when** you want to explore markets, discover events, or fetch metadata +before committing to a specific instrument: + +```python +# No loader needed - just query the API directly +market = await PolymarketDataLoader.query_market_by_slug("some-market") +event = await PolymarketDataLoader.query_event_by_slug("some-event") +``` + +**Use `fetch_*` when** you have a loader instance and want to fetch data using its +configured HTTP client (for coordinated rate limiting across multiple calls): + +```python +loader = await PolymarketDataLoader.from_market_slug("some-market") + +# All fetch calls share the loader's HTTP client +markets = await loader.fetch_markets(active=True, limit=100) +events = await loader.fetch_events(active=True) +details = await loader.fetch_market_details(condition_id) +``` + +### Finding markets + +Use the provided utility scripts to discover active markets: + +```bash +# List all active markets +python nautilus_trader/adapters/polymarket/scripts/active_markets.py + +# List BTC and ETH UpDown markets specifically +python nautilus_trader/adapters/polymarket/scripts/list_updown_markets.py +``` + +### Basic usage + +The recommended way to create a loader is using the factory classmethods, which handle +all the API calls and instrument creation automatically: + +```python +import asyncio + +from nautilus_trader.adapters.polymarket import PolymarketDataLoader + +async def main(): + # Create loader from market slug (recommended) + loader = await PolymarketDataLoader.from_market_slug("gta-vi-released-before-june-2026") + + # Loader is ready to use with instrument and token_id set + print(loader.instrument) + print(loader.token_id) + +asyncio.run(main()) +``` + +For events with multiple markets (e.g., temperature buckets), use `from_event_slug`: + +```python +# Returns a list of loaders, one per market in the event +loaders = await PolymarketDataLoader.from_event_slug("highest-temperature-in-nyc-on-january-26") +``` + +### Discovering markets and events + +Use `fetch_markets()` and `fetch_events()` to discover available markets programmatically: + +```python +loader = await PolymarketDataLoader.from_market_slug("any-market") + +# List active markets +markets = await loader.fetch_markets(active=True, closed=False, limit=100) +for market in markets: + print(f"{market['slug']}: {market['question']}") + +# List active events +events = await loader.fetch_events(active=True, limit=50) +for event in events: + print(f"{event['slug']}: {event['title']}") + +# Get all markets within a specific event +event_markets = await loader.get_event_markets("highest-temperature-in-nyc-on-january-26") +``` + +For quick exploration without creating a loader, use the static `query_*` methods +(see [Method naming conventions](#method-naming-conventions) above). + +### Fetching trade history + +The `load_trades()` convenience method fetches and parses historical trades in one step: + +```python +import pandas as pd + +# Load all available trades +trades = await loader.load_trades() + +# Or filter by time range (client-side filtering) +end = pd.Timestamp.now(tz="UTC") +start = end - pd.Timedelta(hours=24) + +trades = await loader.load_trades( + start=start, + end=end, +) +``` + +Alternatively, you can fetch and parse separately using the lower-level methods: + +```python +condition_id = loader.condition_id + +# Fetch raw trades from the Polymarket Data API +raw_trades = await loader.fetch_trades(condition_id=condition_id) + +# Parse to NautilusTrader TradeTicks +trades = loader.parse_trades(raw_trades) +``` + +Trade data is sourced from the [Polymarket Data API](https://data-api.polymarket.com/trades), +which provides real execution data including price, size, side, and on-chain transaction hash. + +### Complete backtest example + +A complete working example is available at `examples/backtest/polymarket_simple_quoter.py`: + +```python +import asyncio +from decimal import Decimal + +from nautilus_trader.adapters.polymarket import POLYMARKET_VENUE +from nautilus_trader.adapters.polymarket import PolymarketDataLoader +from nautilus_trader.backtest.config import BacktestEngineConfig +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.examples.strategies.ema_cross_long_only import EMACrossLongOnly +from nautilus_trader.examples.strategies.ema_cross_long_only import EMACrossLongOnlyConfig +from nautilus_trader.model.currencies import USDC_POS +from nautilus_trader.model.data import BarType +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.model.identifiers import TraderId +from nautilus_trader.model.objects import Money + +async def run_backtest(): + # Initialize loader and fetch market data + loader = await PolymarketDataLoader.from_market_slug("gta-vi-released-before-june-2026") + instrument = loader.instrument + + # Load historical trades from the Polymarket Data API + trades = await loader.load_trades() + + # Configure and run backtest + config = BacktestEngineConfig(trader_id=TraderId("BACKTESTER-001")) + engine = BacktestEngine(config=config) + + engine.add_venue( + venue=POLYMARKET_VENUE, + oms_type=OmsType.NETTING, + account_type=AccountType.CASH, + base_currency=USDC_POS, + starting_balances=[Money(10_000, USDC_POS)], + ) + + engine.add_instrument(instrument) + engine.add_data(trades) + + bar_type = BarType.from_str(f"{instrument.id}-100-TICK-LAST-INTERNAL") + strategy_config = EMACrossLongOnlyConfig( + instrument_id=instrument.id, + bar_type=bar_type, + trade_size=Decimal("20"), + ) + + strategy = EMACrossLongOnly(config=strategy_config) + engine.add_strategy(strategy=strategy) + engine.run() + + # Display results + print(engine.trader.generate_account_report(POLYMARKET_VENUE)) + +# Run the backtest +asyncio.run(run_backtest()) +``` + +**Run the complete example**: + +```bash +python examples/backtest/polymarket_simple_quoter.py +``` + +### Helper functions + +The adapter provides utility functions for working with Polymarket identifiers: + +```python +from nautilus_trader.adapters.polymarket import get_polymarket_instrument_id + +# Create NautilusTrader InstrumentId from Polymarket identifiers +instrument_id = get_polymarket_instrument_id( + condition_id="0xcccb7e7613a087c132b69cbf3a02bece3fdcb824c1da54ae79acc8d4a562d902", + token_id="8441400852834915183759801017793514978104486628517653995211751018945988243154" +) +``` + +## Contributing + +:::info +For additional features or to contribute to the Polymarket adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/integrations/tardis.md b/nautilus-docs/docs/integrations/tardis.md new file mode 100644 index 0000000..0a1fc91 --- /dev/null +++ b/nautilus-docs/docs/integrations/tardis.md @@ -0,0 +1,695 @@ +# Tardis + +Tardis provides granular data for cryptocurrency markets including tick-by-tick order book snapshots & updates, +trades, open interest, funding rates, options chains and liquidations data for leading crypto exchanges. + +NautilusTrader provides an integration with the Tardis API and data formats, enabling access. +The capabilities of this adapter include: + +- `TardisCSVDataLoader`: Reads Tardis-format CSV files and converts them into Nautilus data, with support for both bulk loading and memory-efficient streaming. +- `TardisMachineClient`: Supports live streaming and historical replay of data from the Tardis Machine WebSocket server - converting messages into Nautilus data. +- `TardisHttpClient`: Requests instrument definition metadata from the Tardis HTTP API, parsing it into Nautilus instrument definitions. +- `TardisDataClient`: Provides a live data client for subscribing to data streams from a Tardis Machine WebSocket server. +- `TardisInstrumentProvider`: Provides instrument definitions from Tardis through the HTTP instrument metadata API. +- **Data pipeline functions**: Enables replay of historical data from Tardis Machine and writes it to the Nautilus Parquet format, including direct catalog integration for data management (see below). + +:::info +A Tardis API key is required for the adapter to operate correctly. See also [environment variables](#environment-variables). +::: + +## Overview + +This adapter is implemented in Rust, with optional Python bindings for ease of use in Python-based workflows. +It does not require any external Tardis client library dependencies. + +:::info +There is **no** need for additional installation steps for `tardis`. +The core components of the adapter are compiled as static libraries and automatically linked during the build process. +::: + +## Tardis documentation + +Tardis provides extensive user [documentation](https://docs.tardis.dev/). +We recommend also referring to the Tardis documentation in conjunction with this NautilusTrader integration guide. + +## Supported formats + +Tardis provides *normalized* market data, a unified format consistent across all supported exchanges. +This normalization is highly valuable because it allows a single parser to handle data from any [Tardis-supported exchange](#venues), reducing development time and complexity. +As a result, NautilusTrader will not support exchange-native market data formats, as it would be inefficient to implement separate parsers for each exchange at this stage. + +The following normalized Tardis formats are supported by NautilusTrader: + +| Tardis format | Nautilus data type | +|:----------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------| +| [book_change](https://docs.tardis.dev/api/tardis-machine#book_change) | `OrderBookDelta` | +| [book_snapshot_*](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) | `OrderBookDepth10` or `OrderBookDeltas` (see [book snapshot output](#book-snapshot-output)) | +| [quote](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) | `QuoteTick` | +| [quote_10s](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) | `QuoteTick` | +| [trade](https://docs.tardis.dev/api/tardis-machine#trade) | `Trade` | +| [trade_bar_*](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) | `Bar` | +| [instrument](https://docs.tardis.dev/api/instruments-metadata-api) | `CurrencyPair`, `CryptoFuture`, `CryptoPerpetual`, `OptionContract` | +| [derivative_ticker](https://docs.tardis.dev/api/tardis-machine#derivative_ticker) | `FundingRateUpdate` | +| [disconnect](https://docs.tardis.dev/api/tardis-machine#disconnect) | *Not applicable* | + +**Notes:** + +- [quote](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) is an alias for [book_snapshot_1_0ms](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit). +- [quote_10s](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) is an alias for [book_snapshot_1_10s](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit). +- Both quote, quote\_10s, and one-level snapshots are parsed as `QuoteTick`. + +:::info +See also the Tardis [normalized market data APIs](https://docs.tardis.dev/api/tardis-machine#normalized-market-data-apis). +::: + +## Bars + +The adapter will automatically convert [Tardis trade bar interval and suffix](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) to Nautilus `BarType`s. +This includes the following: + +| Tardis suffix | Nautilus bar aggregation | +|:-------------------------------------------------------------------------------------------------------------|:----------------------------| +| [ms](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - milliseconds | `MILLISECOND` | +| [s](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - seconds | `SECOND` | +| [m](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - minutes | `MINUTE` | +| [ticks](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - number of ticks | `TICK` | +| [vol](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - volume size | `VOLUME` | + +## Symbology and normalization + +The Tardis integration ensures compatibility with NautilusTrader’s crypto exchange adapters +by consistently normalizing symbols. Typically, NautilusTrader uses the native exchange naming conventions +provided by Tardis. However, for certain exchanges, raw symbols are adjusted to adhere to the Nautilus symbology normalization, as outlined below: + +### Common rules + +- All symbols are converted to uppercase. +- Market type suffixes are appended with a hyphen for some exchanges (see [exchange-specific normalizations](#exchange-specific-normalizations)). +- Original exchange symbols are preserved in the Nautilus instrument definitions `raw_symbol` field. + +### Exchange-specific normalizations + +- **Binance**: Nautilus appends the suffix `-PERP` to all perpetual symbols. +- **Bybit**: Nautilus uses specific product category suffixes, including `-SPOT`, `-LINEAR`, `-INVERSE`, `-OPTION`. +- **dYdX**: Nautilus appends the suffix `-PERP` to all perpetual symbols. +- **Gate.io**: Nautilus appends the suffix `-PERP` to all perpetual symbols. + +For detailed symbology documentation per exchange: + +- [Binance symbology](./binance.md#symbology) +- [Bybit symbology](./bybit.md#symbology) +- [dYdX symbology](./dydx.md#symbology) + +## Venues + +Some exchanges on Tardis are partitioned into multiple venues. +The table below outlines the mappings between Nautilus venues and corresponding Tardis exchanges, as well as the exchanges that Tardis supports: + +| Nautilus venue | Tardis exchange(s) | +|:------------------------|:------------------------------------------------------| +| `ASCENDEX` | `ascendex` | +| `BINANCE` | `binance`, `binance-dex`, `binance-european-options`, `binance-futures`, `binance-jersey`, `binance-options` | +| `BINANCE_DELIVERY` | `binance-delivery` (*COIN-margined contracts*) | +| `BINANCE_US` | `binance-us` | +| `BITFINEX` | `bitfinex`, `bitfinex-derivatives` | +| `BITFLYER` | `bitflyer` | +| `BITGET` | `bitget`, `bitget-futures` | +| `BITMEX` | `bitmex` | +| `BITNOMIAL` | `bitnomial` | +| `BITSTAMP` | `bitstamp` | +| `BLOCKCHAIN_COM` | `blockchain-com` | +| `BYBIT` | `bybit`, `bybit-options`, `bybit-spot` | +| `COINBASE` | `coinbase` | +| `COINBASE_INTX` | `coinbase-international` | +| `COINFLEX` | `coinflex` (*for historical research*) | +| `CRYPTO_COM` | `crypto-com`, `crypto-com-derivatives` | +| `CRYPTOFACILITIES` | `cryptofacilities` | +| `DELTA` | `delta` | +| `DERIBIT` | `deribit` | +| `DYDX` | `dydx` | +| `DYDX_V4` | `dydx-v4` | +| `FTX` | `ftx`, `ftx-us` (*historical research*) | +| `GATE_IO` | `gate-io`, `gate-io-futures` | +| `GEMINI` | `gemini` | +| `HITBTC` | `hitbtc` | +| `HUOBI` | `huobi`, `huobi-dm`, `huobi-dm-linear-swap`, `huobi-dm-options` | +| `HUOBI_DELIVERY` | `huobi-dm-swap` | +| `HYPERLIQUID` | `hyperliquid` | +| `KRAKEN` | `kraken` | +| `KUCOIN` | `kucoin`, `kucoin-futures` | +| `MANGO` | `mango` | +| `OKCOIN` | `okcoin` | +| `OKEX` | `okex`, `okex-futures`, `okex-options`, `okex-spreads`, `okex-swap` | +| `PHEMEX` | `phemex` | +| `POLONIEX` | `poloniex` | +| `SERUM` | `serum` (*historical research*) | +| `STAR_ATLAS` | `star-atlas` | +| `UPBIT` | `upbit` | +| `WOO_X` | `woo-x` | + +## Environment variables + +The following environment variables are used by Tardis and NautilusTrader. + +- `TM_API_KEY`: API key for the Tardis Machine. +- `TARDIS_API_KEY`: API key for NautilusTrader Tardis clients. +- `TARDIS_MACHINE_WS_URL` (optional): WebSocket URL for the `TardisMachineClient` in NautilusTrader. +- `TARDIS_BASE_URL` (optional): Base URL for the `TardisHttpClient` in NautilusTrader. +- `NAUTILUS_PATH` (optional): Parent directory containing the `catalog/` subdirectory for writing replay data in the Nautilus catalog format. + +## Running Tardis Machine historical replays + +The [Tardis Machine Server](https://docs.tardis.dev/api/tardis-machine) is a locally runnable server +with built-in data caching, providing both tick-level historical and consolidated real-time cryptocurrency market data through HTTP and WebSocket APIs. + +You can perform complete Tardis Machine WebSocket replays of historical data and output the results +in Nautilus Parquet format, using either Python or Rust. Since the function is implemented in Rust, +performance is consistent whether run from Python or Rust, letting you choose based on your preferred workflow. + +The end-to-end `run_tardis_machine_replay` data pipeline function uses a specified [configuration](#configuration) to execute the following steps: + +- Connect to the Tardis Machine server. +- Request and parse all necessary instrument definitions from the [Tardis instruments metadata](https://docs.tardis.dev/api/instruments-metadata-api) HTTP API. +- Stream all requested instruments and data types for the specified time ranges from the Tardis Machine server. +- For each instrument, data type and date (UTC), generate a `.parquet` file in the catalog-compatible format. +- Disconnect from the Tardis Machine server, and terminate the program. + +**File Naming Convention** + +Files are written one per day, per instrument, using ISO 8601 timestamp ranges that clearly indicate the exact time span of data: + +- **Format**: `{start_timestamp}_{end_timestamp}.parquet` +- **Example**: `2023-10-01T00-00-00-000000000Z_2023-10-01T23-59-59-999999999Z.parquet` +- **Structure**: `data/{data_type}/{instrument_id}/{filename}` + +This format is fully compatible with the Nautilus data catalog, enabling querying, consolidation, and data management operations. + +:::note +You can request data for the first day of each month without an API key. For all other dates, a Tardis Machine API key is required. +::: + +This process is optimized for direct output to a Nautilus Parquet data catalog. +Ensure that the `NAUTILUS_PATH` environment variable is set to the parent directory containing the `catalog/` subdirectory. +Parquet files will then be organized under `/catalog/data/` in the expected subdirectories corresponding to data type and instrument. + +If no `output_path` is specified in the configuration file and the `NAUTILUS_PATH` environment variable is unset, the system will default to the current working directory. + +### Procedure + +First, ensure the `tardis-machine` docker container is running. Use the following command: + +```bash +docker run -p 8000:8000 -p 8001:8001 -e "TM_API_KEY=YOUR_API_KEY" -d tardisdev/tardis-machine +``` + +This command starts the `tardis-machine` server without a persistent local cache, which may affect performance. +For improved performance, consider running the server with a persistent volume. Refer to the [Tardis Docker documentation](https://docs.tardis.dev/api/tardis-machine#docker) for details. + +### Configuration + +Next, ensure you have a configuration JSON file available. + +**Configuration JSON format** + +| Field | Type | Description | Default | +|:------------------------|:------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------| +| `tardis_ws_url` | string (optional) | The Tardis Machine WebSocket URL. | If `null` then will use the `TARDIS_MACHINE_WS_URL` env var. | +| `normalize_symbols` | bool (optional) | If Nautilus [symbol normalization](#symbology-and-normalization) should be applied. | If `null` then will default to `true`. | +| `output_path` | string (optional) | The output directory path to write Nautilus Parquet data to. | If `null` then will use the `NAUTILUS_PATH` env var, otherwise the current working directory. | +| `book_snapshot_output` | string (optional) | Output format for `book_snapshot_*` data: `"deltas"` or `"depth10"`. See [book snapshot output](#book-snapshot-output). | If `null` then will default to `"deltas"`. | +| `ws_proxy_url` | string (optional) | Optional WebSocket proxy URL. | If `null` then no proxy is used. | +| `options` | JSON[] | An array of [ReplayNormalizedRequestOptions](https://docs.tardis.dev/api/tardis-machine#replay-normalized-options) objects. | + +An example configuration file, `example_config.json`, is available [here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_config.json): + +```json +{ + "tardis_ws_url": "ws://localhost:8001", + "output_path": null, + "options": [ + { + "exchange": "bitmex", + "symbols": [ + "xbtusd", + "ethusd" + ], + "data_types": [ + "trade" + ], + "from": "2019-10-01", + "to": "2019-10-02" + } + ] +} +``` + +### Book snapshot output + +The `book_snapshot_output` configuration option controls how Tardis `book_snapshot_*` messages (e.g., `book_snapshot_5_100ms`, `book_snapshot_10_1s`) are converted and stored. + +| Value | Nautilus Type | Output Directory | Description | +|:----------|:--------------------|:----------------------|:------------------------------------------------------------| +| `deltas` | `OrderBookDeltas` | `order_book_deltas/` | Individual price level updates with snapshot flag set (default) | +| `depth10` | `OrderBookDepth10` | `order_book_depths/` | Periodic depth snapshots with up to 10 price levels | + +**When to use each format:** + +- **`deltas` (default)**: Best when you need to reconstruct the full order book state or when working with `book_change` data. Each price level becomes a separate delta record. +- **`depth10`**: Best for strategies that need periodic order book snapshots. More memory-efficient as each snapshot is a single record containing all levels. Snapshots with more than 10 levels will have only the first 10 preserved. + +**Avoiding file overwrites:** + +When downloading both `book_snapshot_*` and `book_change` data for the same instrument and date range, using `depth10` format ensures they are written to separate directories (`order_book_depths/` vs `order_book_deltas/`), preventing file overwrites. + +Example configuration with explicit format: + +```json +{ + "tardis_ws_url": "ws://localhost:8001", + "book_snapshot_output": "depth10", + "options": [ + { + "exchange": "binance-futures", + "symbols": ["btcusdt"], + "data_types": ["book_snapshot_5_100ms", "book_change"], + "from": "2024-01-01", + "to": "2024-01-02" + } + ] +} +``` + +### Python Replays + +To run a replay in Python, create a script similar to the following: + +```python +import asyncio + +from nautilus_trader.core import nautilus_pyo3 + + +async def run(): + config_filepath = Path("YOUR_CONFIG_FILEPATH") + await nautilus_pyo3.run_tardis_machine_replay(str(config_filepath.resolve())) + + +if __name__ == "__main__": + asyncio.run(run()) +``` + +### Rust Replays + +To run a replay in Rust, create a binary similar to the following: + +```rust +use std::path::PathBuf; + +use nautilus_adapters::tardis::replay::run_tardis_machine_replay_from_config; + +#[tokio::main] +async fn main() { + nautilus_common::logging::ensure_logging_initialized(); + + let config_filepath = PathBuf::from("YOUR_CONFIG_FILEPATH"); + run_tardis_machine_replay_from_config(&config_filepath).await; +} +``` + +Logging defaults to INFO level. To enable debug logging, export the following environment variable: + +```bash +export NAUTILUS_LOG=debug +``` + +A working example binary can be found [here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_replay.rs). + +This can also be run using cargo: + +```bash +cargo run --bin tardis-replay +``` + +## Loading Tardis CSV data + +Tardis-format CSV data can be loaded using either Python or Rust. The loader reads the CSV text data +from disk and parses it into Nautilus data. Since the loader is implemented in Rust, performance remains +consistent regardless of whether you run it from Python or Rust, allowing you to choose based on your preferred workflow. + +You can also optionally specify a `limit` parameter for the `load_*` functions/methods to control the maximum number of rows loaded. + +:::note +Loading mixed-instrument CSV files is challenging due to precision requirements and is not recommended. Use single-instrument CSV files instead (see below). +::: + +### Loading CSV Data in Python + +You can load Tardis-format CSV data in Python using the `TardisCSVDataLoader`. +When loading data, you can optionally specify the instrument ID but must specify both the price precision, and size precision. +Providing the instrument ID improves loading performance, while specifying the precisions is required, as they cannot be inferred from the text data alone. + +To load the data, create a script similar to the following: + +```python +from nautilus_trader.adapters.tardis import TardisCSVDataLoader +from nautilus_trader.model import InstrumentId + + +instrument_id = InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") +loader = TardisCSVDataLoader( + price_precision=1, + size_precision=0, + instrument_id=instrument_id, +) + +filepath = Path("YOUR_CSV_DATA_PATH") +limit = None + +deltas = loader.load_deltas(filepath, limit) +``` + +### Loading CSV Data in Rust + +You can load Tardis-format CSV data in Rust using the loading functions found [here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/src/csv/mod.rs). +When loading data, you can optionally specify the instrument ID but must specify both the price precision and size precision. +Providing the instrument ID improves loading performance, while specifying the precisions is required, as they cannot be inferred from the text data alone. + +For a complete example, see the [example binary here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_csv.rs). + +To load the data, you can use code similar to the following: + +```rust +use std::path::Path; + +use nautilus_adapters::tardis; +use nautilus_model::identifiers::InstrumentId; + +#[tokio::main] +async fn main() { + // You must specify precisions and the CSV filepath + let price_precision = 1; + let size_precision = 0; + let filepath = Path::new("YOUR_CSV_DATA_PATH"); + + // Optionally specify an instrument ID and/or limit + let instrument_id = InstrumentId::from("BTC-PERPETUAL.DERIBIT"); + let limit = None; + + // Consider propagating any parsing error depending on your workflow + let _deltas = tardis::csv::load_deltas( + filepath, + price_precision, + size_precision, + Some(instrument_id), + limit, + ) + .unwrap(); +} +``` + +## Streaming Tardis CSV Data + +For memory-efficient processing of large CSV files, the Tardis integration provides streaming capabilities that load and process data in configurable chunks rather than loading entire files into memory at once. This is particularly useful for processing multi-gigabyte CSV files without exhausting system memory. + +The streaming functionality is available for all supported Tardis data types: + +- Order book deltas (`stream_deltas`). +- Quote ticks (`stream_quotes`). +- Trade ticks (`stream_trades`). +- Order book depth snapshots (`stream_depth10`). + +### Streaming CSV Data in Python + +The `TardisCSVDataLoader` provides streaming methods that yield chunks of data as iterators. Each method accepts a `chunk_size` parameter that controls how many records are read from the CSV file per chunk: + +```python +from nautilus_trader.adapters.tardis import TardisCSVDataLoader +from nautilus_trader.model import InstrumentId + +instrument_id = InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") +loader = TardisCSVDataLoader( + price_precision=1, + size_precision=0, + instrument_id=instrument_id, +) + +filepath = Path("large_trades_file.csv") +chunk_size = 100_000 # Process 100,000 records per chunk (default) + +# Stream trade ticks in chunks +for chunk in loader.stream_trades(filepath, chunk_size): + print(f"Processing chunk with {len(chunk)} trades") + # Process each chunk - only this chunk is in memory + for trade in chunk: + # Your processing logic here + pass +``` + +### Streaming Order Book Data + +For order book data, streaming is available for both deltas and depth snapshots: + +```python +# Stream order book deltas +for chunk in loader.stream_deltas(filepath): + print(f"Processing {len(chunk)} deltas") + # Process delta chunk + +# Stream depth10 snapshots (specify levels: 5 or 25) +for chunk in loader.stream_depth10(filepath, levels=5): + print(f"Processing {len(chunk)} depth snapshots") + # Process depth chunk +``` + +### Streaming Quote Data + +Quote data can be streamed similarly: + +```python +# Stream quote ticks +for chunk in loader.stream_quotes(filepath): + print(f"Processing {len(chunk)} quotes") + # Process quote chunk +``` + +### Memory Efficiency Benefits + +The streaming approach provides significant memory efficiency advantages: + +- **Controlled Memory Usage**: Only one chunk is loaded in memory at a time. +- **Scalable Processing**: Can process files larger than available RAM. +- **Configurable Chunk Sizes**: Tune `chunk_size` based on your system's memory and performance requirements (default 100,000). + +:::warning +When using streaming with precision inference (not providing explicit precisions), the inferred precision may differ from bulk loading the entire file. +This is because precision inference works within chunk boundaries, and different chunks may contain values with different precision requirements. +For deterministic precision behavior, provide explicit `price_precision` and `size_precision` parameters when calling streaming methods. +::: + +### Streaming CSV Data in Rust + +The underlying streaming functionality is implemented in Rust and can be used directly: + +```rust +use std::path::Path; +use nautilus_adapters::tardis::csv::{stream_trades, stream_deltas}; +use nautilus_model::identifiers::InstrumentId; + +#[tokio::main] +async fn main() { + let filepath = Path::new("large_trades_file.csv"); + let chunk_size = 100_000; + let price_precision = Some(1); + let size_precision = Some(0); + let instrument_id = Some(InstrumentId::from("BTC-PERPETUAL.DERIBIT")); + + // Stream trades in chunks + let stream = stream_trades( + filepath, + chunk_size, + price_precision, + size_precision, + instrument_id, + ).unwrap(); + + for chunk_result in stream { + match chunk_result { + Ok(chunk) => { + println!("Processing chunk with {} trades", chunk.len()); + // Process chunk + } + Err(e) => { + eprintln!("Error processing chunk: {}", e); + break; + } + } + } +} +``` + +## Requesting instrument definitions + +You can request instrument definitions in both Python and Rust using the `TardisHttpClient`. +This client interacts with the [Tardis instruments metadata API](https://docs.tardis.dev/api/instruments-metadata-api) to request and parse instrument metadata into Nautilus instruments. + +The `TardisHttpClient` constructor accepts optional parameters for `api_key`, `base_url`, and `timeout_secs` (default is 60 seconds). + +The client provides methods to retrieve either a specific `instrument`, or all `instruments` available on a particular exchange. +Ensure that you use Tardis’s lower-kebab casing convention when referring to a [Tardis-supported exchange](https://api.tardis.dev/v1/exchanges). + +:::note +A Tardis API key is required to access the instruments metadata API. +::: + +### Requesting Instruments in Python + +To request instrument definitions in Python, create a script similar to the following: + +```python +import asyncio + +from nautilus_trader.core import nautilus_pyo3 + + +async def run(): + http_client = nautilus_pyo3.TardisHttpClient() + + instrument = await http_client.instrument("bitmex", "xbtusd") + print(f"Received: {instrument}") + + instruments = await http_client.instruments("bitmex") + print(f"Received: {len(instruments)} instruments") + + +if __name__ == "__main__": + asyncio.run(run()) +``` + +### Requesting Instruments in Rust + +To request instrument definitions in Rust, use code similar to the following. +For a complete example, see the [example binary here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_http.rs). + +```rust +use nautilus_tardis::{ + enums::TardisExchange, + http::client::TardisHttpClient, +}; + +#[tokio::main] +async fn main() { + nautilus_common::logging::ensure_logging_initialized(); + + let client = TardisHttpClient::new(None, None, None, true).unwrap(); + + // Tardis instrument definitions + let resp = client + .instruments_info(TardisExchange::Bitmex, Some("XBTUSD"), None) + .await; + println!("Received: {resp:?}"); + + // Nautilus instrument definitions + let resp = client + .instruments(TardisExchange::Bitmex, Some("XBTUSD"), None, None, None, None, None, None) + .await; + println!("Received: {resp:?}"); +} +``` + +## Instrument provider + +The `TardisInstrumentProvider` requests and parses instrument definitions from Tardis through the HTTP instrument metadata API. +Since there are multiple [Tardis-supported exchanges](#venues), when loading all instruments, +you must filter for the desired venues using an `InstrumentProviderConfig`: + +```python +from nautilus_trader.config import InstrumentProviderConfig + +# See supported venues https://nautilustrader.io/docs/nightly/integrations/tardis#venues +venues = {"BINANCE", "BYBIT"} +filters = {"venues": frozenset(venues)} +instrument_provider_config = InstrumentProviderConfig(load_all=True, filters=filters) +``` + +You can also load specific instrument definitions in the usual way: + +```python +from nautilus_trader.config import InstrumentProviderConfig + +instrument_ids = [ + InstrumentId.from_str("BTCUSDT-PERP.BINANCE"), # Will use the 'binance-futures' exchange + InstrumentId.from_str("BTCUSDT.BINANCE"), # Will use the 'binance' exchange +] +instrument_provider_config = InstrumentProviderConfig(load_ids=instrument_ids) +``` + +### Option exchange filtering + +The instrument provider automatically filters out option-specific exchanges (such as `binance-options`, `binance-european-options`, `bybit-options`, `okex-options`, and `huobi-dm-options`) when the `instrument_type` filter is not provided or does not include `"option"`. + +To explicitly load option instruments, include `"option"` in the `instrument_type` filter: + +```python +from nautilus_trader.config import InstrumentProviderConfig + +venues = {"BINANCE", "BYBIT"} +filters = { + "venues": frozenset(venues), + "instrument_type": {"option"}, # Explicitly request options +} +instrument_provider_config = InstrumentProviderConfig(load_all=True, filters=filters) +``` + +This filtering mechanism prevents unnecessary API calls to option exchanges when they are not needed, improving performance and reducing API usage. + +:::note +Instruments must be available in the cache for all subscriptions. +For simplicity, it's recommended to load all instruments for the venues you intend to subscribe to. +::: + +## Live data client + +The `TardisDataClient` enables integration of a Tardis Machine with a running NautilusTrader system. +It supports subscriptions to the following data types: + +- `OrderBookDelta` (L2 granularity from Tardis, includes all changes or full-depth snapshots) +- `OrderBookDepth10` (L2 granularity from Tardis, provides snapshots up to 10 levels) +- `QuoteTick` +- `TradeTick` +- `Bar` (trade bars with [Tardis-supported bar aggregations](#bars)) +- `FundingRateUpdate` (from derivative_ticker messages) + +### Data WebSockets + +The main `TardisMachineClient` data WebSocket manages all stream subscriptions received during the initial connection phase, +up to the duration specified by `ws_connection_delay_secs`. For any additional subscriptions made +after this period, a new `TardisMachineClient` is created. This approach optimizes performance by +allowing the main WebSocket to handle potentially hundreds of subscriptions in a single stream if +they are provided at startup. + +When an initial subscription delay is set with `ws_connection_delay_secs`, unsubscribing from any +of these streams will not actually remove the subscription from the Tardis Machine stream, as selective +unsubscription is not supported by Tardis. However, the component will still unsubscribe from message +bus publishing as expected. + +All subscriptions made after any initial delay will behave normally, fully unsubscribing from the +Tardis Machine stream when requested. + +:::tip +If you anticipate frequent subscription and unsubscription of data, it is recommended to set +`ws_connection_delay_secs` to zero. This will create a new client for each initial subscription, +allowing them to be later closed individually upon unsubscription. +::: + +## Limitations and considerations + +The following limitations and considerations are currently known: + +- Historical data requests are not supported, as each would require a minimum one-day replay from the Tardis Machine, potentially with a filter. This approach is neither practical nor efficient. + +## Contributing + +:::info +For additional features or to contribute to the Tardis adapter, please see our +[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). +::: diff --git a/nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md b/nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md new file mode 100644 index 0000000..9e276d0 --- /dev/null +++ b/nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md @@ -0,0 +1,279 @@ +# AX Exchange - FX Perpetual Mean Reversion + +This tutorial walks through backtesting a **Bollinger Band mean reversion** strategy on +**EURUSD-PERP** (EUR/USD perpetual) using [AX Exchange](https://architect.exchange) instrument +definitions and [TrueFX](https://www.truefx.com) spot FX data as a proxy. + +## Introduction + +Mean reversion strategies assume that prices tend to return to a statistical average after +deviating from it. **Bollinger Bands** provide a volatility-adaptive envelope around a moving +average: the upper and lower bands expand in volatile markets and contract in quiet ones. +When price touches a band, it may be overextended relative to recent history. + +This strategy adds a **Relative Strength Index (RSI)** filter as confirmation. A touch of the +lower band alone is not sufficient to buy - RSI must also indicate oversold conditions. This +two-indicator approach reduces whipsaws in trending markets. + +For demonstration purposes, NautilusTrader ships with a `BBMeanReversion` example strategy +that is intentionally simple (no alpha advantage). + +### Why proxy data? + +AX Exchange is a new venue and is not yet covered by most historical data vendors. +[TrueFX](https://www.truefx.com) provides free institutional-grade spot FX tick data sourced +from Integral and Jefferies liquidity pools. EUR/USD spot data serves as a representative +proxy for backtesting an AX EURUSD-PERP strategy. + +## Prerequisites + +- **NautilusTrader** installed (see the [installation guide](../getting_started/installation.md)). +- **TrueFX account** (free): Sign up at [truefx.com](https://www.truefx.com) to access + historical tick data downloads. + +## Data preparation + +### Download TrueFX EUR/USD tick data + +1. Go to the [TrueFX historical downloads page](https://www.truefx.com/truefx-historical-downloads/). +2. Select **EUR/USD** and your desired month (e.g., December 2025). +3. Download and extract the CSV file (e.g., `EURUSD-2025-12.csv`). + +The raw TrueFX format has **no headers**. Columns are: `pair, timestamp, bid, ask`. + +### Load and prepare the data + +Use pandas to load the CSV and parse timestamps, then process through +`QuoteTickDataWrangler` which auto-renames `bid`/`ask` columns: + +```python +from pathlib import Path + +import pandas as pd + +from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler + +data_path = Path("EURUSD-2025-12.csv") + +df = pd.read_csv( + data_path, + header=None, + names=["pair", "timestamp", "bid", "ask"], +) +df["timestamp"] = pd.to_datetime(df["timestamp"], format="%Y%m%d %H:%M:%S.%f") +df = df.set_index("timestamp") +df = df[["bid", "ask"]] + +wrangler = QuoteTickDataWrangler(instrument=EURUSD_PERP) # defined below +ticks = wrangler.process(df) +``` + +The wrangler produces `QuoteTick` objects tagged with the instrument ID. These ticks drive +bar aggregation internally - 1-minute MID bars will be built from the quote tick stream. + +## Instrument definition + +Since we are using proxy data, we define the EURUSD-PERP instrument manually as a +`PerpetualContract`. The multiplier of 1000 means each contract represents 1000 EUR notional: + +```python +from decimal import Decimal + +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AssetClass +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.identifiers import Symbol +from nautilus_trader.model.instruments import PerpetualContract +from nautilus_trader.model.objects import Price +from nautilus_trader.model.objects import Quantity + +instrument_id = InstrumentId.from_str("EURUSD-PERP.AX") + +EURUSD_PERP = PerpetualContract( + instrument_id=instrument_id, + raw_symbol=Symbol("EURUSD-PERP"), + underlying="EUR", + asset_class=AssetClass.FX, + quote_currency=USD, + settlement_currency=USD, + is_inverse=False, + price_precision=5, + size_precision=0, + price_increment=Price.from_str("0.00001"), + size_increment=Quantity.from_int(1), + multiplier=Quantity.from_int(1000), + lot_size=Quantity.from_int(1), + margin_init=Decimal("0.05"), + margin_maint=Decimal("0.025"), + maker_fee=Decimal("0.0002"), + taker_fee=Decimal("0.0005"), + ts_event=0, + ts_init=0, +) +``` + +Fees are explicit backtest assumptions and should be set deliberately. Check the +[AX Exchange documentation](https://docs.architect.exchange/) for current rates. + +## Strategy overview + +The `BBMeanReversion` strategy works as follows: + +1. **Wait for warm-up**: Both indicators must be initialized before trading. +2. **Exit check (first)**: If long and close >= BB middle band → close position (mean + reversion target reached). If short and close <= BB middle band → close position. +3. **Entry signals**: If close <= BB lower band AND RSI < buy threshold → buy. If + close >= BB upper band AND RSI > sell threshold → sell. Existing positions in the + opposite direction are closed before entering. + +### Configuration + +| Parameter | Value | Description | +| -------------------- | ------ | ---------------------------------------------------- | +| `bb_period` | `20` | 20-bar lookback for Bollinger Bands. | +| `bb_std` | `2.0` | 2 standard deviations for band width. | +| `rsi_period` | `14` | 14-bar lookback for RSI. | +| `rsi_buy_threshold` | `0.30` | RSI below 0.30 confirms oversold (range 0-1). | +| `rsi_sell_threshold` | `0.70` | RSI above 0.70 confirms overbought (range 0-1). | +| `trade_size` | `1` | 1 contract per trade (1000 EUR notional). | + +:::tip +NautilusTrader RSI outputs values in the range [0.0, 1.0], not [0, 100]. Set thresholds +accordingly - 0.30 corresponds to the traditional RSI level of 30. +::: + +## Backtest setup + +### Configure the engine + +```python +from nautilus_trader.backtest.config import BacktestEngineConfig +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.config import LoggingConfig +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.model.identifiers import TraderId +from nautilus_trader.model.identifiers import Venue +from nautilus_trader.model.objects import Money + +config = BacktestEngineConfig( + trader_id=TraderId("BACKTESTER-001"), + logging=LoggingConfig(log_level="INFO"), +) + +engine = BacktestEngine(config=config) +``` + +### Add the venue + +AX Exchange uses margin accounts with netting position management: + +```python +AX = Venue("AX") + +engine.add_venue( + venue=AX, + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, + base_currency=USD, + starting_balances=[Money(100_000, USD)], +) +``` + +### Add instrument, data, and strategy + +The bar type `EURUSD-PERP.AX-1-MINUTE-MID-INTERNAL` tells the engine to build 1-minute +bars from the mid-price of the quote tick stream: + +```python +from nautilus_trader.examples.strategies.bb_mean_reversion import BBMeanReversion +from nautilus_trader.examples.strategies.bb_mean_reversion import BBMeanReversionConfig +from nautilus_trader.model.data import BarType + +bar_type = BarType.from_str("EURUSD-PERP.AX-1-MINUTE-MID-INTERNAL") + +strategy_config = BBMeanReversionConfig( + instrument_id=instrument_id, + bar_type=bar_type, + trade_size=Decimal("1"), + bb_period=20, + bb_std=2.0, + rsi_period=14, + rsi_buy_threshold=0.30, + rsi_sell_threshold=0.70, +) + +strategy = BBMeanReversion(config=strategy_config) + +engine.add_instrument(EURUSD_PERP) +engine.add_data(ticks) +engine.add_strategy(strategy) +``` + +### Run the backtest + +```python +engine.run() +``` + +## Results + +After the run completes, generate reports to analyze performance: + +```python +import pandas as pd + +with pd.option_context( + "display.max_rows", 100, + "display.max_columns", None, + "display.width", 300, +): + print(engine.trader.generate_account_report(AX)) + print(engine.trader.generate_order_fills_report()) + print(engine.trader.generate_positions_report()) +``` + +Clean up when done: + +```python +engine.reset() +engine.dispose() +``` + +## Complete script + +The complete script is available as +[`architect_ax_mean_reversion.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_mean_reversion.py) +in the examples directory. + +## Next steps + +- **Tune parameters**: Experiment with `bb_period`, `bb_std`, and RSI thresholds to + understand their effect on trade frequency and PnL. +- **Try different pairs**: Download GBP/USD or USD/JPY data from TrueFX and define the + corresponding perpetual contract. +- **Add stop losses**: Extend the strategy with stop-loss orders to limit downside on + positions that move against you. +- **Go live on AX sandbox**: Once you are satisfied with backtest results, connect to the + AX sandbox environment for paper trading. See the + [AX Exchange integration guide](../integrations/architect_ax.md) for setup instructions. + +## Running live + +The same strategy used in this backtest can be run live with no code changes - only a +launch script is needed. NautilusTrader's architecture means your strategy is +venue-agnostic: switching from backtest to live is a configuration change, not a rewrite. + +See the complete live example: +[`ax_mean_reversion.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/architect_ax/ax_mean_reversion.py) + +For connection setup and API key configuration, refer to the +[AX Exchange integration guide](../integrations/architect_ax.md). + +## Further reading + +- [AX Exchange mean reversion backtest example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_mean_reversion.py) +- [`BBMeanReversion` strategy source](https://github.com/nautechsystems/nautilus_trader/tree/develop/nautilus_trader/examples/strategies/bb_mean_reversion.py) +- [AX Exchange gold book imbalance tutorial](ax_gold_book_imbalance.md) +- [Architect Exchange documentation](https://docs.architect.exchange/) diff --git a/nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md b/nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md new file mode 100644 index 0000000..015da31 --- /dev/null +++ b/nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md @@ -0,0 +1,306 @@ +# AX Exchange - Gold Perpetual Book Imbalance + +This tutorial walks through backtesting an **order book imbalance** strategy on +**XAU-PERP** (gold perpetual) using [AX Exchange](https://architect.exchange) instrument +definitions and [Databento](https://databento.com) CME gold futures data as a proxy. + +## Introduction + +Order book imbalance is a canonical microstructure signal used in high-frequency and +short-term trading. When there is significantly more volume resting on one side of the +book than the other, this can signal informed flow and near-term price movement in that +direction. For a deeper dive into the statistical foundations, +see Databento's [blog post on HFT signals with sklearn](https://databento.com/blog/hft-sklearn-python) +which demonstrates the predictive power of book imbalance features. + +For demonstration purposes, NautilusTrader ships with an `OrderBookImbalance` example +strategy that is intentionally simple (no alpha advantage). +The strategy monitors the ratio of the smaller to larger side at the top of book, and when +this ratio drops below a configurable threshold it fires a fill-or-kill (FOK) limit order. Because +it only needs top-of-book data, it works with Databento `mbp-1` (market by price best bid/ask) quotes +rather than full depth-of-book, which is significantly cheaper to source. + +### Why proxy data? + +AX Exchange is a new venue and is not yet covered specifically by data vendors like Databento. +CME gold futures (GC) are the most liquid gold derivatives market globally, and +provide representative price action for backtesting gold strategies. We download CME GC +quote data from Databento and replay it through a NautilusTrader backtest with an AX-style +`PerpetualContract` instrument definition. + +## Prerequisites + +- **NautilusTrader** installed (see the [installation guide](../getting_started/installation.md)). +- **Databento API key**: Sign up at [databento.com](https://databento.com) and set the + environment variable: + +```bash +export DATABENTO_API_KEY="your-api-key" +``` + +- **Databento Python client**: Install with `pip install databento`. + +## Data preparation + +### Download CME gold futures quotes + +We use Databento's `mbp-1` schema (top-of-book best bid/ask), which maps directly to +NautilusTrader `QuoteTick` objects. This is simpler and cheaper than downloading full +depth-of-book data. + +We use a Databento **continuous contract** (`GC.v.0`) rather than a specific expiration like +`GCZ4`. Continuous contracts stitch together successive contracts based on a roll rule. +`v.0` tracks the highest-volume contract, which closely mirrors how a perpetual follows +liquidity. The `stype_in="continuous"` parameter tells Databento to resolve the symbol +through its continuous contract mapping. + +```python +import databento as db +from pathlib import Path + +data_path = Path("gc_gold_mbp1.dbn.zst") + +if not data_path.exists(): + client = db.Historical() + data = client.timeseries.get_range( + dataset="GLBX.MDP3", + symbols=["GC.v.0"], + stype_in="continuous", + schema="mbp-1", + start="2024-11-15", + end="2024-11-16", + ) + data.to_file(data_path) +``` + +This downloads one day of top-of-book data for the front-month gold contract. The +file is written once and reused on subsequent runs. The `instrument_id` override in the +loading step below is safe because the continuous contract resolves to a single instrument +at any point in time. + +### Load the data + +Use the `DatabentoDataLoader` to parse the DBN file into Nautilus quote ticks. +We pass `instrument_id` to override the Databento symbology with our AX instrument ID, +so all quote ticks appear to come from XAU-PERP.AX: + +```python +from nautilus_trader.adapters.databento import DatabentoDataLoader +from nautilus_trader.model.identifiers import InstrumentId + +instrument_id = InstrumentId.from_str("XAU-PERP.AX") + +loader = DatabentoDataLoader() +quotes = loader.from_dbn_file( + path=data_path, + instrument_id=instrument_id, +) +``` + +## Instrument definition + +Since we are using proxy data, we define the XAU-PERP instrument manually as a +`PerpetualContract`. The price precision and tick size are set to match the CME source +data, while margin and fee parameters reflect AX Exchange conditions: + +```python +from decimal import Decimal + +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AssetClass +from nautilus_trader.model.identifiers import Symbol +from nautilus_trader.model.instruments import PerpetualContract +from nautilus_trader.model.objects import Price +from nautilus_trader.model.objects import Quantity + +XAU_PERP = PerpetualContract( + instrument_id=instrument_id, + raw_symbol=Symbol("XAU-PERP"), + underlying="XAU", + asset_class=AssetClass.COMMODITY, + quote_currency=USD, + settlement_currency=USD, + is_inverse=False, + price_precision=2, + size_precision=0, + price_increment=Price.from_str("0.01"), + size_increment=Quantity.from_int(1), + multiplier=Quantity.from_int(1), + lot_size=Quantity.from_int(1), + margin_init=Decimal("0.08"), + margin_maint=Decimal("0.04"), + maker_fee=Decimal("0.0002"), + taker_fee=Decimal("0.0005"), + ts_event=0, + ts_init=0, +) +``` + +Fees are explicit backtest assumptions and should be set deliberately. Check the +[AX Exchange documentation](https://docs.architect.exchange/) for current rates. + +## Strategy overview + +The `OrderBookImbalance` strategy works as follows: + +1. **Monitor top of book**: On each quote tick update, compute the ratio of the + smaller side to the larger side (`smaller / larger`). +2. **Check trigger conditions**: If the larger side exceeds `trigger_min_size` and + the ratio falls below `trigger_imbalance_ratio`, a trigger fires. +3. **Determine direction**: If bid size > ask size, the strategy buys at the ask + (anticipating upward pressure). If ask size > bid size, it sells at the bid. +4. **Submit FOK order**: A fill-or-kill limit order is submitted at the opposing + best price, sized to the lesser of the opposing level size and `max_trade_size`. +5. **Cooldown**: A configurable minimum time between triggers prevents overtrading. + +### Configuration + +We use `use_quote_ticks=True` with `book_type="L1_MBP"` since our data is top-of-book +quotes rather than order book deltas: + +```python +from nautilus_trader.examples.strategies.orderbook_imbalance import OrderBookImbalance +from nautilus_trader.examples.strategies.orderbook_imbalance import OrderBookImbalanceConfig + +strategy_config = OrderBookImbalanceConfig( + instrument_id=instrument_id, + max_trade_size=Decimal("10"), + trigger_min_size=1.0, + trigger_imbalance_ratio=0.10, + min_seconds_between_triggers=5.0, + book_type="L1_MBP", + use_quote_ticks=True, +) + +strategy = OrderBookImbalance(config=strategy_config) +``` + +| Parameter | Value | Description | +| ------------------------------ | -------- | --------------------------------------------- | +| `max_trade_size` | `10` | Maximum 10 contracts per order. | +| `trigger_min_size` | `1.0` | Minimum 1 contract on the larger side. | +| `trigger_imbalance_ratio` | `0.10` | Trigger when ratio drops below 10%. | +| `min_seconds_between_triggers` | `5.0` | 5-second cooldown between consecutive trades. | +| `book_type` | `L1_MBP` | Top-of-book data only. | +| `use_quote_ticks` | `True` | Drive the strategy from quote ticks. | + +:::tip +Start with conservative parameters (higher ratio, longer cooldown) and tighten them +as you study the backtest results. A `trigger_imbalance_ratio` of 0.10 means the smaller +side must be less than 10% of the larger side to trigger. +::: + +## Backtest setup + +### Configure the engine + +```python +from nautilus_trader.backtest.config import BacktestEngineConfig +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.config import LoggingConfig +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.model.identifiers import TraderId +from nautilus_trader.model.identifiers import Venue +from nautilus_trader.model.objects import Money + +config = BacktestEngineConfig( + trader_id=TraderId("BACKTESTER-001"), + logging=LoggingConfig(log_level="INFO"), +) + +engine = BacktestEngine(config=config) +``` + +### Add the venue + +AX Exchange uses margin accounts with netting position management: + +```python +AX = Venue("AX") + +engine.add_venue( + venue=AX, + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, + base_currency=USD, + starting_balances=[Money(100_000, USD)], +) +``` + +### Add instrument, data, and strategy + +```python +engine.add_instrument(XAU_PERP) +engine.add_data(quotes) +engine.add_strategy(strategy) +``` + +### Run the backtest + +```python +engine.run() +``` + +## Results + +After the run completes, generate reports to analyze performance: + +```python +import pandas as pd + +with pd.option_context( + "display.max_rows", 100, + "display.max_columns", None, + "display.width", 300, +): + print(engine.trader.generate_account_report(AX)) + print(engine.trader.generate_order_fills_report()) + print(engine.trader.generate_positions_report()) +``` + +Clean up when done: + +```python +engine.reset() +engine.dispose() +``` + +## Complete script + +The complete script is available as +[`architect_ax_book_imbalance.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_book_imbalance.py) +in the examples directory. + +## Next steps + +- **Tune parameters**: Experiment with `trigger_imbalance_ratio`, `min_seconds_between_triggers`, + and `max_trade_size` to understand their effect on fill rates and PnL. +- **Try different data**: Download different time periods or different front-month contracts + to see how the strategy performs across varying market conditions. +- **Go live on AX sandbox**: Once you are satisfied with backtest results, connect to the + AX sandbox environment for paper trading. See the + [AX Exchange integration guide](../integrations/architect_ax.md) for setup instructions. +- **Explore other instruments**: AX offers perpetuals on FX pairs (GBPUSD-PERP, EURUSD-PERP), + metals (XAG-PERP), and more. Adapt this tutorial by downloading the corresponding CME + futures data from Databento. + +## Running live + +The same strategy used in this backtest can be run live with no code changes - only a +launch script is needed. NautilusTrader's architecture means your strategy is +venue-agnostic: switching from backtest to live is a configuration change, not a rewrite. + +See the complete live example: +[`ax_book_imbalance.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/architect_ax/ax_book_imbalance.py) + +For connection setup and API key configuration, refer to the +[AX Exchange integration guide](../integrations/architect_ax.md). + +## Further reading + +- [AX Exchange book imbalance backtest example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_book_imbalance.py) +- [`OrderBookImbalance` strategy source](https://github.com/nautechsystems/nautilus_trader/tree/develop/nautilus_trader/examples/strategies/orderbook_imbalance.py) +- [Architect Exchange documentation](https://docs.architect.exchange/) +- [Databento: HFT signals with sklearn](https://databento.com/blog/hft-sklearn-python) diff --git a/nautilus-docs/docs/tutorials/backtest_binance_orderbook.py b/nautilus-docs/docs/tutorials/backtest_binance_orderbook.py new file mode 100644 index 0000000..f770a02 --- /dev/null +++ b/nautilus-docs/docs/tutorials/backtest_binance_orderbook.py @@ -0,0 +1,190 @@ +# %% [markdown] +# # Backtest: Binance OrderBook data +# +# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/backtest_binance_orderbook.py). + +# %% [markdown] +# ## Overview +# +# This tutorial sets up the data catalog and a `BacktestNode` to backtest an `OrderBookImbalance` strategy on order book data. You need to bring your own Binance order book data. + +# %% [markdown] +# ## Prerequisites +# +# - Python 3.12+ installed +# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) + +# %% [markdown] +# ## Imports +# +# We'll start with all of our imports for the remainder of this tutorial: + +# %% +import shutil +from decimal import Decimal +from pathlib import Path + +import pandas as pd + +from nautilus_trader.adapters.binance.loaders import BinanceOrderBookDeltaDataLoader +from nautilus_trader.backtest.node import BacktestDataConfig +from nautilus_trader.backtest.node import BacktestEngineConfig +from nautilus_trader.backtest.node import BacktestNode +from nautilus_trader.backtest.node import BacktestRunConfig +from nautilus_trader.backtest.node import BacktestVenueConfig +from nautilus_trader.config import ImportableStrategyConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.core.datetime import dt_to_unix_nanos +from nautilus_trader.model import OrderBookDelta +from nautilus_trader.persistence.catalog import ParquetDataCatalog +from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# %% [markdown] +# ## Loading data + +# %% +# Path to your data directory, using user /Downloads as an example +DATA_DIR = "~/Downloads" + +# %% +data_path = Path(DATA_DIR).expanduser() / "Data" / "Binance" +raw_files = [f for f in data_path.iterdir() if f.is_file()] +assert raw_files, f"Unable to find any data files in directory {data_path}" +raw_files + +# %% +# First we'll load the initial order book snapshot +path_snap = data_path / "BTCUSDT_T_DEPTH_2022-11-01_depth_snap.csv" +df_snap = BinanceOrderBookDeltaDataLoader.load(path_snap) +df_snap.head() + +# %% +# Load the order book updates, limiting to 1 million rows +path_update = data_path / "BTCUSDT_T_DEPTH_2022-11-01_depth_update.csv" +nrows = 1_000_000 +df_update = BinanceOrderBookDeltaDataLoader.load(path_update, nrows=nrows) +df_update.head() + +# %% [markdown] +# ### Process deltas using a wrangler + +# %% +BTCUSDT_BINANCE = TestInstrumentProvider.btcusdt_binance() +wrangler = OrderBookDeltaDataWrangler(BTCUSDT_BINANCE) + +deltas = wrangler.process(df_snap) +deltas += wrangler.process(df_update) +deltas.sort(key=lambda x: x.ts_init) # Ensure data is non-decreasing by `ts_init` +deltas[:10] + +# %% [markdown] +# ### Set up data catalog + +# %% +CATALOG_PATH = Path.cwd() / "catalog" + +# Clear if it already exists, then create fresh +if CATALOG_PATH.exists(): + shutil.rmtree(CATALOG_PATH) +CATALOG_PATH.mkdir() + +catalog = ParquetDataCatalog(CATALOG_PATH) + +# %% +# Write instrument and ticks to catalog +catalog.write_data([BTCUSDT_BINANCE]) +catalog.write_data(deltas) + +# %% +# Confirm the instrument was written +catalog.instruments() + +# %% +# Explore the available data in the catalog +start = dt_to_unix_nanos(pd.Timestamp("2022-11-01", tz="UTC")) +end = dt_to_unix_nanos(pd.Timestamp("2022-11-04", tz="UTC")) + +deltas = catalog.order_book_deltas(start=start, end=end) +print(len(deltas)) +deltas[:10] + +# %% [markdown] +# ## Configure backtest + +# %% +instrument = catalog.instruments()[0] +book_type = "L2_MBP" # Data book type must match venue book type + +data_configs = [ + BacktestDataConfig( + catalog_path=CATALOG_PATH, + data_cls=OrderBookDelta, + instrument_id=instrument.id, + # start_time=start, # Run across all data + # end_time=end, # Run across all data + ), +] + +venues_configs = [ + BacktestVenueConfig( + name="BINANCE", + oms_type="NETTING", + account_type="CASH", + base_currency=None, + starting_balances=["20 BTC", "100000 USDT"], + book_type=book_type, # <-- Venues book type + ), +] + +strategies = [ + ImportableStrategyConfig( + strategy_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalance", + config_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalanceConfig", + config={ + "instrument_id": instrument.id, + "book_type": book_type, + "max_trade_size": Decimal("1.000"), + "min_seconds_between_triggers": 1.0, + }, + ), +] + +config = BacktestRunConfig( + engine=BacktestEngineConfig( + strategies=strategies, + logging=LoggingConfig(log_level="ERROR"), + ), + data=data_configs, + venues=venues_configs, +) + +config + +# %% [markdown] +# ## Run the backtest + +# %% +node = BacktestNode(configs=[config]) + +result = node.run() + +# %% +result + +# %% +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.model import Venue + + +engine: BacktestEngine = node.get_engine(config.id) + +engine.trader.generate_order_fills_report() + +# %% +engine.trader.generate_positions_report() + +# %% +engine.trader.generate_account_report(Venue("BINANCE")) diff --git a/nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py b/nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py new file mode 100644 index 0000000..6f70f75 --- /dev/null +++ b/nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py @@ -0,0 +1,184 @@ +# %% [markdown] +# # Backtest: Bybit OrderBook data +# +# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/backtest_bybit_orderbook.py). + +# %% [markdown] +# ## Overview +# +# This tutorial sets up the data catalog and a `BacktestNode` to backtest an `OrderBookImbalance` strategy on order book data. This example requires order book depth data from Bybit. +# + +# %% [markdown] +# ## Prerequisites +# +# - Python 3.12+ installed +# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) + +# %% [markdown] +# ## Imports +# +# We'll start with all of our imports for the remainder of this tutorial: + +# %% +import shutil +from decimal import Decimal +from pathlib import Path + +import pandas as pd + +from nautilus_trader.adapters.bybit.loaders import BybitOrderBookDeltaDataLoader +from nautilus_trader.backtest.node import BacktestDataConfig +from nautilus_trader.backtest.node import BacktestEngineConfig +from nautilus_trader.backtest.node import BacktestNode +from nautilus_trader.backtest.node import BacktestRunConfig +from nautilus_trader.backtest.node import BacktestVenueConfig +from nautilus_trader.config import ImportableStrategyConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.core.datetime import dt_to_unix_nanos +from nautilus_trader.model import OrderBookDelta +from nautilus_trader.persistence.catalog import ParquetDataCatalog +from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# %% [markdown] +# ## Loading data + +# %% +# Path to your data directory, using user /Downloads as an example +DATA_DIR = "~/Downloads" + +# %% +data_path = Path(DATA_DIR).expanduser() / "Data" / "Bybit" +raw_files = [f for f in data_path.iterdir() if f.is_file()] +assert raw_files, f"Unable to find any data files in directory {data_path}" +raw_files + +# %% +# We'll use orderbook depth 500 data provided by Bybit with limit of 1000000 rows +path_update = data_path / "2024-12-01_XRPUSDT_ob500.data.zip" +nrows = 1_000_000 +df_raw = BybitOrderBookDeltaDataLoader.load(path_update, nrows=nrows) +df_raw.head() + +# %% [markdown] +# ### Process deltas using a wrangler + +# %% +XRPUSDT_BYBIT = TestInstrumentProvider.xrpusdt_linear_bybit() +wrangler = OrderBookDeltaDataWrangler(XRPUSDT_BYBIT) + +deltas = wrangler.process(df_raw) +deltas.sort(key=lambda x: x.ts_init) # Ensure data is non-decreasing by `ts_init` +deltas[:10] + +# %% [markdown] +# ### Set up data catalog + +# %% +CATALOG_PATH = Path.cwd() / "catalog" + +# Clear if it already exists, then create fresh +if CATALOG_PATH.exists(): + shutil.rmtree(CATALOG_PATH) +CATALOG_PATH.mkdir() + +catalog = ParquetDataCatalog(CATALOG_PATH) + +# %% +# Write instrument and ticks to catalog +catalog.write_data([XRPUSDT_BYBIT]) +catalog.write_data(deltas) + +# %% +# Confirm the instrument was written +catalog.instruments() + +# %% +# Explore the available data in the catalog +start = dt_to_unix_nanos(pd.Timestamp("2022-11-01", tz="UTC")) +end = dt_to_unix_nanos(pd.Timestamp("2022-11-04", tz="UTC")) + +deltas = catalog.order_book_deltas(start=start, end=end) +print(len(deltas)) +deltas[:10] + +# %% [markdown] +# ## Configure backtest + +# %% +instrument = catalog.instruments()[0] +book_type = "L2_MBP" # Data book type must match venue book type + +data_configs = [ + BacktestDataConfig( + catalog_path=CATALOG_PATH, + data_cls=OrderBookDelta, + instrument_id=instrument.id, + # start_time=start, # Run across all data + # end_time=end, # Run across all data + ), +] + +venues_configs = [ + BacktestVenueConfig( + name="BYBIT", + oms_type="NETTING", + account_type="CASH", + base_currency=None, + starting_balances=["200000 XRP", "100000 USDT"], + book_type=book_type, # <-- Venues book type + ), +] + +strategies = [ + ImportableStrategyConfig( + strategy_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalance", + config_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalanceConfig", + config={ + "instrument_id": instrument.id, + "book_type": book_type, + "max_trade_size": Decimal("1.000"), + "min_seconds_between_triggers": 1.0, + }, + ), +] + +config = BacktestRunConfig( + engine=BacktestEngineConfig( + strategies=strategies, + logging=LoggingConfig(log_level="ERROR"), + ), + data=data_configs, + venues=venues_configs, +) + +config + +# %% [markdown] +# ## Run the backtest + +# %% +node = BacktestNode(configs=[config]) + +result = node.run() + +# %% +result + +# %% +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.model import Venue + + +engine: BacktestEngine = node.get_engine(config.id) + +engine.trader.generate_order_fills_report() + +# %% +engine.trader.generate_positions_report() + +# %% +engine.trader.generate_account_report(Venue("BYBIT")) diff --git a/nautilus-docs/docs/tutorials/backtest_fx_bars.py b/nautilus-docs/docs/tutorials/backtest_fx_bars.py new file mode 100644 index 0000000..74f766f --- /dev/null +++ b/nautilus-docs/docs/tutorials/backtest_fx_bars.py @@ -0,0 +1,164 @@ +# %% [markdown] +# # Backtest: FX bar data +# +# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/backtest_fx_bars.py). + +# %% [markdown] +# ## Overview +# +# This tutorial runs through how to set up a `BacktestEngine` (low-level API) for a single 'one-shot' backtest run using FX bar data. + +# %% [markdown] +# ## Prerequisites +# +# - Python 3.12+ installed +# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) + +# %% [markdown] +# ## Imports +# +# We'll start with all of our imports for the remainder of this tutorial. + +# %% +from decimal import Decimal + +from nautilus_trader.backtest.config import BacktestEngineConfig +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.backtest.models import FillModel +from nautilus_trader.backtest.modules import FXRolloverInterestConfig +from nautilus_trader.backtest.modules import FXRolloverInterestModule +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import RiskEngineConfig +from nautilus_trader.examples.strategies.ema_cross import EMACross +from nautilus_trader.examples.strategies.ema_cross import EMACrossConfig +from nautilus_trader.model import BarType +from nautilus_trader.model import Money +from nautilus_trader.model import Venue +from nautilus_trader.model.currencies import JPY +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler +from nautilus_trader.test_kit.providers import TestDataProvider +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# %% [markdown] +# ## Set up backtest engine + +# %% +# Initialize a backtest configuration +config = BacktestEngineConfig( + trader_id="BACKTESTER-001", + logging=LoggingConfig(log_level="ERROR"), + risk_engine=RiskEngineConfig( + bypass=True, # Example of bypassing pre-trade risk checks for backtests + ), +) + +# Build backtest engine +engine = BacktestEngine(config=config) + +# %% [markdown] +# ## Add simulation module +# +# We can optionally plug in a module to simulate rollover interest. The data is available from pre-packaged test data. + +# %% +provider = TestDataProvider() +interest_rate_data = provider.read_csv("short-term-interest.csv") +config = FXRolloverInterestConfig(interest_rate_data) +fx_rollover_interest = FXRolloverInterestModule(config=config) + +# %% [markdown] +# ## Add fill model + +# %% [markdown] +# For this backtest we'll use a simple probabilistic fill model. + +# %% +fill_model = FillModel( + prob_fill_on_limit=0.2, + prob_slippage=0.5, + random_seed=42, +) + +# %% [markdown] +# ## Add venue + +# %% [markdown] +# For this backtest we need a single trading venue which will be a simulated FX ECN. + +# %% +SIM = Venue("SIM") +engine.add_venue( + venue=SIM, + oms_type=OmsType.HEDGING, # Venue will generate position IDs + account_type=AccountType.MARGIN, + base_currency=None, # Multi-currency account + starting_balances=[Money(1_000_000, USD), Money(10_000_000, JPY)], + fill_model=fill_model, + modules=[fx_rollover_interest], +) + +# %% [markdown] +# ## Add instruments and data + +# %% [markdown] +# Now we can add instruments and data. For this backtest we'll pre-process bid and ask side bar data into quotes using a `QuoteTickDataWrangler`. + +# %% +# Add instruments +USDJPY_SIM = TestInstrumentProvider.default_fx_ccy("USD/JPY", SIM) +engine.add_instrument(USDJPY_SIM) + +# Add data +wrangler = QuoteTickDataWrangler(instrument=USDJPY_SIM) +ticks = wrangler.process_bar_data( + bid_data=provider.read_csv_bars("fxcm/usdjpy-m1-bid-2013.csv"), + ask_data=provider.read_csv_bars("fxcm/usdjpy-m1-ask-2013.csv"), +) +engine.add_data(ticks) + +# %% [markdown] +# ## Configure strategy + +# %% [markdown] +# Next we'll configure and initialize a simple `EMACross` strategy we'll use for the backtest. + +# %% +# Configure your strategy +config = EMACrossConfig( + instrument_id=USDJPY_SIM.id, + bar_type=BarType.from_str("USD/JPY.SIM-5-MINUTE-BID-INTERNAL"), + fast_ema_period=10, + slow_ema_period=20, + trade_size=Decimal(1_000_000), +) + +# Instantiate and add your strategy +strategy = EMACross(config=config) +engine.add_strategy(strategy=strategy) + +# %% [markdown] +# ## Run backtest +# +# Run the backtest. The engine logs a post-analysis report when it finishes processing all data. + +# %% +engine.run() + +# %% [markdown] +# ## Generating reports +# +# Produce reports for post-run analysis. + +# %% +engine.trader.generate_account_report(SIM) + +# %% +engine.trader.generate_order_fills_report() + +# %% +engine.trader.generate_positions_report() diff --git a/nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md b/nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md new file mode 100644 index 0000000..8fa4e35 --- /dev/null +++ b/nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md @@ -0,0 +1,541 @@ +# BitMEX - Grid Market Making with Deadman's Switch + +This tutorial walks through backtesting a grid market making strategy on BitMEX using free +historical quote tick data from [Tardis.dev](https://tardis.dev), then running it live +using the Rust-native `LiveNode`. The key differentiator covered here is BitMEX's +**deadman's switch**, a server-side safety mechanism that automatically cancels all open +orders if your client loses connectivity. + +## Introduction + +### Why BitMEX for grid market making? + +BitMEX is one of the deepest and most liquid Bitcoin derivatives venues, with a live order +book going back to 2014. The `XBTUSD` inverse perpetual swap is among the most-traded +instruments in crypto derivatives. Its thick order book and predictable spread behaviour +make it a natural venue for grid market making. + +Two features make BitMEX particularly well-suited for automated market making: + +1. **Deadman's switch** (`cancelAllAfter`): BitMEX maintains a server-side countdown timer. + Your client refreshes it periodically. If the connection drops and the timer expires, + BitMEX cancels all open orders on your behalf, protecting you from stranded quotes. + +2. **Submit/cancel broadcaster**: The adapter can fan out order submissions and + cancellations across multiple independent HTTP connections simultaneously, with the first + successful response short-circuiting the rest. This provides redundancy against transient + network failures. + +### Deadman's switch mechanics + +When `deadmans_switch_timeout_secs` is set in the execution client config, a background +task runs continuously: + +``` +timeout = 60s → refresh interval = timeout / 4 = 15s + + t=0s Strategy starts, cancelAllAfter(60000ms) sent + t=15s Refresh: cancelAllAfter(60000ms) sent (resets timer) + t=30s Refresh: cancelAllAfter(60000ms) sent + t=45s Refresh: cancelAllAfter(60000ms) sent + ↓ + Connectivity lost at t=50s (last refresh was at t=45s) + ↓ + t=105s Server timer fires → BitMEX cancels all open orders +``` + +For market making specifically, stranded quotes are a serious risk: if your software crashes +while holding grid orders around the mid-price, price can move against those orders before +they are cancelled. The deadman's switch caps the window of exposure at `timeout` seconds, +regardless of why connectivity was lost. + +## Prerequisites + +- **NautilusTrader** installed (see the [installation guide](../getting_started/installation.md)). +- **Rust toolchain** (`cargo`) for the live example. Install from [rustup.rs](https://rustup.rs/). +- **BitMEX account**: sign up at [bitmex.com](https://www.bitmex.com/) and generate an API key + with order management permissions. For testing, use the + [BitMEX testnet](https://testnet.bitmex.com/). + +### Environment variables + +```bash +# Mainnet +export BITMEX_API_KEY="your-api-key" +export BITMEX_API_SECRET="your-api-secret" + +# Testnet +export BITMEX_TESTNET_API_KEY="your-testnet-api-key" +export BITMEX_TESTNET_API_SECRET="your-testnet-api-secret" +``` + +Alternatively, place these in a `.env` file in the project root (loaded automatically via `dotenvy`). + +## Backtesting with Tardis free quote data + +BitMEX does not offer historical market data via its own API beyond recent trade history. +[Tardis.dev](https://tardis.dev) captures and archives tick-level BitMEX data from March 2019 +onward in its native WebSocket format. The **first day of each month is freely downloadable** +without an API key (enough for a representative backtest run). + +The grid market maker subscribes to best-bid/ask quotes, so the `quotes` dataset is the +right source: it records every change to the top of book. + +### Download the data + +```bash +curl -LO https://datasets.tardis.dev/v1/bitmex/quotes/2024/01/01/XBTUSD.csv.gz +``` + +This downloads January 1 2024 XBTUSD quote data. No API key required. + +:::tip +Full historical data (all dates) requires a paid Tardis API key. Use the +[Tardis download utility](https://docs.tardis.dev/downloadable-csv-files) for bulk fetches. +::: + +### Load the data + +`TardisCSVDataLoader` parses the `.csv.gz` file directly (no decompression needed) and +returns a list of `QuoteTick` objects: + +```python +from nautilus_trader.adapters.tardis.loaders import TardisCSVDataLoader +from nautilus_trader.model.identifiers import InstrumentId + +instrument_id = InstrumentId.from_str("XBTUSD.BITMEX") + +loader = TardisCSVDataLoader(instrument_id=instrument_id) +quotes = loader.load_quotes("XBTUSD.csv.gz") +``` + +The `instrument_id` override ensures every tick is tagged `XBTUSD.BITMEX` regardless of +what appears in the CSV. + +### Instrument definition + +Since we are loading the data directly (not through the live BitMEX adapter), we define the +`XBTUSD` instrument manually. XBTUSD is an **inverse perpetual**: prices are quoted in USD +but the contract is margined and settled in BTC. One contract equals 1 USD of notional +exposure. + +```python +from decimal import Decimal + +from nautilus_trader.model.currencies import BTC +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.enums import AssetClass +from nautilus_trader.model.identifiers import Symbol +from nautilus_trader.model.instruments import PerpetualContract +from nautilus_trader.model.objects import Price +from nautilus_trader.model.objects import Quantity + +XBTUSD = PerpetualContract( + instrument_id=instrument_id, + raw_symbol=Symbol("XBTUSD"), + underlying="XBT", + asset_class=AssetClass.CRYPTOCURRENCY, + base_currency=BTC, + quote_currency=USD, + settlement_currency=BTC, + is_inverse=True, + price_precision=1, # $0.5 tick → one decimal place + size_precision=0, # integer contracts + price_increment=Price.from_str("0.5"), + size_increment=Quantity.from_int(1), + multiplier=Quantity.from_int(1), # 1 USD per contract + lot_size=Quantity.from_int(1), + margin_init=Decimal("0.01"), # 1% initial margin = 100x max leverage + margin_maint=Decimal("0.005"), + maker_fee=Decimal("-0.00025"), # maker rebate + taker_fee=Decimal("0.00075"), + ts_event=0, + ts_init=0, +) +``` + +Fee rates are explicit backtest assumptions. Check +[bitmex.com/app/fees](https://www.bitmex.com/app/fees) for current rates. + +### Backtest engine setup + +XBTUSD is BTC-margined, so the starting balance is in BTC: + +```python +from nautilus_trader.backtest.config import BacktestEngineConfig +from nautilus_trader.backtest.engine import BacktestEngine +from nautilus_trader.config import LoggingConfig +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.model.identifiers import TraderId +from nautilus_trader.model.identifiers import Venue +from nautilus_trader.model.objects import Money + +config = BacktestEngineConfig( + trader_id=TraderId("BACKTESTER-001"), + logging=LoggingConfig(log_level="INFO"), +) +engine = BacktestEngine(config=config) + +BITMEX = Venue("BITMEX") +engine.add_venue( + venue=BITMEX, + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, + base_currency=BTC, + starting_balances=[Money(1, BTC)], +) + +engine.add_instrument(XBTUSD) +engine.add_data(quotes) +``` + +### Strategy configuration + +```python +from nautilus_trader.examples.strategies.grid_market_maker import GridMarketMaker +from nautilus_trader.examples.strategies.grid_market_maker import GridMarketMakerConfig + +strategy_config = GridMarketMakerConfig( + instrument_id=instrument_id, + max_position=Quantity.from_int(300), # 300 USD contracts max exposure + trade_size=Quantity.from_int(100), # 100 USD contracts per level + num_levels=3, + grid_step_bps=100, # 1% between levels + skew_factor=0.5, + requote_threshold_bps=10, +) +strategy = GridMarketMaker(config=strategy_config) +engine.add_strategy(strategy) +``` + +### Run and review results + +```python +import pandas as pd + +engine.run() + +with pd.option_context("display.max_rows", 100, "display.max_columns", None, "display.width", 300): + print(engine.trader.generate_account_report(BITMEX)) + print(engine.trader.generate_order_fills_report()) + print(engine.trader.generate_positions_report()) + +engine.reset() +engine.dispose() +``` + +The complete script is available as +[`bitmex_grid_market_maker.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/bitmex_grid_market_maker.py) +in the examples directory. + +## Live trading: GridMarketMaker with deadman's switch + +Once you have validated data loading and strategy mechanics in the backtest, the same +configuration runs live using the Rust `LiveNode`. The `GridMarketMaker` strategy is +implemented natively in Rust for maximum throughput. + +### Environment setup + +Credentials are loaded automatically from environment variables when not set explicitly in +the config: + +```bash +# Testnet (recommended for initial setup) +export BITMEX_TESTNET_API_KEY="your-key" +export BITMEX_TESTNET_API_SECRET="your-secret" +``` + +```ini +# Or use a .env file at the project root +BITMEX_TESTNET_API_KEY=your-key +BITMEX_TESTNET_API_SECRET=your-secret +``` + +### Code walkthrough + +The complete `main()` function from +[`node_grid_mm.rs`](https://github.com/nautechsystems/nautilus_trader/tree/develop/crates/adapters/bitmex/examples/node_grid_mm.rs): + +```rust +#[tokio::main] +async fn main() -> Result<(), Box> { + dotenvy::dotenv().ok(); // Load .env file if present + + let use_testnet = true; // Set false for mainnet + + let environment = Environment::Live; + let trader_id = TraderId::from("TESTER-001"); + let instrument_id = InstrumentId::from("XBTUSD.BITMEX"); + + // Minimal data client: just selects testnet or mainnet endpoints + let data_config = BitmexDataClientConfig { + use_testnet, + ..Default::default() + }; + + // Execution client with deadman's switch enabled + let exec_config = BitmexExecFactoryConfig::new( + trader_id, + BitmexExecClientConfig { + use_testnet, + deadmans_switch_timeout_secs: Some(60), // Cancels all orders after 60s without refresh + ..Default::default() + }, + ); + + let data_factory = BitmexDataClientFactory::new(); + let exec_factory = BitmexExecutionClientFactory::new(); + + let log_config = LoggerConfig { + stdout_level: LevelFilter::Info, + ..Default::default() + }; + + // Builder wires up logging, data and execution clients, and node options + let mut node = LiveNode::builder(trader_id, environment)? + .with_logging(log_config) + .add_data_client(None, Box::new(data_factory), Box::new(data_config))? + .add_exec_client(None, Box::new(exec_factory), Box::new(exec_config))? + .with_reconciliation(true) // Resume state across restarts + .with_reconciliation_lookback_mins(2880) // Look back 2 days (2880 min) + .with_delay_post_stop_secs(5) // Grace period for pending cancel/close events + .build()?; + + // Grid configuration: XBTUSD, 300 USD max position, 3 levels at 100 bps each + let config = GridMarketMakerConfig::new(instrument_id, Quantity::from("300")) + .with_num_levels(3) + .with_grid_step_bps(100) // 1% between levels + .with_skew_factor(0.5) // shift grid 0.5 price units per unit of inventory + .with_requote_threshold_bps(10); // requote when mid moves more than 0.1% + let strategy = GridMarketMaker::new(config); + + node.add_strategy(strategy)?; + node.run().await?; + + Ok(()) +} +``` + +Configuration points: + +- **`deadmans_switch_timeout_secs: Some(60)`**: enables the deadman's switch with a 60-second + timeout. The background task refreshes every 15 seconds (`timeout / 4`). +- **`with_reconciliation(true)`**: reconciles open orders and positions on startup by + querying the BitMEX REST API, allowing the strategy to resume correctly after a restart. +- **`with_reconciliation_lookback_mins(2880)`**: looks back 2 days when reconciling order history. +- **`with_delay_post_stop_secs(5)`**: allows 5 seconds after strategy stop for pending + cancel/fill events to arrive before the node exits. + +### Deadman's switch in context + +During normal operation the deadman's switch is invisible: the background task silently +refreshes the server-side timer. Its value becomes apparent in failure scenarios: + +``` +Normal operation: + ┌─────────────────────────────────────────────────────────┐ + │ Strategy running │ + │ t=0s cancelAllAfter(60_000ms) ──────────────► BitMEX │ + │ t=15s cancelAllAfter(60_000ms) ──────────────► BitMEX │ + │ t=30s cancelAllAfter(60_000ms) ──────────────► BitMEX │ + └─────────────────────────────────────────────────────────┘ + +Connectivity loss: + ┌──────────────────────────────────────────────────────────┐ + │ t=40s Network failure, no more refreshes sent │ + │ t=100s BitMEX timer fires → all open orders cancelled │ + │ (60s after the last successful refresh at t=40s) │ + └──────────────────────────────────────────────────────────┘ +``` + +Unlike dYdX where short-term order expiry provides a similar automatic cleanup, BitMEX +uses GTC orders (no expiry). Without the deadman's switch, a crashed client could leave +grid orders resting in the book indefinitely. + +### BitMEX-specific considerations + +#### GTC orders and post-only + +BitMEX grid orders use `GTC` (Good-Till-Cancelled) time-in-force combined with +`ParticipateDoNotInitiate` (post-only). Post-only ensures every order enters the book as +a maker order; if a grid price has moved through the book by the time it reaches the +matching engine, the order is rejected rather than filling as a taker. + +This differs from the dYdX setup where short-term orders provide automatic expiry every +~8 seconds. On BitMEX, the requote cycle is driven entirely by mid-price movement +(`requote_threshold_bps`) rather than order expiry. + +#### Order quantization + +All price and size quantization for BitMEX instruments is handled automatically by the +adapter. No manual rounding or conversion is needed in strategy code. + +#### Inverse perpetual accounting + +Because XBTUSD is inverse (BTC-margined), PnL is in BTC. A grid that captures a $1 spread +on a $42,000 BTC price earns approximately 1/42,000 BTC per fill. Account for this when +sizing `max_position` and `trade_size`. + +### Run the example + +```bash +cargo run --example bitmex-grid-mm --package nautilus-bitmex +``` + +### Graceful shutdown + +Press **Ctrl+C** to stop the node. The shutdown sequence: + +1. SIGINT received, trader stops, `on_stop()` fires. +2. Strategy cancels all orders and closes positions. +3. 5-second grace period (`delay_post_stop_secs`) processes residual events. +4. Deadman's switch background task stops. +5. Clients disconnect, node exits. + +## Configuration + +### GridMarketMaker parameters + +| Parameter | Type | Default | Description | +| ----------------------- | -------------- | ---------- | ------------------------------------------------------------------------ | +| `instrument_id` | `InstrumentId` | *required* | Instrument to trade (e.g., `XBTUSD.BITMEX`). | +| `max_position` | `Quantity` | *required* | Maximum net exposure in contracts (long or short). | +| `trade_size` | `Quantity` | `None` | Size per grid level. If `None`, uses instrument's `min_quantity` or 1.0. | +| `num_levels` | `usize` | `3` | Number of buy and sell levels. | +| `grid_step_bps` | `u32` | `10` | Grid spacing in basis points (100 = 1%). | +| `skew_factor` | `f64` | `0.0` | How aggressively to shift the grid based on net inventory. | +| `requote_threshold_bps` | `u32` | `5` | Minimum mid-price move (bps) before re-quoting. | +| `expire_time_secs` | `Option` | `None` | Order expiry in seconds. Use `None` for GTC on BitMEX. | +| `on_cancel_resubmit` | `bool` | `false` | Resubmit grid on next quote after an unexpected cancel. | + +### Deadman's switch parameter + +| Parameter | Type | Description | +| ----------------------------- | ----------------- | ------------------------------------------------------------------------------------------------- | +| `deadmans_switch_timeout_secs`| `Option` | Server-side cancel timer in seconds. Refresh interval = `timeout / 4` (minimum 1s). `None` disables the feature. | + +**Recommended value**: `60`. This gives a 15-second refresh interval and a 60-second window +before BitMEX fires the timer. Lower values reduce the exposure window but increase API +call frequency; higher values reduce overhead but extend the window. + +### Choosing grid parameters + +**`grid_step_bps`**: XBTUSD has tight spreads. Start wider (50–100 bps) to ensure fills +before tightening. Each level captures half the step as spread (buy fills $1 below mid, +sells $1 above on a 200 bps total spread). + +**`skew_factor`**: Start at `0.0` (no skew). A value of `0.5` shifts the grid by 0.5 price +units per unit of net position. For XBTUSD, this is 0.5 USD per contract; with max_position +of 300, full skew is ±150 price units. + +**`requote_threshold_bps`**: 10 bps (0.1%) is a reasonable starting point for XBTUSD. +Too low causes excessive cancel/replace churn; too high leaves orders stale during fast moves. + +## Event flow + +``` +LiveNode starts + │ + ├── connect() → REST: load instruments; WebSocket: subscribe channels + │ + ├── deadman's switch task starts + │ └── cancelAllAfter(timeout_ms) sent every timeout/4 seconds + │ + ├── on_start() + │ └── subscribe_quotes(XBTUSD.BITMEX) + │ + ├── on_quote() [repeated] + │ ├── Calculate mid-price + │ ├── Check should_requote(): skip if within threshold + │ ├── cancel_all_orders(): record IDs in pending_self_cancels + │ ├── Compute grid with inventory skew + │ └── Submit GTC post-only limit orders + │ + ├── on_order_filled() + │ └── Remove from pending_self_cancels; position/skew update + │ + ├── on_order_canceled() + │ ├── Self-cancel? → no action + │ └── Unexpected cancel? → reset last_quoted_mid (triggers requote) + │ + └── on_stop() + ├── cancel_all_orders() + ├── close_all_positions() + ├── unsubscribe_quotes() + └── deadman's switch task stops +``` + +## Monitoring and understanding output + +### Key log messages + +| Log message | Meaning | +| ----------------------------------------------------------------------- | --------------------------------------------------------- | +| `Requoting grid: mid=X, last_mid=Y` | Mid-price moved beyond threshold, refreshing grid. | +| `Starting dead man's switch: timeout=60s, refresh_interval=15s` | Deadman's switch armed at node start. | +| `Dead man's switch heartbeat failed: ...` | Transient network issue; switch will retry next interval. | +| `Disarming dead man's switch` | Switch stopped cleanly during shutdown. | +| `benign cancel error, treating as success` | Cancel for an already-filled or cancelled order (normal). | +| `Reconciling orders from last 2880 minutes` | Startup reconciliation loading prior state. | + +### Expected behaviour patterns + +1. **Startup**: Instruments load, reconciliation queries prior orders, WebSocket connects, + first quote triggers initial grid. +2. **Steady state**: Grid persists across ticks; requotes only when mid moves beyond threshold. +3. **Fills**: Position updates, skew adjusts on next requote. +4. **Shutdown**: All orders cancelled, positions closed, deadman's switch stops. +5. **Restart**: Reconciliation restores open order state; strategy resumes from prior grid. + +## Customization tips + +### High vs low volatility + +| Condition | Adjustment | +| --------------- | -------------------------------------------------------------------------- | +| High volatility | Wider `grid_step_bps` (100–200), fewer `num_levels`, lower `skew_factor`. | +| Low volatility | Tighter `grid_step_bps` (20–50), more `num_levels`, higher `skew_factor`. | +| Thin liquidity | Increase `requote_threshold_bps` to reduce cancel frequency. | + +### Enabling the submit broadcaster + +For production deployments, enable the submit broadcaster to provide redundant order +submission across multiple HTTP connections: + +```rust +let exec_config = BitmexExecFactoryConfig::new( + trader_id, + BitmexExecClientConfig { + use_testnet: false, + deadmans_switch_timeout_secs: Some(60), + submitter_pool_size: Some(2), // two parallel submission paths + canceller_pool_size: Some(2), // two parallel cancel paths + ..Default::default() + }, +); +``` + +With `submitter_pool_size=2`, each order submission fans out to two HTTP clients in +parallel; the first successful response wins. This reduces the probability of a missed +submission due to a transient network failure on a single path. + +### Mainnet toggle + +Change a single flag to switch networks: + +```rust +let use_testnet = false; // true for testnet +``` + +All endpoints and credential environment variables are resolved automatically. + +## Further reading + +- [BitMEX integration guide](../integrations/bitmex.md): full adapter reference. +- [dYdX grid market maker tutorial](./dydx_grid_market_maker.md): comparison with + short-term order expiry as an alternative to the deadman's switch. +- [Tardis downloadable CSV files](https://docs.tardis.dev/downloadable-csv-files): full + schema documentation for `incremental_book_L2` and other data types. +- [BitMEX API documentation](https://www.bitmex.com/app/apiOverview): `cancelAllAfter` + endpoint and order management reference. diff --git a/nautilus-docs/docs/tutorials/databento_data_catalog.py b/nautilus-docs/docs/tutorials/databento_data_catalog.py new file mode 100644 index 0000000..157bf1e --- /dev/null +++ b/nautilus-docs/docs/tutorials/databento_data_catalog.py @@ -0,0 +1,221 @@ +# %% [markdown] +# # Databento data catalog +# +# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. +# +# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/databento_data_catalog.py). + +# %% [markdown] +# ## Overview +# +# This tutorial walks through setting up a Nautilus Parquet data catalog with various Databento schemas. + +# %% [markdown] +# ## Prerequisites +# +# - Python 3.12+ installed +# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) +# - [databento](https://pypi.org/project/databento/) Python client library installed to make data requests (`uv pip install databento`) +# - [Databento](https://databento.com) account + +# %% [markdown] +# ## Requesting data + +# %% [markdown] +# We'll use a Databento historical client for the rest of this tutorial. You can either initialize one by passing your Databento API key to the constructor, or implicitly use the `DATABENTO_API_KEY` environment variable (as shown). + +# %% +import databento as db + + +client = db.Historical() # Uses the DATABENTO_API_KEY environment variable + +# %% [markdown] +# **Every historical streaming request from `timeseries.get_range` incurs a cost (even for the same data), so**: +# - Check the cost before making a request +# - Avoid requesting the same data twice +# - Write responses to disk as zstd compressed DBN files + +# %% [markdown] +# Use the metadata [get_cost endpoint](https://databento.com/docs/api-reference-historical/metadata/metadata-get-cost?historical=python&live=python) to quote the cost before each request. Only request data that does not already exist on disk. +# +# The response is in USD, displayed as fractional cents. + +# %% [markdown] +# The following request is for a small amount of data (as used in this Medium article [Building high-frequency trading signals in Python with Databento and sklearn](https://databento.com/blog/hft-sklearn-python)) to demonstrate the workflow. + +# %% +from pathlib import Path + +from databento import DBNStore + +# %% [markdown] +# We'll prepare a directory for the raw Databento DBN format data, which we'll use for the rest of the tutorial. + +# %% +DATABENTO_DATA_DIR = Path("databento") +DATABENTO_DATA_DIR.mkdir(exist_ok=True) + +# %% +# Request cost quote (USD) - this endpoint is 'free' +client.metadata.get_cost( + dataset="GLBX.MDP3", + symbols=["ES.n.0"], + stype_in="continuous", + schema="mbp-10", + start="2023-12-06T14:30:00", + end="2023-12-06T20:30:00", +) + +# %% [markdown] +# Use the historical API to request the data used in the Medium article. + +# %% +path = DATABENTO_DATA_DIR / "es-front-glbx-mbp10.dbn.zst" + +if not path.exists(): + # Request data + client.timeseries.get_range( + dataset="GLBX.MDP3", + symbols=["ES.n.0"], + stype_in="continuous", + schema="mbp-10", + start="2023-12-06T14:30:00", + end="2023-12-06T20:30:00", + path=path, # <-- Passing a `path` writes the data to disk + ) + +# %% [markdown] +# Read the data from disk and convert to a pandas.DataFrame + +# %% +data = DBNStore.from_file(path) + +df = data.to_df() +df + +# %% [markdown] +# ## Write to data catalog + +# %% +import shutil +from pathlib import Path + +from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader +from nautilus_trader.model import InstrumentId +from nautilus_trader.persistence.catalog import ParquetDataCatalog + +# %% +CATALOG_PATH = Path.cwd() / "catalog" + +# Clear if it already exists +if CATALOG_PATH.exists(): + shutil.rmtree(CATALOG_PATH) +CATALOG_PATH.mkdir() + +# Create a catalog instance +catalog = ParquetDataCatalog(CATALOG_PATH) + +# %% [markdown] +# Use a `DatabentoDataLoader` to decode and load the data into Nautilus objects. + +# %% +loader = DatabentoDataLoader() + +# %% [markdown] +# Load Rust pyo3 objects by setting `as_legacy_cython=False` (more efficient than legacy Cython objects). +# +# Passing an `instrument_id` is optional but speeds up loading by skipping symbology mapping. If provided, use the Nautilus `symbol.venue` format (e.g., "ES.GLBX"). + +# %% +path = DATABENTO_DATA_DIR / "es-front-glbx-mbp10.dbn.zst" + +# Option 1 (recommended): Let the loader infer the instrument ID from DBN metadata +depth10 = loader.from_dbn_file( + path=path, + as_legacy_cython=False, +) + +# Option 2: Explicitly specify a valid Nautilus instrument ID (symbol.venue format) +# instrument_id = InstrumentId.from_str("ESZ3.GLBX") # E-mini S&P December 2023 futures on Globex +# depth10 = loader.from_dbn_file( +# path=path, +# instrument_id=instrument_id, +# as_legacy_cython=False, +# ) + +# %% +# Write data to catalog (this takes ~20 seconds or ~250,000/second for writing MBP-10 at the moment) +catalog.write_data(depth10) + +# %% +# Test reading from catalog +depths = catalog.order_book_depth10() +len(depths) + +# %% [markdown] +# ## Preparing a month of AAPL trades + +# %% [markdown] +# Now we'll expand on this workflow by preparing a month of AAPL trades on the Nasdaq exchange using the Databento `trade` schema, which will translate to Nautilus `TradeTick` objects. + +# %% +# Request cost quote (USD) - this endpoint is 'free' +client.metadata.get_cost( + dataset="XNAS.ITCH", + symbols=["AAPL"], + schema="trades", + start="2024-01", +) + +# %% [markdown] +# Pass a `path` parameter when requesting historical data to write it to disk. + +# %% +path = DATABENTO_DATA_DIR / "aapl-xnas-202401.trades.dbn.zst" + +if not path.exists(): + # Request data + client.timeseries.get_range( + dataset="XNAS.ITCH", + symbols=["AAPL"], + schema="trades", + start="2024-01", + path=path, # <-- Passing a `path` parameter + ) + +# %% [markdown] +# Read the data from disk and convert to a pandas.DataFrame + +# %% +data = DBNStore.from_file(path) + +df = data.to_df() +df + +# %% [markdown] +# We'll use an `InstrumentId` of `"AAPL.XNAS"`, where XNAS is the ISO 10383 MIC (Market Identifier Code) for the Nasdaq venue. +# +# Passing an `instrument_id` speeds up loading by skipping symbology mapping. Setting `as_legacy_cython=False` is more efficient when writing to the catalog. + +# %% +instrument_id = InstrumentId.from_str("AAPL.XNAS") + +trades = loader.from_dbn_file( + path=path, + instrument_id=instrument_id, + as_legacy_cython=False, +) + +# %% [markdown] +# Here we organize data as one file per month. A file per day works equally well. + +# %% +# Write data to catalog +catalog.write_data(trades) + +# %% +trades = catalog.trade_ticks([instrument_id]) + +# %% +len(trades) diff --git a/nautilus-docs/docs/tutorials/dydx_grid_market_maker.md b/nautilus-docs/docs/tutorials/dydx_grid_market_maker.md new file mode 100644 index 0000000..2b1e6c0 --- /dev/null +++ b/nautilus-docs/docs/tutorials/dydx_grid_market_maker.md @@ -0,0 +1,597 @@ +# dYdX - Grid Market Making + +This tutorial walks through running a grid market making strategy on dYdX v4 using the +Rust-native `LiveNode`. By the end, you will have a working grid quoter that places symmetric +limit orders around the mid-price, skews the grid to manage inventory, and continuously +requotes as short-term orders cycle through expiry. + +## Introduction + +### What is grid market making? + +A grid market maker maintains a ladder of resting buy and sell limit orders at fixed price +intervals around the current mid-price. When an order fills, the strategy profits from the +spread between buy and sell levels. The approach is conceptually simple but requires careful +inventory management to avoid accumulating a large directional position. + +### Inventory skewing (Avellaneda-Stoikov inspired) + +The strategy implements inventory-based skewing: when the position grows long, the entire grid +shifts downward (cheaper buys, cheaper sells) to encourage selling. When the position grows +short, the grid shifts upward. This is inspired by the Avellaneda-Stoikov framework for +optimal market making, adapted to a discrete grid. + +### Why dYdX v4? + +dYdX v4 is well-suited for market making strategies because: + +- **Short-term orders** (~20s expiry) provide low-latency placement without on-chain storage. +- **~0.5s block times** give fast confirmation cycles. +- **No gas fees for cancellations**: short-term order cancels are free (GTB replay protection). +- **On-chain order book**: deterministic matching within each block. +- **Batch cancel**: cancel all short-term orders in a single `MsgBatchCancel` call. + +## Prerequisites + +### Funded dYdX account + +You need a dYdX account with USDC collateral. See the +[Testnet setup](../integrations/dydx.md#testnet-setup) section in the integration guide +for instructions on creating and funding a testnet account. + +### Environment variables + +```bash +# For mainnet +export DYDX_PRIVATE_KEY="0x..." +export DYDX_WALLET_ADDRESS="dydx1..." + +# For testnet +export DYDX_TESTNET_PRIVATE_KEY="0x..." +export DYDX_TESTNET_WALLET_ADDRESS="dydx1..." +``` + +## Strategy overview + +### Geometric grid pricing + +The strategy uses a **geometric grid** where each level is a fixed percentage (basis points) +away from the mid-price: + +``` +Buy level N: mid × (1 - bps/10000)^N - skew +Sell level N: mid × (1 + bps/10000)^N - skew +``` + +Where `skew = skew_factor × net_position`. + +For a 3-level grid with `grid_step_bps=100` (1%) around a mid of 1000.00: + +``` + Sell 3: 1030.30 + Sell 2: 1020.10 + Sell 1: 1010.00 + ─── Mid: 1000.00 ─── + Buy 1: 990.00 + Buy 2: 980.10 + Buy 3: 970.30 +``` + +With inventory skew (long 2 units, `skew_factor=1.0`), the entire grid shifts down by 2.0: + +``` + Sell 3: 1028.30 + Sell 2: 1018.10 + Sell 1: 1008.00 + ─── Mid: 1000.00 ─── + Buy 1: 988.00 + Buy 2: 978.10 + Buy 3: 968.30 +``` + +### Inventory management + +The strategy enforces position limits through two mechanisms: + +1. **`max_position`**: Hard cap on net exposure (long or short). When the projected exposure + from adding another grid level would exceed this cap, that level is skipped. +2. **Projected exposure tracking**: Before placing each level, the strategy tracks the + worst-case per-side exposure (current position + all pending orders) to prevent over-committing. + +Because `cancel_all_orders()` is asynchronous, pending orders may still fill between the cancel +request and acknowledgement. The strategy accounts for this by tracking worst-case per-side +exposure (current position + all pending buy/sell orders) before placing new grid levels. This +prevents momentary over-exposure during cancel-requote transitions. + +### Requote threshold + +The `requote_threshold_bps` parameter controls how much the mid-price must move (in basis points) +before the strategy cancels all existing orders and places a fresh grid. This creates a +trade-off: + +- **Lower threshold** (e.g., 5 bps): More responsive to price moves, but generates more + cancel/place transactions. +- **Higher threshold** (e.g., 50 bps): Fewer transactions, but orders may sit further from + the current price. + +## Configuration + +| Parameter | Type | Default | Description | +| ----------------------- | -------------- | ---------- | ------------------------------------------------------------------------ | +| `instrument_id` | `InstrumentId` | *required* | Instrument to trade (e.g., `ETH-USD-PERP.DYDX`). | +| `max_position` | `Quantity` | *required* | Maximum net exposure (long or short). | +| `trade_size` | `Quantity` | `None` | Size per grid level. If `None`, uses instrument's `min_quantity` or 1.0. | +| `num_levels` | `usize` | `3` | Number of buy and sell levels. | +| `grid_step_bps` | `u32` | `10` | Grid spacing in basis points (10 = 0.1%). | +| `skew_factor` | `f64` | `0.0` | How aggressively to shift the grid based on inventory. | +| `requote_threshold_bps` | `u32` | `5` | Minimum mid-price move in bps before re-quoting. | +| `expire_time_secs` | `Option` | `None` | Order expiry in seconds. Uses GTD when set, GTC otherwise. | +| `on_cancel_resubmit` | `bool` | `false` | Resubmit grid on next quote after an unexpected cancel. | + +### Choosing parameters + +**`grid_step_bps`**: Start wider (50-100 bps) in volatile markets, tighter (5-20 bps) in calm +conditions. Wider grids capture more spread per fill but fill less frequently. + +**`skew_factor`**: Start at `0.0` (no skew) and increase gradually. A value of `0.5` means +each unit of position shifts the grid by 0.5 price units. Too aggressive a skew can cause +the grid to move entirely above or below the mid-price. + +**`expire_time_secs`**: For dYdX short-term orders, set to `8` seconds. This fits within +the 40-block (~20s) short-term window, giving the orders time to rest while keeping them +in the fast short-term path. When `None`, orders use GTC (long-term path). + +**`on_cancel_resubmit`**: Resubmits the grid on the next quote tick after any unexpected +cancel (e.g. self-trade prevention, risk limits). Note that short-term order expiry is +silent and does not generate cancel events, so the grid refreshes naturally via continuous +requoting, not through this flag. + +## dYdX-specific considerations + +### Short-term order expiry + +When `expire_time_secs=8`, orders are classified as short-term by the adapter: + +1. The adapter checks: `8 seconds < max_short_term_secs (40 blocks × ~0.5s = ~20s)`. +2. Since it fits, the order is submitted as short-term with `GoodTilBlock = current_height + N`. +3. The order expires silently after ~8 seconds if not filled. + +This is the recommended configuration for market making because: + +- Short-term orders have lower latency. +- No gas fees for expiry (GTB replay protection handles it). +- Continuous requoting naturally replaces expired orders. + +See the [Order classification](../integrations/dydx.md#order-classification) section +in the integration guide for full details. + +### Unexpected cancels and `on_cancel_resubmit` + +The `pending_self_cancels` set distinguishes between self-initiated and unexpected cancels: + +1. When the strategy calls `cancel_all_orders()`, it records all open order IDs in + `pending_self_cancels`. +2. When `on_order_canceled` fires: + - If the order ID is in `pending_self_cancels`, it's a self-cancel and no action is needed. + - If not, it's an unexpected cancel (e.g. self-trade prevention or risk limits). + Reset `last_quoted_mid` to trigger a full grid resubmission on the next quote. + +This prevents the strategy from re-quoting unnecessarily during its own cancel waves while +still responding to unexpected cancels. + +`on_order_filled` also removes the order from `pending_self_cancels`. If an order fills +before the cancel acknowledgement arrives, this prevents stale entries from accumulating +in the set. + +### Order quantization + +All price and size quantization for dYdX markets is handled automatically by the adapter's +`OrderMessageBuilder`. No manual rounding or conversion is needed. See +[Price and size quantization](../integrations/dydx.md#price-and-size-quantization) for details. + +### Post-only orders + +All grid orders are submitted with `post_only=true`. This ensures every order enters the +book as a maker order (never crosses the spread). If a grid price has moved through the +book by the time it reaches the matching engine, the order is rejected rather than filling +as a taker. This guarantees maker fee rates and prevents unintended crossing during +requote transitions. + +## Running and stopping + +### Environment setup + +Credentials can be set via environment variables or a `.env` file in the project root +(loaded automatically via `dotenvy`): + +```bash +# Export directly +export DYDX_PRIVATE_KEY="0x..." +export DYDX_WALLET_ADDRESS="dydx1..." +``` + +```bash +# Or use a .env file (alternative to shell exports) +DYDX_PRIVATE_KEY=0x... +DYDX_WALLET_ADDRESS=dydx1... +``` + +### Run the example + +```bash +cargo run --example dydx-grid-mm --package nautilus-dydx +``` + +### Graceful shutdown + +Press **Ctrl+C** to stop the node. The shutdown sequence: + +1. SIGINT received, trader stops, `on_stop()` fires. +2. Strategy cancels all orders and closes positions. +3. 5-second grace period (`delay_post_stop_secs`) processes residual events. +4. Clients disconnect, node exits. + +## Code walkthrough + +### Node setup + +The complete `main()` function from the example (`node_grid_mm.rs`): + +```rust +#[tokio::main] +async fn main() -> Result<(), Box> { + dotenvy::dotenv().ok(); // Load .env file if present + + // Configuration + let is_testnet = false; + let network = if is_testnet { + DydxNetwork::Testnet + } else { + DydxNetwork::Mainnet + }; + + let environment = Environment::Live; + let trader_id = TraderId::from("TESTER-001"); + let account_id = AccountId::from("DYDX-001"); + let node_name = "DYDX-GRID-MM-001".to_string(); + let instrument_id = InstrumentId::from("ETH-USD-PERP.DYDX"); + + // Load credentials from environment (testnet/mainnet-aware) + let private_key_env = if is_testnet { + "DYDX_TESTNET_PRIVATE_KEY" + } else { + "DYDX_PRIVATE_KEY" + }; + let private_key = get_env_option(private_key_env); + let wallet_env = if is_testnet { + "DYDX_TESTNET_WALLET_ADDRESS" + } else { + "DYDX_WALLET_ADDRESS" + }; + let wallet_address = get_env_option(wallet_env); + + if private_key.is_none() && wallet_address.is_none() { + return Err( + format!("Set {private_key_env} or {wallet_env} environment variable").into(), + ); + } + + // Minimal data client config: is_testnet selects the correct endpoints + let data_config = DydxDataClientConfig { + is_testnet, + ..Default::default() + }; + + // Execution client with trader ID, network, credentials, and rate limiting + let exec_config = DYDXExecClientConfig { + trader_id, + account_id, + network, + private_key, + wallet_address, + subaccount_number: 0, + grpc_endpoint: None, + grpc_urls: vec![], + ws_endpoint: None, + http_endpoint: None, + authenticator_ids: vec![], + http_timeout_secs: Some(30), + max_retries: Some(3), + retry_delay_initial_ms: Some(1000), + retry_delay_max_ms: Some(10000), + grpc_rate_limit_per_second: Some(4), // Conservative for public providers + }; + + let data_factory = DydxDataClientFactory::new(); + let exec_factory = DydxExecutionClientFactory::new(); + + let log_config = LoggerConfig { + stdout_level: LevelFilter::Info, + ..Default::default() + }; + + // Builder pattern wires up logging, data/execution clients, and node options + let mut node = LiveNode::builder(trader_id, environment)? + .with_name(node_name) + .with_logging(log_config) + .add_data_client(None, Box::new(data_factory), Box::new(data_config))? + .add_exec_client(None, Box::new(exec_factory), Box::new(exec_config))? + .with_reconciliation(false) // Disabled for simplicity; enable in production + // to resume state across restarts + .with_delay_post_stop_secs(5) // Grace period for pending cancel/close events + .build()?; + + // Strategy configuration and registration + let config = GridMarketMakerConfig::new(instrument_id, Quantity::from("0.10")) + .with_num_levels(3) + .with_grid_step_bps(100) + .with_skew_factor(0.5) + .with_requote_threshold_bps(10) + .with_expire_time_secs(8) + .with_on_cancel_resubmit(true); + let strategy = GridMarketMaker::new(config); + + node.add_strategy(strategy)?; + node.run().await?; + + Ok(()) +} +``` + +Key configuration points: + +- **`dotenvy::dotenv().ok()`**: loads a `.env` file from the project root (if present). +- **`with_reconciliation(false)`**: disabled for simplicity; enable in production to resume + state across restarts. +- **`with_delay_post_stop_secs(5)`**: grace period for pending cancel/close events to finalize + during shutdown. + +### Event flow + +``` +LiveNode starts + │ + ├── connect() → HTTP: load instruments, WebSocket: subscribe channels + │ + ├── on_start() + │ └── subscribe_quotes(ETH-USD-PERP.DYDX) + │ + ├── on_quote() [repeated] + │ ├── Calculate mid-price + │ ├── Check should_requote(): skip if within threshold + │ ├── cancel_all_orders(): record IDs in pending_self_cancels + │ ├── Compute grid with inventory skew + │ └── Submit limit orders (GTD, expire in 8s) + │ + ├── on_order_filled() + │ └── Remove from pending_self_cancels + │ + ├── on_order_canceled() + │ ├── Self-cancel? → no action + │ └── Protocol cancel? → reset last_quoted_mid (triggers requote) + │ + └── on_stop() + ├── cancel_all_orders() + ├── close_all_positions() + └── unsubscribe_quotes() +``` + +## Strategy internals + +This section shows the key Rust code from `grid_mm.rs` so you can see exactly how the +strategy works without reading the full source. + +### Trade size resolution (`on_start`) + +When the strategy starts, it resolves the trade size from the instrument cache. The fallback +chain is: config value → instrument `min_quantity` → `1.0`: + +```rust +fn on_start(&mut self) -> anyhow::Result<()> { + let instrument_id = self.config.instrument_id; + let (price_precision, size_precision, min_quantity) = { + let cache = self.cache(); + let instrument = cache + .instrument(&instrument_id) + .expect("Instrument should be in cache"); + ( + instrument.price_precision(), + instrument.size_precision(), + instrument.min_quantity(), + ) + }; + self.price_precision = price_precision; + + // Resolve trade_size from instrument when not explicitly provided + if self.trade_size.is_none() { + self.trade_size = + Some(min_quantity.unwrap_or_else(|| Quantity::new(1.0, size_precision))); + } + + self.subscribe_quotes(instrument_id, None, None); + Ok(()) +} +``` + +### Quote handler (`on_quote`, abbreviated) + +This is the heart of the strategy. On each quote tick it computes the mid-price, +checks whether a requote is needed, cancels stale orders, computes worst-case exposure, +and places new grid orders with GTD + post_only: + +```rust +fn on_quote(&mut self, quote: &QuoteTick) -> anyhow::Result<()> { + let mid_f64 = (quote.bid_price.as_f64() + quote.ask_price.as_f64()) / 2.0; + let mid = Price::new(mid_f64, self.price_precision); + + if !self.should_requote(mid) { + return Ok(()); // Mid hasn't moved enough, keep existing grid + } + + // ... record open order IDs in pending_self_cancels (for on_cancel_resubmit) ... + + self.cancel_all_orders(instrument_id, None, None)?; + + // Compute worst-case per-side exposure (position + all pending orders) + // since cancels are async and pending orders may still fill + let (net_position, worst_long, worst_short) = { /* ... */ }; + + let grid = self.grid_orders(mid, net_position, worst_long, worst_short); + + if grid.is_empty() { + return Ok(()); // Don't advance requote anchor when fully constrained + } + + // Compute time-in-force from config + let (tif, expire_time) = match self.config.expire_time_secs { + Some(secs) => { + let now_ns = self.core.clock().timestamp_ns(); + let expire_ns = now_ns + secs * 1_000_000_000; + (Some(TimeInForce::Gtd), Some(expire_ns)) + } + None => (None, None), + }; + + for (side, price) in grid { + let order = self.core.order_factory().limit( + instrument_id, + side, + trade_size, + price, + tif, + expire_time, + Some(true), // post_only + // ... remaining None fields ... + ); + self.submit_order(order, None, None)?; + } + + self.last_quoted_mid = Some(mid); + Ok(()) +} +``` + +### Grid pricing (`grid_orders`) + +Computes geometric grid prices and enforces max_position per-level. This is the function +behind the ASCII diagrams in the [Strategy overview](#geometric-grid-pricing) section: + +```rust +fn grid_orders( + &self, + mid: Price, + net_position: f64, + worst_long: Decimal, + worst_short: Decimal, +) -> Vec<(OrderSide, Price)> { + let mid_f64 = mid.as_f64(); + let skew_f64 = self.config.skew_factor * net_position; + let pct = self.config.grid_step_bps as f64 / 10_000.0; + let trade_size = self.trade_size + .expect("trade_size should be resolved in on_start") + .as_decimal(); + let max_pos = self.config.max_position.as_decimal(); + let mut projected_long = worst_long; + let mut projected_short = worst_short; + let mut orders = Vec::new(); + + for level in 1..=self.config.num_levels { + let buy_price = Price::new( + mid_f64 * (1.0 - pct).powi(level as i32) - skew_f64, + precision, + ); + let sell_price = Price::new( + mid_f64 * (1.0 + pct).powi(level as i32) - skew_f64, + precision, + ); + + // Only place buy if projected long exposure stays within max_position + if projected_long + trade_size <= max_pos { + orders.push((OrderSide::Buy, buy_price)); + projected_long += trade_size; + } + + // Only place sell if projected short exposure stays within max_position + if projected_short - trade_size >= -max_pos { + orders.push((OrderSide::Sell, sell_price)); + projected_short -= trade_size; + } + } + + orders +} +``` + +## Monitoring and understanding output + +### Key log messages + +| Log message | Meaning | +| --------------------------------------------------- | -------------------------------------------------- | +| `Requoting grid: mid=X, last_mid=Y` | Mid-price moved beyond threshold, refreshing grid. | +| `Submit short-term order N` | Order submitted via short-term broadcast path. | +| `BatchCancel N short-term orders` | Batch cancel executed for expired/stale orders. | +| `benign cancel error, treating as success` | Cancel for already-filled/expired order (normal). | +| `Sequence mismatch detected, will resync and retry` | Cosmos SDK sequence error, auto-recovering. | + +### Expected behavior patterns + +1. **Startup**: Instruments load, WebSocket connects, first quote triggers initial grid. +2. **Steady state**: Grid persists across ticks; requotes only on significant mid-price moves. +3. **Fills**: Position updates, skew adjusts, next requote shifts grid. +4. **Expiry**: Short-term orders expire silently after ~8s; grid naturally refreshes on the next requote. +5. **Shutdown**: All orders cancelled, positions closed, WebSocket disconnected. + +## Customization tips + +### High vs low volatility + +| Condition | Adjustment | +| --------------- | ------------------------------------------------------------------------- | +| High volatility | Wider `grid_step_bps` (100-200), fewer `num_levels`, lower `skew_factor`. | +| Low volatility | Tighter `grid_step_bps` (10-30), more `num_levels`, higher `skew_factor`. | +| Thin liquidity | Increase `requote_threshold_bps` to reduce cancel frequency. | + +### Multiple instruments + +Run separate `GridMarketMaker` instances for each instrument. Each instance manages its own +grid, position tracking, and cancel state independently: + +```rust +let btc_config = GridMarketMakerConfig::new( + InstrumentId::from("BTC-USD-PERP.DYDX"), + Quantity::from("0.001"), +) +.with_strategy_id(StrategyId::from("GRID_MM-BTC")) +.with_order_id_tag("BTC".to_string()) +.with_grid_step_bps(50); + +let eth_config = GridMarketMakerConfig::new( + InstrumentId::from("ETH-USD-PERP.DYDX"), + Quantity::from("0.10"), +) +.with_strategy_id(StrategyId::from("GRID_MM-ETH")) +.with_order_id_tag("ETH".to_string()) +.with_grid_step_bps(100); + +node.add_strategy(GridMarketMaker::new(btc_config))?; +node.add_strategy(GridMarketMaker::new(eth_config))?; +``` + +### Mainnet vs testnet toggle + +Change a single flag to switch networks: + +```rust +let is_testnet = true; // false for mainnet +let network = if is_testnet { DydxNetwork::Testnet } else { DydxNetwork::Mainnet }; +``` + +All endpoints, chain IDs, and credential environment variables are resolved automatically +based on this flag. + +## Further reading + +- [dYdX v4 Integration Guide](../integrations/dydx.md): full adapter reference. +- [dYdX Protocol Documentation](https://docs.dydx.xyz/): official protocol docs. +- [Order types](https://docs.dydx.xyz/concepts/trading/orders): protocol-level order mechanics. diff --git a/nautilus-docs/docs/tutorials/index.md b/nautilus-docs/docs/tutorials/index.md new file mode 100644 index 0000000..5f589d0 --- /dev/null +++ b/nautilus-docs/docs/tutorials/index.md @@ -0,0 +1,15 @@ +# Tutorials + +Step-by-step walkthroughs demonstrating specific features and workflows. + +:::info +Each tutorial is a Jupytext percent-format Python file in the docs [tutorials directory](https://github.com/nautechsystems/nautilus_trader/tree/develop/docs/tutorials). You can run them directly as scripts or open them as notebooks with Jupytext. +::: + +:::tip + +- Make sure you are using the tutorial docs that match your NautilusTrader version: +- **Latest**: These docs are built from the HEAD of the `master` branch and work with the latest stable release. See . +- **Nightly**: These docs are built from the HEAD of the `nightly` branch and work with bleeding-edge and experimental features. See . + +::: diff --git a/nautilus-docs/docs/tutorials/loading_external_data.py b/nautilus-docs/docs/tutorials/loading_external_data.py new file mode 100644 index 0000000..4c47559 --- /dev/null +++ b/nautilus-docs/docs/tutorials/loading_external_data.py @@ -0,0 +1,123 @@ +# %% [markdown] +# # Loading External Data +# +# This tutorial demonstrates how to load external data into the `ParquetDataCatalog`, and then use this to run a one-shot backtest using a `BacktestNode`. + +# %% +import shutil +from decimal import Decimal +from pathlib import Path + +import pandas as pd + +from nautilus_trader.backtest.node import BacktestDataConfig +from nautilus_trader.backtest.node import BacktestEngineConfig +from nautilus_trader.backtest.node import BacktestNode +from nautilus_trader.backtest.node import BacktestRunConfig +from nautilus_trader.backtest.node import BacktestVenueConfig +from nautilus_trader.config import ImportableStrategyConfig +from nautilus_trader.core.datetime import dt_to_unix_nanos +from nautilus_trader.model import BarType +from nautilus_trader.model import QuoteTick +from nautilus_trader.persistence.catalog import ParquetDataCatalog +from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler +from nautilus_trader.test_kit.providers import CSVTickDataLoader +from nautilus_trader.test_kit.providers import TestInstrumentProvider + +# %% +DATA_DIR = "~/Downloads/Data/" + +# %% +path = Path(DATA_DIR).expanduser() / "HISTDATA" +raw_files = [ + f for f in path.iterdir() if f.is_file() and (f.suffix == ".csv" or f.name.endswith(".csv.gz")) +] +assert raw_files, f"Unable to find any data files in directory {path}" +raw_files + +# %% +# Load the first data file into a pandas DataFrame +df = CSVTickDataLoader.load(raw_files[0], index_col=0, datetime_format="%Y%m%d %H%M%S%f") +df.columns = ["bid_price", "ask_price"] + +# Process quotes using a wrangler +EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD") +wrangler = QuoteTickDataWrangler(EURUSD) + +ticks = wrangler.process(df) + +# %% +CATALOG_PATH = Path.cwd() / "catalog" + +# Clear if it already exists, then create fresh +if CATALOG_PATH.exists(): + shutil.rmtree(CATALOG_PATH) +CATALOG_PATH.mkdir() + +catalog = ParquetDataCatalog(CATALOG_PATH) + +# %% +catalog.write_data([EURUSD]) +catalog.write_data(ticks) + +# %% +# Verify instruments written to catalog +catalog.instruments() + +# %% +start = dt_to_unix_nanos(pd.Timestamp("2020-01-03", tz="UTC")) +end = dt_to_unix_nanos(pd.Timestamp("2020-01-04", tz="UTC")) + +ticks = catalog.quote_ticks(instrument_ids=[EURUSD.id.value], start=start, end=end) +ticks[:10] + +# %% +instrument = catalog.instruments()[0] + +venue_configs = [ + BacktestVenueConfig( + name="SIM", + oms_type="HEDGING", + account_type="MARGIN", + base_currency="USD", + starting_balances=["1000000 USD"], + ), +] + +data_configs = [ + BacktestDataConfig( + catalog_path=str(catalog.path), + data_cls=QuoteTick, + instrument_id=instrument.id, + start_time=start, + end_time=end, + ), +] + +strategies = [ + ImportableStrategyConfig( + strategy_path="nautilus_trader.examples.strategies.ema_cross:EMACross", + config_path="nautilus_trader.examples.strategies.ema_cross:EMACrossConfig", + config={ + "instrument_id": instrument.id, + "bar_type": BarType.from_str(f"{instrument.id.value}-15-MINUTE-BID-INTERNAL"), + "fast_ema_period": 10, + "slow_ema_period": 20, + "trade_size": Decimal(1_000_000), + }, + ), +] + +config = BacktestRunConfig( + engine=BacktestEngineConfig(strategies=strategies), + data=data_configs, + venues=venue_configs, +) + +# %% +node = BacktestNode(configs=[config]) + +[result] = node.run() + +# %% +result From abf92273c35e6ab09037fa64d6da9841b6693e0a Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 00:35:36 +0200 Subject: [PATCH 02/25] updating the skill --- nautilus-docs/SKILL.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 84e3132..f78998e 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -1,15 +1,15 @@ --- name: nautilus-docs description: > - Official NautilusTrader documentation — concepts, API reference, integration guides, - tutorials. Use when needing detailed explanations of NautilusTrader architecture, - component behavior, adapter configuration, or API signatures beyond what the - nautilus-trader skill covers. + NautilusTrader trading platform — strategies, backtesting, live deployment, order management, + market data, Rust native binaries. Use when writing, reviewing, or debugging NautilusTrader + code (Python or Rust), configuring venue adapters, or answering questions about the platform. +argument-hint: "[topic or question]" --- -# NautilusTrader Official Docs +# NautilusTrader v1.224.0 -Read the relevant doc before answering questions about platform behavior. All paths relative to `docs/`. +Read the relevant doc before generating code or answering questions. Use `${CLAUDE_SKILL_DIR}/docs/` for doc file paths, `${CLAUDE_SKILL_DIR}/battle_tested.md` for non-obvious patterns. ## Doc Navigator @@ -83,9 +83,10 @@ Read the relevant doc before answering questions about platform behavior. All pa | Databento data catalog | [databento_data_catalog.py](docs/tutorials/databento_data_catalog.py) | | Loading external data | [loading_external_data.py](docs/tutorials/loading_external_data.py) | -## Battle-Tested Patterns +## Supporting Files -Non-obvious tricks from live testing — NOT in official docs: [battle_tested.md](battle_tested.md). Covers: on_start() ordering, microprice, inventory skew, signal pipeline, timer polling, FillModel, frozen_account inversion, F_LAST rule, venue gotchas, performance checklist. +- **[battle_tested.md](battle_tested.md)** — Non-obvious patterns verified against live exchanges. Load when: writing on_start() ordering, market making, signal pipelines, backtest config, venue-specific gotchas, performance optimization. +- **[REBUILD.md](REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. ## Rust Standalone Binary From a743a5e46589ca468cc459c8facd6a720d0a7b60 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 01:06:16 +0200 Subject: [PATCH 03/25] submodule docs --- .gitmodules | 4 + nautilus-docs/SKILL.md | 106 +- .../docs/api_reference/_static/custom.css | 14 - .../docs/api_reference/accounting.md | 45 - .../docs/api_reference/adapters/betfair.md | 109 - .../docs/api_reference/adapters/binance.md | 143 -- .../docs/api_reference/adapters/bybit.md | 69 - .../docs/api_reference/adapters/databento.md | 79 - .../docs/api_reference/adapters/dydx.md | 59 - .../docs/api_reference/adapters/index.md | 23 - .../adapters/interactive_brokers.md | 117 - .../docs/api_reference/adapters/okx.md | 59 - .../docs/api_reference/adapters/polymarket.md | 69 - .../docs/api_reference/adapters/tardis.md | 59 - nautilus-docs/docs/api_reference/analysis.md | 29 - nautilus-docs/docs/api_reference/backtest.md | 61 - nautilus-docs/docs/api_reference/cache.md | 29 - nautilus-docs/docs/api_reference/common.md | 59 - nautilus-docs/docs/api_reference/conf.py | 122 - nautilus-docs/docs/api_reference/config.md | 101 - nautilus-docs/docs/api_reference/core.md | 55 - nautilus-docs/docs/api_reference/data.md | 45 - nautilus-docs/docs/api_reference/execution.md | 75 - nautilus-docs/docs/api_reference/index.md | 8 - .../docs/api_reference/indicators.md | 61 - nautilus-docs/docs/api_reference/live.md | 61 - .../docs/api_reference/model/book.md | 9 - .../docs/api_reference/model/data.md | 9 - .../docs/api_reference/model/events.md | 29 - .../docs/api_reference/model/identifiers.md | 9 - .../docs/api_reference/model/index.md | 23 - .../docs/api_reference/model/instruments.md | 141 -- .../docs/api_reference/model/objects.md | 9 - .../docs/api_reference/model/orders.md | 93 - .../docs/api_reference/model/position.md | 9 - .../docs/api_reference/model/tick_scheme.md | 29 - .../docs/api_reference/persistence.md | 37 - nautilus-docs/docs/api_reference/portfolio.md | 21 - nautilus-docs/docs/api_reference/risk.md | 21 - .../docs/api_reference/serialization.md | 21 - nautilus-docs/docs/api_reference/system.md | 13 - nautilus-docs/docs/api_reference/trading.md | 37 - nautilus-docs/docs/concepts/actors.md | 342 --- nautilus-docs/docs/concepts/adapters.md | 194 -- nautilus-docs/docs/concepts/architecture.md | 641 ----- nautilus-docs/docs/concepts/backtesting.md | 1636 ------------- nautilus-docs/docs/concepts/cache.md | 539 ----- nautilus-docs/docs/concepts/custom_data.md | 396 --- nautilus-docs/docs/concepts/data.md | 1892 --------------- nautilus-docs/docs/concepts/execution.md | 504 ---- nautilus-docs/docs/concepts/greeks.md | 321 --- nautilus-docs/docs/concepts/index.md | 113 - nautilus-docs/docs/concepts/instruments.md | 541 ----- nautilus-docs/docs/concepts/live.md | 597 ----- nautilus-docs/docs/concepts/logging.md | 470 ---- nautilus-docs/docs/concepts/message_bus.md | 473 ---- nautilus-docs/docs/concepts/options.md | 291 --- nautilus-docs/docs/concepts/order_book.md | 246 -- nautilus-docs/docs/concepts/orders.md | 838 ------- nautilus-docs/docs/concepts/overview.md | 255 -- nautilus-docs/docs/concepts/portfolio.md | 169 -- nautilus-docs/docs/concepts/positions.md | 460 ---- nautilus-docs/docs/concepts/reports.md | 435 ---- nautilus-docs/docs/concepts/strategies.md | 692 ------ nautilus-docs/docs/concepts/value_types.md | 303 --- nautilus-docs/docs/concepts/visualization.md | 571 ----- .../docs/dev_templates/criterion_template.rs | 52 - .../docs/dev_templates/iai_template.rs | 33 - .../docs/developer_guide/adapters.md | 2131 ----------------- .../docs/developer_guide/benchmarking.md | 180 -- .../docs/developer_guide/coding_standards.md | 141 -- nautilus-docs/docs/developer_guide/docs.md | 134 -- .../docs/developer_guide/environment_setup.md | 399 --- nautilus-docs/docs/developer_guide/ffi.md | 142 -- nautilus-docs/docs/developer_guide/index.md | 27 - nautilus-docs/docs/developer_guide/python.md | 98 - .../docs/developer_guide/releases.md | 271 --- nautilus-docs/docs/developer_guide/rust.md | 1265 ---------- .../docs/developer_guide/spec_data_testing.md | 854 ------- .../docs/developer_guide/spec_exec_testing.md | 1690 ------------- .../docs/developer_guide/test_datasets.md | 282 --- nautilus-docs/docs/developer_guide/testing.md | 289 --- .../getting_started/backtest_high_level.py | 251 -- .../getting_started/backtest_low_level.py | 220 -- nautilus-docs/docs/getting_started/index.md | 75 - .../docs/getting_started/installation.md | 361 --- .../docs/getting_started/quickstart.py | 211 -- .../docs/integrations/architect_ax.md | 400 ---- nautilus-docs/docs/integrations/betfair.md | 574 ----- nautilus-docs/docs/integrations/binance.md | 857 ------- nautilus-docs/docs/integrations/bitmex.md | 871 ------- nautilus-docs/docs/integrations/blockchain.md | 92 - nautilus-docs/docs/integrations/bybit.md | 673 ------ nautilus-docs/docs/integrations/databento.md | 881 ------- nautilus-docs/docs/integrations/deribit.md | 648 ----- nautilus-docs/docs/integrations/dydx.md | 803 ------- .../docs/integrations/hyperliquid.md | 494 ---- nautilus-docs/docs/integrations/ib.md | 2058 ---------------- nautilus-docs/docs/integrations/index.md | 60 - nautilus-docs/docs/integrations/kraken.md | 555 ----- nautilus-docs/docs/integrations/okx.md | 675 ------ nautilus-docs/docs/integrations/polymarket.md | 801 ------- nautilus-docs/docs/integrations/tardis.md | 695 ------ .../docs/tutorials/ax_fx_mean_reversion.md | 279 --- .../docs/tutorials/ax_gold_book_imbalance.md | 306 --- .../tutorials/backtest_binance_orderbook.py | 190 -- .../tutorials/backtest_bybit_orderbook.py | 184 -- .../docs/tutorials/backtest_fx_bars.py | 164 -- .../tutorials/bitmex_grid_market_maker.md | 541 ----- .../docs/tutorials/databento_data_catalog.py | 221 -- .../docs/tutorials/dydx_grid_market_maker.md | 597 ----- nautilus-docs/docs/tutorials/index.md | 15 - .../docs/tutorials/loading_external_data.py | 123 - nautilus-docs/{ => references}/REBUILD.md | 17 +- .../{ => references}/battle_tested.md | 0 nautilus-docs/references/upstream | 1 + 116 files changed, 67 insertions(+), 37904 deletions(-) create mode 100644 .gitmodules delete mode 100644 nautilus-docs/docs/api_reference/_static/custom.css delete mode 100644 nautilus-docs/docs/api_reference/accounting.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/betfair.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/binance.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/bybit.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/databento.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/dydx.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/index.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/interactive_brokers.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/okx.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/polymarket.md delete mode 100644 nautilus-docs/docs/api_reference/adapters/tardis.md delete mode 100644 nautilus-docs/docs/api_reference/analysis.md delete mode 100644 nautilus-docs/docs/api_reference/backtest.md delete mode 100644 nautilus-docs/docs/api_reference/cache.md delete mode 100644 nautilus-docs/docs/api_reference/common.md delete mode 100644 nautilus-docs/docs/api_reference/conf.py delete mode 100644 nautilus-docs/docs/api_reference/config.md delete mode 100644 nautilus-docs/docs/api_reference/core.md delete mode 100644 nautilus-docs/docs/api_reference/data.md delete mode 100644 nautilus-docs/docs/api_reference/execution.md delete mode 100644 nautilus-docs/docs/api_reference/index.md delete mode 100644 nautilus-docs/docs/api_reference/indicators.md delete mode 100644 nautilus-docs/docs/api_reference/live.md delete mode 100644 nautilus-docs/docs/api_reference/model/book.md delete mode 100644 nautilus-docs/docs/api_reference/model/data.md delete mode 100644 nautilus-docs/docs/api_reference/model/events.md delete mode 100644 nautilus-docs/docs/api_reference/model/identifiers.md delete mode 100644 nautilus-docs/docs/api_reference/model/index.md delete mode 100644 nautilus-docs/docs/api_reference/model/instruments.md delete mode 100644 nautilus-docs/docs/api_reference/model/objects.md delete mode 100644 nautilus-docs/docs/api_reference/model/orders.md delete mode 100644 nautilus-docs/docs/api_reference/model/position.md delete mode 100644 nautilus-docs/docs/api_reference/model/tick_scheme.md delete mode 100644 nautilus-docs/docs/api_reference/persistence.md delete mode 100644 nautilus-docs/docs/api_reference/portfolio.md delete mode 100644 nautilus-docs/docs/api_reference/risk.md delete mode 100644 nautilus-docs/docs/api_reference/serialization.md delete mode 100644 nautilus-docs/docs/api_reference/system.md delete mode 100644 nautilus-docs/docs/api_reference/trading.md delete mode 100644 nautilus-docs/docs/concepts/actors.md delete mode 100644 nautilus-docs/docs/concepts/adapters.md delete mode 100644 nautilus-docs/docs/concepts/architecture.md delete mode 100644 nautilus-docs/docs/concepts/backtesting.md delete mode 100644 nautilus-docs/docs/concepts/cache.md delete mode 100644 nautilus-docs/docs/concepts/custom_data.md delete mode 100644 nautilus-docs/docs/concepts/data.md delete mode 100644 nautilus-docs/docs/concepts/execution.md delete mode 100644 nautilus-docs/docs/concepts/greeks.md delete mode 100644 nautilus-docs/docs/concepts/index.md delete mode 100644 nautilus-docs/docs/concepts/instruments.md delete mode 100644 nautilus-docs/docs/concepts/live.md delete mode 100644 nautilus-docs/docs/concepts/logging.md delete mode 100644 nautilus-docs/docs/concepts/message_bus.md delete mode 100644 nautilus-docs/docs/concepts/options.md delete mode 100644 nautilus-docs/docs/concepts/order_book.md delete mode 100644 nautilus-docs/docs/concepts/orders.md delete mode 100644 nautilus-docs/docs/concepts/overview.md delete mode 100644 nautilus-docs/docs/concepts/portfolio.md delete mode 100644 nautilus-docs/docs/concepts/positions.md delete mode 100644 nautilus-docs/docs/concepts/reports.md delete mode 100644 nautilus-docs/docs/concepts/strategies.md delete mode 100644 nautilus-docs/docs/concepts/value_types.md delete mode 100644 nautilus-docs/docs/concepts/visualization.md delete mode 100644 nautilus-docs/docs/dev_templates/criterion_template.rs delete mode 100644 nautilus-docs/docs/dev_templates/iai_template.rs delete mode 100644 nautilus-docs/docs/developer_guide/adapters.md delete mode 100644 nautilus-docs/docs/developer_guide/benchmarking.md delete mode 100644 nautilus-docs/docs/developer_guide/coding_standards.md delete mode 100644 nautilus-docs/docs/developer_guide/docs.md delete mode 100644 nautilus-docs/docs/developer_guide/environment_setup.md delete mode 100644 nautilus-docs/docs/developer_guide/ffi.md delete mode 100644 nautilus-docs/docs/developer_guide/index.md delete mode 100644 nautilus-docs/docs/developer_guide/python.md delete mode 100644 nautilus-docs/docs/developer_guide/releases.md delete mode 100644 nautilus-docs/docs/developer_guide/rust.md delete mode 100644 nautilus-docs/docs/developer_guide/spec_data_testing.md delete mode 100644 nautilus-docs/docs/developer_guide/spec_exec_testing.md delete mode 100644 nautilus-docs/docs/developer_guide/test_datasets.md delete mode 100644 nautilus-docs/docs/developer_guide/testing.md delete mode 100644 nautilus-docs/docs/getting_started/backtest_high_level.py delete mode 100644 nautilus-docs/docs/getting_started/backtest_low_level.py delete mode 100644 nautilus-docs/docs/getting_started/index.md delete mode 100644 nautilus-docs/docs/getting_started/installation.md delete mode 100644 nautilus-docs/docs/getting_started/quickstart.py delete mode 100644 nautilus-docs/docs/integrations/architect_ax.md delete mode 100644 nautilus-docs/docs/integrations/betfair.md delete mode 100644 nautilus-docs/docs/integrations/binance.md delete mode 100644 nautilus-docs/docs/integrations/bitmex.md delete mode 100644 nautilus-docs/docs/integrations/blockchain.md delete mode 100644 nautilus-docs/docs/integrations/bybit.md delete mode 100644 nautilus-docs/docs/integrations/databento.md delete mode 100644 nautilus-docs/docs/integrations/deribit.md delete mode 100644 nautilus-docs/docs/integrations/dydx.md delete mode 100644 nautilus-docs/docs/integrations/hyperliquid.md delete mode 100644 nautilus-docs/docs/integrations/ib.md delete mode 100644 nautilus-docs/docs/integrations/index.md delete mode 100644 nautilus-docs/docs/integrations/kraken.md delete mode 100644 nautilus-docs/docs/integrations/okx.md delete mode 100644 nautilus-docs/docs/integrations/polymarket.md delete mode 100644 nautilus-docs/docs/integrations/tardis.md delete mode 100644 nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md delete mode 100644 nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md delete mode 100644 nautilus-docs/docs/tutorials/backtest_binance_orderbook.py delete mode 100644 nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py delete mode 100644 nautilus-docs/docs/tutorials/backtest_fx_bars.py delete mode 100644 nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md delete mode 100644 nautilus-docs/docs/tutorials/databento_data_catalog.py delete mode 100644 nautilus-docs/docs/tutorials/dydx_grid_market_maker.md delete mode 100644 nautilus-docs/docs/tutorials/index.md delete mode 100644 nautilus-docs/docs/tutorials/loading_external_data.py rename nautilus-docs/{ => references}/REBUILD.md (96%) rename nautilus-docs/{ => references}/battle_tested.md (100%) create mode 160000 nautilus-docs/references/upstream diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4d67b92 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "nautilus-docs/references/upstream"] + path = nautilus-docs/references/upstream + url = https://github.com/nautechsystems/nautilus_trader.git + shallow = true diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index f78998e..c5597d2 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -9,7 +9,7 @@ argument-hint: "[topic or question]" # NautilusTrader v1.224.0 -Read the relevant doc before generating code or answering questions. Use `${CLAUDE_SKILL_DIR}/docs/` for doc file paths, `${CLAUDE_SKILL_DIR}/battle_tested.md` for non-obvious patterns. +Read the relevant doc before generating code or answering questions. Docs are a git submodule (sparse checkout, `docs/` only) from [nautilus_trader](https://github.com/nautechsystems/nautilus_trader). Use `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for doc file paths. ## Doc Navigator @@ -17,76 +17,76 @@ Read the relevant doc before generating code or answering questions. Use `${CLAU | Topic | Doc | |-------|-----| -| Architecture / overview | [architecture.md](docs/concepts/architecture.md), [overview.md](docs/concepts/overview.md) | -| Actor development | [actors.md](docs/concepts/actors.md) | -| Strategy development | [strategies.md](docs/concepts/strategies.md) | -| Backtesting | [backtesting.md](docs/concepts/backtesting.md) | -| Live trading | [live.md](docs/concepts/live.md) | -| Order types & execution | [orders.md](docs/concepts/orders.md), [execution.md](docs/concepts/execution.md) | -| Order book | [order_book.md](docs/concepts/order_book.md) | -| Data types & custom data | [data.md](docs/concepts/data.md), [custom_data.md](docs/concepts/custom_data.md) | -| Instruments | [instruments.md](docs/concepts/instruments.md) | -| Value types (Price, Quantity, Money) | [value_types.md](docs/concepts/value_types.md) | -| Positions & PnL | [positions.md](docs/concepts/positions.md) | -| Cache | [cache.md](docs/concepts/cache.md) | -| MessageBus | [message_bus.md](docs/concepts/message_bus.md) | -| Portfolio | [portfolio.md](docs/concepts/portfolio.md) | -| Options & Greeks | [options.md](docs/concepts/options.md), [greeks.md](docs/concepts/greeks.md) | -| Logging | [logging.md](docs/concepts/logging.md) | -| Reports & analysis | [reports.md](docs/concepts/reports.md) | -| Visualization | [visualization.md](docs/concepts/visualization.md) | -| Adapter development | [concepts/adapters.md](docs/concepts/adapters.md), [developer_guide/adapters.md](docs/developer_guide/adapters.md) | +| Architecture / overview | [architecture.md](references/upstream/docs/concepts/architecture.md), [overview.md](references/upstream/docs/concepts/overview.md) | +| Actor development | [actors.md](references/upstream/docs/concepts/actors.md) | +| Strategy development | [strategies.md](references/upstream/docs/concepts/strategies.md) | +| Backtesting | [backtesting.md](references/upstream/docs/concepts/backtesting.md) | +| Live trading | [live.md](references/upstream/docs/concepts/live.md) | +| Order types & execution | [orders.md](references/upstream/docs/concepts/orders.md), [execution.md](references/upstream/docs/concepts/execution.md) | +| Order book | [order_book.md](references/upstream/docs/concepts/order_book.md) | +| Data types & custom data | [data.md](references/upstream/docs/concepts/data.md), [custom_data.md](references/upstream/docs/concepts/custom_data.md) | +| Instruments | [instruments.md](references/upstream/docs/concepts/instruments.md) | +| Value types (Price, Quantity, Money) | [value_types.md](references/upstream/docs/concepts/value_types.md) | +| Positions & PnL | [positions.md](references/upstream/docs/concepts/positions.md) | +| Cache | [cache.md](references/upstream/docs/concepts/cache.md) | +| MessageBus | [message_bus.md](references/upstream/docs/concepts/message_bus.md) | +| Portfolio | [portfolio.md](references/upstream/docs/concepts/portfolio.md) | +| Options & Greeks | [options.md](references/upstream/docs/concepts/options.md), [greeks.md](references/upstream/docs/concepts/greeks.md) | +| Logging | [logging.md](references/upstream/docs/concepts/logging.md) | +| Reports & analysis | [reports.md](references/upstream/docs/concepts/reports.md) | +| Visualization | [visualization.md](references/upstream/docs/concepts/visualization.md) | +| Adapter development | [concepts/adapters.md](references/upstream/docs/concepts/adapters.md), [developer_guide/adapters.md](references/upstream/docs/developer_guide/adapters.md) | ### Venue Integrations | Venue | Doc | |-------|-----| -| Binance | [binance.md](docs/integrations/binance.md) | -| Bybit | [bybit.md](docs/integrations/bybit.md) | -| OKX | [okx.md](docs/integrations/okx.md) | -| dYdX | [dydx.md](docs/integrations/dydx.md) | -| Deribit | [deribit.md](docs/integrations/deribit.md) | -| Hyperliquid | [hyperliquid.md](docs/integrations/hyperliquid.md) | -| Kraken | [kraken.md](docs/integrations/kraken.md) | -| Interactive Brokers | [ib.md](docs/integrations/ib.md) | -| Betfair | [betfair.md](docs/integrations/betfair.md) | -| Polymarket | [polymarket.md](docs/integrations/polymarket.md) | -| Databento | [databento.md](docs/integrations/databento.md) | -| Tardis | [tardis.md](docs/integrations/tardis.md) | -| BitMEX | [bitmex.md](docs/integrations/bitmex.md) | -| AX Exchange | [architect_ax.md](docs/integrations/architect_ax.md) | +| Binance | [binance.md](references/upstream/docs/integrations/binance.md) | +| Bybit | [bybit.md](references/upstream/docs/integrations/bybit.md) | +| OKX | [okx.md](references/upstream/docs/integrations/okx.md) | +| dYdX | [dydx.md](references/upstream/docs/integrations/dydx.md) | +| Deribit | [deribit.md](references/upstream/docs/integrations/deribit.md) | +| Hyperliquid | [hyperliquid.md](references/upstream/docs/integrations/hyperliquid.md) | +| Kraken | [kraken.md](references/upstream/docs/integrations/kraken.md) | +| Interactive Brokers | [ib.md](references/upstream/docs/integrations/ib.md) | +| Betfair | [betfair.md](references/upstream/docs/integrations/betfair.md) | +| Polymarket | [polymarket.md](references/upstream/docs/integrations/polymarket.md) | +| Databento | [databento.md](references/upstream/docs/integrations/databento.md) | +| Tardis | [tardis.md](references/upstream/docs/integrations/tardis.md) | +| BitMEX | [bitmex.md](references/upstream/docs/integrations/bitmex.md) | +| AX Exchange | [architect_ax.md](references/upstream/docs/integrations/architect_ax.md) | ### Dev / Setup | Topic | Doc | |-------|-----| -| Installation | [installation.md](docs/getting_started/installation.md) | -| Quickstart | [quickstart.py](docs/getting_started/quickstart.py) | -| Environment setup | [environment_setup.md](docs/developer_guide/environment_setup.md) | -| Rust development | [rust.md](docs/developer_guide/rust.md) | -| Testing | [testing.md](docs/developer_guide/testing.md) | -| Coding standards | [coding_standards.md](docs/developer_guide/coding_standards.md) | -| Benchmarking | [benchmarking.md](docs/developer_guide/benchmarking.md) | -| FFI | [ffi.md](docs/developer_guide/ffi.md) | +| Installation | [installation.md](references/upstream/docs/getting_started/installation.md) | +| Quickstart | [quickstart.py](references/upstream/docs/getting_started/quickstart.py) | +| Environment setup | [environment_setup.md](references/upstream/docs/developer_guide/environment_setup.md) | +| Rust development | [rust.md](references/upstream/docs/developer_guide/rust.md) | +| Testing | [testing.md](references/upstream/docs/developer_guide/testing.md) | +| Coding standards | [coding_standards.md](references/upstream/docs/developer_guide/coding_standards.md) | +| Benchmarking | [benchmarking.md](references/upstream/docs/developer_guide/benchmarking.md) | +| FFI | [ffi.md](references/upstream/docs/developer_guide/ffi.md) | ### Tutorials | Tutorial | Doc | |----------|-----| -| FX mean reversion (AX) | [ax_fx_mean_reversion.md](docs/tutorials/ax_fx_mean_reversion.md) | -| Gold book imbalance (AX) | [ax_gold_book_imbalance.md](docs/tutorials/ax_gold_book_imbalance.md) | -| dYdX grid market maker | [dydx_grid_market_maker.md](docs/tutorials/dydx_grid_market_maker.md) | -| BitMEX grid market maker | [bitmex_grid_market_maker.md](docs/tutorials/bitmex_grid_market_maker.md) | -| Backtest Binance orderbook | [backtest_binance_orderbook.py](docs/tutorials/backtest_binance_orderbook.py) | -| Backtest Bybit orderbook | [backtest_bybit_orderbook.py](docs/tutorials/backtest_bybit_orderbook.py) | -| Backtest FX bars | [backtest_fx_bars.py](docs/tutorials/backtest_fx_bars.py) | -| Databento data catalog | [databento_data_catalog.py](docs/tutorials/databento_data_catalog.py) | -| Loading external data | [loading_external_data.py](docs/tutorials/loading_external_data.py) | +| FX mean reversion (AX) | [ax_fx_mean_reversion.md](references/upstream/docs/tutorials/ax_fx_mean_reversion.md) | +| Gold book imbalance (AX) | [ax_gold_book_imbalance.md](references/upstream/docs/tutorials/ax_gold_book_imbalance.md) | +| dYdX grid market maker | [dydx_grid_market_maker.md](references/upstream/docs/tutorials/dydx_grid_market_maker.md) | +| BitMEX grid market maker | [bitmex_grid_market_maker.md](references/upstream/docs/tutorials/bitmex_grid_market_maker.md) | +| Backtest Binance orderbook | [backtest_binance_orderbook.py](references/upstream/docs/tutorials/backtest_binance_orderbook.py) | +| Backtest Bybit orderbook | [backtest_bybit_orderbook.py](references/upstream/docs/tutorials/backtest_bybit_orderbook.py) | +| Backtest FX bars | [backtest_fx_bars.py](references/upstream/docs/tutorials/backtest_fx_bars.py) | +| Databento data catalog | [databento_data_catalog.py](references/upstream/docs/tutorials/databento_data_catalog.py) | +| Loading external data | [loading_external_data.py](references/upstream/docs/tutorials/loading_external_data.py) | ## Supporting Files -- **[battle_tested.md](battle_tested.md)** — Non-obvious patterns verified against live exchanges. Load when: writing on_start() ordering, market making, signal pipelines, backtest config, venue-specific gotchas, performance optimization. -- **[REBUILD.md](REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. +- **[battle_tested.md](references/battle_tested.md)** — Non-obvious patterns verified against live exchanges. Load when: writing on_start() ordering, market making, signal pipelines, backtest config, venue-specific gotchas, performance optimization. +- **[REBUILD.md](references/REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. ## Rust Standalone Binary diff --git a/nautilus-docs/docs/api_reference/_static/custom.css b/nautilus-docs/docs/api_reference/_static/custom.css deleted file mode 100644 index 3c75676..0000000 --- a/nautilus-docs/docs/api_reference/_static/custom.css +++ /dev/null @@ -1,14 +0,0 @@ -body[data-theme="dark"] h1, -body[data-theme="dark"] h2, -body[data-theme="dark"] h3, -body[data-theme="dark"] h4, -body[data-theme="dark"] h5, -body[data-theme="dark"] h6, -body[data-theme="dark"] .sidebar-title, -body[data-theme="dark"] .toctree-l1 > a { - color: #e6edf3; -} - -.bottom-of-page .related-information { - display: none; -} diff --git a/nautilus-docs/docs/api_reference/accounting.md b/nautilus-docs/docs/api_reference/accounting.md deleted file mode 100644 index 43b9777..0000000 --- a/nautilus-docs/docs/api_reference/accounting.md +++ /dev/null @@ -1,45 +0,0 @@ -# Accounting - -```{eval-rst} -.. automodule:: nautilus_trader.accounting -``` - -```{eval-rst} -.. automodule:: nautilus_trader.accounting.accounts.cash - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.accounting.accounts.margin - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.accounting.calculators - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.accounting.factory - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.accounting.manager - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/betfair.md b/nautilus-docs/docs/api_reference/adapters/betfair.md deleted file mode 100644 index 96394a3..0000000 --- a/nautilus-docs/docs/api_reference/adapters/betfair.md +++ /dev/null @@ -1,109 +0,0 @@ -# Betfair - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Client - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Common - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.common - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data Types - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.data_types - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## OrderBook - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.orderbook - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Sockets - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.betfair.sockets - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/binance.md b/nautilus-docs/docs/api_reference/adapters/binance.md deleted file mode 100644 index 3f37fdc..0000000 --- a/nautilus-docs/docs/api_reference/adapters/binance.md +++ /dev/null @@ -1,143 +0,0 @@ -# Binance - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Enums - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.common.enums - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Types - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.common.types - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Futures - -### Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.futures.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Enums - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.futures.enums - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.futures.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.futures.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Types - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.futures.types - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Spot - -### Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.spot.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Enums - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.spot.enums - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.spot.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -### Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.binance.spot.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/bybit.md b/nautilus-docs/docs/api_reference/adapters/bybit.md deleted file mode 100644 index 4d5fccd..0000000 --- a/nautilus-docs/docs/api_reference/adapters/bybit.md +++ /dev/null @@ -1,69 +0,0 @@ -# Bybit - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Enums - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit.common.enums - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.bybit.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/databento.md b/nautilus-docs/docs/api_reference/adapters/databento.md deleted file mode 100644 index 67b8ead..0000000 --- a/nautilus-docs/docs/api_reference/adapters/databento.md +++ /dev/null @@ -1,79 +0,0 @@ -# Databento - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Enums - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.enums - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Types - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.types - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Loaders - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.loaders - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.databento.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/dydx.md b/nautilus-docs/docs/api_reference/adapters/dydx.md deleted file mode 100644 index f079d1c..0000000 --- a/nautilus-docs/docs/api_reference/adapters/dydx.md +++ /dev/null @@ -1,59 +0,0 @@ -# dYdX - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.dydx - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.dydx.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.dydx.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.dydx.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.dydx.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.dydx.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/index.md b/nautilus-docs/docs/api_reference/adapters/index.md deleted file mode 100644 index 4ed2be4..0000000 --- a/nautilus-docs/docs/api_reference/adapters/index.md +++ /dev/null @@ -1,23 +0,0 @@ -# Adapters - -```{eval-rst} -.. automodule:: nautilus_trader.adapters -``` - -```{eval-rst} -.. toctree:: - :maxdepth: 2 - :glob: - :titlesonly: - :hidden: - - betfair.md - binance.md - bybit.md - databento.md - dydx.md - interactive_brokers.md - okx.md - polymarket.md - tardis.md -``` diff --git a/nautilus-docs/docs/api_reference/adapters/interactive_brokers.md b/nautilus-docs/docs/api_reference/adapters/interactive_brokers.md deleted file mode 100644 index aa7a1c3..0000000 --- a/nautilus-docs/docs/api_reference/adapters/interactive_brokers.md +++ /dev/null @@ -1,117 +0,0 @@ -# Interactive Brokers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Client - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.client.client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Common - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.common - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Gateway - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.gateway - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Historical - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.historical.client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Parsing - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.parsing.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.parsing.instruments - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.interactive_brokers.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/okx.md b/nautilus-docs/docs/api_reference/adapters/okx.md deleted file mode 100644 index d28a00a..0000000 --- a/nautilus-docs/docs/api_reference/adapters/okx.md +++ /dev/null @@ -1,59 +0,0 @@ -# OKX - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.okx - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.okx.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.okx.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.okx.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.okx.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.okx.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/polymarket.md b/nautilus-docs/docs/api_reference/adapters/polymarket.md deleted file mode 100644 index cac4450..0000000 --- a/nautilus-docs/docs/api_reference/adapters/polymarket.md +++ /dev/null @@ -1,69 +0,0 @@ -# Polymarket - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Enums - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket.common.enums - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.polymarket.execution - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/adapters/tardis.md b/nautilus-docs/docs/api_reference/adapters/tardis.md deleted file mode 100644 index dd4b6b7..0000000 --- a/nautilus-docs/docs/api_reference/adapters/tardis.md +++ /dev/null @@ -1,59 +0,0 @@ -# Tardis - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.tardis - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Loaders - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.tardis.loaders - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Config - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.tardis.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Providers - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.tardis.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Factories - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.tardis.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.adapters.tardis.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/analysis.md b/nautilus-docs/docs/api_reference/analysis.md deleted file mode 100644 index 77cd33c..0000000 --- a/nautilus-docs/docs/api_reference/analysis.md +++ /dev/null @@ -1,29 +0,0 @@ -# Analysis - -```{eval-rst} -.. automodule:: nautilus_trader.analysis -``` - -```{eval-rst} -.. automodule:: nautilus_trader.analysis.analyzer - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.analysis.reporter - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.analysis.statistic - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/backtest.md b/nautilus-docs/docs/api_reference/backtest.md deleted file mode 100644 index 2d7af89..0000000 --- a/nautilus-docs/docs/api_reference/backtest.md +++ /dev/null @@ -1,61 +0,0 @@ -# Backtest - -```{eval-rst} -.. automodule:: nautilus_trader.backtest -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.data_client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.execution_client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.models - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.modules - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.node - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.results - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/cache.md b/nautilus-docs/docs/api_reference/cache.md deleted file mode 100644 index 5a26d96..0000000 --- a/nautilus-docs/docs/api_reference/cache.md +++ /dev/null @@ -1,29 +0,0 @@ -# Cache - -```{eval-rst} -.. automodule:: nautilus_trader.cache -``` - -```{eval-rst} -.. automodule:: nautilus_trader.cache.cache - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.cache.database - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.cache.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/common.md b/nautilus-docs/docs/api_reference/common.md deleted file mode 100644 index 80ec5c2..0000000 --- a/nautilus-docs/docs/api_reference/common.md +++ /dev/null @@ -1,59 +0,0 @@ -# Common - -```{eval-rst} -.. automodule:: nautilus_trader.common -``` - -```{eval-rst} -.. automodule:: nautilus_trader.common.actor - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.common.factories - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Component - -```{eval-rst} -.. automodule:: nautilus_trader.common.component - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Executor - -```{eval-rst} -.. automodule:: nautilus_trader.common.executor - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Generators - -```{eval-rst} -.. automodule:: nautilus_trader.common.generators - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.common.providers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/conf.py b/nautilus-docs/docs/api_reference/conf.py deleted file mode 100644 index 61b1d6b..0000000 --- a/nautilus-docs/docs/api_reference/conf.py +++ /dev/null @@ -1,122 +0,0 @@ -# ------------------------------------------------------------------------------------------------- -# Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. -# https://nautechsystems.io -# -# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ------------------------------------------------------------------------------------------------- - -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# - -import nautilus_trader - - -# -- Project information ----------------------------------------------------- -project = "NautilusTrader" -author = "Nautech Systems Pty Ltd." -copyright = "2015-2026 Nautech Systems Pty Ltd" -version = nautilus_trader.__version__ - -# -- General configuration --------------------------------------------------- -extensions = [ - "myst_parser", - "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", - "sphinx.ext.napoleon", - "sphinx_comments", -] - -html_theme = "furo" -html_title = "NautilusTrader Python API" -html_favicon = "https://nautilustrader.io/docs/img/shell.ico" - -html_theme_options = { - "dark_css_variables": { - "color-brand-primary": "#33bca4", - "color-brand-content": "#33bca4", - "color-background-primary": "#0d1117", - "color-background-secondary": "#0e1316", - "color-background-border": "#23282c", - "color-foreground-primary": "#b7b7b7", - "color-foreground-secondary": "#cdcdcd", - "color-highlight-on-target": "#171c20", - "color-api-name": "#33bca4", - "color-api-pre-name": "#33bca4", - }, - "light_css_variables": { - "color-brand-primary": "#007e68", - "color-brand-content": "#007e68", - }, - "footer_icons": [], - "source_repository": "https://github.com/nautechsystems/nautilus_trader", - "source_branch": "master", - "source_directory": "docs/api_reference/", -} - -html_show_sphinx = False -html_static_path = ["_static"] -html_css_files = ["custom.css"] - -comments_config = {"hypothesis": False, "utterances": False} -exclude_patterns = ["**.ipynb_checkpoints", ".DS_Store", "Thumbs.db", "_build"] -source_suffix = [".rst", ".md"] - -myst_enable_extensions = [ - "colon_fence", - "dollarmath", - "fieldlist", - "linkify", - "substitution", - "tasklist", -] -myst_url_schemes = ("mailto", "http", "https") -suppress_warnings = ["myst.domains"] - -add_module_names = False -todo_include_todos = False - -autosummary_generate = True -autodoc_member_order = "bysource" -autoclass_content = "class" -autodoc_class_signature = "separated" - -# -- Extension configuration ------------------------------------------------- -autodoc_default_options = { - "members": True, - "undoc-members": False, - "private-members": False, - "exclude-members": "__init__,__new__", - "show-inheritance": True, - "class-signature": "separated", -} - -# -- Napoleon settings ------------------------------------------------------- -napoleon_google_docstring = False -napoleon_numpy_docstring = True -napoleon_include_init_with_doc = False -napoleon_include_private_with_doc = False -napoleon_include_special_with_doc = False -napoleon_use_admonition_for_examples = True -napoleon_use_admonition_for_notes = True -napoleon_use_admonition_for_references = True -napoleon_use_ivar = False -napoleon_use_param = True -napoleon_use_rtype = True diff --git a/nautilus-docs/docs/api_reference/config.md b/nautilus-docs/docs/api_reference/config.md deleted file mode 100644 index f33ee8f..0000000 --- a/nautilus-docs/docs/api_reference/config.md +++ /dev/null @@ -1,101 +0,0 @@ -# Config - -## Backtest - -```{eval-rst} -.. automodule:: nautilus_trader.backtest.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Cache - -```{eval-rst} -.. automodule:: nautilus_trader.cache.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Common - -```{eval-rst} -.. automodule:: nautilus_trader.common.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Data - -```{eval-rst} -.. automodule:: nautilus_trader.data.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Execution - -```{eval-rst} -.. automodule:: nautilus_trader.execution.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Live - -```{eval-rst} -.. automodule:: nautilus_trader.live.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Persistence - -```{eval-rst} -.. automodule:: nautilus_trader.persistence.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Risk - -```{eval-rst} -.. automodule:: nautilus_trader.risk.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## System - -```{eval-rst} -.. automodule:: nautilus_trader.system.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Trading - -```{eval-rst} -.. automodule:: nautilus_trader.trading.config - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/core.md b/nautilus-docs/docs/api_reference/core.md deleted file mode 100644 index 3a6a546..0000000 --- a/nautilus-docs/docs/api_reference/core.md +++ /dev/null @@ -1,55 +0,0 @@ -# Core - -```{eval-rst} -.. automodule:: nautilus_trader.core -``` - -## Datetime - -```{eval-rst} -.. automodule:: nautilus_trader.core.datetime - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Finite-State Machine (FSM) - -```{eval-rst} -.. automodule:: nautilus_trader.core.fsm - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Message - -```{eval-rst} -.. automodule:: nautilus_trader.core.message - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Stats - -```{eval-rst} -.. automodule:: nautilus_trader.core.stats - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## UUID - -```{eval-rst} -.. automodule:: nautilus_trader.core.uuid - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/data.md b/nautilus-docs/docs/api_reference/data.md deleted file mode 100644 index 03ef65f..0000000 --- a/nautilus-docs/docs/api_reference/data.md +++ /dev/null @@ -1,45 +0,0 @@ -# Data - -```{eval-rst} -.. automodule:: nautilus_trader.data -``` - -## Aggregation - -```{eval-rst} -.. automodule:: nautilus_trader.data.aggregation - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Client - -```{eval-rst} -.. automodule:: nautilus_trader.data.client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Engine - -```{eval-rst} -.. automodule:: nautilus_trader.data.engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Messages - -```{eval-rst} -.. automodule:: nautilus_trader.data.messages - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/execution.md b/nautilus-docs/docs/api_reference/execution.md deleted file mode 100644 index 801daad..0000000 --- a/nautilus-docs/docs/api_reference/execution.md +++ /dev/null @@ -1,75 +0,0 @@ -# Execution - -```{eval-rst} -.. automodule:: nautilus_trader.execution -``` - -## Components - -```{eval-rst} -.. automodule:: nautilus_trader.execution.algorithm - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.execution.client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.execution.emulator - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.execution.engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.execution.manager - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.execution.matching_core - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Messages - -```{eval-rst} -.. automodule:: nautilus_trader.execution.messages - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -## Reports - -```{eval-rst} -.. automodule:: nautilus_trader.execution.reports - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/index.md b/nautilus-docs/docs/api_reference/index.md deleted file mode 100644 index 4507125..0000000 --- a/nautilus-docs/docs/api_reference/index.md +++ /dev/null @@ -1,8 +0,0 @@ -# Python API - -Auto-generated from the latest source using [Sphinx](https://www.sphinx-doc.org/en/master/). - -- **Latest**: Built from the `develop` branch (latest stable release). -- **Nightly**: Built from the `nightly` branch (bleeding-edge features in development). - -Select the version from the **Versions** drop-down menu (top left). diff --git a/nautilus-docs/docs/api_reference/indicators.md b/nautilus-docs/docs/api_reference/indicators.md deleted file mode 100644 index 4962dfe..0000000 --- a/nautilus-docs/docs/api_reference/indicators.md +++ /dev/null @@ -1,61 +0,0 @@ -# Indicators - -```{eval-rst} -.. automodule:: nautilus_trader.indicators -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.averages - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.fuzzy_candlesticks - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.momentum - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.spread_analyzer - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.trend - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.volatility - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.indicators.volume - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/live.md b/nautilus-docs/docs/api_reference/live.md deleted file mode 100644 index b73736b..0000000 --- a/nautilus-docs/docs/api_reference/live.md +++ /dev/null @@ -1,61 +0,0 @@ -# Live - -```{eval-rst} -.. automodule:: nautilus_trader.live -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.data_client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.data_engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.execution_client - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.execution_engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.risk_engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.node - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.live.node_builder - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/book.md b/nautilus-docs/docs/api_reference/model/book.md deleted file mode 100644 index 4ff3983..0000000 --- a/nautilus-docs/docs/api_reference/model/book.md +++ /dev/null @@ -1,9 +0,0 @@ -# Order Book - -```{eval-rst} -.. automodule:: nautilus_trader.model.book - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/data.md b/nautilus-docs/docs/api_reference/model/data.md deleted file mode 100644 index 8fd5065..0000000 --- a/nautilus-docs/docs/api_reference/model/data.md +++ /dev/null @@ -1,9 +0,0 @@ -# Data - -```{eval-rst} -.. automodule:: nautilus_trader.model.data - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/events.md b/nautilus-docs/docs/api_reference/model/events.md deleted file mode 100644 index 22d6705..0000000 --- a/nautilus-docs/docs/api_reference/model/events.md +++ /dev/null @@ -1,29 +0,0 @@ -# Events - -```{eval-rst} -.. automodule:: nautilus_trader.model.events -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.events.account - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.events.order - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.events.position - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/identifiers.md b/nautilus-docs/docs/api_reference/model/identifiers.md deleted file mode 100644 index 5bfb715..0000000 --- a/nautilus-docs/docs/api_reference/model/identifiers.md +++ /dev/null @@ -1,9 +0,0 @@ -# Identifiers - -```{eval-rst} -.. automodule:: nautilus_trader.model.identifiers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/index.md b/nautilus-docs/docs/api_reference/model/index.md deleted file mode 100644 index 70ef14a..0000000 --- a/nautilus-docs/docs/api_reference/model/index.md +++ /dev/null @@ -1,23 +0,0 @@ -# Model - -```{eval-rst} -.. automodule:: nautilus_trader.model -``` - -```{eval-rst} -.. toctree:: - :maxdepth: 2 - :glob: - :titlesonly: - :hidden: - - book.md - data.md - events.md - identifiers.md - instruments.md - objects.md - orders.md - position.md - tick_scheme.md -``` diff --git a/nautilus-docs/docs/api_reference/model/instruments.md b/nautilus-docs/docs/api_reference/model/instruments.md deleted file mode 100644 index 520c5d0..0000000 --- a/nautilus-docs/docs/api_reference/model/instruments.md +++ /dev/null @@ -1,141 +0,0 @@ -# Instruments - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.betting - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.binary_option - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.cfd - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.commodity - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.crypto_future - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.crypto_option - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.crypto_perpetual - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.currency_pair - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.equity - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.futures_contract - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.futures_spread - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.index - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.perpetual_contract - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.option_contract - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.option_spread - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.synthetic - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.instruments.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/objects.md b/nautilus-docs/docs/api_reference/model/objects.md deleted file mode 100644 index 42fa6ba..0000000 --- a/nautilus-docs/docs/api_reference/model/objects.md +++ /dev/null @@ -1,9 +0,0 @@ -# Objects - -```{eval-rst} -.. automodule:: nautilus_trader.model.objects - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/orders.md b/nautilus-docs/docs/api_reference/model/orders.md deleted file mode 100644 index 6906107..0000000 --- a/nautilus-docs/docs/api_reference/model/orders.md +++ /dev/null @@ -1,93 +0,0 @@ -# Orders - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.market - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.limit - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.stop_market - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.stop_limit - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.market_to_limit - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.market_if_touched - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.limit_if_touched - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.trailing_stop_market - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.trailing_stop_limit - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.list - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.orders.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/position.md b/nautilus-docs/docs/api_reference/model/position.md deleted file mode 100644 index cfefed0..0000000 --- a/nautilus-docs/docs/api_reference/model/position.md +++ /dev/null @@ -1,9 +0,0 @@ -# Position - -```{eval-rst} -.. automodule:: nautilus_trader.model.position - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/model/tick_scheme.md b/nautilus-docs/docs/api_reference/model/tick_scheme.md deleted file mode 100644 index 345f040..0000000 --- a/nautilus-docs/docs/api_reference/model/tick_scheme.md +++ /dev/null @@ -1,29 +0,0 @@ -# Tick Scheme - -```{eval-rst} -.. automodule:: nautilus_trader.model.tick_scheme -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.tick_scheme.implementations.fixed - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.tick_scheme.implementations.tiered - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.model.tick_scheme.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/persistence.md b/nautilus-docs/docs/api_reference/persistence.md deleted file mode 100644 index 9b1109a..0000000 --- a/nautilus-docs/docs/api_reference/persistence.md +++ /dev/null @@ -1,37 +0,0 @@ -# Persistence - -```{eval-rst} -.. automodule:: nautilus_trader.persistence -``` - -```{eval-rst} -.. automodule:: nautilus_trader.persistence.catalog.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.persistence.catalog.parquet - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.persistence.wranglers - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.persistence.writer - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/portfolio.md b/nautilus-docs/docs/api_reference/portfolio.md deleted file mode 100644 index ada9683..0000000 --- a/nautilus-docs/docs/api_reference/portfolio.md +++ /dev/null @@ -1,21 +0,0 @@ -# Portfolio - -```{eval-rst} -.. automodule:: nautilus_trader.portfolio -``` - -```{eval-rst} -.. automodule:: nautilus_trader.portfolio.portfolio - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.portfolio.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/risk.md b/nautilus-docs/docs/api_reference/risk.md deleted file mode 100644 index 13b78f2..0000000 --- a/nautilus-docs/docs/api_reference/risk.md +++ /dev/null @@ -1,21 +0,0 @@ -# Risk - -```{eval-rst} -.. automodule:: nautilus_trader.risk -``` - -```{eval-rst} -.. automodule:: nautilus_trader.risk.engine - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.risk.sizing - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/serialization.md b/nautilus-docs/docs/api_reference/serialization.md deleted file mode 100644 index 11b3cf3..0000000 --- a/nautilus-docs/docs/api_reference/serialization.md +++ /dev/null @@ -1,21 +0,0 @@ -# Serialization - -```{eval-rst} -.. automodule:: nautilus_trader.serialization -``` - -```{eval-rst} -.. automodule:: nautilus_trader.serialization.serializer - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.serialization.base - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/system.md b/nautilus-docs/docs/api_reference/system.md deleted file mode 100644 index 4dc885e..0000000 --- a/nautilus-docs/docs/api_reference/system.md +++ /dev/null @@ -1,13 +0,0 @@ -# System - -```{eval-rst} -.. automodule:: nautilus_trader.system -``` - -```{eval-rst} -.. automodule:: nautilus_trader.system.kernel - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/api_reference/trading.md b/nautilus-docs/docs/api_reference/trading.md deleted file mode 100644 index 190ab69..0000000 --- a/nautilus-docs/docs/api_reference/trading.md +++ /dev/null @@ -1,37 +0,0 @@ -# Trading - -```{eval-rst} -.. automodule:: nautilus_trader.trading -``` - -```{eval-rst} -.. automodule:: nautilus_trader.trading.controller - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.trading.filters - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.trading.strategy - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` - -```{eval-rst} -.. automodule:: nautilus_trader.trading.trader - :show-inheritance: - :inherited-members: - :members: - :member-order: bysource -``` diff --git a/nautilus-docs/docs/concepts/actors.md b/nautilus-docs/docs/concepts/actors.md deleted file mode 100644 index 6e89871..0000000 --- a/nautilus-docs/docs/concepts/actors.md +++ /dev/null @@ -1,342 +0,0 @@ -# Actors - -An `Actor` receives data, handles events, and manages state. The `Strategy` class extends Actor -with order management capabilities. - -**Key capabilities**: - -- Data subscription and requests (market data, custom data). -- Event handling and publishing. -- Timers and alerts. -- Cache and portfolio access. -- Logging. - -## Basic example - -Actors support configuration through a pattern similar to strategies. - -```python -from nautilus_trader.config import ActorConfig -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Bar, BarType -from nautilus_trader.common.actor import Actor - - -class MyActorConfig(ActorConfig): - instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" - bar_type: BarType # example value: "ETHUSDT-PERP.BINANCE-15-MINUTE[LAST]-INTERNAL" - lookback_period: int = 10 - - -class MyActor(Actor): - def __init__(self, config: MyActorConfig) -> None: - super().__init__(config) - - # Custom state variables - self.count_of_processed_bars: int = 0 - - def on_start(self) -> None: - # Subscribe to bars matching the configured bar type - self.subscribe_bars(self.config.bar_type) - - def on_bar(self, bar: Bar) -> None: - self.count_of_processed_bars += 1 -``` - -## Lifecycle - -Actors follow a defined state machine through their lifecycle: - -```mermaid -stateDiagram-v2 - [*] --> PRE_INITIALIZED - PRE_INITIALIZED --> READY : register() - READY --> STARTING : start() - STARTING --> RUNNING : on_start() - RUNNING --> STOPPING : stop() - STOPPING --> STOPPED : on_stop() - STOPPED --> RUNNING : resume() - RUNNING --> DEGRADING : degrade() - DEGRADING --> DEGRADED : on_degrade() - DEGRADED --> RUNNING : resume() - RUNNING --> FAULTING : fault() - FAULTING --> FAULTED : on_fault() - RUNNING --> DISPOSED : dispose() -``` - -Override these methods to hook into lifecycle events: - -| Method | When called | -|-----------------|---------------------------------------------------------------------| -| `on_start()` | Actor is starting (subscribe to data here). | -| `on_stop()` | Actor is stopping (cancel timers, clean up resources). | -| `on_resume()` | Actor is resuming from a stopped state. | -| `on_reset()` | Reset indicators and internal state (called between backtest runs). | -| `on_degrade()` | Actor is entering a degraded state (partial functionality). | -| `on_fault()` | Actor has encountered a fault. | -| `on_dispose()` | Actor is being disposed (final cleanup). | - -## Timers and alerts - -Actors have access to a clock for scheduling: - -```python -def on_start(self) -> None: - # Set a recurring timer (fires every 5 seconds) - self.clock.set_timer("my_timer", timedelta(seconds=5)) - - # Set a one-time alert - self.clock.set_alert("my_alert", self.clock.utc_now() + timedelta(minutes=1)) - -def on_stop(self) -> None: - # Cancel timers to prevent resource leaks across stop/resume cycles - self.clock.cancel_timer("my_timer") - -def on_timer(self, event: TimeEvent) -> None: - if event.name == "my_timer": - self.log.info("Timer fired!") - -def on_alert(self, event: TimeEvent) -> None: - if event.name == "my_alert": - self.log.info("Alert triggered!") -``` - -## System access - -Actors have access to core system components: - -| Property | Description | -|-------------------|----------------------------------------------------------| -| `self.cache` | Shared state for instruments, orders, positions, etc. | -| `self.portfolio` | Portfolio state and calculations. | -| `self.clock` | Current time and timer/alert scheduling. | -| `self.log` | Structured logging. | -| `self.msgbus` | Publish/subscribe to custom messages. | - -For custom messaging between components, see the [Message Bus](message_bus.md) guide. - -## Data handling and callbacks - -The system uses different callback handlers depending on whether data is historical or real-time. -Understanding the relationship between data *requests/subscriptions* and their handlers is key. - -### Historical vs real-time data - -The system distinguishes between two data flows: - -1. **Historical data** (from *requests*): - - Obtained through methods like `request_bars()`, `request_quote_ticks()`, etc. - - Processed through the `on_historical_data()` handler. - - Used for initial data loading and historical analysis. - -2. **Real-time data** (from *subscriptions*): - - Obtained through methods like `subscribe_bars()`, `subscribe_quote_ticks()`, etc. - - Processed through specific handlers like `on_bar()`, `on_quote_tick()`, etc. - - Used for live data processing. - -### Callback handlers - -Different data operations map to these handlers: - -| Operation | Category | Handler | Purpose | -|--------------------------------------|------------|--------------------------|---------------------------------------------------| -| `subscribe_data()` | Real-time | `on_data()` | Live data updates. | -| `subscribe_instrument()` | Real-time | `on_instrument()` | Live instrument definition updates. | -| `subscribe_instruments()` | Real-time | `on_instrument()` | Live instrument definition updates (for venue). | -| `subscribe_order_book_deltas()` | Real-time | `on_order_book_deltas()` | Live order book deltas. | -| `subscribe_order_book_depth()` | Real-time | `on_order_book_depth()` | Live order book depth snapshots. | -| `subscribe_order_book_at_interval()` | Real-time | `on_order_book()` | Live order book snapshots at intervals. | -| `subscribe_quote_ticks()` | Real-time | `on_quote_tick()` | Live quote updates. | -| `subscribe_trade_ticks()` | Real-time | `on_trade_tick()` | Live trade updates. | -| `subscribe_mark_prices()` | Real-time | `on_mark_price()` | Live mark price updates. | -| `subscribe_index_prices()` | Real-time | `on_index_price()` | Live index price updates. | -| `subscribe_bars()` | Real-time | `on_bar()` | Live bar updates. | -| `subscribe_funding_rates()` | Real-time | `on_funding_rate()` | Live funding rate updates. | -| `subscribe_instrument_status()` | Real-time | `on_instrument_status()` | Live instrument status updates. | -| `subscribe_instrument_close()` | Real-time | `on_instrument_close()` | Live instrument close updates. | -| `subscribe_option_greeks()` | Real-time | `on_option_greeks()` | Live option greeks updates. | -| `subscribe_option_chain()` | Real-time | `on_option_chain()` | Live option chain slice snapshots. | -| `subscribe_order_fills()` | Real-time | `on_order_filled()` | Live order fill events for an instrument. | -| `subscribe_order_cancels()` | Real-time | `on_order_canceled()` | Live order cancel events for an instrument. | -| `request_data()` | Historical | `on_historical_data()` | Historical data processing. | -| `request_order_book_deltas()` | Historical | `on_historical_data()` | Historical order book deltas. | -| `request_order_book_depth()` | Historical | `on_historical_data()` | Historical order book depth. | -| `request_order_book_snapshot()` | Historical | `on_historical_data()` | Historical order book snapshot. | -| `request_instrument()` | Historical | `on_instrument()` | Instrument definition. | -| `request_instruments()` | Historical | `on_instrument()` | Instrument definitions. | -| `request_quote_ticks()` | Historical | `on_historical_data()` | Historical quotes processing. | -| `request_trade_ticks()` | Historical | `on_historical_data()` | Historical trades processing. | -| `request_bars()` | Historical | `on_historical_data()` | Historical bars processing. | -| `request_aggregated_bars()` | Historical | `on_historical_data()` | Historical aggregated bars (on-the-fly). | -| `request_funding_rates()` | Historical | `on_historical_data()` | Historical funding rates processing. | - -### Example - -This example shows both historical and real-time data handling: - -```python -from nautilus_trader.common.actor import Actor -from nautilus_trader.config import ActorConfig -from nautilus_trader.core.data import Data -from nautilus_trader.model import Bar, BarType -from nautilus_trader.model import ClientId, InstrumentId - - -class MyActorConfig(ActorConfig): - instrument_id: InstrumentId # example value: "AAPL.XNAS" - bar_type: BarType # example value: "AAPL.XNAS-1-MINUTE-LAST-EXTERNAL" - - -class MyActor(Actor): - def __init__(self, config: MyActorConfig) -> None: - super().__init__(config) - self.bar_type = config.bar_type - - def on_start(self) -> None: - # Request historical data - will be processed by on_historical_data() handler - self.request_bars( - bar_type=self.bar_type, - # Many optional parameters - start=None, # pd.Timestamp | None - end=None, # pd.Timestamp | None - callback=None, # Callable[[UUID4], None] | None - update_catalog_mode=None, # UpdateCatalogMode | None - params=None, # dict[str, Any] | None - ) - - # Subscribe to real-time data - will be processed by on_bar() handler - self.subscribe_bars( - bar_type=self.bar_type, - # Many optional parameters - client_id=None, # ClientId, optional - params=None, # dict[str, Any], optional - ) - - def on_historical_data(self, data: Data) -> None: - # Handle historical data (from requests) - if isinstance(data, Bar): - self.log.info(f"Received historical bar: {data}") - - def on_bar(self, bar: Bar) -> None: - # Handle real-time bar updates (from subscriptions) - self.log.info(f"Received real-time bar: {bar}") -``` - -Separating historical and real-time handlers lets you apply different processing logic -based on context. For example: - -- Use historical data to initialize indicators or establish baseline metrics. -- Process real-time data differently for live trading decisions. -- Apply different validation or logging for historical vs real-time data. - -:::tip -When debugging data flow issues, check that you're looking at the correct handler for your data source. -If you're not seeing data in `on_bar()` but see log messages about receiving bars, check `on_historical_data()` -as the data might be coming from a request rather than a subscription. -::: - -## Order fill subscriptions - -Actors can subscribe to order fill events for specific instruments using `subscribe_order_fills()`. -This is useful for monitoring trading activity, fill analysis, or tracking execution quality. - -When subscribed, the handler `on_order_filled()` receives all fills for the specified instrument, -regardless of which strategy or component generated the original order. - -### Example - -```python -from nautilus_trader.common.actor import Actor -from nautilus_trader.config import ActorConfig -from nautilus_trader.model import InstrumentId -from nautilus_trader.model.events import OrderFilled - - -class MyActorConfig(ActorConfig): - instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" - - -class FillMonitorActor(Actor): - def __init__(self, config: MyActorConfig) -> None: - super().__init__(config) - self.fill_count = 0 - self.total_volume = 0.0 - - def on_start(self) -> None: - # Subscribe to all fills for the instrument - self.subscribe_order_fills(self.config.instrument_id) - - def on_order_filled(self, event: OrderFilled) -> None: - # Handle order fill events - self.fill_count += 1 - self.total_volume += float(event.last_qty) - - self.log.info( - f"Fill received: {event.order_side} {event.last_qty} @ {event.last_px}, " - f"Total fills: {self.fill_count}, Volume: {self.total_volume}" - ) - - def on_stop(self) -> None: - # Unsubscribe from fills - self.unsubscribe_order_fills(self.config.instrument_id) -``` - -:::note -Order fill subscriptions use the message bus only and do not involve the data engine. -The `on_order_filled()` handler receives events only while the actor is running. -::: - -## Order cancel subscriptions - -Actors can subscribe to order cancel events for specific instruments using `subscribe_order_cancels()`. -This is useful for monitoring cancellations or tracking order lifecycle events. - -When subscribed, the handler `on_order_canceled()` receives all cancels for the specified instrument, -regardless of which strategy or component generated the original order. - -### Example - -```python -from nautilus_trader.common.actor import Actor -from nautilus_trader.config import ActorConfig -from nautilus_trader.model import InstrumentId -from nautilus_trader.model.events import OrderCanceled - - -class MyActorConfig(ActorConfig): - instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" - - -class CancelMonitorActor(Actor): - def __init__(self, config: MyActorConfig) -> None: - super().__init__(config) - self.cancel_count = 0 - - def on_start(self) -> None: - # Subscribe to all cancels for the instrument - self.subscribe_order_cancels(self.config.instrument_id) - - def on_order_canceled(self, event: OrderCanceled) -> None: - # Handle order cancel events - self.cancel_count += 1 - - self.log.info( - f"Cancel received: {event.client_order_id}, " - f"Total cancels: {self.cancel_count}" - ) - - def on_stop(self) -> None: - # Unsubscribe from cancels - self.unsubscribe_order_cancels(self.config.instrument_id) -``` - -:::note -Order cancel subscriptions use the message bus only and do not involve the data engine. -The `on_order_canceled()` handler receives events only while the actor is running. -::: - -## Related guides - -- [Strategies](strategies.md) - Strategies extend actors with order management capabilities. -- [Data](data.md) - Data types and subscriptions available to actors. -- [Message Bus](message_bus.md) - The messaging system actors use for communication. diff --git a/nautilus-docs/docs/concepts/adapters.md b/nautilus-docs/docs/concepts/adapters.md deleted file mode 100644 index 859ee9d..0000000 --- a/nautilus-docs/docs/concepts/adapters.md +++ /dev/null @@ -1,194 +0,0 @@ -# Adapters - -Adapters integrate data providers and trading venues into NautilusTrader. -They can be found in the top-level `adapters` subpackage. - -An adapter typically comprises these components: - -```mermaid -flowchart LR - subgraph Venue ["Trading Venue"] - API[REST API] - WS[WebSocket] - end - - subgraph Adapter ["Adapter"] - HTTP[HttpClient] - WSC[WebSocketClient] - IP[InstrumentProvider] - DC[DataClient] - EC[ExecutionClient] - end - - subgraph Core ["Nautilus Core"] - DE[DataEngine] - EE[ExecutionEngine] - end - - API <--> HTTP - WS <--> WSC - HTTP --> IP - HTTP --> DC - HTTP --> EC - WSC --> DC - WSC --> EC - DC <--> DE - EC <--> EE -``` - -| Component | Purpose | -|----------------------|------------------------------------------------------------| -| `HttpClient` | REST API communication. | -| `WebSocketClient` | Real-time streaming connection. | -| `InstrumentProvider` | Loads and parses instrument definitions from the venue. | -| `DataClient` | Handles market data subscriptions and requests. | -| `ExecutionClient` | Handles order submission, modification, and cancellation. | - -## Instrument providers - -Instrument providers parse venue API responses into Nautilus `Instrument` objects. - -An `InstrumentProvider` serves two use cases: - -- Standalone discovery of available instruments for research or backtesting -- Runtime loading in a `sandbox` or `live` [environment context](architecture.md#environment-contexts) - for actors and strategies - -### Research and backtesting - -Here is an example of discovering the current instruments for the Binance Futures testnet: - -```python -import asyncio -import os - -from nautilus_trader.adapters.binance.common.enums import BinanceAccountType -from nautilus_trader.adapters.binance import get_cached_binance_http_client -from nautilus_trader.adapters.binance.futures.providers import BinanceFuturesInstrumentProvider -from nautilus_trader.common.component import LiveClock - - -async def main(): - clock = LiveClock() - - client = get_cached_binance_http_client( - clock=clock, - account_type=BinanceAccountType.USDT_FUTURES, - api_key=os.getenv("BINANCE_FUTURES_TESTNET_API_KEY"), - api_secret=os.getenv("BINANCE_FUTURES_TESTNET_API_SECRET"), - is_testnet=True, - ) - - provider = BinanceFuturesInstrumentProvider( - client=client, - account_type=BinanceAccountType.USDT_FUTURES, - ) - - await provider.load_all_async() - - # Access loaded instruments - instruments = provider.list_all() - print(f"Loaded {len(instruments)} instruments") - - -if __name__ == "__main__": - asyncio.run(main()) -``` - -### Live trading - -Each integration handles this differently. An `InstrumentProvider` within a `TradingNode` -generally offers two loading behaviors: - -- Load all instruments on start: - -```python -from nautilus_trader.config import InstrumentProviderConfig - -InstrumentProviderConfig(load_all=True) -``` - -- Load only the instruments specified in configuration: - -```python -InstrumentProviderConfig(load_ids=["BTCUSDT-PERP.BINANCE", "ETHUSDT-PERP.BINANCE"]) -``` - -## Data clients - -Data clients handle market data subscriptions and requests for a venue. They connect to venue APIs -and normalize incoming data into Nautilus types. - -### Requesting data - -Actors and strategies can request data using built-in methods. Data returns via callbacks: - -```python -from nautilus_trader.model import Instrument, InstrumentId -from nautilus_trader.trading.strategy import Strategy - - -class MyStrategy(Strategy): - def on_start(self) -> None: - # Request an instrument definition - self.request_instrument(InstrumentId.from_str("BTCUSDT-PERP.BINANCE")) - - # Request historical bars - self.request_bars(BarType.from_str("BTCUSDT-PERP.BINANCE-1-HOUR-LAST-EXTERNAL")) - - def on_instrument(self, instrument: Instrument) -> None: - self.log.info(f"Received instrument: {instrument.id}") - - def on_historical_data(self, data) -> None: - self.log.info(f"Received historical data: {data}") -``` - -### Subscribing to data - -For real-time data, use subscription methods: - -```python -def on_start(self) -> None: - # Subscribe to live trade updates - self.subscribe_trade_ticks(InstrumentId.from_str("BTCUSDT-PERP.BINANCE")) - - # Subscribe to live bars - self.subscribe_bars(BarType.from_str("BTCUSDT-PERP.BINANCE-1-MINUTE-LAST-EXTERNAL")) - -def on_trade_tick(self, tick: TradeTick) -> None: - self.log.info(f"Trade: {tick}") - -def on_bar(self, bar: Bar) -> None: - self.log.info(f"Bar: {bar}") -``` - -:::tip -See the [Actors](actors.md) documentation for a complete reference of available -request and subscription methods with their corresponding callbacks. -::: - -## Execution clients - -Execution clients handle order management for a venue. They translate Nautilus order commands -into venue-specific API calls and process execution reports back into Nautilus events. - -Key responsibilities: - -- Submit, modify, and cancel orders. -- Process fills and execution reports. -- Reconcile order state with the venue. -- Handle account and position updates. - -The `ExecutionEngine` routes commands to the appropriate -execution client based on the order's venue. See the [Execution](execution.md) guide for details -on order management from a strategy perspective. - -:::tip -For building a custom adapter, see the [Adapter Developer Guide](../developer_guide/adapters.md). -::: - -## Related guides - -- [Live Trading](live.md) - Configure and run live trading with adapters. -- [Execution](execution.md) - Order execution through adapters. -- [Data](data.md) - Market data provided by adapters. diff --git a/nautilus-docs/docs/concepts/architecture.md b/nautilus-docs/docs/concepts/architecture.md deleted file mode 100644 index f0faeb2..0000000 --- a/nautilus-docs/docs/concepts/architecture.md +++ /dev/null @@ -1,641 +0,0 @@ -# Architecture - -This guide covers the architectural principles and structure of NautilusTrader: - -- Design philosophy and quality attributes. -- Core components and how they interact. -- Environment contexts (backtest, sandbox, live). -- Framework organization and code structure. - -:::note -Throughout the documentation, the term *"Nautilus system boundary"* refers to operations within -the runtime of a single Nautilus node (also known as a "trader instance"). -::: - -## Design philosophy - -The major architectural techniques and design patterns employed by NautilusTrader are: - -- [Domain driven design (DDD)](https://en.wikipedia.org/wiki/Domain-driven_design) -- [Event-driven architecture](https://en.wikipedia.org/wiki/Event-driven_programming) -- [Messaging patterns](https://en.wikipedia.org/wiki/Messaging_pattern) (Pub/Sub, Req/Rep, point-to-point) -- [Ports and adapters](https://en.wikipedia.org/wiki/Hexagonal_architecture_(software)) -- [Crash-only design](#crash-only-design) - -These techniques help achieve certain architectural quality attributes. - -### Quality attributes - -Architectural decisions are often a trade-off between competing priorities. -The following quality attributes guide design and architectural decisions, -roughly in order of weighting. - -- Reliability -- Performance -- Modularity -- Testability -- Maintainability -- Deployability - -### Assurance-driven engineering - -NautilusTrader is incrementally adopting a high-assurance mindset: critical code -paths should carry executable invariants that verify behaviour matches the -business requirements. Practically this means we: - -- Identify the components whose failure has the highest blast radius (core - domain types, risk and execution flows) and write down their invariants in - plain language. -- Codify those invariants as executable checks (unit tests, property tests, - fuzzers, static assertions) that run in CI, keeping the feedback loop light. -- Prefer zero-cost safety techniques built into Rust (ownership, `Result` - surfaces, `panic = abort`) and add targeted formal tools only where they pay - for themselves. -- Track “assurance debt” alongside feature work so new integrations extend the - safety net rather than bypass it. - -This approach preserves the platform’s delivery cadence while giving -high-stakes flows the additional scrutiny they need. - -Further reading: [High Assurance Rust](https://highassurance.rs/). - -### Crash-only design - -NautilusTrader draws inspiration from [crash-only design](https://en.wikipedia.org/wiki/Crash-only_software) -principles, particularly for handling unrecoverable faults. The core insight is that systems which -can recover cleanly from crashes are more robust than those with separate (and rarely tested) -graceful shutdown paths. - -Key principles: - -- **Unified recovery path** - Startup and crash recovery share the same code path, ensuring it is well-tested. -- **Externalized state** - Critical state is meant to be persisted externally when configured, reducing data-loss risk; durability depends on the backing store. -- **Fast restart** - The system is designed to restart quickly after a crash, minimizing downtime. -- **Idempotent operations** - Operations are designed to be safely retried after restart. -- **Fail-fast for unrecoverable errors** - Data corruption or invariant violations trigger immediate termination rather than attempting to continue in a compromised state. - -:::note -The system does provide graceful shutdown flows (`stop`, `dispose`) for normal operation. These -tear down clients, persist state, and flush writers. The crash-only philosophy applies specifically -to *unrecoverable faults* where attempting graceful cleanup could cause further damage. -::: - -This design complements the [fail-fast policy](#data-integrity-and-fail-fast-policy), where -unrecoverable errors result in immediate process termination. - -**References:** - -- [Crash-Only Software](https://www.usenix.org/conference/hotos-ix/crash-only-software) - Candea & Fox, HotOS 2003 (original research paper) -- [Microreboot—A Technique for Cheap Recovery](https://www.usenix.org/conference/osdi-04/microreboot—-technique-cheap-recovery) - Candea et al., OSDI 2004 -- [The properties of crash-only software](https://brooker.co.za/blog/2012/01/22/crash-only.html) - Marc Brooker's blog -- [Crash-only software: More than meets the eye](https://lwn.net/Articles/191059/) - LWN.net article -- [Recovery-Oriented Computing (ROC) Project](http://roc.cs.berkeley.edu/) - UC Berkeley/Stanford research - -### Data integrity and fail-fast policy - -NautilusTrader prioritizes data integrity over availability for trading operations. The system employs -a strict fail-fast policy for arithmetic operations and data handling to prevent silent data corruption -that could lead to incorrect trading decisions. - -#### Fail-fast principles - -The system will fail fast (panic or return an error) when encountering: - -- Arithmetic overflow or underflow in operations on timestamps, prices, or quantities that exceed valid ranges. -- Invalid data during deserialization including NaN, Infinity, or out-of-range values in market data or configuration. -- Type conversion failures such as negative values where only positive values are valid (timestamps, quantities). -- Malformed input parsing for prices, timestamps, or precision values. - -Rationale: - -In trading systems, corrupt data is worse than no data. A single incorrect price, timestamp, or quantity -can cascade through the system, resulting in: - -- Incorrect position sizing or risk calculations. -- Orders placed at wrong prices. -- Backtests producing misleading results. -- Silent financial losses. - -By crashing immediately on invalid data, NautilusTrader aims to provide: - -1. **No silent corruption** - The fail-fast policy is intended to prevent invalid data from propagating; this relies on checks covering the inputs. -2. **Immediate feedback** - Issues are discovered during development and testing, not in production. -3. **Audit trail** - Crash logs clearly identify the source of invalid data. -4. **Deterministic behavior** - With deterministic ordering and configuration, the same invalid input should trigger the same failure; nondeterministic sources can vary outcomes. - -#### When fail-fast applies - -Panics are used for: - -- Programmer errors (logic bugs, incorrect API usage). -- Data that violates fundamental invariants (negative timestamps, NaN prices). -- Arithmetic that would silently produce incorrect results. - -Results or Options are used for: - -- Expected runtime failures (network errors, file I/O). -- Business logic validation (order constraints, risk limits). -- User input validation. -- Library APIs exposed to downstream crates where callers need explicit error handling without relying on panics for control flow. - -#### Example scenarios - -```rust -// CORRECT: Panics on overflow - prevents data corruption -let total_ns = timestamp1 + timestamp2; // Panics if result > u64::MAX - -// CORRECT: Rejects NaN during deserialization -let price = serde_json::from_str("NaN"); // Error: "must be finite" - -// CORRECT: Explicit overflow handling when needed -let total_ns = timestamp1.checked_add(timestamp2)?; // Returns Option -``` - -This policy is implemented throughout the core types (`UnixNanos`, `Price`, `Quantity`, etc.) -and helps NautilusTrader maintain strong data correctness for production trading. - -In production deployments, the system is typically configured with `panic = abort` in release builds, -ensuring that any panic results in a clean process termination that can be handled by process supervisors -or orchestration systems. This aligns with the [crash-only design](#crash-only-design) principle, where unrecoverable errors -lead to immediate restart rather than attempting to continue in a potentially corrupted state. - -## System architecture - -The NautilusTrader codebase is actually both a framework for composing trading - systems, and a set of default system implementations which can operate in various -[environment contexts](#environment-contexts). - -![Architecture](https://github.com/nautechsystems/nautilus_trader/blob/develop/assets/architecture-overview.png?raw=true "architecture") - -### Core components - -Several core components work together to form the trading system: - -#### `NautilusKernel` - -The central orchestration component responsible for: - -- Initializing and managing all system components. -- Configuring the messaging infrastructure. -- Maintaining environment-specific behaviors. -- Coordinating shared resources and lifecycle management. -- Providing a unified entry point for system operations. - -#### `MessageBus` - -The backbone of inter-component communication, implementing: - -- **Publish/Subscribe patterns**: For broadcasting events and data to multiple consumers. -- **Request/Response communication**: For operations requiring acknowledgment. -- **Command/Event messaging**: For triggering actions and notifying state changes. -- **Optional state persistence**: Using Redis for durability and restart capabilities. - -#### `Cache` - -High-performance in-memory storage system that: - -- Stores instruments, accounts, orders, positions, and more. -- Provides performant fetching capabilities for trading components. -- Maintains consistent state across the system. -- Supports both read and write operations with optimized access patterns. - -#### `DataEngine` - -Processes and routes market data throughout the system: - -- Handles multiple data types (quotes, trades, bars, order books, custom data, and more). -- Routes data to appropriate consumers based on subscriptions. -- Manages data flow from external sources to internal components. - -#### `ExecutionEngine` - -Manages order lifecycle and execution: - -- Routes trading commands to the appropriate adapter clients. -- Tracks order and position states. -- Coordinates with risk management systems. -- Handles execution reports and fills from venues. -- Handles reconciliation of external execution state. - -#### `RiskEngine` - -Provides risk management: - -- Pre-trade risk checks and validation. -- Position and exposure monitoring. -- Real-time risk calculations. -- Configurable risk rules and limits. - -### Environment contexts - -An environment context in NautilusTrader defines the type of data and trading venue you work with. -Understanding these contexts matters for backtesting, development, and live trading. - -Here are the available environments you can work with: - -- `Backtest`: Historical data with simulated venues. -- `Sandbox`: Real-time data with simulated venues. -- `Live`: Real-time data with live venues (paper trading or real accounts). - -### Common core - -The platform has been designed to share as much common code between backtest, sandbox and live trading systems as possible. -This is formalized in the `system` subpackage, where you will find the `NautilusKernel` class, -providing a common core system 'kernel'. - -The *ports and adapters* architectural style enables modular components to be integrated into the -core system, providing various hooks for user-defined or custom component implementations. - -### Data and execution flow patterns - -Understanding how data and execution flow through the system helps when working with the platform: - -#### Data flow pattern - -1. **External data ingestion**: Market data enters via venue-specific `DataClient` adapters where it is normalized. -2. **Data processing**: The `DataEngine` handles data processing for internal components. -3. **Caching**: Processed data is stored in the `Cache` for fast access. -4. **Event publishing**: Data events are published to the `MessageBus`. -5. **Consumer delivery**: Subscribed components (Actors, Strategies) receive relevant data events. - -#### Execution flow pattern - -1. **Command generation**: Strategies create trading commands. -2. **Command publishing**: Commands are sent through the `MessageBus`. -3. **Risk validation**: The `RiskEngine` validates trading commands against configured risk rules. -4. **Execution routing**: The `ExecutionEngine` routes commands to appropriate venues. -5. **External submission**: The `ExecutionClient` submits orders to external trading venues. -6. **Event flow back**: Order events (fills, cancellations) flow back through the system. -7. **State updates**: Portfolio and position states update based on execution events. - -#### Component state management - -All components follow a finite state machine pattern. The `ComponentState` enum defines both stable states and transitional states: - -```mermaid -stateDiagram-v2 - [*] --> PRE_INITIALIZED - - PRE_INITIALIZED --> READY : register() - - READY --> STARTING : start() - STARTING --> RUNNING - - RUNNING --> STOPPING : stop() - STOPPING --> STOPPED - - STOPPED --> STARTING : start() - STOPPED --> RESETTING : reset() - RESETTING --> READY - - RUNNING --> RESUMING : resume() - RESUMING --> RUNNING - - RUNNING --> DEGRADING : degrade() - DEGRADING --> DEGRADED - - DEGRADED --> STOPPING : stop() - DEGRADED --> FAULTING : fault() - - RUNNING --> FAULTING : fault() - FAULTING --> FAULTED - - STOPPED --> DISPOSING : dispose() - FAULTED --> DISPOSING : dispose() - DISPOSING --> DISPOSED - - DISPOSED --> [*] -``` - -**Stable states:** - -- **PRE_INITIALIZED**: Component is instantiated but not yet ready to fulfill its specification. -- **READY**: Component is configured and able to be started. -- **RUNNING**: Component is operating normally and can fulfill its specification. -- **STOPPED**: Component has successfully stopped. -- **DEGRADED**: Component has degraded and may not meet its full specification. -- **FAULTED**: Component has shut down due to a detected fault. -- **DISPOSED**: Component has shut down and released all of its resources. - -**Transitional states:** - -- **STARTING**: Component is executing its actions on `start`. -- **STOPPING**: Component is executing its actions on `stop`. -- **RESUMING**: Component is being started again after its initial start. -- **RESETTING**: Component is executing its actions on `reset`. -- **DISPOSING**: Component is executing its actions on `dispose`. -- **DEGRADING**: Component is executing its actions on `degrade`. -- **FAULTING**: Component is executing its actions on `fault`. - -Transitional states are brief intermediate states that occur during state transitions. Components should not remain in transitional states for extended periods. - -#### Actor vs Component traits - -At the Rust implementation level, the system distinguishes between two complementary traits: - -```mermaid -classDiagram - class Actor { - <> - +id() Ustr - +handle(message) - } - - class Component { - <> - +component_id() ComponentId - +state() ComponentState - +register() - +start() - +stop() - +reset() - +dispose() - } - - class ActorRegistry { - +insert(actor) - +get(id) ActorRef - } - - class ComponentRegistry { - +insert(component) - +get(id) ComponentRef - } - - Actor <|.. Throttler : implements - Actor <|.. Strategy : implements - Component <|.. Strategy : implements - Component <|.. DataEngine : implements - Component <|.. ExecutionEngine : implements - - ActorRegistry --> Actor : manages - ComponentRegistry --> Component : manages - - class Throttler { - Actor only - } - - class Strategy { - Actor + Component - } - - class DataEngine { - Component only - } - - class ExecutionEngine { - Component only - } -``` - -**`Actor` trait** - Message dispatch: - -- Provides the `handle` method for receiving messages dispatched through the actor registry. -- Enables type-safe lookup and message dispatch by actor ID. -- Used by components that need to receive targeted messages (strategies, throttlers). - -**`Component` trait** - Lifecycle management: - -- Manages state transitions (`start`, `stop`, `reset`, `dispose`). -- Provides registration with the system kernel (`register`). -- Tracks component state via the finite state machine described above. -- Used by all system components that need lifecycle management. - -:::note -All components can publish and subscribe to messages via the `MessageBus` directly - this is independent of the `Actor` trait. The `Actor` trait specifically enables the registry-based message dispatch pattern where messages are routed to a specific actor by ID. -::: - -This separation allows: - -- **Actor-only**: Lightweight message handlers without lifecycle (e.g., `Throttler`). -- **Component-only**: System infrastructure with lifecycle but using direct MessageBus pub/sub (e.g., `DataEngine`, `ExecutionEngine`). -- **Both traits**: Trading strategies that need lifecycle management AND targeted message dispatch. - -The traits are managed by separate registries to support their different access patterns - lifecycle methods are called sequentially, while message handlers may be invoked re-entrantly during callbacks. - -### Messaging - -For modularity and loose coupling, an efficient `MessageBus` passes messages (data, commands, and events) between components. - -#### Threading model - -Within a node, the *kernel* consumes and dispatches messages on a single thread. The kernel encompasses: - -- The `MessageBus` and actor callback dispatch. -- Strategy logic and order management. -- Risk engine checks and execution coordination. -- Cache reads and writes. - -This single-threaded core provides deterministic event ordering and helps maintain backtest-live parity, -though live inputs and latency can still cause behavioral differences. Components consume messages -synchronously in a pattern *similar* to the [actor model](https://en.wikipedia.org/wiki/Actor_model). - -:::note -Of interest is the LMAX exchange architecture, which achieves award winning performance running on -a single thread. You can read about their *disruptor* pattern based architecture in [this interesting article](https://martinfowler.com/articles/lmax.html) by Martin Fowler. -::: - -Background services use separate threads or async runtimes: - -- **Network I/O** - WebSocket connections, REST clients, and async data feeds. -- **Persistence** - DataFusion queries and database operations via multi-threaded Tokio runtime. -- **Adapters** - Async adapter operations via thread pool executors. - -These services communicate results back to the kernel via the `MessageBus`. The bus itself is thread-local, -so each thread has its own instance, with cross-thread communication occurring through channels that -ultimately deliver events to the single-threaded core. - -## Framework organization - -The codebase organizes into layers of abstraction, grouped into logical subpackages -of cohesive concepts. You can navigate to the documentation for each subpackage -from the left nav menu. - -### Core / low-level - -- `core`: Constants, functions and low-level components used throughout the framework. -- `common`: Common parts for assembling the frameworks various components. -- `network`: Low-level base components for networking clients. -- `serialization`: Serialization base components and serializer implementations. -- `model`: Defines a rich trading domain model. - -### Components - -- `accounting`: Different account types and account management machinery. -- `adapters`: Integration adapters for the platform including brokers and exchanges. -- `analysis`: Components relating to trading performance statistics and analysis. -- `cache`: Provides common caching infrastructure. -- `data`: The data stack and data tooling for the platform. -- `execution`: The execution stack for the platform. -- `indicators`: A set of efficient indicators and analyzers. -- `persistence`: Data storage, cataloging and retrieval, mainly to support backtesting. -- `portfolio`: Portfolio management functionality. -- `risk`: Risk specific components and tooling. -- `trading`: Trading domain specific components and tooling. - -### System implementations - -- `backtest`: Backtesting componentry as well as a backtest engine and node implementations. -- `live`: Live engine and client implementations as well as a node for live trading. -- `system`: The core system kernel common between `backtest`, `sandbox`, `live` [environment contexts](#environment-contexts). - -## Code structure - -The foundation of the codebase is the `crates` directory, containing a collection of Rust crates including a C foreign function interface (FFI) generated by `cbindgen`. - -The bulk of the production code resides in the `nautilus_trader` directory, which contains a collection of Python/Cython subpackages and modules. - -Python bindings for the Rust core are provided by statically linking the Rust libraries to the C extension modules generated by Cython at compile time (effectively extending the CPython API). - -### Dependency flow - -```mermaid -flowchart TB - subgraph trader["nautilus_trader
Python / Cython"] - end - - subgraph core["crates
Rust"] - end - - trader -->|"C API"| core -``` - -### Rust crates - -The `crates/` directory contains the Rust implementation organized into focused crates with clear dependency boundaries. -Feature flags control optional functionality - for example, `streaming` enables persistence for catalog-based data streaming, -and `cloud` enables cloud storage backends (S3, Azure, GCP). - -Dependency flow (arrows point to dependencies): - -```mermaid -flowchart BT - subgraph Foundation - core - model - common - system - trading - end - - subgraph Infrastructure - serialization - network - cryptography - persistence - end - - subgraph Engines - data - execution - portfolio - risk - end - - subgraph Runtime - live - backtest - end - - adapters - pyo3 - - model --> core - common --> core - common --> model - system --> common - trading --> common - serialization --> model - network --> common - network --> cryptography - persistence --> serialization - data --> common - execution --> common - portfolio --> common - risk --> portfolio - live --> system - live --> trading - backtest --> system - backtest --> persistence - adapters --> live - adapters --> network - pyo3 --> adapters -``` - -**Crate categories:** - -| Category | Crates | Purpose | -|----------------|-----------------------------------------------------------|----------------------------------------------------------| -| Foundation | `core`, `model`, `common`, `system`, `trading` | Primitives, domain model, kernel, actor & strategy base. | -| Engines | `data`, `execution`, `portfolio`, `risk` | Core trading engine components. | -| Infrastructure | `serialization`, `network`, `cryptography`, `persistence` | Encoding, networking, signing, storage. | -| Runtime | `live`, `backtest` | Environment-specific node implementations. | -| External | `adapters/*` | Venue and data integrations. | -| Bindings | `pyo3` | Python bindings. | - -**Feature flags:** - -| Feature | Crates | Effect | -|-------------|----------------------------|------------------------------------------------------------| -| `streaming` | `data`, `system`, `live` | Enables `persistence` dependency for catalog streaming. | -| `cloud` | `persistence` | Enables cloud storage backends (S3, Azure, GCP, HTTP). | -| `python` | most crates | Enables PyO3 bindings (auto-enables `streaming`, `cloud`). | -| `defi` | `common`, `model`, `data` | Enables DeFi/blockchain data types. | - -:::note -Both Rust and Cython are build dependencies. The binary wheels produced from a build do not require -Rust or Cython to be installed at runtime. -::: - -### Type safety - -The platform design prioritizes software correctness and safety. - -The Rust codebase under `crates/` relies on the `rustc` compiler's guarantees for safe code. -Any `unsafe` blocks are explicit opt-outs where we must uphold the required invariants ourselves -(see the Rust section of the [Developer Guide](../developer_guide/rust.md)); overall memory and type safety -depend on those invariants holding. - -Cython provides type safety at the C level at both compile time, and runtime: - -:::info -If you pass an argument with an invalid type to a Cython implemented module with typed parameters, -then you will receive a `TypeError` at runtime. -::: - -If a function or method's parameter is not explicitly typed to accept `None`, passing `None` as an -argument will result in a `ValueError` at runtime. - -:::warning -The above exceptions are not explicitly documented to prevent excessive bloating of the docstrings. -::: - -### Errors and exceptions - -The documentation aims to cover all possible exceptions that NautilusTrader code -can raise, and the conditions that trigger them. - -:::warning -There may be other undocumented exceptions which can be raised by Python's standard -library, or from third party library dependencies. -::: - -### Processes and threads - -:::warning[One node per process] -Running multiple `TradingNode` or `BacktestNode` instances **concurrently** in the same process is not supported due to global singleton state: - -- **Backtest force-stop flag** - The `_FORCE_STOP` global flag is shared across all engines in the process. -- **Logger mode and timestamps** - The logging subsystem uses global state; backtests flip between static and real-time modes. -- **Runtime singletons** - Global Tokio runtime, callback registries, and other `OnceLock` instances are process-wide. - -**Sequential execution** of multiple nodes (one after another with proper disposal between runs) is fully supported and used in the test suite. - -For production deployments, add multiple strategies to a **single TradingNode** within a process. -For parallel execution or workload isolation, run each node in its own separate process. -::: - -## Related guides - -- [Overview](overview.md) - High-level introduction to NautilusTrader. -- [Message Bus](message_bus.md) - Core messaging infrastructure. diff --git a/nautilus-docs/docs/concepts/backtesting.md b/nautilus-docs/docs/concepts/backtesting.md deleted file mode 100644 index bb48b23..0000000 --- a/nautilus-docs/docs/concepts/backtesting.md +++ /dev/null @@ -1,1636 +0,0 @@ -# Backtesting - -Backtesting simulates trading using a specific system implementation. The system comprises the -built-in engines, `Cache`, [MessageBus](message_bus.md), `Portfolio`, [Actors](actors.md), -[Strategies](strategies.md), [Execution Algorithms](execution.md), and user-defined modules. -A `BacktestEngine` processes a stream of historical data. When the stream is exhausted, the -engine produces results and performance metrics for analysis. - -NautilusTrader offers two API levels for backtesting: - -- **High-level API**: Uses a `BacktestNode` and configuration objects (`BacktestEngine`s are used internally). -- **Low-level API**: Uses a `BacktestEngine` directly with more "manual" setup. - -## Choosing an API level - -Consider using the **low-level** API when: - -- Your entire data stream can be processed within the available machine resources (e.g., RAM). -- You prefer not to store data in the Nautilus-specific Parquet format. -- You have a specific need or preference to retain raw data in its original format (e.g., CSV, binary, etc.). -- You require fine-grained control over the `BacktestEngine`, such as the ability to re-run backtests on identical datasets while swapping out components (e.g., actors or strategies) or adjusting parameter configurations. - -Consider using the **high-level** API when: - -- Your data stream exceeds available memory, requiring streaming data in batches. -- You want the performance and convenience of the `ParquetDataCatalog` for storing data in the Nautilus-specific Parquet format. -- You value the flexibility and functionality of passing configuration objects to define and manage multiple backtest runs across various engines simultaneously. - -## Low-level API - -The low-level API centers around a `BacktestEngine`, where inputs are initialized and added manually via a Python script. -An instantiated `BacktestEngine` can accept the following: - -- Lists of `Data` objects, which are automatically sorted into monotonic order based on `ts_init`. -- Multiple venues, manually initialized. -- Multiple actors, manually initialized and added. -- Multiple execution algorithms, manually initialized and added. - -This approach offers detailed control over the backtesting process, allowing you to manually configure each component. - -### Loading large datasets efficiently - -When working with large amounts of data across multiple instruments, the way you load data -can significantly impact performance. - -#### The performance consideration - -By default, `BacktestEngine.add_data()` sorts the entire data stream (existing data + newly -added data) on each call when `sort=True` (the default). This means: - -- First call with 1M bars: sorts 1M bars. -- Second call with 1M bars: sorts 2M bars. -- Third call with 1M bars: sorts 3M bars. -- And so on... - -This repeated sorting of increasingly large datasets can become a bottleneck when loading -data for multiple instruments. - -#### Optimization strategies - -**Strategy 1: Defer sorting until the end (recommended for multiple instruments)** - -```python -from nautilus_trader.backtest.engine import BacktestEngine - -engine = BacktestEngine() - -# Setup venue and instruments -engine.add_venue(...) -engine.add_instrument(instrument1) -engine.add_instrument(instrument2) -engine.add_instrument(instrument3) - -# Load all data WITHOUT sorting on each call -engine.add_data(instrument1_bars, sort=False) -engine.add_data(instrument2_bars, sort=False) -engine.add_data(instrument3_bars, sort=False) - -# Sort once at the end - much more efficient! -engine.sort_data() - -# Now run your backtest -engine.add_strategy(strategy) -engine.run() -``` - -**Strategy 2: Collect and add in a single batch** - -```python -# Collect all data first -all_bars = [] -all_bars.extend(instrument1_bars) -all_bars.extend(instrument2_bars) -all_bars.extend(instrument3_bars) - -# Add once with sorting -engine.add_data(all_bars, sort=True) -``` - -**Strategy 3: Use streaming API for very large datasets** - -For datasets that don't fit in memory, there are two streaming approaches: - -**Automatic chunking** - supply a generator that yields batches. The engine pulls chunks -lazily during a single `run()` call: - -```python -def data_generator(): - # Yield chunks of data (each chunk is a list of Data objects) - yield load_chunk_1() - yield load_chunk_2() - yield load_chunk_3() - -engine.add_data_iterator( - data_name="my_data_stream", - generator=data_generator(), -) - -engine.run() # Chunks are consumed on-demand -``` - -**Manual chunking** - load and run each batch yourself. This is the pattern -used internally by `BacktestNode` and gives full control over batch boundaries: - -```python -engine.add_strategy(strategy) - -for batch in data_batches: - engine.add_data(batch) - engine.run(streaming=True) - engine.clear_data() - -engine.end() # Finalize: flushes remaining timers, stops engines, produces results -``` - -:::note -In streaming mode, timer advancement stops when data exhausts for each batch. Timers scheduled -past the last data point (e.g. bar aggregation intervals) are deferred until more data arrives -or `end()` is called, which flushes up to the `end` boundary from the last `run()` call. -::: - -:::tip[Performance impact] -For a backtest with 10 instruments, each with 1M bars: - -- Sorting on each call: ~10 sorts of increasing size (1M, 2M, 3M, ... 10M bars). -- Sorting once at the end: 1 sort of 10M bars. - -The deferred sorting approach can be **significantly faster** for large datasets. -::: - -### Data loading contract - -The `BacktestEngine` enforces important invariants to ensure data integrity: - -**Requirements:** - -- All data must be sorted before calling `run()`. -- When using `sort=False`, you **must** call `sort_data()` before running. -- The engine validates this and raises `RuntimeError` if unsorted data is detected. -- Calling `sort_data()` multiple times is safe (idempotent). - -**Safety guarantees:** - -- Data lists are always copied internally to prevent external mutations from affecting engine state. -- You can safely clear or modify data lists after passing them to `add_data()`. -- Adding data with `sort=True` makes it immediately available for backtesting. - -This design ensures data integrity while enabling performance optimizations for large datasets. - -## High-level API - -The high-level API centers around a `BacktestNode`, which orchestrates the management of multiple `BacktestEngine` instances, -each defined by a `BacktestRunConfig`. Multiple configurations can be bundled into a list and processed by the node in one run. - -Each `BacktestRunConfig` object consists of the following: - -- A list of `BacktestDataConfig` objects. -- A list of `BacktestVenueConfig` objects. -- A list of `ImportableActorConfig` objects. -- A list of `ImportableStrategyConfig` objects. -- A list of `ImportableExecAlgorithmConfig` objects. -- An optional `ImportableControllerConfig` object. -- An optional `BacktestEngineConfig` object, with a default configuration if not specified. - -## Repeated runs - -When conducting multiple backtest runs, it's important to understand how components reset to avoid unexpected behavior. - -### BacktestEngine.reset() - -The `.reset()` method returns all stateful fields to their **initial value**, except for data and instruments which persist. - -**What gets reset:** - -- All trading state (orders, positions, account balances). -- Strategy instances are removed (you must re-add strategies before the next run). -- Engine counters and timestamps. - -**What persists:** - -- Data added via `.add_data()` (use `.clear_data()` to remove). -- Instruments (must match the persisted data). -- Venue configurations. - -**Instrument handling:** - -For `BacktestEngine`, instruments persist across resets by default (because data persists and instruments must match data). -This is configured via `CacheConfig.drop_instruments_on_reset=False` in the default `BacktestEngineConfig`. - -### Approaches for multiple backtest runs - -There are two main approaches for running multiple backtests: - -#### 1. Use BacktestNode (recommended for production) - -The high-level API is designed for multiple backtest runs with different configurations: - -```python -from nautilus_trader.backtest.node import BacktestNode -from nautilus_trader.config import BacktestRunConfig - -# Define multiple run configurations -configs = [ - BacktestRunConfig(...), # Run 1 - BacktestRunConfig(...), # Run 2 - BacktestRunConfig(...), # Run 3 -] - -# Execute all runs -node = BacktestNode(configs=configs) -results = node.run() -``` - -Each run gets a fresh engine with clean state - no reset() needed. - -#### 2. Use BacktestEngine.reset() - -For fine-grained control with the low-level API: - -```python -from nautilus_trader.backtest.engine import BacktestEngine - -engine = BacktestEngine() - -# Setup once -engine.add_venue(...) -engine.add_instrument(ETHUSDT) -engine.add_data(data) - -# Run 1 -engine.add_strategy(strategy1) -engine.run() - -# Reset and run 2 - instruments and data persist -engine.reset() -engine.add_strategy(strategy2) -engine.run() - -# Reset and run 3 -engine.reset() -engine.add_strategy(strategy3) -engine.run() -``` - -:::note -Instruments and data persist across resets by default for `BacktestEngine`, making parameter optimizations straightforward. -::: - -:::tip[Best practices] - -- **For production backtesting:** Use `BacktestNode` with configuration objects. -- **For parameter optimizations:** Use `BacktestEngine.reset()` to run multiple strategies against the same data. -- **For quick experiments:** Either approach works - choose based on individual use case. - -::: - -## Data - -Data provided for backtesting drives the execution flow. Since a variety of data types can be used, -it's crucial that your venue configurations align with the data being provided for backtesting. -Mismatches between data and configuration can lead to unexpected behavior during execution. - -NautilusTrader is primarily designed and optimized for order book data, which provides -a complete representation of every price level or order in the market, reflecting the real-time behavior of a trading venue. -This provides the greatest execution granularity and realism. However, if granular order book data is either not -available or necessary, then the platform has the capability of processing market data in the following descending order of detail: - -```mermaid -flowchart LR - L3["L3 Order Book
(market-by-order)"] - L2["L2 Order Book
(market-by-price)"] - L1["L1 Quotes
(top of book)"] - T["Trades"] - B["Bars"] - - L3 --> L2 --> L1 --> T --> B - - style L3 fill:#2d5a3d,color:#fff - style L2 fill:#3d6a4d,color:#fff - style L1 fill:#4d7a5d,color:#fff - style T fill:#5d8a6d,color:#fff - style B fill:#6d9a7d,color:#fff -``` - -1. **Order Book Data/Deltas (L3 market-by-order)**: - - Full market depth with visibility of all individual orders. - -2. **Order Book Data/Deltas (L2 market-by-price)**: - - Market depth visibility across all price levels. - -3. **Quote Ticks (L1 market-by-price)**: - - Top of book only - best bid and ask prices and sizes. - -4. **Trade Ticks**: - - Actual executed trades. - -5. **Bars**: - - Aggregated trading activity over fixed time intervals (e.g., 1-minute, 1-hour, 1-day). - -### Choosing data: cost vs. accuracy - -For many trading strategies, bar data (e.g., 1-minute) can be sufficient for backtesting and strategy development. This is -particularly important because bar data is typically much more accessible and cost-effective compared to tick or order book data. - -Given this practical reality, Nautilus is designed to support bar-based backtesting with advanced features -that maximize simulation accuracy, even when working with lower granularity data. - -:::tip -For some trading strategies, it can be practical to start development with bar data to validate core trading ideas. -If the strategy looks promising, but is more sensitive to precise execution timing (e.g., requires fills at specific prices -between OHLC levels, or uses tight take-profit/stop-loss levels), you can then invest in higher granularity data -for more accurate validation. -::: - -## Venues - -When initializing a venue for backtesting, you must specify its internal order `book_type` for execution processing from the following options: - -- `L1_MBP`: Level 1 market-by-price (default). Only the top level of the order book is maintained. -- `L2_MBP`: Level 2 market-by-price. Order book depth is maintained, with a single order aggregated per price level. -- `L3_MBO`: Level 3 market-by-order. Order book depth is maintained, with all individual orders tracked as provided by the data. - -The `book_type` determines which data types the matching engine uses to update book -state and drive execution. Data types not applicable for a given `book_type` are -ignored for book and price updates, though precision validation still applies and -the engine clock still advances. Strategies always receive all subscribed data via -the data engine regardless of `book_type`. - -| Data Type | L1_MBP | L2_MBP | L3_MBO | -| ------------------ | ----------------- | ----------------- | ----------------- | -| `QuoteTick` | Updates book | *Ignored* | *Ignored* | -| `TradeTick` | Triggers matching | Triggers matching | Triggers matching | -| `Bar` | Updates book | *Ignored* | *Ignored* | -| `OrderBookDelta` | *Ignored* | Updates book | Updates book | -| `OrderBookDeltas` | *Ignored* | Updates book | Updates book | -| `OrderBookDepth10` | Updates book | Updates book | Updates book | - -:::note -The granularity of the data must match the specified order `book_type`. Nautilus -cannot generate higher granularity data (L2 or L3) from lower-level data such as -quotes, trades, or bars. -::: - -:::warning -If you specify `L2_MBP` or `L3_MBO` as the venue’s `book_type`, quotes and bars -will not update the book. Ensure you provide order book delta data, otherwise -orders may appear as though they are never filled. -::: - -:::warning -When using `L1_MBP` (the default), order book deltas are ignored by the matching -engine. If you subscribe to order book deltas, set the venue `book_type` to -`L2_MBP` or `L3_MBO`. This also applies to sandbox execution, where the matching -engine uses the same `book_type` configuration. -::: - -## Execution - -### Data and message sequencing - -In the main backtesting loop, new market data is processed for order execution before being dispatched to actors/strategies via the data engine. - -#### Main loop flow - -For each data point the engine runs three phases: - -- **Exchange processes data.** The simulated exchange updates its order book from - the incoming market data and iterates the matching engine. This fills any existing - orders that now match against the new market state. -- **Strategy receives data.** The data engine dispatches the data point to actors - and strategies via their callbacks (e.g. `on_quote_tick`, `on_bar`). Strategies - may submit, cancel, or modify orders during these callbacks. -- **Settle venues.** The engine drains all queued venue commands and then iterates - matching engines to fill newly submitted orders. This loop repeats until no - pending commands remain, so cascading orders (e.g. a hedge submitted from - `on_order_filled`) settle within the same timestamp. - -```mermaid -sequenceDiagram - participant BL as Backtest Loop - participant Exch as SimulatedExchange - participant ME as MatchingEngine - participant DE as DataEngine - participant Stgy as Strategy - - BL->>BL: next data point (ts=T) - - rect rgb(240, 248, 255) - note right of BL: Phase 1 - Exchange processes data - BL->>Exch: process_quote_tick / process_bar - Exch->>ME: update book + iterate() - note right of ME: Matches existing orders
against new market state - end - - rect rgb(245, 255, 245) - note right of BL: Phase 2 - Strategy receives data - BL->>DE: process(data) - DE->>Stgy: on_quote_tick() / on_bar() - Stgy-->>Exch: submit_order (queued or immediate) - end - - rect rgb(255, 248, 240) - note right of BL: Phase 3 - Settle venues - BL->>BL: _process_and_settle_venues(T) - BL->>Exch: _drain_commands(T) - note right of Exch: Processes queued commands,
adds orders to matching core - BL->>ME: _core.iterate(T) - note right of ME: Matches newly added orders
against current market state - note right of ME: Fills may trigger strategy callbacks
that enqueue further commands,
repeats until no pending commands - BL->>Exch: run simulation modules - BL->>Exch: check instrument expirations - end -``` - -Timer events use the same settle mechanism but batch by timestamp: all callbacks at -timestamp T execute first, then venues are settled for T before advancing to T+1. - -#### Command settling - -When an order fill triggers a strategy callback that submits additional orders (e.g., a stop-loss submitted -in `on_order_filled`), those cascading commands are settled within the same timestamp/event cycle. The engine -repeatedly drains venue command queues and any newly generated commands until no commands remain pending -for the current timestamp. Simulation modules are run only once per cycle, after all commands have settled. - -When a `LatencyModel` is configured, commands are placed in the venue's inflight queue with a future -timestamp derived from the simulated latency. The settle loop considers inflight commands that are due -at the current timestamp as pending, so zero-latency or same-tick latency configurations still settle -correctly. Commands with future timestamps are deferred and processed when the engine reaches that time. - -### Fill modeling philosophy - -NautilusTrader treats historical order book and trade data as **immutable** during backtesting. What happened in the market is preserved exactly as recorded. Fills never modify the underlying book state. - -This addresses a gap in academic literature: most research focuses on live market dynamics where the book actually evolves. Historical backtesting with frozen snapshots is a distinct engineering problem: how do we simulate realistic fills against data that doesn't change in response to our orders? - -**Design choices:** - -- **Immutable historical data**: Order book and trade data are never modified. -- **Optional consumption tracking**: When `liquidity_consumption=True`, the engine tracks consumed liquidity per price level to prevent duplicate fills. See [Order book immutability](#order-book-immutability) for configuration. -- **Deterministic results**: The same backtest with the same data and configuration produces identical results when probabilistic fill models use a fixed `random_seed`. - -### Fill price determination - -The matching engine determines fill prices based on order type, book type, and market state. - -#### L2/L3 order book data - -With full order book depth, fills are determined by actual book simulation: - -| Order Type | Fill Price | -| ---------------------- | ----------------------------------------------------------- | -| `MARKET` | Walks the book, filling at each price level (taker). | -| `MARKET_TO_LIMIT` | Walks the book, filling at each price level (taker). | -| `LIMIT` | Order's limit price when matched (maker). | -| `STOP_MARKET` | Walks the book when triggered. | -| `STOP_LIMIT` | Order's limit price when triggered and matched. | -| `MARKET_IF_TOUCHED` | Walks the book when triggered. | -| `LIMIT_IF_TOUCHED` | Order's limit price when triggered. | -| `TRAILING_STOP_MARKET` | Walks the book when activated and triggered. | -| `TRAILING_STOP_LIMIT` | Order's limit price when activated, triggered, and matched. | - -With L2/L3 data, market-type orders may partially fill across multiple price levels if insufficient liquidity exists at the top of book. -Limit-type orders act as resting orders after triggering and may remain unfilled if the market doesn't reach the limit price. -`MARKET_TO_LIMIT` fills as a taker first, then rests any remaining quantity as a limit order at its first fill price. - -#### L1 order book data (quotes, trades, bars) - -With only top-of-book data, the same book simulation is used with a single-level book: - -| Order Type | BUY Fill Price | SELL Fill Price | -| ---------------------- | -------------- | --------------- | -| `MARKET` | Best ask | Best bid | -| `MARKET_TO_LIMIT` | Best ask | Best bid | -| `LIMIT` | Limit price | Limit price | -| `STOP_MARKET` | Best ask | Best bid | -| `STOP_LIMIT` | Limit price | Limit price | -| `MARKET_IF_TOUCHED` | Best ask | Best bid | -| `LIMIT_IF_TOUCHED` | Limit price | Limit price | -| `TRAILING_STOP_MARKET` | Best ask | Best bid | -| `TRAILING_STOP_LIMIT` | Limit price | Limit price | - -With L1 data, the simulated book has a single price level. Orders fill against the available size at that level. If an order has remaining quantity after exhausting top-of-book liquidity, market and marketable limit-style orders will slip one tick to fill the residual. - -For bar data specifically, `STOP_MARKET` and `TRAILING_STOP_MARKET` orders may fill at the trigger price rather than best ask/bid when the bar moves through the trigger during its high/low processing. See [Stop order fill behavior with bar data](#stop-order-fill-behavior-with-bar-data) for details. - -:::note -Fill models can alter these fill prices. See the [Fill models](#fill-models) section for details on configuring execution simulation. -::: - -#### Order type semantics - -- **Market execution**: Fill at current market price (bid/ask). This models real exchange behavior where these orders execute at the best available price after triggering. Exception: with bar data, `STOP_MARKET` and `TRAILING_STOP_MARKET` orders triggered during H/L processing fill at the trigger price (see below). -- **Limit execution**: Fill at the order's limit price when matched. Provides price guarantee but may not fill if the market doesn't reach the limit. - -#### Stop order fill behavior with bar data - -When backtesting with bar data only (no tick data), the matching engine distinguishes between two scenarios for `STOP_MARKET` and `TRAILING_STOP_MARKET` orders: - -**Gap scenario** (bar opens past trigger): -When a bar's open price gaps past the trigger price, the stop triggers immediately and fills at the market price (the open). This models real exchange behavior where stop-market orders provide no price guarantee during gaps. - -Example - SELL `STOP_MARKET` with trigger at 100: - -- Previous bar closes at 105. -- Next bar opens at 90 (overnight gap down). -- Stop triggers at open and fills at 90. - -**Move-through scenario** (bar moves through trigger): -When a bar opens normally and then its high or low moves through the trigger price, the stop fills at the trigger price. Since we only have OHLC data, we assume the market moved smoothly through the trigger and the order would have filled there. - -Example - SELL `STOP_MARKET` with trigger at 100: - -- Bar opens at 102 (no gap). -- Bar low reaches 98, moving through trigger at 100. -- Stop fills at 100 (the trigger price). - -This behavior caps potential slippage during orderly market moves while still modeling gap slippage accurately. For tick-level precision, use quote or trade tick data instead of bars. - -### Price protection - -Price protection defines an exchange-calculated price boundary that prevents marketable orders from -executing at excessively aggressive prices. This models exchanges like Binance and CME that implement -protection mechanisms for market and stop-market orders. - -**Configuration:** - -```python -from nautilus_trader.backtest.config import BacktestVenueConfig - -venue_config = BacktestVenueConfig( - name="BINANCE", - oms_type="NETTING", - account_type="MARGIN", - starting_balances=["100_000 USDT"], - price_protection_points=100, # 100 points = 1.00 offset for 2-decimal instruments -) -``` - -**How it works:** - -The matching engine calculates the protection boundary from the current best bid/ask at fill time: - -- **BUY orders**: `protection_price = ask + (points × price_increment)` -- **SELL orders**: `protection_price = bid - (points × price_increment)` - -The engine filters out fills beyond the protection boundary. For example, with `price_protection_points=100` -on an instrument with `price_increment=0.01`: - -- Best ask is 1001.00. -- Protection price = 1001.00 + (100 × 0.01) = 1002.00. -- A BUY market order fills only at prices ≤ 1002.00. -- Liquidity at 1003.00 or higher is filtered, leaving the order partially filled. - -**Trigger-time semantics:** - -The engine computes protection at fill time, not order submission time: - -- **Market orders**: Protection computed immediately when the order processes. -- **Stop-market orders**: Protection computed when the stop triggers, using the bid/ask at that moment. - -This design allows stop orders to be submitted even when the opposite side of the book is empty, -since the engine computes protection later when the stop triggers. - -**Order types affected:** - -- `MARKET` -- `STOP_MARKET` - -Limit orders are unaffected since they already define a price boundary. - -:::note -Set `price_protection_points=0` to disable price protection (default behavior). -::: - -### Slippage and spread handling - -When backtesting with different types of data, Nautilus implements specific handling for slippage and spread simulation: - -For L2 (market-by-price) or L3 (market-by-order) data, slippage is simulated with high accuracy by: - -- Filling orders against actual order book levels. -- Matching available size at each price level sequentially. -- Maintaining realistic order book depth impact (per order fill). - -For L1 data types (e.g., L1 order book, trades, quotes, bars), slippage is handled through the `FillModel`: - -**Per-fill slippage** (`prob_slippage`): - -- Applies to each fill when using an L1 book with a configured `FillModel`. -- Affects all order types (market, limit, stop, etc.). -- When triggered, moves the fill price one tick against the order direction. -- Example: With `prob_slippage=0.5`, a BUY order has 50% chance of filling one tick above the best ask. - -:::note -When backtesting with bar data, be aware that the reduced granularity of price information affects the slippage mechanism. -For the most realistic backtesting results, consider using higher granularity data sources such as L2 or L3 order book data when available. -::: - -#### How simulation varies by data type - -The behavior of the `FillModel` adapts based on the order book type being used: - -**L2/L3 order book data** - -With full order book depth, the `FillModel` focuses purely on simulating queue position for limit orders through `prob_fill_on_limit`. -The order book itself handles slippage naturally based on available liquidity at each price level. - -- `prob_fill_on_limit` is active - simulates queue position. -- `prob_slippage` is not used - real order book depth determines price impact. - -:::warning -The historical order book is immutable during backtesting. Book depth is **not** decremented after fills. -By default (`liquidity_consumption=False`), the same liquidity can be consumed repeatedly within an iteration. -Enable `liquidity_consumption=True` to track consumed liquidity per price level. Consumption resets when fresh -data arrives at that level. See [Order book immutability](#order-book-immutability) for details. -::: - -**L1 order book data** - -With only best bid/ask prices available, the `FillModel` provides additional simulation: - -- `prob_fill_on_limit` is active - simulates queue position. -- `prob_slippage` is active - simulates basic price impact since we lack real depth information. - -**Bar/Quote/Trade data** - -When using less granular data, the same behaviors apply as L1: - -- `prob_fill_on_limit` is active - simulates queue position. -- `prob_slippage` is active - simulates basic price impact. - -#### Important considerations - -- **Partial fills**: With L2/L3 data, fills are limited to available liquidity at each price level. With L1 data, the full order quantity fills at the single available level. -- **Consumption tracking**: See [Order book immutability](#order-book-immutability) for details on preventing duplicate fills. - -### Order book immutability - -Historical order book data is immutable during backtesting. When your order fills against book liquidity, -the book state remains unchanged. This preserves historical data integrity. - -The matching engine can optionally use **per-level consumption tracking** to prevent duplicate fills while -allowing fills when fresh liquidity arrives. This behavior is controlled by the `liquidity_consumption` -configuration option. - -**Configuration:** - -```python -from nautilus_trader.backtest.config import BacktestVenueConfig - -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="CASH", - starting_balances=["100_000 USD"], - liquidity_consumption=True, # Enable consumption tracking (default: False) -) -``` - -- `liquidity_consumption=False` (default): Each iteration fills against the full book liquidity independently. - Simpler behavior, assumes you're a small participant whose orders don't meaningfully impact available liquidity. -- `liquidity_consumption=True`: Tracks consumed liquidity per price level. Prevents the same - displayed liquidity from generating multiple fills. Resets when fresh data arrives at that level. - -**How consumption tracking works (when enabled):** - -For each price level, the engine maintains: - -- `original_size`: The book's quantity when tracking began. -- `consumed`: How much has been filled against this level. - -When processing a fill: - -1. Check if the book's current size at this level matches `original_size` -2. If different (fresh data arrived), reset the entry: `original_size = current_size`, `consumed = 0` -3. Calculate `available = original_size - consumed` -4. After filling, increment `consumed` by the fill quantity - -**Example:** - -1. Order book shows 100 units at ask 100.00. Engine tracks: `(original=100, consumed=0)`. -2. Your BUY order fills 30 units. Engine updates: `(original=100, consumed=30)`. Available = 70. -3. Another BUY order attempts 50 units. Available = 70, so it fills 50. `(original=100, consumed=80)`. -4. A delta updates ask 100.00 to 120 units. Engine resets: `(original=120, consumed=0)`. -5. New orders can now fill against the fresh 120 units. - -**Passive limit order fills on L1 data:** - -With L1 data (quotes, trades, bars), the book has only a single price level per side. When the market -moves through a passive (MAKER) limit order's price, the engine must decide how to handle remaining -order quantity after exhausting displayed liquidity. - -| `liquidity_consumption` | Behavior when market moves through passive limit | -| ----------------------- | ----------------------------------------------------------------------------------------------- | -| `False` (default) | Fill entire order at limit price. Assumes market movement implies sufficient liquidity existed. | -| `True` | Fill only against displayed liquidity. Order remains open for subsequent fills. | - -**Example scenario** (`liquidity_consumption=True`): - -1. Quote shows ask 100.10 with 50 units. -2. You place BUY LIMIT at 100.05 for 1000 units (passive, resting below ask). -3. Next quote shows ask 100.00 with 30 units (market moved through your limit). -4. Order fills 30 units against displayed liquidity. 970 units remain open. -5. Next quote shows ask 99.95 with 200 units. -6. Order fills another 200 units. 770 units remain open. -7. Fills continue as fresh liquidity arrives at crossed price levels. - -This behavior provides conservative fill simulation: your order only fills against liquidity -actually observed in the data, rather than inferring liquidity from price movements. - -**Trade tick liquidity:** - -Trade ticks provide evidence of executable liquidity at the trade price. When a trade occurs at a price level -not reflected in the current book, the engine can use the trade quantity as available liquidity, subject to -the same consumption tracking rules (when enabled). - -**Trade consumption seeding:** - -When using L2/L3 book data and a trade tick triggers order matching (e.g., triggering a resting stop order), -the trade itself consumed liquidity from the book. Before simulating fills for triggered orders, the engine -pre-seeds the consumption maps with the trade's consumed volume. This prevents triggered orders from filling -against liquidity that the triggering trade already consumed. This seeding is skipped for L1 books, where the -trade tick has already updated the single top-of-book level directly. - -For example, if the book has 10 units at the best ask and a BUY trade of size 8 triggers a stop market BUY -for 5 units, the stop order sees only 2 units remaining at best ask (10 - 8) and must fill the remaining -3 units at the next price level. Without this seeding, the stop would incorrectly fill all 5 units at the -best ask price. - -The engine uses a timestamp guard to avoid double-counting: if the book's most recent update (`ts_last`) -is newer than the trade's event time (`ts_event`), seeding is skipped. This handles exchanges like Binance -where depth deltas arrive before the corresponding trade tick, so the book already reflects the consumed -liquidity, so additional seeding would over-penalize fills. - -:::note -As the `FillModel` continues to evolve, future versions may introduce more sophisticated simulation of order execution dynamics, including: - -- Variable slippage based on order size. -- More complex queue position modeling. - -::: - -#### Known limitations - -**No queue position within a level**: Consumption tracking determines *how much* liquidity remains at a level, -but doesn't model *where* your order sits in the queue relative to other participants. Use `prob_fill_on_limit` -to simulate queue position probabilistically. - -**Trade-driven fills are opportunistic**: When trade ticks indicate liquidity at a price not in the book, -the engine uses this as fill evidence. However, this represents liquidity that existed momentarily and may -not reflect sustained availability. - -### Trade based execution - -Trade tick data triggers order fills by default (`trade_execution=True`). A trade tick indicates that liquidity -was accessed at the trade price, allowing resting limit orders to match. This mirrors the default behavior -for bar data (`bar_execution=True`). - -Advanced users who want to isolate execution to L1 book data only (quotes or order book updates) can disable -trade-based execution: - -```python -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="CASH", - starting_balances=["100_000 USD"], - trade_execution=False, # Disable trade-based fills -) -``` - -When `trade_execution=False` or `bar_execution=False`, the respective data types skip order matching -and maintenance operations (GTD order expiry, trailing stop activation, instrument expiration checks). -Quote ticks always trigger maintenance, so this is typically acceptable when using multiple data types. - -The matching engine uses a "transient override" mechanism: during the matching process, it temporarily adjusts -the matching core's Best Bid (for BUYER trades) or Best Ask (for SELLER trades) toward the trade price. This allows -resting orders on the passive side to cross the spread and fill. Note: the underlying order book data is never -modified (it remains immutable); only the matching core's internal price references are adjusted. - -**Fill determination:** - -When a trade tick triggers order matching, the engine determines fills as follows: - -1. **Book reflects trade price**: If the order book has liquidity at the trade price, fills use book depth (standard behavior). -2. **Book doesn't reflect trade price**: If the book's liquidity is at a different price, the engine uses a "trade-driven fill" at the order's limit price, capped to `min(order.leaves_qty, trade.size)`. - -This ensures that when a trade prints through the spread but the book hasn't updated, fills are bounded by what the trade tick actually evidences. When `liquidity_consumption=False` (default), the same trade size can fill multiple orders within an iteration. When `liquidity_consumption=True`, consumption tracking applies to trade-driven fills as well. Repeated fills at the same trade price will be bounded by consumed liquidity until fresh data arrives. - -**Restoration behavior:** - -After matching, the core's bid/ask are only restored to their original values if the trade price improved them -(moved them away from the spread): - -- **SELLER trade**: Ask is restored only if trade price was below the original ask. -- **BUYER trade**: Bid is restored only if trade price was above the original bid. - -If the trade price didn't improve the quote (e.g., a SELLER trade at or above the ask), the core retains -the trade price. This means repeated trades at or beyond the spread can progressively move the core's bid/ask. - -**Fill price:** - -- **SELLER trade at P**: The engine sets the core's Best Ask to P (if P < current ask). Resting BUY LIMIT orders at P or higher will fill at their limit price (if book doesn't have that level) or at book prices (if book does). -- **BUYER trade at P**: The engine sets the core's Best Bid to P (if P > current bid). Resting SELL LIMIT orders at P or lower will fill at their limit price (if book doesn't have that level) or at book prices (if book does). - -This conservative approach ensures fills occur at the order's limit price rather than potentially better trade prices. For example, a BUY LIMIT at 100.05 triggered by a SELLER trade at 100.00 will fill at 100.05, not 100.00. - -:::tip -Combine trade data with book or quote data for best results: book/quote data establishes the baseline spread, -while trade ticks trigger execution for orders that might be inside the spread or ahead of the quote updates. -::: - -#### Understanding trade tick aggressor sides - -A common source of confusion is the `aggressor_side` field on trade ticks: - -- **SELLER trade**: A seller aggressed, selling into the bid. This provides evidence of fill-able liquidity for **BUY** orders at the trade price. -- **BUYER trade**: A buyer aggressed, buying from the ask. This provides evidence of fill-able liquidity for **SELL** orders at the trade price. - -In other words, trade ticks trigger fills for orders on the **opposite** side of the aggressor. A SELLER trade at 100.00 can fill your resting BUY LIMIT at 100.00, but cannot fill your SELL LIMIT, since the trade already represents someone else selling. - -#### Combining L2 book data with trade ticks - -When using L2 order book data (e.g., 100ms throttled depth snapshots) combined with trade tick data: - -1. **Book updates establish the spread**: Each book delta/snapshot updates the matching engine's view of available liquidity at each price level. - -2. **Trade ticks provide execution evidence**: Trade ticks indicate that liquidity was accessed at a specific price, potentially between book snapshots. - -3. **Fill quantity determination**: When a trade triggers a fill: - - If the book already reflects liquidity at the trade price, fills use book depth - - If the trade price is inside the spread (not in the current book), fills are capped by `min(order.leaves_qty, trade.size)` - -4. **Timing considerations**: With throttled book data (e.g., 100ms), the book may lag behind trades. A trade at a price not yet reflected in the book will use trade-driven fill logic. - -**Common misconception**: Users sometimes expect every trade tick to trigger fills. Remember: - -- Only trades on the **opposite** side can fill your orders. -- SELLER trades → potential BUY fills. -- BUYER trades → potential SELL fills. -- Book UPDATE events move the market but only trigger fills if prices cross your order. - -#### Queue position tracking - -When `queue_position=True` is enabled alongside `trade_execution=True`, the matching engine simulates -queue position for limit orders. This provides more realistic fill behavior by tracking how many -orders are "ahead" of your order at a given price level. - -**How it works:** - -1. **Order placement**: When a LIMIT order is accepted, the engine snapshots the current same-side - book depth at the order's price level. This represents the orders ahead in the queue. - -2. **Trade ticks**: When trade ticks occur at the order's price level, the "quantity ahead" is - decremented by the trade size. Only trades on the correct side affect the queue (BUYER trades - decrement queue for SELL orders, SELLER trades decrement queue for BUY orders). Trades with - `NO_AGGRESSOR` (common in historical datasets lacking aggressor metadata) affect both sides. - This is pessimistic but prevents orders from stalling indefinitely. - -3. **Fill eligibility**: The order becomes eligible to fill only when the quantity ahead reaches zero. - On the tick that clears the queue, only the excess volume (trade size minus queue ahead) is - available for fill, preventing overfill. - -4. **Price level DELETE**: If the order book level is deleted (BookAction.DELETE), the queue clears - immediately, making the order fill-eligible. UPDATE actions are ignored (queue unchanged). - -5. **Order modification**: If the order is modified (price or quantity change), the queue position - resets. The order moves to the back of the queue at its new price level. - -**Configuration:** - -```python -from nautilus_trader.backtest.config import BacktestVenueConfig - -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="MARGIN", - starting_balances=["100_000 USD"], - trade_execution=True, # Required for queue_position - queue_position=True, # Enable queue position tracking -) -``` - -**Example scenario:** - -1. Order book shows 100 units at bid 100.00. -2. You place a BUY LIMIT at 100.00 for 50 units. Queue ahead = 100. -3. SELLER trade of 80 units at 100.00 → queue ahead = 20. No fill yet. -4. SELLER trade of 30 units at 100.00 → queue clears with 10 excess. Fill = 10 units. -5. Next SELLER trade of 50 units → fill remaining 40 units. - -**Limitations:** - -- Only applies to `LIMIT` orders. Stop-limit and limit-if-touched orders are not tracked in this implementation. -- Queue position is per-order, not shared across multiple orders at the same price. -- The queue snapshot is based on book state at order acceptance time. -- Trades with `NO_AGGRESSOR` decrement queue for both sides, which may cause orders to fill sooner than in reality (pessimistic for queue estimation, but prevents stalling). - -**L1 quote-based mode:** - -When using `BookType.L1_MBP` (top-of-book quotes only), queue position tracking uses -trade ticks to decrement the queue (the same mechanism as L2/L3), while quote ticks -handle price-move detection and deferred snapshot resolution. - -- **Trade ticks**: Trades at the order's price level decrement the queue ahead by the trade - size, identical to L2/L3 behavior. Only trades on the correct aggressor side affect the - queue (SELLER trades decrement queue for BUY orders, BUYER trades for SELL orders). -- **Price moves away**: If the bid drops below a BUY order's price (or ask rises above a - SELL order's price), the order's price level has been "crossed" and the queue clears to zero, - making the order fill-eligible on the next matching trade. -- **Price moves toward**: If the bid rises (or ask drops), the level at the order's price was - not consumed, so queue positions are preserved. -- **Price returns to a level**: When the price returns after moving away, the queue ahead is - capped at the new displayed size if it was previously larger. -- **Orders behind BBO (pending)**: When a limit order is placed behind the best bid/ask - (e.g., BUY below best bid), the queue snapshot is deferred because L1 data has no visible - depth at that level. Fills are blocked until the BBO reaches the order's price, at which - point the queue is snapshotted from the displayed size. Pending orders are also resolved - when trades cross through their price level. - -L1 mode uses the same configuration: set `queue_position=True` with `book_type=BookType.L1_MBP`. -This provides a lightweight alternative to full L2/L3 data when only top-of-book quotes are -available. - -:::note -Queue position tracking provides a heuristic simulation of queue dynamics. Real exchange queue -behavior depends on many factors (order priority rules, hidden orders, etc.) that cannot be -perfectly reconstructed from historical data. -::: - -### Bar based execution - -Bar data provides a summary of market activity with four key prices for each time period (assuming bars are aggregated by trades): - -- **Open**: opening price (first trade) -- **High**: highest price traded -- **Low**: lowest price traded -- **Close**: closing price (last trade) - -While this gives us an overview of price movement, we lose some important information that we'd have with more granular data: - -- We don't know in what order the market hit the high and low prices. -- We can't see exactly when prices changed within the time period. -- We don't know the actual sequence of trades that occurred. - -This is why Nautilus processes bar data through a system that attempts to maintain -the most realistic yet conservative market behavior possible, despite these limitations. -At its core, the platform always maintains an order book simulation - even when you provide less -granular data such as quotes, trades, or bars (although the simulation will only have a top level book). - -:::warning -When using bars for execution simulation (enabled by default with `bar_execution=True` in venue configurations), -Nautilus strictly expects the initialization timestamp (`ts_init`) of each bar to represent its **closing time**. -This ensures accurate chronological processing, prevents look-ahead bias, and aligns market updates (Open → High → Low → Close) with the moment the bar is complete. - -The event timestamp (`ts_event`) can represent either the open or close time of the bar: - -- If `ts_event` is at the **close**, ensure `ts_init_delta=0` when processing bars (default). -- If `ts_event` is at the **open**, set `ts_init_delta` equal to the bar's duration to shift `ts_init` to the close. - -::: - -#### Bar timestamp convention - -If your data source provides bars timestamped at the **opening time** (common in some providers), you need to ensure `ts_init` is set to the closing time for correct execution simulation. There are two approaches: - -**Approach 1: Adjust data timestamps (recommended)** - -- Use adapter-specific configurations like `bars_timestamp_on_close=True` (e.g., for Bybit or Databento adapters) to handle this automatically during data ingestion. -- For custom data, manually shift the timestamps by the bar duration before loading (e.g., add 1 minute for `1-MINUTE` bars). -- This approach is clearest because the data itself reflects the close time. - -**Approach 2: Use `ts_init_delta` parameter** - -- When calling `BarDataWrangler.process()`, set `ts_init_delta` to the bar's duration in nanoseconds (e.g., `60_000_000_000` for 1-minute bars). -- The wrangler computes `ts_init = ts_event + ts_init_delta`, shifting execution timing to the close. -- Use this when you cannot or prefer not to modify source data timestamps. - -Always verify your data's timestamp convention with a small sample to avoid simulation inaccuracies. Incorrect timestamp handling can lead to look-ahead bias and unrealistic backtest results. - -#### Processing bar data - -Even when you provide bar data, Nautilus maintains an internal order book for each instrument, as a real venue would. - -1. **Time processing**: - - Nautilus has a specific way of handling the timing of bar data *for execution* that's crucial for accurate simulation. - - The initialization timestamp (`ts_init`) is used for execution timing and must represent the close time of the bar. This approach is most logical because it represents the moment when the bar is fully formed and its aggregation is complete. - - The event timestamp (`ts_event`) represents when the data event occurred and may differ from `ts_init` depending on your data source: - - If your bars are timestamped at the **close** (the recommended default), use `ts_init_delta=0` in `BarDataWrangler` so that `ts_init = ts_event`. - - If your bars are timestamped at the **open**, set `ts_init_delta` to the bar's duration in nanoseconds (e.g., 60_000_000_000 for 1-minute bars) to shift `ts_init` to the close time. - - The platform ensures all events happen in the correct sequence based on `ts_init`, preventing any possibility of look-ahead bias in your backtests. - -:::note[Exceptions for bar execution] -Bars will **not** be processed for execution (and will not update the order book) in the following cases: - -- **Internally aggregated bars**: Bars with `AggregationSource.INTERNAL` are skipped to avoid processing bars that are derived from already-processed tick data. -- **Non-L1 book types**: When the venue's `book_type` is configured as `L2_MBP` or `L3_MBO`, bar data is ignored for execution processing, as bars are derived from top-of-book prices only. - -In these cases, bars will still be received by strategies for analytics and decision-making, but they won't trigger order matching or update the simulated order book. -::: - -2. **Price processing**: - - The platform converts each bar's OHLC prices into a sequence of market updates. - - By default, updates follow the order: Open → High → Low → Close (configurable via `bar_adaptive_high_low_ordering`). - - If you provide multiple timeframes (like both 1-minute and 5-minute bars), the platform uses the more granular data for highest accuracy. - -3. **Executions**: - - When you place orders, they interact with the simulated order book as they would on a real venue. - - For MARKET orders, execution happens at the current simulated market price plus any configured latency. - - For LIMIT orders working in the market, they'll execute if any of the bar's prices reach or cross your limit price (see below). - - The matching engine continuously processes orders as OHLC prices move, rather than waiting for complete bars. - -#### OHLC prices simulation - -During backtest execution, each bar is converted into a sequence of four price points: - -1. Opening price -2. High price *(Order between High/Low is configurable. See `bar_adaptive_high_low_ordering` below.)* -3. Low price -4. Closing price - -The trading volume for that bar is **split evenly** among these four points (25% each), with any -remainder added to the closing price trade to preserve total volume. In marginal cases, if the -bar's volume divided by 4 is less than the instrument's minimum `size_increment`, we use the -minimum `size_increment` per price point to ensure valid market activity (e.g., 1 contract for -CME group exchanges). - -How these price points are sequenced can be controlled via the `bar_adaptive_high_low_ordering` parameter when configuring a venue. - -Nautilus supports two modes of bar processing: - -1. **Fixed ordering** (`bar_adaptive_high_low_ordering=False`, default) - - Processes every bar in a fixed sequence: `Open → High → Low → Close`. - - Simple and deterministic approach. - -2. **Adaptive ordering** (`bar_adaptive_high_low_ordering=True`) - - Uses bar structure to estimate likely price path: - - If Open is closer to High: processes as `Open → High → Low → Close`. - - If Open is closer to Low: processes as `Open → Low → High → Close`. - - [Research](https://gist.github.com/stefansimik/d387e1d9ff784a8973feca0cde51e363) shows this approach achieves ~75-85% accuracy in predicting correct High/Low sequence (compared to statistical ~50% accuracy with fixed ordering). - - This is particularly important when both take-profit and stop-loss levels occur within the same bar - as the sequence determines which order fills first. - -Here's how to configure adaptive bar ordering for a venue, including account setup: - -```python -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.model.enums import OmsType, AccountType -from nautilus_trader.model import Money, Currency - -# Initialize the backtest engine -engine = BacktestEngine() - -# Add a venue with adaptive bar ordering and required account settings -engine.add_venue( - venue=venue, # Your Venue identifier, e.g., Venue("BINANCE") - oms_type=OmsType.NETTING, - account_type=AccountType.CASH, - starting_balances=[Money(10_000, Currency.from_str("USDT"))], - bar_adaptive_high_low_ordering=True, # Enable adaptive ordering of High/Low bar prices -) -``` - -### Internal bar aggregation timing - -When aggregating time bars internally from tick data, the data engine uses timers to close bars at -interval boundaries. A timing edge case occurs when data arrives at the exact bar close timestamp: the -timer may fire before processing boundary data. - -Configure `time_bars_build_delay` in `DataEngineConfig` to delay bar close timers: - -```python -from nautilus_trader.config import BacktestEngineConfig -from nautilus_trader.data.config import DataEngineConfig - -config = BacktestEngineConfig( - data_engine=DataEngineConfig( - time_bars_build_delay=1, # Microseconds - ), -) -``` - -:::tip -A small delay (1 microsecond) ensures boundary data is processed before the bar closes. -Useful when tick data clusters at round interval timestamps. -::: - -:::note -Only affects internally aggregated bars (`AggregationSource.INTERNAL`). -::: - -### Timer-only backtests - -The backtest engine supports running with timers but no market data. This is useful for scheduled -operations or testing timer-based logic. Timers fire in chronological order, and timer callbacks -can dynamically add data via `add_data_iterator()` which will be processed in sequence. - -:::warning -Data added by timer callbacks at the exact start time should have timestamps **after** the start time. -The engine reads the first data point before processing start-time timers, so dynamically added data -with timestamps at or before the start time may not be processed in the expected order. -::: - -### Fill models - -Fill models simulate order execution dynamics during backtesting. They address a fundamental challenge: -*even with perfect historical market data, we can't fully simulate how orders may have interacted -with other market participants in real-time*. - -The base `FillModel` provides probabilistic parameters for queue position and slippage simulation. -Subclasses can override `get_orderbook_for_fill_simulation()` to generate synthetic order books -for more sophisticated liquidity modeling. - -#### Available fill models - -| Model | Description | Use Case | -| ---------------------------- | ------------------------------------------------------- | -------------------------------------------- | -| `FillModel` | Base model with probabilistic fill/slippage parameters. | Simple queue position and slippage. | -| `BestPriceFillModel` | Fills at best price with unlimited liquidity. | Testing basic strategy logic optimistically. | -| `OneTickSlippageFillModel` | Forces exactly one tick of slippage on all orders. | Conservative slippage testing. | -| `TwoTierFillModel` | 10 contracts at best price, remainder one tick worse. | Basic market depth simulation. | -| `ThreeTierFillModel` | 50/30/20 contracts across three price levels. | More realistic depth simulation. | -| `ProbabilisticFillModel` | 50% chance best price, 50% chance one tick slippage. | Randomized execution quality. | -| `SizeAwareFillModel` | Different execution based on order size (≤10 vs >10). | Size-dependent market impact. | -| `LimitOrderPartialFillModel` | Max 5 contracts fill per price touch. | Queue position via partial fills. | -| `MarketHoursFillModel` | Wider spreads during low liquidity periods. | Session-aware execution. | -| `VolumeSensitiveFillModel` | Liquidity based on recent trading volume. | Volume-adaptive depth. | -| `CompetitionAwareFillModel` | Only percentage of visible liquidity available. | Multi-participant competition. | - -#### Configuring fill models - -**Using the base FillModel with probabilistic parameters:** - -```python -from nautilus_trader.backtest.config import BacktestVenueConfig -from nautilus_trader.backtest.config import ImportableFillModelConfig - -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="CASH", - starting_balances=["100_000 USD"], - fill_model=ImportableFillModelConfig( - fill_model_path="nautilus_trader.backtest.models:FillModel", - config_path="nautilus_trader.backtest.config:FillModelConfig", - config={ - "prob_fill_on_limit": 0.2, # Chance a limit order fills when price matches - "prob_slippage": 0.5, # Chance of 1-tick slippage (L1 data only) - "random_seed": 42, # Optional: Set for reproducible results - }, - ), -) -``` - -**Using an order book simulation model:** - -```python -from nautilus_trader.backtest.config import BacktestVenueConfig -from nautilus_trader.backtest.config import ImportableFillModelConfig - -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="CASH", - starting_balances=["100_000 USD"], - fill_model=ImportableFillModelConfig( - fill_model_path="nautilus_trader.backtest.models:ThreeTierFillModel", - ), -) -``` - -#### Probabilistic parameters (base FillModel) - -**prob_fill_on_limit** (default: `1.0`) - -Simulates queue position by controlling the probability of a limit order filling when its price level is touched (but not crossed). - -- `0.0`: Never fills at touch (back of queue). -- `0.5`: 50% chance of filling (middle of queue). -- `1.0`: Always fills at touch (front of queue). - -**prob_slippage** (default: `0.0`) - -Simulates price slippage on each fill. Only applies to L1 data types (quotes, trades, bars) where real depth is unavailable. Affects all order types when executing as takers. - -- `0.0`: No slippage (fills at best price). -- `0.5`: 50% chance of one tick slippage per fill. -- `1.0`: Always slips one tick. - -#### Order book simulation models - -These models override the `get_orderbook_for_fill_simulation()` method to generate synthetic order books -representing expected market liquidity. The matching engine fills orders against this simulated book. - -**How it works:** - -1. Before processing a fill, the matching engine calls `get_orderbook_for_fill_simulation()`. -2. If the model returns a synthetic order book, fills execute against that book's liquidity. -3. If the model returns `None`, standard fill logic applies. - -:::note -When a custom fill model provides a simulated order book, the `liquidity_consumption` tracking is **not** applied. -Custom fill models are expected to manage their own liquidity simulation within the returned order book. -Liquidity consumption tracking only affects the built-in fill logic (when `get_orderbook_for_fill_simulation()` returns `None`). -::: - -**Example: ThreeTierFillModel** - -This model creates a book with liquidity distributed across three price levels: - -- 50 contracts at best price -- 30 contracts one tick worse -- 20 contracts two ticks worse - -A 100-contract market order would fill partially at each level, experiencing realistic price impact. - -**Creating custom fill models:** - -```python -from nautilus_trader.backtest.models import FillModel -from nautilus_trader.model.book import OrderBook, BookOrder -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.core.rust.model import BookType - -class MyCustomFillModel(FillModel): - def get_orderbook_for_fill_simulation( - self, - instrument, - order, - best_bid, - best_ask, - ): - book = OrderBook( - instrument_id=instrument.id, - book_type=BookType.L2_MBP, - ) - - # Add custom liquidity based on your market model - # ... - - return book -``` - -### Precision requirements and invariants - -The matching engine enforces strict precision invariants to ensure data integrity throughout the fill pipeline. -All prices and quantities must match the instrument's configured precision (`price_precision` and `size_precision`). -Mismatches raise a `RuntimeError` immediately, preventing silent corruption of fill quantities. - -| Data/Operation | Field | Required Precision | Validation Location | -| -------------- | ------------------------------ | ---------------------------- | --------------------------- | -| `QuoteTick` | `bid_price`, `ask_price` | `instrument.price_precision` | `process_quote_tick` | -| `QuoteTick` | `bid_size`, `ask_size` | `instrument.size_precision` | `process_quote_tick` | -| `TradeTick` | `price` | `instrument.price_precision` | `process_trade_tick` | -| `TradeTick` | `size` | `instrument.size_precision` | `process_trade_tick` | -| `Bar` | `open`, `high`, `low`, `close` | `instrument.price_precision` | `process_bar` | -| `Bar` | `volume` (base units) | `instrument.size_precision` | `process_bar` | -| `Order` | `quantity` | `instrument.size_precision` | `process_order` | -| `Order` | `price` | `instrument.price_precision` | `process_order` | -| `Order` | `trigger_price` | `instrument.price_precision` | `process_order` | -| `Order` | `activation_price`\* | `instrument.price_precision` | `process_order` | -| Order update | `quantity` | `instrument.size_precision` | `update_order` | -| Order update | `price`, `trigger_price` | `instrument.price_precision` | `update_order` | -| Fill | `fill_qty` | `instrument.size_precision` | `apply_fills`, `fill_order` | -| Fill | `fill_px` | `instrument.price_precision` | `apply_fills` | - -\*`activation_price` is immutable after order submission. - -:::warning -`Bar.volume` must be in **base currency units**. Some data providers report quote-currency volume; -convert to base units before loading (divide by price or use provider-specific fields). -::: - -:::tip -If you encounter a precision mismatch error, align your data to the instrument: - -```python -# Align price/quantity to instrument precision -price = instrument.make_price(raw_price) -qty = instrument.make_qty(raw_qty) -``` - -Also verify that: - -1. The instrument definition matches your data source's precision. -2. Data was not inadvertently rounded or truncated during loading. -3. Custom data loaders preserve the original precision metadata. - -::: - -## Account types - -When you attach a venue to the engine, either for live trading or a backtest, you must pick one of three accounting modes by passing the `account_type` parameter: - -| Account type | Typical use-case | What the engine locks | -| ------------ | ------------------------------------------------ | ------------------------------------------------------------------------- | -| Cash | Spot trading (e.g., BTC/USDT, stocks). | Notional value for every position a pending order would open. | -| Margin | Derivatives or any product that allows leverage. | Initial margin for each order plus maintenance margin for open positions. | -| Betting | Sports betting, bookmaking. | Stake required by the venue; no leverage. | - -Example of adding a `CASH` account for a backtest venue: - -```python -from nautilus_trader.adapters.binance import BINANCE_VENUE -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.model.currencies import USDT -from nautilus_trader.model.enums import OmsType, AccountType -from nautilus_trader.model import Money, Currency - -# Initialize the backtest engine -engine = BacktestEngine() - -# Add a CASH account for the venue -engine.add_venue( - venue=BINANCE_VENUE, # Create or reference a Venue identifier - oms_type=OmsType.NETTING, - account_type=AccountType.CASH, - starting_balances=[Money(10_000, USDT)], -) -``` - -### Cash accounts - -Cash accounts settle trades in full; there is no leverage and therefore no concept of margin. - -### Margin accounts - -A *margin account* supports trading of instruments requiring margin, such as futures or leveraged products. -It tracks account balances, calculates required margins, and manages leverage to ensure sufficient collateral for positions and orders. - -**Key concepts**: - -- **Leverage**: Amplifies trading exposure relative to account equity. Higher leverage increases potential returns and risks. -- **Initial Margin**: Collateral required to submit an order to open a position. -- **Maintenance Margin**: Minimum collateral required to maintain an open position. -- **Locked Balance**: Funds reserved as collateral, unavailable for new orders or withdrawals. - -:::note -Reduce-only orders **do not** contribute to `balance_locked` in cash accounts, -nor do they add to initial margin in margin accounts, as they can only reduce existing exposure. -::: - -### Betting accounts - -Betting accounts are specialised for venues where you stake an amount to win or lose a fixed payout (some prediction markets, sports books, etc.). -The engine locks only the stake required by the venue; leverage and margin are not applicable. - -## Margin models - -NautilusTrader provides flexible margin calculation models to accommodate different venue types and trading scenarios. - -### Overview - -Different venues and brokers have varying approaches to calculating margin requirements: - -- **Traditional Brokers** (Interactive Brokers, TD Ameritrade): Fixed margin percentages regardless of leverage. -- **Crypto Exchanges** (Binance, some others): Leverage may reduce margin requirements. -- **Futures Exchanges** (CME, ICE): Fixed margin amounts per contract. - -### Available models - -#### StandardMarginModel - -Uses fixed percentages without leverage division, matching traditional broker behavior. - -**Formula:** - -```python -# Fixed percentages - leverage ignored -margin = notional * instrument.margin_init -``` - -- Initial Margin = `notional_value * instrument.margin_init` -- Maintenance Margin = `notional_value * instrument.margin_maint` - -**Use cases:** - -- Traditional brokers (Interactive Brokers, TD Ameritrade). -- Futures exchanges (CME, ICE). -- Forex brokers with fixed margin requirements. - -#### LeveragedMarginModel - -Divides margin requirements by leverage. - -**Formula:** - -```python -# Leverage reduces margin requirements -adjusted_notional = notional / leverage -margin = adjusted_notional * instrument.margin_init -``` - -- Initial Margin = `(notional_value / leverage) * instrument.margin_init` -- Maintenance Margin = `(notional_value / leverage) * instrument.margin_maint` - -**Use cases:** - -- Crypto exchanges that reduce margin with leverage. -- Venues where leverage affects margin requirements. - -### Usage - -#### Programmatic configuration - -```python -from nautilus_trader.backtest.models import LeveragedMarginModel -from nautilus_trader.backtest.models import StandardMarginModel -from nautilus_trader.test_kit.stubs.execution import TestExecStubs - -# Create account -account = TestExecStubs.margin_account() - -# Set standard model for traditional brokers -standard_model = StandardMarginModel() -account.set_margin_model(standard_model) - -# Or use leveraged model for crypto exchanges -leveraged_model = LeveragedMarginModel() -account.set_margin_model(leveraged_model) -``` - -#### Backtest configuration - -```python -from nautilus_trader.backtest.config import BacktestVenueConfig -from nautilus_trader.backtest.config import MarginModelConfig - -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="MARGIN", - starting_balances=["1_000_000 USD"], - margin_model=MarginModelConfig(model_type="standard"), # Options: 'standard', 'leveraged' -) -``` - -#### Available model types - -- `"leveraged"`: Margin reduced by leverage (default). -- `"standard"`: Fixed percentages (traditional brokers). -- Custom class path: `"my_package.my_module.MyMarginModel"`. - -#### Default behavior - -By default, `MarginAccount` uses `LeveragedMarginModel`. - -#### Real-world example - -**EUR/USD Trading Scenario:** - -- **Instrument**: EUR/USD -- **Quantity**: 100,000 EUR -- **Price**: 1.10000 -- **Notional Value**: $110,000 -- **Leverage**: 50x -- **Instrument Margin Init**: 3% - -**Margin calculations:** - -| Model | Calculation | Result | Percentage | -| --------- | ---------------------- | ------ | ---------- | -| Standard | $110,000 × 0.03 | $3,300 | 3.00% | -| Leveraged | ($110,000 ÷ 50) × 0.03 | $66 | 0.06% | - -**Account balance impact:** - -- **Account Balance**: $10,000. -- **Standard Model**: Cannot trade (requires $3,300 margin). -- **Leveraged Model**: Can trade (requires only $66 margin). - -### Real-world scenarios - -#### Interactive Brokers EUR/USD futures - -```python -# IB requires fixed margin regardless of leverage -account.set_margin_model(StandardMarginModel()) -margin = account.calculate_margin_init(instrument, quantity, price) -# Result: Fixed percentage of notional value -``` - -#### Binance crypto trading - -```python -# Binance may reduce margin with leverage -account.set_margin_model(LeveragedMarginModel()) -margin = account.calculate_margin_init(instrument, quantity, price) -# Result: Margin reduced by leverage factor -``` - -### Model selection - -#### Using the default model - -The default `LeveragedMarginModel` works out of the box: - -```python -account = TestExecStubs.margin_account() -margin = account.calculate_margin_init(instrument, quantity, price) -``` - -#### Using the standard model - -For traditional broker behavior: - -```python -account.set_margin_model(StandardMarginModel()) -margin = account.calculate_margin_init(instrument, quantity, price) -``` - -### Custom models - -You can create custom margin models by inheriting from `MarginModel`. Custom models receive configuration through the `MarginModelConfig`: - -```python -from nautilus_trader.backtest.models import MarginModel -from nautilus_trader.backtest.config import MarginModelConfig - -class RiskAdjustedMarginModel(MarginModel): - def __init__(self, config: MarginModelConfig): - """Initialize with configuration parameters.""" - self.risk_multiplier = Decimal(str(config.config.get("risk_multiplier", 1.0))) - self.use_leverage = config.config.get("use_leverage", False) - - def calculate_margin_init(self, instrument, quantity, price, leverage, use_quote_for_inverse=False): - notional = instrument.notional_value(quantity, price, use_quote_for_inverse) - if self.use_leverage: - adjusted_notional = notional.as_decimal() / leverage - else: - adjusted_notional = notional.as_decimal() - margin = adjusted_notional * instrument.margin_init * self.risk_multiplier - return Money(margin, instrument.quote_currency) - - def calculate_margin_maint(self, instrument, side, quantity, price, leverage, use_quote_for_inverse=False): - return self.calculate_margin_init(instrument, quantity, price, leverage, use_quote_for_inverse) -``` - -#### Using custom models - -**Programmatic:** - -```python -from nautilus_trader.backtest.config import MarginModelConfig -from nautilus_trader.backtest.config import MarginModelFactory - -config = MarginModelConfig( - model_type="my_package.my_module:RiskAdjustedMarginModel", - config={"risk_multiplier": 1.5, "use_leverage": False} -) - -custom_model = MarginModelFactory.create(config) -account.set_margin_model(custom_model) -``` - -### High-level backtest API configuration - -When using the high-level backtest API, you can specify margin models in your venue configuration using `MarginModelConfig`: - -```python -from nautilus_trader.backtest.config import MarginModelConfig -from nautilus_trader.backtest.config import BacktestVenueConfig -from nautilus_trader.config import BacktestRunConfig - -# Configure venue with specific margin model -venue_config = BacktestVenueConfig( - name="SIM", - oms_type="NETTING", - account_type="MARGIN", - starting_balances=["1_000_000 USD"], - margin_model=MarginModelConfig( - model_type="standard" # Use standard model for traditional broker simulation - ), -) - -# Use in backtest configuration -config = BacktestRunConfig( - venues=[venue_config], - # ... other config -) -``` - -#### Configuration examples - -**Standard model (traditional brokers):** - -```python -margin_model=MarginModelConfig(model_type="standard") -``` - -**Leveraged model (default):** - -```python -margin_model=MarginModelConfig(model_type="leveraged") # Default -``` - -**Custom model with configuration:** - -```python -margin_model=MarginModelConfig( - model_type="my_package.my_module:CustomMarginModel", - config={ - "risk_multiplier": 1.5, - "use_leverage": False, - "volatility_threshold": 0.02, - } -) -``` - -The margin model will be automatically applied to the simulated exchange during backtest execution. - -## Related guides - -- [Strategies](strategies.md) - Develop strategies to backtest. -- [Visualization](visualization.md) - Generate tearsheets from backtest results. -- [Reports](reports.md) - Analyze backtest performance data. diff --git a/nautilus-docs/docs/concepts/cache.md b/nautilus-docs/docs/concepts/cache.md deleted file mode 100644 index 438c9f3..0000000 --- a/nautilus-docs/docs/concepts/cache.md +++ /dev/null @@ -1,539 +0,0 @@ -# Cache - -The `Cache` is a central in-memory database that stores and manages all trading-related data, -from market data to order history to custom calculations. - -The Cache serves multiple purposes: - -1. **Stores market data**: - - Stores recent market history (e.g., order books, quotes, trades, bars). - - Gives you access to both current and historical market data for your strategy. - -2. **Tracks trading data**: - - Maintains complete `Order` history and current execution state. - - Tracks all `Position`s and `Account` information. - - Stores `Instrument` definitions and `Currency` information. - -3. **Stores custom data**: - - You can store any user-defined objects or data in the `Cache` for later use. - - Enables data sharing between different strategies. - -## How caching works - -**Built-in types**: - -- The system automatically adds data to the `Cache` as it flows through. -- In live contexts, the engine applies updates asynchronously, so you might see a brief delay between an event and its appearance in the `Cache`. -- All data flows through the `Cache` before reaching your strategy’s callbacks – see the diagram below: - -```mermaid -flowchart LR - data[Data] - engine[DataEngine] - cache[Cache] - callback["Strategy callback:
on_data(...)"] - - data --> engine --> cache --> callback -``` - -### Basic example - -Within a strategy, you can access the `Cache` through `self.cache`. Here’s a typical example: - -:::note -Within a `Strategy` class, `self` refers to the strategy instance. -::: - -```python -def on_bar(self, bar: Bar) -> None: - # Current bar is provided in the parameter 'bar' - - # Get historical bars from Cache - last_bar = self.cache.bar(self.bar_type, index=0) # Last bar (practically the same as the 'bar' parameter) - previous_bar = self.cache.bar(self.bar_type, index=1) # Previous bar - third_last_bar = self.cache.bar(self.bar_type, index=2) # Third last bar - - # Get current position information - if self.last_position_opened_id is not None: - position = self.cache.position(self.last_position_opened_id) - if position.is_open: - # Check position details - current_pnl = position.unrealized_pnl - - # Get all open orders for our instrument - open_orders = self.cache.orders_open(instrument_id=self.instrument_id) -``` - -## Configuration - -Use the `CacheConfig` class to configure the `Cache` behavior and capacity. -You can provide this configuration either to a `BacktestEngine` or a `TradingNode`, depending on your [environment context](architecture.md#environment-contexts). - -Here's a basic example of configuring the `Cache`: - -```python -from nautilus_trader.config import CacheConfig, BacktestEngineConfig, TradingNodeConfig - -# For backtesting -engine_config = BacktestEngineConfig( - cache=CacheConfig( - tick_capacity=10_000, # Store last 10,000 ticks per instrument - bar_capacity=5_000, # Store last 5,000 bars per bar type - ), -) - -# For live trading -node_config = TradingNodeConfig( - cache=CacheConfig( - tick_capacity=10_000, - bar_capacity=5_000, - ), -) -``` - -:::tip -By default, the `Cache` keeps the last 10,000 bars for each bar type and 10,000 trade ticks per instrument. -These limits provide a good balance between memory usage and data availability. Increase them if your strategy needs more historical data. -::: - -### Configuration options - -The `CacheConfig` class supports these parameters: - -```python -from nautilus_trader.config import CacheConfig - -cache_config = CacheConfig( - database: DatabaseConfig | None = None, # Database configuration for persistence - encoding: str = "msgpack", # Data encoding format ('msgpack' or 'json') - timestamps_as_iso8601: bool = False, # Store timestamps as ISO8601 strings - buffer_interval_ms: int | None = None, # Buffer interval for batch operations - bulk_read_batch_size: int | None = None, # Batch size for bulk reads (e.g., MGET) - use_trader_prefix: bool = True, # Use trader prefix in keys - use_instance_id: bool = False, # Include instance ID in keys - flush_on_start: bool = False, # Clear database on startup - drop_instruments_on_reset: bool = True, # Clear instruments on reset - tick_capacity: int = 10_000, # Maximum ticks stored per instrument - bar_capacity: int = 10_000, # Maximum bars stored per each bar-type -) -``` - -:::note -Each bar type maintains its own separate capacity. For example, if you're using both 1-minute and 5-minute bars, each stores up to `bar_capacity` bars. -When `bar_capacity` is reached, the `Cache` automatically removes the oldest data. -::: - -### Database configuration - -For persistence between system restarts, you can configure a database backend. - -When is it useful to use persistence? - -- **Long-running systems**: If you want your data to survive system restarts, upgrading, or unexpected failures, having a database configuration helps to pick up exactly where you left off. -- **Historical insights**: When you need to preserve past trading data for detailed post-analysis or audits. -- **Multi-node or distributed setups**: If multiple services or nodes need to access the same state, a persistent store helps ensure shared and consistent data. - -```python -from nautilus_trader.config import DatabaseConfig - -config = CacheConfig( - database=DatabaseConfig( - type="redis", # Database type - host="localhost", # Database host - port=6379, # Database port - timeout=2, # Connection timeout (seconds) - ), -) -``` - -## Using the cache - -### Accessing market data - -The `Cache` provides a full interface for accessing order books, quotes, trades, and bars. -All market data in the cache uses reverse indexing, so the most recent entry sits at index 0. - -#### Bar access - -```python -# Get a list of all cached bars for a bar type -bars = self.cache.bars(bar_type) # Returns list[Bar] or an empty list if no bars found - -# Get the most recent bar -latest_bar = self.cache.bar(bar_type) # Returns Bar or None if no such object exists - -# Get a specific historical bar by index (0 = most recent) -second_last_bar = self.cache.bar(bar_type, index=1) # Returns Bar or None if no such object exists - -# Check if bars exist and get count -bar_count = self.cache.bar_count(bar_type) # Returns number of bars in cache for the specified bar type -has_bars = self.cache.has_bars(bar_type) # Returns bool indicating if any bars exist for the specified bar type -``` - -#### Quote ticks - -```python -# Get quotes -quotes = self.cache.quote_ticks(instrument_id) # Returns list[QuoteTick] or an empty list if no quotes found -latest_quote = self.cache.quote_tick(instrument_id) # Returns QuoteTick or None if no such object exists -second_last_quote = self.cache.quote_tick(instrument_id, index=1) # Returns QuoteTick or None if no such object exists - -# Check quote availability -quote_count = self.cache.quote_tick_count(instrument_id) # Returns the number of quotes in cache for this instrument -has_quotes = self.cache.has_quote_ticks(instrument_id) # Returns bool indicating if any quotes exist for this instrument -``` - -#### Trade ticks - -```python -# Get trades -trades = self.cache.trade_ticks(instrument_id) # Returns list[TradeTick] or an empty list if no trades found -latest_trade = self.cache.trade_tick(instrument_id) # Returns TradeTick or None if no such object exists -second_last_trade = self.cache.trade_tick(instrument_id, index=1) # Returns TradeTick or None if no such object exists - -# Check trade availability -trade_count = self.cache.trade_tick_count(instrument_id) # Returns the number of trades in cache for this instrument -has_trades = self.cache.has_trade_ticks(instrument_id) # Returns bool indicating if any trades exist -``` - -#### Order book - -```python -# Get current order book -book = self.cache.order_book(instrument_id) # Returns OrderBook or None if no such object exists - -# Check if order book exists -has_book = self.cache.has_order_book(instrument_id) # Returns bool indicating if an order book exists - -# Get count of order book updates -update_count = self.cache.book_update_count(instrument_id) # Returns the number of updates received -``` - -#### Price access - -```python -from nautilus_trader.core.rust.model import PriceType - -# Get current price by type; Returns Price or None. -price = self.cache.price( - instrument_id=instrument_id, - price_type=PriceType.MID, # Options: BID, ASK, MID, LAST -) -``` - -#### Bar types - -```python -from nautilus_trader.core.rust.model import PriceType, AggregationSource - -# Get all available bar types for an instrument; Returns list[BarType]. -bar_types = self.cache.bar_types( - instrument_id=instrument_id, - price_type=PriceType.LAST, # Options: BID, ASK, MID, LAST - aggregation_source=AggregationSource.EXTERNAL, -) -``` - -#### Simple example - -```python -class MarketDataStrategy(Strategy): - def on_start(self): - # Subscribe to 1-minute bars - self.bar_type = BarType.from_str(f"{self.instrument_id}-1-MINUTE-LAST-EXTERNAL") # example of instrument_id = "EUR/USD.FXCM" - self.subscribe_bars(self.bar_type) - - def on_bar(self, bar: Bar) -> None: - bars = self.cache.bars(self.bar_type)[:3] - if len(bars) < 3: # Wait until we have at least 3 bars - return - - # Access last 3 bars for analysis - current_bar = bars[0] # Most recent bar - prev_bar = bars[1] # Second to last bar - prev_prev_bar = bars[2] # Third to last bar - - # Get latest quote and trade - latest_quote = self.cache.quote_tick(self.instrument_id) - latest_trade = self.cache.trade_tick(self.instrument_id) - - if latest_quote is not None: - current_spread = latest_quote.ask_price - latest_quote.bid_price - self.log.info(f"Current spread: {current_spread}") -``` - -### Trading objects - -The `Cache` provides access to all trading objects within the system, including: - -- Orders -- Positions -- Accounts -- Instruments - -#### Orders - -You can access and query orders through multiple methods, with flexible filtering options by venue, strategy, instrument, and order side. - -##### Basic order access - -```python -# Get a specific order by its client order ID -order = self.cache.order(ClientOrderId("O-123")) - -# Get all orders in the system -orders = self.cache.orders() - -# Get orders filtered by specific criteria -orders_for_venue = self.cache.orders(venue=venue) # All orders for a specific venue -orders_for_strategy = self.cache.orders(strategy_id=strategy_id) # All orders for a specific strategy -orders_for_instrument = self.cache.orders(instrument_id=instrument_id) # All orders for an instrument -``` - -##### Order state queries - -```python -# Get orders by their current state -open_orders = self.cache.orders_open() # Orders currently active at the venue -closed_orders = self.cache.orders_closed() # Orders that have completed their lifecycle -emulated_orders = self.cache.orders_emulated() # Orders being simulated locally by the system -inflight_orders = self.cache.orders_inflight() # Orders submitted (or modified) to venue, but not yet confirmed - -# Check specific order states -exists = self.cache.order_exists(client_order_id) # Checks if an order with the given ID exists in the cache -is_open = self.cache.is_order_open(client_order_id) # Checks if an order is currently open -is_closed = self.cache.is_order_closed(client_order_id) # Checks if an order is closed -is_emulated = self.cache.is_order_emulated(client_order_id) # Checks if an order is being simulated locally -is_inflight = self.cache.is_order_inflight(client_order_id) # Checks if an order is submitted or modified, but not yet confirmed -``` - -##### Order statistics - -```python -# Get counts of orders in different states -open_count = self.cache.orders_open_count() # Number of open orders -closed_count = self.cache.orders_closed_count() # Number of closed orders -emulated_count = self.cache.orders_emulated_count() # Number of emulated orders -inflight_count = self.cache.orders_inflight_count() # Number of inflight orders -total_count = self.cache.orders_total_count() # Total number of orders in the system - -# Get filtered order counts -buy_orders_count = self.cache.orders_open_count(side=OrderSide.BUY) # Number of currently open BUY orders -venue_orders_count = self.cache.orders_total_count(venue=venue) # Total number of orders for a given venue -``` - -#### Positions - -The `Cache` maintains a record of all positions and offers several ways to query them. - -##### Position access - -```python -# Get a specific position by its ID -position = self.cache.position(PositionId("P-123")) - -# Get positions by their state -all_positions = self.cache.positions() # All positions in the system -open_positions = self.cache.positions_open() # All currently open positions -closed_positions = self.cache.positions_closed() # All closed positions - -# Get positions filtered by various criteria -venue_positions = self.cache.positions(venue=venue) # Positions for a specific venue -instrument_positions = self.cache.positions(instrument_id=instrument_id) # Positions for a specific instrument -strategy_positions = self.cache.positions(strategy_id=strategy_id) # Positions for a specific strategy -long_positions = self.cache.positions(side=PositionSide.LONG) # All long positions -``` - -##### Position state queries - -```python -# Check position states -exists = self.cache.position_exists(position_id) # Checks if a position with the given ID exists -is_open = self.cache.is_position_open(position_id) # Checks if a position is open -is_closed = self.cache.is_position_closed(position_id) # Checks if a position is closed - -# Get position and order relationships -orders = self.cache.orders_for_position(position_id) # All orders related to a specific position -position = self.cache.position_for_order(client_order_id) # Find the position associated with a specific order -``` - -##### Position statistics - -```python -# Get position counts in different states -open_count = self.cache.positions_open_count() # Number of currently open positions -closed_count = self.cache.positions_closed_count() # Number of closed positions -total_count = self.cache.positions_total_count() # Total number of positions in the system - -# Get filtered position counts -long_positions_count = self.cache.positions_open_count(side=PositionSide.LONG) # Number of open long positions -instrument_positions_count = self.cache.positions_total_count(instrument_id=instrument_id) # Number of positions for a given instrument -``` - -#### Accounts - -```python -# Access account information -account = self.cache.account(account_id) # Retrieve account by ID -account = self.cache.account_for_venue(venue) # Retrieve account for a specific venue -account_id = self.cache.account_id(venue) # Retrieve account ID for a venue -accounts = self.cache.accounts() # Retrieve all accounts in the cache -``` - -#### Purging cached state - -The cache exposes explicit maintenance hooks that remove closed or stale objects while preserving safety checks: - -- `purge_closed_orders(ts_now, buffer_secs=0, purge_from_database=False)` drops closed orders that have been inactive for at least `buffer_secs`. Linked contingency orders remain until every dependent child is closed. -- `purge_closed_positions(ts_now, buffer_secs=0, purge_from_database=False)` removes positions that have stayed closed beyond the buffer window and deletes associated indices. -- `purge_account_events(ts_now, lookback_secs=0, purge_from_database=False)` trims account event history outside the lookback window and can cascade deletes to the backing database. - -Key safeguards: - -- Open orders and positions are never purged; the cache logs a warning and leaves the item intact. -- Linked orders keep parents in the cache until all children have closed, preventing premature removal of contingency chains. -- Indices and reverse lookups are cleaned alongside the primary object to avoid dangling references. -- Database deletions occur only when `purge_from_database=True` and a cache database is configured, ensuring in-memory purges do not silently erase persisted data. - -Use the trading clock (for example, `self.clock.timestamp_ns()`) when supplying `ts_now`. Set `purge_from_database=True` only when you intend to delete persisted records from Redis or PostgreSQL as well. In live trading these methods run automatically when the execution engine is configured with purge intervals; see [Memory management](live.md#memory-management) for the scheduler settings. - -#### Instruments and currencies - -##### Instruments - -```python -# Get instrument information -instrument = self.cache.instrument(instrument_id) # Retrieve a specific instrument by its ID -all_instruments = self.cache.instruments() # Retrieve all instruments in the cache - -# Get filtered instruments -venue_instruments = self.cache.instruments(venue=venue) # Instruments for a specific venue -instruments_by_underlying = self.cache.instruments(underlying="ES") # Instruments by underlying - -# Get instrument identifiers -instrument_ids = self.cache.instrument_ids() # Get all instrument IDs -venue_instrument_ids = self.cache.instrument_ids(venue=venue) # Get instrument IDs for a specific venue -``` - -##### Currencies - -```python -# Get currency information -currency = self.cache.load_currency("USD") # Loads currency data for USD -``` - ---- - -### Custom data - -The `Cache` can also store and retrieve custom data types in addition to built-in market data and trading objects. -Use it to share any user-defined data between system components, primarily actors and strategies. - -#### Basic storage and retrieval - -```python -# Call this code inside Strategy methods (`self` refers to Strategy) - -# Store data -self.cache.add(key="my_key", value=b"some binary data") - -# Retrieve data -stored_data = self.cache.get("my_key") # Returns bytes or None -``` - -For more complex use cases, the `Cache` can store custom data objects that inherit from the `nautilus_trader.core.Data` base class. - -:::warning -The `Cache` is not designed to be a full database replacement. For large datasets or complex querying needs, consider using a dedicated database system. -::: - -## Best practices and common questions - -### Cache vs. portfolio usage - -The `Cache` and `Portfolio` components serve different but complementary purposes in NautilusTrader: - -**Cache**: - -- Maintains the historical knowledge and current state of the trading system. -- Updates immediately when local state changes (for example, initializing an order before submission). -- Updates asynchronously as external events occur (for example, when an order fills). -- Provides a complete history of trading activity and market data. -- Keeps every event the strategy receives in the cache. - -**Portfolio**: - -- Aggregates position, exposure, and account information. -- Provides current state without history. - -**Example**: - -```python -class MyStrategy(Strategy): - def on_position_changed(self, event: PositionEvent) -> None: - # Use Cache when you need historical perspective - position_history = self.cache.position_snapshots(event.position_id) - - # Use Portfolio when you need current real-time state - current_exposure = self.portfolio.net_exposure(event.instrument_id) -``` - -### Cache vs. strategy variables - -Choosing between storing data in the `Cache` versus strategy variables depends on your specific needs: - -**Cache storage**: - -- Use for data that needs to be shared between strategies. -- Best for data that needs to persist between system restarts. -- Acts as a central database accessible to all components. -- Ideal for state that needs to survive strategy resets. - -**Strategy variables**: - -- Use for strategy-specific calculations. -- Better for temporary values and intermediate results. -- Provides faster access and better encapsulation. -- Best for data that only your strategy needs. - -**Example**: - -The following example shows how you might store data in the `Cache` so multiple strategies can access the same information. - -```python -import pickle - -class MyStrategy(Strategy): - def on_start(self): - # Prepare data you want to share with other strategies - shared_data = { - "last_reset": self.clock.timestamp_ns(), - "trading_enabled": True, - # Include any other fields that you want other strategies to read - } - - # Store it in the cache with a descriptive key - # This way, multiple strategies can call self.cache.get("shared_strategy_info") - # to retrieve the same data - self.cache.add("shared_strategy_info", pickle.dumps(shared_data)) -``` - -Another strategy can retrieve the cached data as follows: - -```python -import pickle - -class AnotherStrategy(Strategy): - def on_start(self): - # Load the shared data from the same key - data_bytes = self.cache.get("shared_strategy_info") - if data_bytes is not None: - shared_data = pickle.loads(data_bytes) - self.log.info(f"Shared data retrieved: {shared_data}") -``` - -## Related guides - -- [Data](data.md) - Data types stored in the cache. -- [Strategies](strategies.md) - Strategies access cache for market data and state. -- [Reports](reports.md) - Generate reports from cached data. diff --git a/nautilus-docs/docs/concepts/custom_data.md b/nautilus-docs/docs/concepts/custom_data.md deleted file mode 100644 index f92c7c4..0000000 --- a/nautilus-docs/docs/concepts/custom_data.md +++ /dev/null @@ -1,396 +0,0 @@ -# Custom Data - -Nautilus Trader supports custom data authored in Python and Rust, and moves -that data through the same runtime, persistence, and query pipeline used -by the rest of the platform. - -This document explains how custom data is: - -- Registered at runtime. -- Wrapped across the Python/Rust boundary. -- Serialized to and from Arrow/Parquet. -- Routed through actors and strategies. - -## Goals - -The custom-data architecture satisfies the following requirements: - -- Let users define custom data in pure Python without writing Rust code. -- Let Rust-defined custom data use native Rust JSON and Arrow handlers. -- Preserve a single user-facing `CustomData` wrapper at the PyO3 boundary. -- Support persistence in `ParquetDataCatalogV2` using dynamic type registration - instead of hardcoded schemas. -- Make custom data routable through the normal data-engine, actor, and strategy - subscription flow. - -## High-level model - -There are two supported authoring modes: - -| Mode | Example | Registration path | Encode/decode path | Wrapper backend | -|-------------------|----------------------------------------------------|---------------------------------------------------------|---------------------------------|---------------------------| -| Pure Python | `@customdataclass_pyo3` class | `register_custom_data_class(...)` | Python callback + Arrow C FFI | `PythonCustomDataWrapper` | -| Same-binary Rust | `#[custom_data]` or `#[custom_data(pyo3)]` type | `ensure_custom_data_registered::()` and native extractor | Native Rust | Native Rust payload | - -Both modes converge on the same outer PyO3 `CustomData` wrapper and the same -`DataType` identity model. - -## End-to-end flow - -```mermaid -sequenceDiagram - participant U as User code - participant P as Python layer - participant R as Rust model/catalog - participant G as Global DataRegistry - participant S as Storage - - U->>P: define class/type - U->>P: register_custom_data_class(...) or module init - P->>R: install type registration - R->>G: store JSON/Arrow/extractor handlers - - U->>P: CustomData(data_type, data) - P->>R: write_custom_data([...]) - R->>G: lookup encoder by type_name - G-->>R: encoder - R->>S: write RecordBatch to Parquet - - U->>P: query(type_name, ...) - P->>R: query catalog - R->>S: read RecordBatch + metadata - R->>G: lookup decoder by type_name - G-->>R: decoder - R-->>P: CustomData wrappers - P-->>U: typed data via .data -``` - -## Core components - -### `DataRegistry` - -`crates/model/src/data/registry.rs` is the central runtime registry module for -custom data in the main process. Registration uses atomic `DashMap::entry()` so -that concurrent `register_*` and `ensure_*` calls do not race. - -The module contains several `OnceLock`-initialized `DashMap` singletons: - -- JSON deserializers keyed by `type_name`. -- Arrow schemas, encoders, and decoders keyed by `type_name`. -- Python extractors that convert a Python object into - `Arc`. -- Rust extractor factories that produce Python extractors for same-binary types. - -Instead of hardcoding every type into the main binary, Nautilus resolves -handlers at runtime using the `type_name` stored in `DataType` and Parquet -metadata. - -### `CustomData` - -The outer PyO3 `CustomData` wrapper is the common container that crosses the -FFI boundary. - -Constructor signature: `CustomData(data_type, data)` -- the `DataType` comes -first, then the inner payload. - -It contains: - -- A `DataType`. -- An inner custom payload implementing `CustomDataTrait` (wrapped in - `Arc`). - -Timestamps (`ts_event`, `ts_init`) are delegated to the inner -`CustomDataTrait` implementation and exposed as properties on the wrapper. - -On the Python side, `CustomData` exposes value semantics: `__eq__` and -`__repr__` are implemented (equality uses the Rust `PartialEq` logic). -Instances are intentionally unhashable so that equality remains consistent with -the inner payload comparison. - -This wrapper is shared across both custom-data modes. User code interacts with -one API even though the underlying payload may be: - -- A Python-backed wrapper. -- A same-binary Rust value. - -#### `CustomData` JSON envelope - -When serialized to JSON (e.g. for `to_json_bytes` / `from_json_bytes`, SQL -cache, or Redis), `CustomData` uses a single canonical envelope so that -deserialization does not depend on user payload field names: - -- `type`: The custom type name (from `CustomDataTrait::type_name`). -- `data_type`: An object with `type_name`, `metadata`, and optional - `identifier`. -- `payload`: The inner payload only (the result of `CustomDataTrait::to_json` - parsed as a value). Registered deserializers receive only this value in - `from_json`, so user structs can use any field names (including `value`) - without conflicting with wrapper metadata. - -This envelope is produced by Rust `CustomData` serialization and consumed by -`DataRegistry` when deserializing custom data from JSON. - -### `DataType` - -`DataType` identifies custom data for routing and persistence. - -Constructor: `DataType(type_name, metadata=None, identifier=None)`. - -It includes: - -- `type_name`. -- Optional `metadata`. -- Optional `identifier` (used only for catalog pathing, not for routing or - equality). - -Equality, hashing, and topic routing are derived from `type_name` and -`metadata` only. Two `DataType` values with the same type name and metadata but -different identifiers compare equal and publish to the same message bus topic. -The `identifier` affects only the storage path under -`data/custom//`. - -Custom-data storage and queries use `DataType`, not just the bare Rust/Python -class name. This allows the same logical type to be stored under different -metadata or identifiers while still decoding through the same registered -handler. - -## Registration architecture - -Registration bridges the gap between Python objects and Rust trait objects. - -```mermaid -flowchart TD - A[User-defined custom type] --> B{Mode} - B --> C[Pure Python] - B --> D[Same-binary Rust] - - C --> F[register_custom_data_class] - D --> G[ensure_custom_data_registered and native extractor] - - F --> I[Python callbacks registered] - G --> J[Native JSON and Arrow handlers registered] - - I --> L[Main-process DataRegistry] - J --> L -``` - -### Pure Python registration - -When Python code calls `register_custom_data_class(MyType)`: - -1. The type is registered in the Python serialization layer for JSON and Arrow - support. -2. Rust registers a Python extractor that wraps Python instances as - `PythonCustomDataWrapper`. -3. Rust registers Arrow schema/encode/decode callbacks in `DataRegistry`. - -This path is flexible and user-friendly, but Arrow encoding and reconstruction -rely on Python callbacks. - -### Same-binary Rust registration - -For Rust types defined inside Nautilus: - -1. `#[custom_data]` or `#[custom_data(pyo3)]` generates the necessary trait, - JSON, and Arrow implementations. -2. `ensure_custom_data_registered::()` inserts native schema/encoder/decoder - handlers into `DataRegistry`. -3. For PyO3-exposed types, a native extractor can convert Python instances back - into the concrete Rust type rather than a Python fallback wrapper. - -This path stays fully native in Rust for encode/decode. - -### Registration precedence - -`register_custom_data_class(...)` resolves types in the following order: - -1. Same-binary native Rust registration. -2. Pure Python fallback registration. - -That ordering preserves the fastest available path for types already known -natively by the main binary. - -## Wrapper backends - -Internally, the outer `CustomData` wrapper can hold different payload -implementations. - -### `PythonCustomDataWrapper` - -Used for pure Python custom data. - -Responsibilities: - -- Stores a reference to the Python object. -- Caches `ts_event`, `ts_init`, and `type_name`. -- Implements `CustomDataTrait`. -- Calls Python methods for JSON and Arrow-related operations under the GIL. - -This is the fallback path when the main process does not have a native Rust -representation for the type. - -### Native same-binary Rust payload - -For Rust types compiled into Nautilus, the inner payload is the concrete Rust -type itself and can be downcast directly from `Arc`. - -No Python callback path is needed for serialization or decode. - -## Persistence architecture - -### Why dynamic Arrow registration is needed - -Built-in Nautilus data types have schemas and encoders known statically to the -Rust binary. Custom data does not. The persistence layer therefore resolves -custom data dynamically using the registered `type_name`. - -### Catalog write flow - -`ParquetDataCatalogV2` expects custom writes to come in as `CustomData` values. - -The custom-data write path: - -1. Extracts `type_name`, `metadata`, and `identifier` from `DataType`. -2. Looks up the Arrow encoder in `DataRegistry`. -3. Encodes the values to a `RecordBatch`. -4. Appends a `data_type` column containing the persisted `DataType`. -5. Attaches `type_name` and metadata to the Arrow schema. -6. Writes the batch to Parquet under the custom-data path. - -The path layout is: - -- `data/custom//` - -Identifiers are normalized before becoming path segments. - -### Catalog read flow - -On query: - -1. The catalog reads matching Parquet files. -2. Extracts `type_name` from schema metadata. -3. Asks `DataRegistry` for the registered decoder. -4. Decodes the `RecordBatch` into `Vec`. -5. Reconstructs `CustomData` with the original `DataType`. - -This makes custom-data query resolution symmetric with write-time registration. -When converting a Feather stream to Parquet (e.g. after a backtest), the -custom-data branch decodes batches and writes them via -`write_custom_data_batch` so that custom data written through the Feather -writer is correctly converted to Parquet. - -## The Arrow C FFI bridge - -Pure Python custom data cannot provide native Rust Arrow encode logic directly. -For those types, Nautilus uses the Arrow C FFI interface to pass `RecordBatch` -data between Python and Rust without serialization overhead. - -```mermaid -sequenceDiagram - participant R as Rust encoder - participant P as Python custom class - participant F as Arrow C FFI structs - participant C as Parquet writer - - R->>P: encode_record_batch_py(items) - P->>P: build pyarrow.RecordBatch - P-->>F: _export_to_c (FFI_ArrowArray + FFI_ArrowSchema) - F-->>R: reconstruct native RecordBatch - R->>C: write Parquet -``` - -### Pure Python encode path - -For pure Python classes: - -1. Rust acquires the GIL. -2. Rust calls `encode_record_batch_py(...)` on the Python class. -3. Python converts objects to a `pyarrow.RecordBatch`. -4. Python exports the batch via `_export_to_c` into Arrow C FFI structs. -5. Rust reconstructs a native `RecordBatch` from the FFI structs and writes it. - -### Pure Python decode path - -For the reverse direction: - -1. Rust converts its `RecordBatch` into Arrow C FFI structs. -2. Python imports the batch via `RecordBatch._import_from_c`. -3. Python calls `decode_record_batch_py(metadata, batch)` on the class. -4. Rust wraps the returned Python objects in `PythonCustomDataWrapper`. - -### Native paths - -The Arrow C FFI bridge is not used for same-binary Rust custom data. Those -types use native Rust encode/decode handlers registered in the main process. - -## Reconstruction on query - -When custom data is loaded back from the catalog, reconstruction depends on the -backend: - -- Same-binary Rust types decode directly to native Rust values. -- Pure Python types reconstruct through the registered Python class using - `from_dict` or `from_json`. - -In all cases the caller receives the same outer `CustomData` wrapper at the -PyO3 API boundary. - -## Runtime integration - -Custom data is not only a persistence feature. It also participates in Nautilus -runtime routing. - -Relevant integrations include: - -- `crates/data/src/engine/mod.rs` publishes `CustomData` through the message - bus. -- `crates/common/src/msgbus/switchboard.rs` derives custom topics from - `DataType`. -- `crates/common/src/actor/*` routes custom data into actor subscriptions. -- `crates/trading/src/python/strategy.rs` exposes custom data to Python - strategy `on_data`. -- `crates/backtest/src/engine.rs` treats `Data::Custom` as - data-engine-delivered input rather than exchange-routed data. - -A registered custom type can be persisted, queried, subscribed to, and consumed -through the same runtime interfaces as other data families. - -## SQL cache and database integration - -The SQL cache/database layer also supports `CustomData`. - -Current behavior: - -- PostgreSQL stores custom data in the `custom` table. -- The stored record includes `data_type`, `metadata`, `identifier`, and full - JSON payload. -- Reads reconstruct `CustomData` using `CustomData::from_json_bytes(...)`. -- Python SQL bindings expose `add_custom_data` and `load_custom_data`. -- Redis cache stores custom data under keys - `custom::` with full `CustomData` JSON as value. -- Redis `add_custom_data` and `load_custom_data` filter by `DataType` - (type_name, metadata, identifier) and return results sorted by `ts_init`; - this is exposed via the PyO3 `RedisCacheDatabase` API. - -## Relationship to legacy Cython custom data - -Legacy Cython `@customdataclass` remains separate from this architecture. - -This document describes the PyO3 custom-data system: - -- PyO3 `CustomData`. -- Dynamic runtime registration. -- Arrow/Parquet persistence. -- Native Rust execution paths. - -Legacy Cython support is intentionally left unchanged. - -## Practical implications - -This architecture gives Nautilus two important properties: - -1. Python-first extensibility for users who only want to write Python. -2. Native Rust performance for built-in or compiled custom types. - -The result is one conceptual custom-data system with two backends, rather than -separate feature silos for Python-only and Rust-only data types. diff --git a/nautilus-docs/docs/concepts/data.md b/nautilus-docs/docs/concepts/data.md deleted file mode 100644 index c286f27..0000000 --- a/nautilus-docs/docs/concepts/data.md +++ /dev/null @@ -1,1892 +0,0 @@ -# Data - -NautilusTrader provides built-in data types for the trading domain: - -- `OrderBookDelta` (L1/L2/L3): Represents the most granular order book updates. -- `OrderBookDeltas` (L1/L2/L3): Batches multiple order book deltas for more efficient processing. -- `OrderBookDepth10`: Aggregated order book snapshot (up to 10 levels per bid and ask side). -- `QuoteTick`: Represents the best bid and ask prices along with their sizes at the top-of-book. -- `TradeTick`: A single trade/match event between counterparties. -- `Bar`: OHLCV (Open, High, Low, Close, Volume) bar/candle, aggregated using a specified *aggregation method*. -- `MarkPriceUpdate`: The current mark price for an instrument (typically used in derivatives trading). -- `IndexPriceUpdate`: The index price for an instrument (underlying price used for mark price calculations). -- `FundingRateUpdate`: The funding rate for perpetual contracts (periodic payments between long and short positions). -- `InstrumentStatus`: An instrument-level status event. -- `InstrumentClose`: The closing price of an instrument. - -NautilusTrader operates primarily on granular order book data for the highest realism -in execution simulations. Backtests can also run on any supported market data type, -depending on the desired simulation fidelity. - -## Order books - -A high-performance order book implemented in Rust is available to maintain order book state based on provided data. - -`OrderBook` instances are maintained per instrument for both backtesting and live trading, with the following book types available: - -- `L3_MBO`: **Market by order (MBO)** or L3 data, uses every order book event at every price level, keyed by order ID. -- `L2_MBP`: **Market by price (MBP)** or L2 data, aggregates order book events by price level. -- `L1_MBP`: **Market by price (MBP)** or L1 data, also known as best bid and offer (BBO), captures only top-level updates. - -:::note -Top-of-book data, such as `QuoteTick`, `TradeTick` and `Bar`, can also be used for backtesting, with markets operating on `L1_MBP` book types. -::: - -### Delta flags and event boundaries - -Each `OrderBookDelta` carries a `flags` field using `RecordFlag` bitmask values -to signal event boundaries to the `DataEngine`: - -- `F_LAST`: Marks the final delta in a logical event group. When `buffer_deltas` - is enabled, the `DataEngine` accumulates deltas and only publishes to - subscribers when it encounters `F_LAST`. Every event group **must** end with - a delta that has `F_LAST` set. -- `F_SNAPSHOT`: Marks deltas that belong to a snapshot (as opposed to an - incremental update). Snapshot sequences begin with a `Clear` action followed - by `Add` deltas reconstructing the full book state. The last delta in a - snapshot has both `F_SNAPSHOT | F_LAST` set. - -:::warning -A missing `F_LAST` on the final delta in an event group causes buffered consumers -to accumulate deltas indefinitely without publishing. This applies to incremental -updates and snapshots alike, including empty book snapshots where only a `Clear` -delta is emitted. -::: - -## Instruments - -NautilusTrader supports a variety of instrument types across spot, derivatives, and specialty markets: - -```mermaid -flowchart TD - I[Instrument Types] - I --> Spot - I --> Derivatives - I --> Other - - Spot --> Equity - Spot --> CurrencyPair - Spot --> Commodity - Spot --> IndexInstrument - - Derivatives --> Futures - Derivatives --> Options - Derivatives --> Cfd - - Futures --> FuturesContract - Futures --> FuturesSpread - Futures --> CryptoFuture - Futures --> CryptoPerpetual - Futures --> PerpetualContract - - Options --> OptionContract - Options --> OptionSpread - Options --> CryptoOption - Options --> BinaryOption - - Other --> BettingInstrument - Other --> SyntheticInstrument -``` - -| Instrument | Description | -|----------------------|----------------------------------------------------------------------------------| -| `Equity` | Generic equity instrument. | -| `CurrencyPair` | Currency pair in a spot/cash market. | -| `Commodity` | Commodity in a spot/cash market. | -| `IndexInstrument` | Spot index (reference price, not directly tradable). | -| `FuturesContract` | Generic deliverable futures contract. | -| `FuturesSpread` | Deliverable futures spread. | -| `CryptoFuture` | Deliverable futures with crypto assets as underlying and settlement. | -| `CryptoPerpetual` | Crypto perpetual futures (perpetual swap). | -| `PerpetualContract` | Asset-class agnostic perpetual swap (any underlying). | -| `OptionContract` | Generic option contract. | -| `OptionSpread` | Generic option spread. | -| `CryptoOption` | Crypto option contract. | -| `BinaryOption` | Binary option instrument. | -| `Cfd` | Contract for Difference (CFD). | -| `BettingInstrument` | Instrument in a betting market. | -| `SyntheticInstrument`| Synthetic instrument with prices derived from component instruments via formula. | - -## Bars and aggregation - -### Introduction to bars - -A *bar* (also known as a candle, candlestick or kline) is a data structure that represents -price and volume information over a specific period, including: - -- Opening price -- Highest price -- Lowest price -- Closing price -- Traded volume (or ticks as a volume proxy) - -The system generates bars using an *aggregation method* that groups data by specific criteria. - -### Purpose of data aggregation - -Data aggregation in NautilusTrader transforms granular market data into structured bars or candles for several reasons: - -- To provide data for technical indicators and strategy development. -- Because time-aggregated data (like minute bars) are often sufficient for many strategies. -- To reduce costs compared to high-frequency L1/L2/L3 market data. - -### Aggregation methods - -The platform implements various aggregation methods: - -| Name | Description | Category | -|:-------------------|:---------------------------------------------------------------------------|:-------------| -| `TICK` | Aggregation of a number of ticks. | Threshold | -| `TICK_IMBALANCE` | Aggregation of the buy/sell imbalance of ticks. | Threshold | -| `TICK_RUNS` | Aggregation of sequential buy/sell runs of ticks. | Information | -| `VOLUME` | Aggregation of traded volume. | Threshold | -| `VOLUME_IMBALANCE` | Aggregation of the buy/sell imbalance of traded volume. | Threshold | -| `VOLUME_RUNS` | Aggregation of sequential runs of buy/sell traded volume. | Information | -| `VALUE` | Aggregation of the notional value of trades (also known as "Dollar bars"). | Threshold | -| `VALUE_IMBALANCE` | Aggregation of the buy/sell imbalance of trading by notional value. | Threshold | -| `VALUE_RUNS` | Aggregation of sequential buy/sell runs of trading by notional value. | Information | -| `RENKO` | Aggregation based on fixed price movements (brick size in ticks). | Threshold | -| `MILLISECOND` | Aggregation of time intervals with millisecond granularity. | Time | -| `SECOND` | Aggregation of time intervals with second granularity. | Time | -| `MINUTE` | Aggregation of time intervals with minute granularity. | Time | -| `HOUR` | Aggregation of time intervals with hour granularity. | Time | -| `DAY` | Aggregation of time intervals with day granularity. | Time | -| `WEEK` | Aggregation of time intervals with week granularity. | Time | -| `MONTH` | Aggregation of time intervals with month granularity. | Time | -| `YEAR` | Aggregation of time intervals with year granularity. | Time | - -### Information-driven bars - -Information-driven bars adapt their sampling frequency to market activity rather than using fixed -intervals. They are based on the concept of *aggressor side* (whether the trade initiator was a -buyer or seller) and come in two families: **imbalance** and **runs**. - -**Imbalance bars** close when the *net* buy/sell activity reaches a threshold. Each trade contributes -a signed value: positive for buyer-initiated trades and negative for seller-initiated. The bar closes -when the absolute imbalance reaches the configured step. This means that opposing trades cancel each -other out, so imbalance bars tend to form more slowly in balanced markets and faster during directional moves. - -**Runs bars** close when *consecutive* activity from the same aggressor side reaches a threshold. -Unlike imbalance bars, runs bars reset their counter when the aggressor side changes. -This makes them sensitive to sustained one-sided pressure rather than net imbalance. - -Both families have three variants based on what is measured: - -| Variant | Imbalance | Runs | What is measured | -|:--------|:-------------------|:--------------|:------------------------------------------| -| Tick | `TICK_IMBALANCE` | `TICK_RUNS` | Number of trades (each trade counts as 1) | -| Volume | `VOLUME_IMBALANCE` | `VOLUME_RUNS` | Traded volume (quantity) | -| Value | `VALUE_IMBALANCE` | `VALUE_RUNS` | Notional value (price x quantity) | - -:::note -Information-driven bars require `TradeTick` data because they need the `aggressor_side` field -to classify each trade. They cannot be aggregated from `QuoteTick` data alone. -::: - -### Types of aggregation - -NautilusTrader implements three distinct data aggregation methods: - -1. **Trade-to-bar aggregation**: Creates bars from `TradeTick` objects (executed trades) - - Use case: For strategies analyzing execution prices or when working directly with trade data. - - Always uses the `LAST` price type in the bar specification. - -2. **Quote-to-bar aggregation**: Creates bars from `QuoteTick` objects (bid/ask prices) - - Use case: For strategies focusing on bid/ask spreads or market depth analysis. - - Uses `BID`, `ASK`, or `MID` price types in the bar specification. - -3. **Bar-to-bar aggregation**: Creates larger-timeframe `Bar` objects from smaller-timeframe `Bar` objects - - Use case: For resampling existing smaller timeframe bars (1-minute) into larger timeframes (5-minute, hourly). - - Always requires the `@` symbol in the specification. - -### Bar types - -NautilusTrader defines a unique *bar type* (`BarType` class) based on the following components: - -- **Instrument ID** (`InstrumentId`): Specifies the particular instrument for the bar. -- **Bar Specification** (`BarSpecification`): - - `step`: Defines the interval or frequency of each bar. - - `aggregation`: Specifies the method used for data aggregation (see the above table). - - `price_type`: Indicates the price basis of the bar (e.g., bid, ask, mid, last). -- **Aggregation Source** (`AggregationSource`): Indicates whether the bar was aggregated internally (within Nautilus). -- or externally (by a trading venue or data provider). - -Bar types can also be classified as either *standard* or *composite*: - -- **Standard**: Generated from granular market data, such as quote-ticks or trade-ticks. -- **Composite**: Derived from a higher-granularity bar type through subsampling (like 5-MINUTE bars aggregate from 1-MINUTE bars). - -### Aggregation sources - -Bar data aggregation can be either *internal* or *external*: - -- `INTERNAL`: The bar is aggregated inside the local Nautilus system boundary. -- `EXTERNAL`: The bar is aggregated outside the local Nautilus system boundary (typically by a trading venue or data provider). - -For bar-to-bar aggregation, the target bar type is always `INTERNAL` (since you're doing the aggregation within NautilusTrader), -but the source bars can be either `INTERNAL` or `EXTERNAL`, i.e., you can aggregate externally provided bars or already -aggregated internal bars. - -### Defining bar types with *string syntax* - -#### Standard bars - -You can define standard bar types from strings using the following convention: - -`{instrument_id}-{step}-{aggregation}-{price_type}-{INTERNAL | EXTERNAL}` - -For example, to define a `BarType` for AAPL trades (last price) on Nasdaq (XNAS) using a 5-minute interval -aggregated from trades locally by Nautilus: - -```python -bar_type = BarType.from_str("AAPL.XNAS-5-MINUTE-LAST-INTERNAL") -``` - -#### Composite bars - -Composite bars are derived by aggregating higher-granularity bars into the desired bar type. To define a composite bar, -use this convention: - -`{instrument_id}-{step}-{aggregation}-{price_type}-INTERNAL@{step}-{aggregation}-{INTERNAL | EXTERNAL}` - -**Notes**: - -- The derived bar type must use an `INTERNAL` aggregation source (since this is how the bar is aggregated). -- The sampled bar type must have a higher granularity than the derived bar type. -- The sampled instrument ID is inferred to match that of the derived bar type. -- Composite bars can be aggregated *from* `INTERNAL` or `EXTERNAL` aggregation sources. - -For example, to define a `BarType` for AAPL trades (last price) on Nasdaq (XNAS) using a 5-minute interval -aggregated locally by Nautilus, from 1-minute interval bars aggregated externally: - -```python -bar_type = BarType.from_str("AAPL.XNAS-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL") -``` - -### Aggregation syntax examples - -The `BarType` string format encodes both the target bar type and, optionally, the source data type: - -``` -{instrument_id}-{step}-{aggregation}-{price_type}-{source}@{step}-{aggregation}-{source} -``` - -The part after the `@` symbol is optional and only used for bar-to-bar aggregation: - -- **Without `@`**: Aggregates from `TradeTick` objects (when price_type is `LAST`) or `QuoteTick` objects (when price_type is `BID`, `ASK`, or `MID`). -- **With `@`**: Aggregates from existing `Bar` objects (specifying the source bar type). - -#### Trade-to-bar example - -```python -def on_start(self) -> None: - # Define a bar type for aggregating from TradeTick objects - # Uses price_type=LAST which indicates TradeTick data as source - bar_type = BarType.from_str("6EH4.XCME-50-VOLUME-LAST-INTERNAL") - - # Request historical data (will receive bars in on_historical_data handler) - self.request_bars(bar_type) - - # Subscribe to live data (will receive bars in on_bar handler) - self.subscribe_bars(bar_type) -``` - -#### Quote-to-bar example - -```python -def on_start(self) -> None: - # Create 1-minute bars from ASK prices (in QuoteTick objects) - bar_type_ask = BarType.from_str("6EH4.XCME-1-MINUTE-ASK-INTERNAL") - - # Create 1-minute bars from BID prices (in QuoteTick objects) - bar_type_bid = BarType.from_str("6EH4.XCME-1-MINUTE-BID-INTERNAL") - - # Create 1-minute bars from MID prices (middle between ASK and BID prices in QuoteTick objects) - bar_type_mid = BarType.from_str("6EH4.XCME-1-MINUTE-MID-INTERNAL") - - # Request historical data and subscribe to live data - self.request_bars(bar_type_ask) # Historical bars processed in on_historical_data - self.subscribe_bars(bar_type_ask) # Live bars processed in on_bar -``` - -#### Bar-to-bar example - -```python -def on_start(self) -> None: - # Create 5-minute bars from 1-minute bars (Bar objects) - # Format: target_bar_type@source_bar_type - # Note: price type (LAST) is only needed on the left target side, not on the source side - bar_type = BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL") - - # Request historical data (processed in on_historical_data(...) handler) - self.request_bars(bar_type) - - # Subscribe to live updates (processed in on_bar(...) handler) - self.subscribe_bars(bar_type) -``` - -#### Advanced bar-to-bar example - -You can create complex aggregation chains where you aggregate from already aggregated bars: - -```python -# First create 1-minute bars from TradeTick objects (LAST indicates TradeTick source) -primary_bar_type = BarType.from_str("6EH4.XCME-1-MINUTE-LAST-INTERNAL") - -# Then create 5-minute bars from 1-minute bars -# Note the @1-MINUTE-INTERNAL part identifying the source bars -intermediate_bar_type = BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-INTERNAL") - -# Then create hourly bars from 5-minute bars -# Note the @5-MINUTE-INTERNAL part identifying the source bars -hourly_bar_type = BarType.from_str("6EH4.XCME-1-HOUR-LAST-INTERNAL@5-MINUTE-INTERNAL") -``` - -### Working with bars: request vs. subscribe - -NautilusTrader provides two distinct operations for working with bars: - -- **`request_bars()`**: Fetches historical data processed by the `on_historical_data()` handler. -- **`subscribe_bars()`**: Establishes a real-time data feed processed by the `on_bar()` handler. - -These methods work together in a typical workflow: - -1. First, `request_bars()` loads historical data to initialize indicators or state of strategy with past market behavior. -2. Then, `subscribe_bars()` ensures the strategy continues receiving new bars as they form in real-time. - -Example usage in `on_start()`: - -```python -def on_start(self) -> None: - # Define bar type - bar_type = BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL") - - # Request historical data to initialize indicators - # These bars will be delivered to the on_historical_data(...) handler in strategy - self.request_bars(bar_type) - - # Subscribe to real-time updates - # New bars will be delivered to the on_bar(...) handler in strategy - self.subscribe_bars(bar_type) - - # Register indicators to receive bar updates (they will be automatically updated) - self.register_indicator_for_bars(bar_type, self.my_indicator) -``` - -Required handlers in your strategy to receive the data: - -```python -def on_historical_data(self, data): - # Processes batches of historical bars from request_bars() - # Note: indicators registered with register_indicator_for_bars - # are updated automatically with historical data - pass - -def on_bar(self, bar): - # Processes individual bars in real-time from subscribe_bars() - # Indicators registered with this bar type will update automatically and they will be updated before this handler is called - pass -``` - -### Historical data requests with aggregation - -When requesting historical bars for backtesting or initializing indicators, you can use the `request_bars()` method, which supports both direct requests and aggregation: - -```python -# Request raw 1-minute bars (aggregated from TradeTick objects as indicated by LAST price type) -self.request_bars(BarType.from_str("6EH4.XCME-1-MINUTE-LAST-EXTERNAL")) - -# Request 5-minute bars aggregated from 1-minute bars -self.request_bars(BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL")) -``` - -If historical aggregated bars are needed, you can use specialized request `request_aggregated_bars()` method: - -```python -# Request bars that are aggregated from historical trade ticks -self.request_aggregated_bars([BarType.from_str("6EH4.XCME-100-VOLUME-LAST-INTERNAL")]) - -# Request bars that are aggregated from other bars -self.request_aggregated_bars([BarType.from_str("6EH4.XCME-5-MINUTE-LAST-INTERNAL@1-MINUTE-EXTERNAL")]) -``` - -### Common pitfalls - -**Register indicators before requesting data**: Ensure indicators are registered before requesting historical data so they get updated properly. - -```python -# Correct order -self.register_indicator_for_bars(bar_type, self.ema) -self.request_bars(bar_type) - -# Incorrect order -self.request_bars(bar_type) # Indicator won't receive historical data -self.register_indicator_for_bars(bar_type, self.ema) -``` - -### Performance considerations - -Bar aggregators track OHLC prices via the fixed-point `Price` type. Threshold comparisons for -tick and volume aggregators use integer arithmetic, while value-based and imbalance/runs aggregators -currently use `f64` for notional value and signed accumulation (these are being migrated to -fixed-point integer arithmetic). The choice of aggregation method has a modest impact on per-update -overhead: - -- **Time bars** are the most efficient for high-throughput data. The aggregator simply accumulates - OHLCV state per update; bar emission is driven by a timer rather than per-tick logic. -- **Threshold bars** (tick, volume, value) add a lightweight counter or accumulator check per update. - Volume and value bars may split a single large trade across multiple bars when it exceeds the - remaining threshold. -- **Information-driven bars** (imbalance, runs) require tracking aggressor side and signed - accumulation per update. The overhead is slightly higher than threshold bars but still minimal. -- **Renko bars** are price-driven and can emit multiple bars from a single large price move. - Otherwise the per-update cost is comparable to threshold bars. -- **Composite bars** (bar-to-bar) are the most efficient way to produce higher-timeframe bars - when lower-timeframe bars are already available, as each input bar represents an already - aggregated period rather than a single tick. - -### Time bar configuration - -Time bar behavior is controlled through `DataEngineConfig`. The following options -apply to all time-based aggregation (millisecond through year): - -| Option | Type | Default | Description | -|:------------------------------------|:-------|:--------------|:------------------------------------------------------------------------------------------------------------------------------------------------| -| `time_bars_interval_type` | `str` | `"left-open"` | `"left-open"`: start excluded, end included. `"right-open"`: start included, end excluded. | -| `time_bars_timestamp_on_close` | `bool` | `True` | When `True`, `ts_event` is the bar close time. When `False`, `ts_event` is the bar open time. | -| `time_bars_skip_first_non_full_bar` | `bool` | `False` | Skip emitting a bar when aggregation starts mid-interval, avoiding partial bars on startup. | -| `time_bars_build_with_no_updates` | `bool` | `True` | When `True`, bars are emitted even if no market updates arrived during the interval. | -| `time_bars_origin_offset` | `dict` | `None` | Maps `BarAggregation` types to `pd.Timedelta` offsets for shifting bar alignment (e.g., align to 09:30 market open). | -| `time_bars_build_delay` | `int` | `0` | Delay in microseconds before building a bar. Useful in backtests to ensure data at bar boundary timestamps is processed before the timer fires. | - -```python -from nautilus_trader.data.config import DataEngineConfig - -config = DataEngineConfig( - time_bars_timestamp_on_close=True, - time_bars_build_with_no_updates=False, - time_bars_skip_first_non_full_bar=True, -) -``` - -## Timestamps - -The platform uses two fundamental timestamp fields that appear across many objects, including market data, orders, and events. -These timestamps serve distinct purposes and help maintain precise timing information throughout the system: - -- `ts_event`: UNIX timestamp (nanoseconds) representing when an event actually occurred. -- `ts_init`: UNIX timestamp (nanoseconds) representing when Nautilus created the internal object representing that event. - -### Examples - -| **Event Type** | **`ts_event`** | **`ts_init`** | -| -----------------| ------------------------------------------------------| --------------| -| `TradeTick` | Time when trade occurred at the exchange. | Time when Nautilus received the trade data. | -| `QuoteTick` | Time when quote occurred at the exchange. | Time when Nautilus received the quote data. | -| `OrderBookDelta` | Time when order book update occurred at the exchange. | Time when Nautilus received the order book update. | -| `Bar` | Time of the bar's closing (exact minute/hour). | Time when Nautilus generated (for internal bars) or received the bar data (for external bars). | -| `OrderFilled` | Time when order was filled at the exchange. | Time when Nautilus received and processed the fill confirmation. | -| `OrderCanceled` | Time when cancellation was processed at the exchange. | Time when Nautilus received and processed the cancellation confirmation. | -| `NewsEvent` | Time when the news was published. | Time when the event object was created (if internal event) or received (if external event) in Nautilus. | -| Custom event | Time when event conditions actually occurred. | Time when the event object was created (if internal event) or received (if external event) in Nautilus. | - -:::note -The `ts_init` field represents a more general concept than "time of reception" for events. -It denotes the timestamp when an object, such as a data point or command, was initialized within Nautilus. -This distinction is important because `ts_init` is not exclusive to "received events". It applies to any internal -initialization process. - -For example, the `ts_init` field is also used for commands, where the concept of reception does not apply. -This broader definition ensures consistent handling of initialization timestamps across various object types in the system. -::: - -### Latency analysis - -The dual timestamp system enables latency analysis within the platform: - -- Latency can be calculated as `ts_init - ts_event`. -- This difference represents total system latency, including network transmission time, processing overhead, and any queueing delays. -- It's important to remember that the clocks producing these timestamps are likely not synchronized. - -### Environment-specific behavior - -#### Backtesting environment - -- Data is ordered by `ts_init` using a stable sort. -- This behavior ensures deterministic processing order and simulates realistic system behavior, including latencies. - -#### Live trading environment - -- The system processes data as it arrives to minimize latency and enable real-time decisions. - - `ts_init` field records the exact moment when data is received by Nautilus in real-time. - - `ts_event` reflects the time the event occurred externally, enabling accurate comparisons between external event timing and system reception. -- We can use the difference between `ts_init` and `ts_event` to detect network or processing delays. - -### Other notes and considerations - -- For data from external sources, `ts_init` is always the same as or later than `ts_event`. -- For data created within Nautilus, `ts_init` and `ts_event` can be the same because the object is initialized at the same time the event happens. -- Not every type with a `ts_init` field necessarily has a `ts_event` field. This reflects cases where: - - The initialization of an object happens at the same time as the event itself. - - The concept of an external event time does not apply. - -#### Persisted data - -The `ts_init` field indicates when the message was originally received. - -## Data flow - -The platform ensures consistency by flowing data through the same pathways across all system [environment contexts](architecture.md#environment-contexts) -(e.g., `backtest`, `sandbox`, `live`). Data is primarily transported via the `MessageBus` to the `DataEngine` -and then distributed to subscribed or registered handlers. - -For users who need more flexibility, the platform also supports the creation of custom data types. -For details on how to implement user-defined data types, see the [Custom Data](#custom-data) section below. - -## Loading data - -NautilusTrader supports data loading and conversion for three main use cases: - -- Providing data for a `BacktestEngine` to run backtests. -- Persisting the Nautilus-specific Parquet format for the data catalog via `ParquetDataCatalog.write_data(...)` to be later used with a `BacktestNode`. -- For research purposes (to ensure data is consistent between research and backtesting). - -Regardless of the destination, the process remains the same: converting diverse external data formats into Nautilus data structures. - -To achieve this, two main components are necessary: - -- A type of DataLoader (normally specific per raw source/format) which can read the data and return a `pd.DataFrame` with the correct schema for the desired Nautilus object. -- A type of DataWrangler (specific per data type) which takes this `pd.DataFrame` and returns a `list[Data]` of Nautilus objects. - -### Data loaders - -Data loader components are typically specific for the raw source/format and per integration. For instance, Binance order book data is stored in its raw CSV file form with -an entirely different format to [Databento Binary Encoding (DBN)](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn) files. - -### Data wranglers - -Data wranglers are implemented per specific Nautilus data type, and can be found in the `nautilus_trader.persistence.wranglers` module. -Currently there exists: - -- `OrderBookDeltaDataWrangler` -- `OrderBookDepth10DataWrangler` -- `QuoteTickDataWrangler` -- `TradeTickDataWrangler` -- `BarDataWrangler` - -:::warning -There are a number of **DataWrangler v2** components, which will take a `pd.DataFrame` typically -with a different fixed width Nautilus Arrow v2 schema, and output PyO3 Nautilus objects which are only compatible with the new version -of the Nautilus core, currently in development. - -**These PyO3 provided data objects are not compatible where the legacy Cython objects are currently used (e.g., adding directly to a `BacktestEngine`).** -::: - -### Fixed-point precision and raw values - -NautilusTrader uses fixed-point arithmetic for `Price` and `Quantity` types for precise financial calculations without floating-point errors. Understanding how raw values work is essential when creating data or working with catalogs. - -#### Raw value requirements - -When constructing `Price` or `Quantity` using `from_raw()`, the raw value **must** be a valid multiple of the scale factor for the given precision. Valid raw values should come from: - -- Accessing the `.raw` field of an existing value (e.g., `price.raw`). -- Using the Nautilus fixed-point conversion functions. -- Values from Nautilus-produced Arrow data. - -:::warning -Raw values that are not valid multiples will cause a panic. The raw value must be divisible by `10^(FIXED_PRECISION - precision)` where `FIXED_PRECISION` is 9 (standard mode) or 16 (high-precision mode). -::: - -#### Legacy catalog data and floating-point errors - -Data written to catalogs using V2 wranglers before 16th December 2025 may contain raw values with floating-point precision errors. This occurred because the wranglers used: - -```python -int(value * FIXED_SCALAR) # Introduces floating-point errors -``` - -For example, `int(0.67068 * 1e9)` produces `670680000000001` instead of the expected `670680000000000`. The correct approach is: - -```python -round(value * 10**precision) * scale # Correct precision-aware conversion -``` - -#### Automatic correction during catalog reads - -To maintain backward compatibility with existing catalog data, the Arrow decode path automatically corrects raw values by rounding them to the nearest valid multiple. This ensures that legacy catalogs continue to work without requiring data migration. - -:::note -This automatic correction adds a small amount of overhead during data decoding. In a future version, once catalogs have been repaired or migrated, this correction will become opt-in. A catalog repair/migration script may be provided to permanently fix legacy data. -::: - -### Transformation pipeline - -**Process flow**: - -1. Raw data (e.g., CSV) is input into the pipeline. -2. DataLoader processes the raw data and converts it into a `pd.DataFrame`. -3. DataWrangler further processes the `pd.DataFrame` to generate a list of Nautilus objects. -4. The Nautilus `list[Data]` is the output of the data loading process. - -The following diagram illustrates how raw data is transformed into Nautilus data structures: - -```mermaid -flowchart LR - raw["Raw data (CSV)"] - loader[DataLoader] - wrangler[DataWrangler] - output["Nautilus list[Data]"] - - raw --> loader - loader -->|"pd.DataFrame"| wrangler - wrangler --> output -``` - -Concretely, this would involve: - -- `BinanceOrderBookDeltaDataLoader.load(...)` which reads CSV files provided by Binance from disk, and returns a `pd.DataFrame`. -- `OrderBookDeltaDataWrangler.process(...)` which takes the `pd.DataFrame` and returns `list[OrderBookDelta]`. - -The following example shows how to accomplish the above in Python: - -```python -from nautilus_trader import TEST_DATA_DIR -from nautilus_trader.adapters.binance.loaders import BinanceOrderBookDeltaDataLoader -from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler -from nautilus_trader.test_kit.providers import TestInstrumentProvider - - -# Load raw data -data_path = TEST_DATA_DIR / "binance" / "btcusdt-depth-snap.csv" -df = BinanceOrderBookDeltaDataLoader.load(data_path) - -# Set up a wrangler -instrument = TestInstrumentProvider.btcusdt_binance() -wrangler = OrderBookDeltaDataWrangler(instrument) - -# Process to a list `OrderBookDelta` Nautilus objects -deltas = wrangler.process(df) -``` - -## Data catalog - -The data catalog is a central store for Nautilus data, persisted in the [Parquet](https://parquet.apache.org) file format. It is the primary data management system for both backtesting and live trading scenarios, providing efficient storage, retrieval, and streaming capabilities for market data. - -### Overview and architecture - -The NautilusTrader data catalog is built on a dual-backend architecture that combines the performance of Rust with the flexibility of Python: - -**Core components:** - -- **ParquetDataCatalog**: The main Python interface for data operations. -- **Rust backend**: High-performance query engine for core data types (OrderBookDelta, QuoteTick, TradeTick, Bar, MarkPriceUpdate). -- **PyArrow backend**: Flexible fallback for custom data types and advanced filtering. -- **fsspec integration**: Support for local and cloud storage (S3, GCS, Azure, etc.). - -**Key benefits**: - -- **Performance**: Rust backend provides optimized query performance for core market data types. -- **Flexibility**: PyArrow backend handles custom data types and complex filtering scenarios. -- **Scalability**: Efficient compression and columnar storage reduce storage costs and improve I/O performance. -- **Cloud native**: Built-in support for cloud storage providers through fsspec. -- **No dependencies**: Self-contained solution requiring no external databases or services. - -**Storage format advantages:** - -- Superior compression ratio and read performance compared to CSV/JSON/HDF5. -- Columnar storage enables efficient filtering and aggregation. -- Schema evolution support for data model changes. -- Cross-language compatibility (Python, Rust, Java, C++, etc.). - -The Arrow schemas used for the Parquet format are primarily single-sourced in the core `persistence` Rust crate, with some legacy schemas available from the `/serialization/arrow/schema.py` module. - -:::note -The current plan is to eventually phase out the Python schemas module, so that all schemas are single sourced in the Rust core for consistency and performance. -::: - -### Initializing - -The data catalog can be initialized from a `NAUTILUS_PATH` environment variable, or by explicitly passing in a path like object. - -:::note[NAUTILUS_PATH environment variable] -The `NAUTILUS_PATH` environment variable should point to the **root** directory containing your Nautilus data. The catalog will automatically append `/catalog` to this path. - -For example: - -- If `NAUTILUS_PATH=/home/user/trading_data`. -- Then the catalog will be located at `/home/user/trading_data/catalog`. - -This is a common pattern when using `ParquetDataCatalog.from_env()` - make sure your `NAUTILUS_PATH` points to the parent directory, not the catalog directory itself. -::: - -The following example shows how to initialize a data catalog where there is pre-existing data already written to disk at the given path. - -```python -from pathlib import Path -from nautilus_trader.persistence.catalog import ParquetDataCatalog - - -CATALOG_PATH = Path.cwd() / "catalog" - -# Create a new catalog instance -catalog = ParquetDataCatalog(CATALOG_PATH) - -# Alternative: Environment-based initialization -catalog = ParquetDataCatalog.from_env() # Uses NAUTILUS_PATH environment variable -``` - -### Filesystem protocols and storage options - -The catalog supports multiple filesystem protocols through fsspec integration, working across local and cloud storage systems. - -#### Supported filesystem protocols - -**Local filesystem (`file`):** - -```python -catalog = ParquetDataCatalog( - path="/path/to/catalog", - fs_protocol="file", # Default protocol -) -``` - -**Amazon S3 (`s3`):** - -```python -catalog = ParquetDataCatalog( - path="s3://my-bucket/nautilus-data/", - fs_protocol="s3", - fs_storage_options={ - "key": "your-access-key-id", - "secret": "your-secret-access-key", - "endpoint_url": "https://s3.amazonaws.com", # Optional custom endpoint - } -) -``` - -**Google Cloud Storage (`gcs`):** - -```python -catalog = ParquetDataCatalog( - path="gcs://my-bucket/nautilus-data/", - fs_protocol="gcs", - fs_storage_options={ - "project": "my-project-id", - "token": "/path/to/service-account.json", # Or "cloud" for default credentials - } -) -``` - -**Azure Blob Storage :** - -`abfs` protocol - -```python -catalog = ParquetDataCatalog( - path="abfs://container@account.dfs.core.windows.net/nautilus-data/", - fs_protocol="abfs", - fs_storage_options={ - "account_name": "your-storage-account", - "account_key": "your-account-key", - # Or use SAS token: "sas_token": "your-sas-token" - } -) -``` - -`az` protocol - -```python -catalog = ParquetDataCatalog( - path="az://container/nautilus-data/", - fs_protocol="az", - fs_storage_options={ - "account_name": "your-storage-account", - "account_key": "your-account-key", - # Or use SAS token: "sas_token": "your-sas-token" - } -) -``` - -#### URI-based initialization - -For convenience, you can use URI strings that automatically parse protocol and storage options: - -```python -# Local filesystem -catalog = ParquetDataCatalog.from_uri("/path/to/catalog") - -# S3 bucket -catalog = ParquetDataCatalog.from_uri("s3://my-bucket/nautilus-data/") - -# With storage options -catalog = ParquetDataCatalog.from_uri( - "s3://my-bucket/nautilus-data/", - fs_storage_options={ - "access_key_id": "your-key", - "secret_access_key": "your-secret" - } -) -``` - -### Writing data - -Store data in the catalog using the `write_data()` method. All Nautilus built-in `Data` objects are supported, and any data which inherits from `Data` can be written. - -```python -# Write a list of data objects -catalog.write_data(quote_ticks) - -# Write with custom timestamp range -catalog.write_data( - trade_ticks, - start=1704067200000000000, # Optional start timestamp override (UNIX nanoseconds) - end=1704153600000000000, # Optional end timestamp override (UNIX nanoseconds) -) - -# Skip disjoint check for overlapping data -catalog.write_data(bars, skip_disjoint_check=True) -``` - -### File naming and data organization - -The catalog automatically generates filenames based on the timestamp range of the data being written. Files are named using the pattern `{start_timestamp}_{end_timestamp}.parquet` where timestamps are in ISO format. - -Data is organized in directories by data type and instrument ID: - -``` -catalog/ -├── data/ -│ ├── quote_ticks/ -│ │ └── eurusd.sim/ -│ │ └── 20240101T000000000000000_20240101T235959999999999.parquet -│ └── trade_ticks/ -│ └── btcusd.binance/ -│ └── 20240101T000000000000000_20240101T235959999999999.parquet -``` - -**Rust backend data types (enhanced performance):** - -The following data types use optimized Rust implementations: - -- `OrderBookDelta`. -- `OrderBookDeltas`. -- `OrderBookDepth10`. -- `QuoteTick`. -- `TradeTick`. -- `Bar`. -- `MarkPriceUpdate`. - -:::warning -By default, data that overlaps with existing files will cause an assertion error to maintain data integrity. Use `skip_disjoint_check=True` in `write_data()` to bypass this check when needed. -::: - -### Reading data - -Use the `query()` method to read data back from the catalog: - -```python -from nautilus_trader.model import QuoteTick, TradeTick - -# Query quote ticks for a specific instrument and time range -quotes = catalog.query( - data_cls=QuoteTick, - identifiers=["EUR/USD.SIM"], - start="2024-01-01T00:00:00Z", - end="2024-01-02T00:00:00Z" -) - -# Query trade ticks with filtering -trades = catalog.query( - data_cls=TradeTick, - identifiers=["BTC/USD.BINANCE"], - start="2024-01-01", - end="2024-01-02", - where="price > 50000" -) -``` - -### `BacktestDataConfig` - data specification for backtests - -The `BacktestDataConfig` class is the primary mechanism for specifying data requirements before a backtest starts. It defines what data should be loaded from the catalog and how it should be filtered and processed during the backtest execution. - -#### Core parameters - -**Required parameters:** - -- `catalog_path`: Path to the data catalog directory. -- `data_cls`: The data type class (e.g., QuoteTick, TradeTick, OrderBookDelta, Bar). - -**Optional parameters:** - -- `catalog_fs_protocol`: Filesystem protocol ('file', 's3', 'gcs', etc.). -- `catalog_fs_storage_options`: Storage-specific options (credentials, region, etc.). -- `instrument_id`: Specific instrument to load data for. -- `instrument_ids`: List of instruments (alternative to single instrument_id). -- `start_time`: Start time for data filtering (ISO string or UNIX nanoseconds). -- `end_time`: End time for data filtering (ISO string or UNIX nanoseconds). -- `filter_expr`: Additional PyArrow filter expressions. -- `client_id`: Client ID for custom data types. -- `metadata`: Additional metadata for data queries. -- `bar_spec`: Bar specification for bar data (e.g., "1-MINUTE-LAST"). -- `bar_types`: List of bar types (alternative to bar_spec). - -#### Basic usage examples - -**Loading quote ticks:** - -```python -from nautilus_trader.config import BacktestDataConfig -from nautilus_trader.model import QuoteTick, InstrumentId - -data_config = BacktestDataConfig( - catalog_path="/path/to/catalog", - data_cls=QuoteTick, - instrument_id=InstrumentId.from_str("EUR/USD.SIM"), - start_time="2024-01-01T00:00:00Z", - end_time="2024-01-02T00:00:00Z", -) -``` - -**Loading multiple instruments:** - -```python -data_config = BacktestDataConfig( - catalog_path="/path/to/catalog", - data_cls=TradeTick, - instrument_ids=["BTC/USD.BINANCE", "ETH/USD.BINANCE"], - start_time="2024-01-01T00:00:00Z", - end_time="2024-01-02T00:00:00Z", -) -``` - -**Loading Bar Data:** - -```python -data_config = BacktestDataConfig( - catalog_path="/path/to/catalog", - data_cls=Bar, - instrument_id=InstrumentId.from_str("AAPL.NASDAQ"), - bar_spec="5-MINUTE-LAST", - start_time="2024-01-01", - end_time="2024-01-31", -) -``` - -#### Advanced configuration examples - -**Cloud Storage with Custom Filtering:** - -```python -data_config = BacktestDataConfig( - catalog_path="s3://my-bucket/nautilus-data/", - catalog_fs_protocol="s3", - catalog_fs_storage_options={ - "key": "your-access-key", - "secret": "your-secret-key", - "region": "us-east-1" - }, - data_cls=OrderBookDelta, - instrument_id=InstrumentId.from_str("BTC/USD.COINBASE"), - start_time="2024-01-01T09:30:00Z", - end_time="2024-01-01T16:00:00Z", -) -``` - -**Custom Data with Client ID:** - -```python -data_config = BacktestDataConfig( - catalog_path="/path/to/catalog", - data_cls="my_package.data.NewsEventData", - client_id="NewsClient", - metadata={"source": "reuters", "category": "earnings"}, - start_time="2024-01-01", - end_time="2024-01-31", -) -``` - -#### Integration with BacktestRunConfig - -The `BacktestDataConfig` objects are integrated into the backtesting framework through `BacktestRunConfig`: - -```python -from nautilus_trader.config import BacktestRunConfig, BacktestVenueConfig - -# Define multiple data configurations -data_configs = [ - BacktestDataConfig( - catalog_path="/path/to/catalog", - data_cls=QuoteTick, - instrument_id="EUR/USD.SIM", - start_time="2024-01-01", - end_time="2024-01-02", - ), - BacktestDataConfig( - catalog_path="/path/to/catalog", - data_cls=TradeTick, - instrument_id="EUR/USD.SIM", - start_time="2024-01-01", - end_time="2024-01-02", - ), -] - -# Create backtest run configuration -run_config = BacktestRunConfig( - venues=[BacktestVenueConfig(name="SIM", oms_type="HEDGING")], - data=data_configs, # List of data configurations - start="2024-01-01T00:00:00Z", - end="2024-01-02T00:00:00Z", -) -``` - -#### Data loading process - -When a backtest runs, the `BacktestNode` processes each `BacktestDataConfig`: - -1. **Catalog Loading**: Creates a `ParquetDataCatalog` instance from the config. -2. **Query Construction**: Builds query parameters from config attributes. -3. **Data Retrieval**: Executes catalog queries using the appropriate backend. -4. **Instrument Loading**: Loads instrument definitions if needed. -5. **Engine Integration**: Adds data to the backtest engine with proper sorting. - -The system automatically handles: - -- Instrument ID resolution and validation. -- Data type validation and conversion. -- Memory-efficient streaming for large datasets. -- Error handling and logging. - -### DataCatalogConfig - on-the-fly data loading - -The `DataCatalogConfig` class provides configuration for on-the-fly data loading scenarios, particularly useful for backtests where the number of possible instruments is vast, -Unlike `BacktestDataConfig` which pre-specifies data for backtests, `DataCatalogConfig` enables flexible catalog access during runtime. -Catalogs defined this way can also be used for requesting historical data. - -#### Core parameters - -**Required Parameters:** - -- `path`: Path to the data catalog directory. - -**Optional Parameters:** - -- `fs_protocol`: Filesystem protocol ('file', 's3', 'gcs', 'azure', etc.). -- `fs_storage_options`: Protocol-specific storage options. -- `name`: Optional name identifier for the catalog configuration. - -#### Basic usage examples - -**Local Catalog Configuration:** - -```python -from nautilus_trader.persistence.config import DataCatalogConfig - -catalog_config = DataCatalogConfig( - path="/path/to/catalog", - fs_protocol="file", - name="local_market_data" -) - -# Convert to catalog instance -catalog = catalog_config.as_catalog() -``` - -**Cloud storage configuration:** - -```python -catalog_config = DataCatalogConfig( - path="s3://my-bucket/market-data/", - fs_protocol="s3", - fs_storage_options={ - "key": "your-access-key", - "secret": "your-secret-key", - "region": "us-west-2", - "endpoint_url": "https://s3.us-west-2.amazonaws.com" - }, - name="cloud_market_data" -) -``` - -#### Integration with live trading - -`DataCatalogConfig` is commonly used in live trading configurations for historical data access: - -```python -from nautilus_trader.config import TradingNodeConfig -from nautilus_trader.persistence.config import DataCatalogConfig - -# Configure catalog for live system -catalog_config = DataCatalogConfig( - path="/data/nautilus/catalog", - fs_protocol="file", - name="historical_data" -) - -# Use in trading node configuration -node_config = TradingNodeConfig( - # ... other configurations - catalog=catalog_config, # Enable historical data access -) -``` - -#### Streaming configuration - -For streaming data to catalogs during live trading or backtesting, use `StreamingConfig`: - -```python -from nautilus_trader.persistence.config import StreamingConfig, RotationMode -import pandas as pd - -streaming_config = StreamingConfig( - catalog_path="/path/to/streaming/catalog", - fs_protocol="file", - flush_interval_ms=1000, # Flush every second - replace_existing=False, - rotation_mode=RotationMode.DAILY, - rotation_interval=pd.Timedelta(hours=1), - max_file_size=1024 * 1024 * 100, # 100MB max file size -) -``` - -#### Use cases - -**Historical Data Analysis:** - -- Load historical data during live trading for strategy calculations. -- Access reference data for instrument lookups. -- Retrieve past performance metrics. - -**Dynamic data loading:** - -- Load data based on runtime conditions. -- Implement custom data loading strategies. -- Support multiple catalog sources. - -**Research and development:** - -- Interactive data exploration in Jupyter notebooks. -- Ad-hoc analysis and backtesting. -- Data quality validation and monitoring. - -### Query system and dual backend architecture - -The catalog's query system uses a dual-backend architecture that selects the query engine based on data type and query parameters. - -#### Backend selection logic - -**Rust backend (high performance):** - -- **Supported Types**: OrderBookDelta, OrderBookDeltas, OrderBookDepth10, QuoteTick, TradeTick, Bar, MarkPriceUpdate. -- **Conditions**: Used when `files` parameter is None (automatic file discovery). -- **Benefits**: Optimized performance, memory efficiency, native Arrow integration. - -**PyArrow backend (flexible):** - -- **Supported Types**: All data types including custom data classes. -- **Conditions**: Used for custom data types or when `files` parameter is specified. -- **Benefits**: Advanced filtering, custom data support, complex query expressions. - -#### Query methods and parameters - -**Core query parameters:** - -```python -catalog.query( - data_cls=QuoteTick, # Data type to query - identifiers=["EUR/USD.SIM"], # Instrument identifiers - start="2024-01-01T00:00:00Z", # Start time (various formats supported) - end="2024-01-02T00:00:00Z", # End time - where="bid > 1.1000", # PyArrow filter expression - files=None, # Specific files (forces PyArrow backend) -) -``` - -**Time format support:** - -- ISO 8601 strings: `"2024-01-01T00:00:00Z"`. -- UNIX nanoseconds: `1704067200000000000` (or ISO format: `"2024-01-01T00:00:00Z"`). -- Pandas Timestamps: `pd.Timestamp("2024-01-01", tz="UTC")`. -- Python datetime objects (timezone-aware recommended). - -**Advanced filtering examples:** - -```python -# Complex PyArrow expressions -catalog.query( - data_cls=TradeTick, - identifiers=["BTC/USD.BINANCE"], - where="price > 50000 AND size > 1.0", - start="2024-01-01", - end="2024-01-02", -) - -# Multiple instruments with metadata filtering -catalog.query( - data_cls=Bar, - identifiers=["AAPL.NASDAQ", "MSFT.NASDAQ"], - where="volume > 1000000", - metadata={"bar_type": "1-MINUTE-LAST"}, -) -``` - -### Catalog operations - -The catalog provides several operation functions for maintaining and organizing data files. These operations help optimize storage, improve query performance, and maintain data integrity. - -#### Reset file names - -Reset parquet file names to match their actual content timestamps. This ensures filename-based filtering works correctly. - -**Reset all files in catalog:** - -```python -# Reset all parquet files in the catalog -catalog.reset_all_file_names() -``` - -**Reset specific data type:** - -```python -# Reset filenames for all quote tick files -catalog.reset_data_file_names(QuoteTick) - -# Reset filenames for specific instrument's trade files -catalog.reset_data_file_names(TradeTick, "BTC/USD.BINANCE") -``` - -#### Consolidate catalog - -Combine multiple small parquet files into larger files to improve query performance and reduce storage overhead. - -**Consolidate entire catalog:** - -```python -# Consolidate all files in the catalog -catalog.consolidate_catalog() - -# Consolidate files within a specific time range -catalog.consolidate_catalog( - start="2024-01-01T00:00:00Z", - end="2024-01-02T00:00:00Z", - ensure_contiguous_files=True -) -``` - -**Consolidate specific data type:** - -```python -# Consolidate all quote tick files -catalog.consolidate_data(QuoteTick) - -# Consolidate specific instrument's files -catalog.consolidate_data( - TradeTick, - identifier="BTC/USD.BINANCE", - start="2024-01-01", - end="2024-01-31" -) -``` - -#### Consolidate catalog by period - -Split data files into fixed time periods for standardized file organization. - -**Consolidate entire catalog by period:** - -```python -import pandas as pd - -# Consolidate all files by 1-day periods -catalog.consolidate_catalog_by_period( - period=pd.Timedelta(days=1) -) - -# Consolidate by 1-hour periods within time range -catalog.consolidate_catalog_by_period( - period=pd.Timedelta(hours=1), - start="2024-01-01T00:00:00Z", - end="2024-01-02T00:00:00Z" -) -``` - -**Consolidate specific data by period:** - -```python -# Consolidate quote data by 4-hour periods -catalog.consolidate_data_by_period( - data_cls=QuoteTick, - period=pd.Timedelta(hours=4) -) - -# Consolidate specific instrument by 30-minute periods -catalog.consolidate_data_by_period( - data_cls=TradeTick, - identifier="EUR/USD.SIM", - period=pd.Timedelta(minutes=30), - start="2024-01-01", - end="2024-01-31" -) -``` - -#### Delete data range - -Remove data within a specified time range for specific data types and instruments. This operation permanently deletes data and handles file intersections intelligently. - -**Delete entire catalog range:** - -```python -# Delete all data within a time range across the entire catalog -catalog.delete_catalog_range( - start="2024-01-01T00:00:00Z", - end="2024-01-02T00:00:00Z" -) - -# Delete all data from the beginning up to a specific time -catalog.delete_catalog_range(end="2024-01-01T00:00:00Z") -``` - -**Delete specific data type:** - -```python -# Delete all quote tick data for a specific instrument -catalog.delete_data_range( - data_cls=QuoteTick, - identifier="BTC/USD.BINANCE" -) - -# Delete trade data within a specific time range -catalog.delete_data_range( - data_cls=TradeTick, - identifier="EUR/USD.SIM", - start="2024-01-01T00:00:00Z", - end="2024-01-31T23:59:59Z" -) -``` - -:::warning -Delete operations permanently remove data and cannot be undone. Files that partially overlap the deletion range are split to preserve data outside the range. -::: - -### Feather streaming and conversion - -The catalog supports streaming data to temporary feather files during backtests, which can then be converted to permanent parquet format for efficient querying. - -**Example: option greeks streaming** - -```python -from option_trader.greeks import GreeksData -from nautilus_trader.persistence.config import StreamingConfig - -# 1. Configure streaming for custom data -streaming = StreamingConfig( - catalog_path=catalog.path, - include_types=[GreeksData], - flush_interval_ms=1000, -) - -# 2. Run backtest with streaming enabled -engine_config = BacktestEngineConfig(streaming=streaming) -results = node.run() - -# 3. Convert streamed data to permanent catalog -catalog.convert_stream_to_data( - results[0].instance_id, - GreeksData, -) - -# 4. Query converted data -greeks_data = catalog.query( - data_cls=GreeksData, - start="2024-01-01", - end="2024-01-31", - where="delta > 0.5", -) -``` - -### Catalog summary - -The NautilusTrader data catalog provides market data management: - -**Core features**: - -- **Dual Backend**: Rust performance + Python flexibility. -- **Multi-Protocol**: Local, S3, GCS, Azure storage. -- **Streaming**: Feather → Parquet conversion pipeline. -- **Operations**: Reset file names, consolidate data, period-based organization. - -**Key use cases**: - -- **Backtesting**: Pre-configured data loading via BacktestDataConfig. -- **Live Trading**: On-demand data access via DataCatalogConfig. -- **Maintenance**: File consolidation and organization operations. -- **Research**: Interactive querying and analysis. - -## Data migrations - -NautilusTrader defines an internal data format specified in the `nautilus_model` crate. -These models are serialized into Arrow record batches and written to Parquet files. -Nautilus backtesting is most efficient when using these Nautilus-format Parquet files. - -However, migrating the data model between [precision modes](../getting_started/installation.md#precision-mode) and schema changes can be challenging. -This guide explains how to handle data migrations using our utility tools. - -### Migration tools - -The `nautilus_persistence` crate provides two key utilities: - -#### `to_json` - -Converts Parquet files to JSON while preserving metadata: - -- Creates two files: - - - `.json`: Contains the deserialized data - - `.metadata.json`: Contains schema metadata and row group configuration - -- Automatically detects data type from filename: - - - `OrderBookDelta` (contains "deltas" or "order_book_delta") - - `QuoteTick` (contains "quotes" or "quote_tick") - - `TradeTick` (contains "trades" or "trade_tick") - - `Bar` (contains "bars") - -#### `to_parquet` - -Converts JSON back to Parquet format: - -- Reads both the data JSON and metadata JSON files. -- Preserves row group sizes from original metadata. -- Uses ZSTD compression. -- Creates `.parquet`. - -### Migration process - -The following migration examples both use trades data (you can also migrate the other data types in the same way). -All commands should be run from the root of the `persistence` crate directory. - -#### Migrating from standard-precision (64-bit) to high-precision (128-bit) - -This example describes a scenario where you want to migrate from standard-precision schema to high-precision schema. - -:::note -If you're migrating from a catalog that used the `Int64` and `UInt64` Arrow data types for prices and sizes, -be sure to check out commit [e284162](https://github.com/nautechsystems/nautilus_trader/commit/e284162cf27a3222115aeb5d10d599c8cf09cf50) -**before** compiling the code that writes the initial JSON. -::: - -**1. Convert from standard-precision Parquet to JSON**: - -```bash -cargo run --bin to_json trades.parquet -``` - -This will create `trades.json` and `trades.metadata.json` files. - -**2. Convert from JSON to high-precision Parquet**: - -Add the `--features high-precision` flag to write data as high-precision (128-bit) schema Parquet. - -```bash -cargo run --features high-precision --bin to_parquet trades.json -``` - -This will create a `trades.parquet` file with high-precision schema data. - -#### Migrating schema changes - -This example describes a scenario where you want to migrate from one schema version to another. - -**1. Convert from old schema Parquet to JSON**: - -Add the `--features high-precision` flag if the source data uses a high-precision (128-bit) schema. - -```bash -cargo run --bin to_json trades.parquet -``` - -This will create `trades.json` and `trades.metadata.json` files. - -**2. Switch to new schema version**: - -```bash -git checkout -``` - -**3. Convert from JSON back to new schema Parquet**: - -```bash -cargo run --features high-precision --bin to_parquet trades.json -``` - -This will create a `trades.parquet` file with the new schema. - -### Best practices - -- Always test migrations with a small dataset first. -- Maintain backups of original files. -- Verify data integrity after migration. -- Perform migrations in a staging environment before applying them to production data. - -## Custom data - -Due to the modular nature of the Nautilus design, it is possible to set up systems -with very flexible data streams, including custom user-defined data types. This -guide covers some possible use cases for this functionality. - -It's possible to create custom data types within the Nautilus system. First you -will need to define your data by subclassing from `Data`. - -:::info -As `Data` holds no state, it is not strictly necessary to call `super().__init__()`. -::: - -```python -from nautilus_trader.core import Data - - -class MyDataPoint(Data): - """ - This is an example of a user-defined data class, inheriting from the base class `Data`. - - The fields `label`, `x`, `y`, and `z` in this class are examples of arbitrary user data. - """ - - def __init__( - self, - label: str, - x: int, - y: int, - z: int, - ts_event: int, - ts_init: int, - ) -> None: - self.label = label - self.x = x - self.y = y - self.z = z - self._ts_event = ts_event - self._ts_init = ts_init - - @property - def ts_event(self) -> int: - """ - UNIX timestamp (nanoseconds) when the data event occurred. - - Returns - ------- - int - - """ - return self._ts_event - - @property - def ts_init(self) -> int: - """ - UNIX timestamp (nanoseconds) when the object was initialized. - - Returns - ------- - int - - """ - return self._ts_init -``` - -The `Data` abstract base class acts as a contract within the system and requires two properties -for all types of data: `ts_event` and `ts_init`. These represent the UNIX nanosecond timestamps -for when the event occurred and when the object was initialized, respectively. - -The recommended approach to satisfy the contract is to assign `ts_event` and `ts_init` -to backing fields, and then implement the `@property` for each as shown above -(for completeness, the docstrings are copied from the `Data` base class). - -:::info -These timestamps enable Nautilus to correctly order data streams for backtests -using monotonically increasing `ts_init` UNIX nanoseconds. -::: - -We can now work with this data type for backtesting and live trading. For instance, -we could now create an adapter which is able to parse and create objects of this -type - and send them back to the `DataEngine` for consumption by subscribers. - -You can publish a custom data type within your actor/strategy using the message bus -in the following way: - -```python -self.publish_data( - DataType(MyDataPoint, metadata={"some_optional_category": 1}), - MyDataPoint(...), -) -``` - -The `metadata` dictionary optionally adds more granular information that is used in the -topic name to publish data with the message bus. - -Extra metadata information can also be passed to a `BacktestDataConfig` configuration object in order to -enrich and describe custom data objects used in a backtesting context: - -```python -from nautilus_trader.config import BacktestDataConfig - -data_config = BacktestDataConfig( - catalog_path=str(catalog.path), - data_cls=MyDataPoint, - metadata={"some_optional_category": 1}, -) -``` - -You can subscribe to custom data types within your actor/strategy in the following way: - -```python -self.subscribe_data( - data_type=DataType(MyDataPoint, - metadata={"some_optional_category": 1}), - client_id=ClientId("MY_ADAPTER"), -) -``` - -The `client_id` provides an identifier to route the data subscription to a specific client. - -This will result in your actor/strategy passing these received `MyDataPoint` -objects to your `on_data` method. You will need to check the type, as this -method acts as a flexible handler for all custom data. - -```python -def on_data(self, data: Data) -> None: - # First check the type of data - if isinstance(data, MyDataPoint): - # Do something with the data -``` - -### Publishing and receiving signal data - -Here is an example of publishing and receiving signal data using the `MessageBus` from an actor or strategy. -A signal is an automatically generated custom data identified by a name containing only one value of a basic type -(str, float, int, bool or bytes). - -```python -self.publish_signal("signal_name", value, ts_event) -self.subscribe_signal("signal_name") - -def on_signal(self, signal): - print("Signal", data) -``` - -### Option greeks example - -This example demonstrates how to create a custom data type for option Greeks, specifically the delta. -By following these steps, you can create custom data types, subscribe to them, publish them, and store -them in the `Cache` or `ParquetDataCatalog` for efficient retrieval. - -```python -import msgspec -from nautilus_trader.core import Data -from nautilus_trader.core.datetime import unix_nanos_to_iso8601 -from nautilus_trader.model import DataType -from nautilus_trader.serialization.base import register_serializable_type -from nautilus_trader.serialization.arrow.serializer import register_arrow -import pyarrow as pa - -from nautilus_trader.model import InstrumentId -from nautilus_trader.core.datetime import dt_to_unix_nanos, unix_nanos_to_dt, format_iso8601 - - -class GreeksData(Data): - def __init__( - self, instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX"), - ts_event: int = 0, - ts_init: int = 0, - delta: float = 0.0, - ) -> None: - self.instrument_id = instrument_id - self._ts_event = ts_event - self._ts_init = ts_init - self.delta = delta - - def __repr__(self): - return (f"GreeksData(ts_init={unix_nanos_to_iso8601(self._ts_init)}, instrument_id={self.instrument_id}, delta={self.delta:.2f})") - - @property - def ts_event(self): - return self._ts_event - - @property - def ts_init(self): - return self._ts_init - - def to_dict(self): - return { - "instrument_id": self.instrument_id.value, - "ts_event": self._ts_event, - "ts_init": self._ts_init, - "delta": self.delta, - } - - @classmethod - def from_dict(cls, data: dict): - return GreeksData(InstrumentId.from_str(data["instrument_id"]), data["ts_event"], data["ts_init"], data["delta"]) - - def to_bytes(self): - return msgspec.msgpack.encode(self.to_dict()) - - @classmethod - def from_bytes(cls, data: bytes): - return cls.from_dict(msgspec.msgpack.decode(data)) - - def to_catalog(self): - return pa.RecordBatch.from_pylist([self.to_dict()], schema=GreeksData.schema()) - - @classmethod - def from_catalog(cls, table: pa.Table): - return [GreeksData.from_dict(d) for d in table.to_pylist()] - - @classmethod - def schema(cls): - return pa.schema( - { - "instrument_id": pa.string(), - "ts_event": pa.int64(), - "ts_init": pa.int64(), - "delta": pa.float64(), - } - ) -``` - -#### Publishing and receiving data - -Here is an example of publishing and receiving data using the `MessageBus` from an actor or strategy: - -```python -register_serializable_type(GreeksData, GreeksData.to_dict, GreeksData.from_dict) - -def publish_greeks(self, greeks_data: GreeksData): - self.publish_data(DataType(GreeksData), greeks_data) - -def subscribe_to_greeks(self): - self.subscribe_data(DataType(GreeksData)) - -def on_data(self, data): - if isinstance(data, GreeksData): - print("Data", data) -``` - -#### Writing and reading data using the cache - -Here is an example of writing and reading data using the `Cache` from an actor or strategy: - -```python -def greeks_key(instrument_id: InstrumentId): - return f"{instrument_id}_GREEKS" - -def cache_greeks(self, greeks_data: GreeksData): - self.cache.add(greeks_key(greeks_data.instrument_id), greeks_data.to_bytes()) - -def greeks_from_cache(self, instrument_id: InstrumentId): - return GreeksData.from_bytes(self.cache.get(greeks_key(instrument_id))) -``` - -#### Writing and reading data using a catalog - -For streaming custom data to feather files or writing it to parquet files in a catalog -(`register_arrow` needs to be used): - -```python -register_arrow(GreeksData, GreeksData.schema(), GreeksData.to_catalog, GreeksData.from_catalog) - -from nautilus_trader.persistence.catalog import ParquetDataCatalog -catalog = ParquetDataCatalog('.') - -catalog.write_data([GreeksData()]) -``` - -### Creating a custom data class automatically - -The `@customdataclass` decorator enables the creation of a custom data class with default -implementations for all the features described above. - -Each method can also be overridden if needed. Here is an example of its usage: - -```python -from nautilus_trader.model.custom import customdataclass - - -@customdataclass -class GreeksTestData(Data): - instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX") - delta: float = 0.0 - - -GreeksTestData( - instrument_id=InstrumentId.from_str("CL.GLBX"), - delta=1000.0, - ts_event=1, - ts_init=2, -) -``` - -#### Python-only custom data with the PyO3 catalog - -To use custom data with the Rust-backed catalog (`ParquetDataCatalogV2` from `nautilus_pyo3`), use the -`@customdataclass_pyo3()` decorator instead of `@customdataclass`. This adds the methods the Rust catalog -expects (JSON and Arrow IPC serialization). After defining your class, register it once. You can pass either -the **type** (recommended) or a **sample instance**: - -```python -from nautilus_trader.core.nautilus_pyo3 import ParquetDataCatalogV2 -from nautilus_trader.core.nautilus_pyo3.model import register_custom_data_class -from nautilus_trader.model.custom import customdataclass_pyo3 - - -@customdataclass_pyo3() -class MarketTickPython: - symbol: str = "" - price: float = 0.0 - volume: int = 0 - - -# Register by type (no instance needed; call once, e.g. at startup) -register_custom_data_class(MarketTickPython) - -catalog = ParquetDataCatalogV2("/path/to/catalog") -catalog.write_custom_data([MarketTickPython(1, 1, "AAPL", 150.5, 1000)]) -result = catalog.query("MarketTickPython", None, None, None, None, None, True) -``` - -See `nautilus_trader.model.custom.customdataclass_pyo3` for details. - -#### Custom data type stub - -For better IDE code suggestions, you can create a `.pyi` -stub file with the proper constructor signature for your custom data types as well as type hints for attributes. -This is particularly useful when the constructor is dynamically generated at runtime, as it allows the IDE to recognize -and provide suggestions for the class's methods and attributes. - -For instance, if you have a custom data class defined in `greeks.py`, you can create a corresponding `greeks.pyi` file -with the following constructor signature: - -```python -from nautilus_trader.core import Data -from nautilus_trader.model import InstrumentId - - -class GreeksData(Data): - instrument_id: InstrumentId - delta: float - - def __init__( - self, - ts_event: int = 0, - ts_init: int = 0, - instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX"), - delta: float = 0.0, - ) -> GreeksData: ... -``` - -## Related guides - -- [Instruments](instruments.md) - Financial instruments referenced by data. -- [Options](options.md) - Option instruments, chain subscriptions, and strike filtering. -- [Greeks](greeks.md) - Venue-provided and locally computed option Greeks. -- [Cache](cache.md) - Data storage and retrieval. -- [Adapters](adapters.md) - Data sources and connectivity. diff --git a/nautilus-docs/docs/concepts/execution.md b/nautilus-docs/docs/concepts/execution.md deleted file mode 100644 index b743041..0000000 --- a/nautilus-docs/docs/concepts/execution.md +++ /dev/null @@ -1,504 +0,0 @@ -# Execution - -NautilusTrader can handle trade execution and order management for multiple strategies and venues -simultaneously (per instance). Several interacting components are involved in execution, making it -important to understand the possible flows of execution messages (commands and events). - -The main execution-related components include: - -- `Strategy` -- `ExecAlgorithm` (execution algorithms) -- `OrderEmulator` -- `RiskEngine` -- `ExecutionEngine` or `LiveExecutionEngine` -- `ExecutionClient` or `LiveExecutionClient` - -## Execution flow - -The `Strategy` base class inherits from `Actor` and contains all common data methods. -It also provides methods for managing orders and trade execution: - -- `submit_order(...)` -- `submit_order_list(...)` -- `modify_order(...)` -- `cancel_order(...)` -- `cancel_orders(...)` -- `cancel_all_orders(...)` -- `close_position(...)` -- `close_all_positions(...)` -- `query_account(...)` -- `query_order(...)` - -These methods create the necessary execution commands and send them on the message bus to the -relevant components (point-to-point). They also publish events such as `OrderInitialized`. - -The general execution flow looks like the following (each arrow indicates movement across the message bus): - -`Strategy` -> `OrderEmulator` -> `ExecAlgorithm` -> `RiskEngine` -> `ExecutionEngine` -> `ExecutionClient` - -The `OrderEmulator` and `ExecAlgorithm`(s) components are optional in the flow, depending on -individual order parameters (as explained below). - -This diagram illustrates message flow (commands and events) across the Nautilus execution components. - -```mermaid -flowchart LR - strategy[Strategy] - emulator[OrderEmulator] - algo[ExecAlgorithm] - risk[RiskEngine] - engine[ExecutionEngine] - client[ExecutionClient] - - strategy <--> emulator - strategy <--> algo - strategy <--> risk - emulator --> risk - algo --> risk - risk <--> engine - engine <--> client -``` - -## Order Management System (OMS) - -An order management system (OMS) type refers to the method used for assigning orders to positions and tracking those positions for an instrument. -OMS types apply to both strategies and venues (simulated and real). Even if a venue doesn't explicitly -state the method in use, an OMS type is always in effect. The OMS type for a component can be specified -using the `OmsType` enum. - -The `OmsType` enum has three variants: - -- `UNSPECIFIED`: The OMS type defaults based on where it is applied (details below) -- `NETTING`: Positions are combined into a single position per instrument ID -- `HEDGING`: Multiple positions per instrument ID are supported (both long and short) - -The table below describes different configuration combinations and their applicable scenarios. -When the strategy and venue OMS types differ, the `ExecutionEngine` handles this by overriding or assigning `position_id` values for received `OrderFilled` events. -A "virtual position" refers to a position ID that exists within the Nautilus system but not on the venue in -reality. - -| Strategy OMS | Venue OMS | Description | -|:-----------------------------|:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| `NETTING` | `NETTING` | The strategy uses the venue's native OMS type, with a single position ID per instrument ID. | -| `HEDGING` | `HEDGING` | The strategy uses the venue's native OMS type, with multiple position IDs per instrument ID (both `LONG` and `SHORT`). | -| `NETTING` | `HEDGING` | The strategy **overrides** the venue's native OMS type. The venue tracks multiple positions per instrument ID, but Nautilus maintains a single position ID. | -| `HEDGING` | `NETTING` | The strategy **overrides** the venue's native OMS type. The venue tracks a single position per instrument ID, but Nautilus maintains multiple position IDs. | - -:::note -Configuring OMS types separately for strategies and venues increases platform complexity but allows -for a wide range of trading styles and preferences (see below). -::: - -OMS config examples: - -- Most cryptocurrency exchanges use a `NETTING` OMS type, representing a single position per market. It may be desirable for a trader to track multiple "virtual" positions for a strategy. -- Some FX ECNs or brokers use a `HEDGING` OMS type, tracking multiple positions both `LONG` and `SHORT`. The trader may only care about the NET position per currency pair. - -:::info -Nautilus does not yet support venue-side hedging modes such as Binance `BOTH` vs. `LONG/SHORT` where the venue nets per direction. -It is advised to keep Binance account configurations as `BOTH` so that a single position is netted. -::: - -### OMS configuration - -If a strategy OMS type is not explicitly set using the `oms_type` configuration option, -it will default to `UNSPECIFIED`. This means the `ExecutionEngine` will not override any venue `position_id`s, -and the OMS type will follow the venue's OMS type. - -:::tip -When configuring a backtest, you can specify the `oms_type` for the venue. For accuracy, match this with the OMS type used by the venue. -::: - -## Risk engine - -The `RiskEngine` is a component of every Nautilus system, including backtest, sandbox, and live environments. -Every order command and event passes through the `RiskEngine` unless specifically bypassed in the `RiskEngineConfig`. - -The `RiskEngine` performs these pre-trade risk checks: - -- Price precisions correct for the instrument. -- Prices are positive (unless an option type instrument) -- Quantity precisions correct for the instrument. -- Below maximum notional for the instrument. -- Within maximum or minimum quantity for the instrument. -- Only reducing position when a `reduce_only` execution instruction is specified for the order. - -If any risk check fails, the system generates an `OrderDenied` event with a human-readable reason, -closing the order. - -### Trading state - -Additionally, the current trading state of a Nautilus system affects order flow. - -The `TradingState` enum has three variants: - -- `ACTIVE`: Operates normally. -- `HALTED`: Does not process further order commands until state changes. -- `REDUCING`: Only processes cancels or commands that reduce open positions. - -See the [`RiskEngineConfig` API Reference](/docs/python-api-latest/config.html#nautilus_trader.risk.config.RiskEngineConfig) for further details. - -## Execution algorithms - -The platform supports custom execution algorithm components and provides built-in algorithms -such as TWAP (Time-Weighted Average Price). - -### TWAP (Time-Weighted Average Price) - -The TWAP algorithm spreads execution evenly over a specified time horizon. It receives a -primary order representing the total size and direction, then spawns smaller child orders -executed at regular intervals. - -This reduces the market impact of the full order size by spreading trade volume over time. - -The algorithm will immediately submit the first order, with the final order submitted being the -primary order at the end of the horizon period. - -Using the TWAP algorithm as an example (found in `/examples/algorithms/twap.py`), this example -demonstrates how to initialize and register a TWAP execution algorithm directly with a -`BacktestEngine` (assuming an engine is already initialized): - -```python -from nautilus_trader.examples.algorithms.twap import TWAPExecAlgorithm - -# `engine` is an initialized BacktestEngine instance -exec_algorithm = TWAPExecAlgorithm() -engine.add_exec_algorithm(exec_algorithm) -``` - -For this particular algorithm, two parameters must be specified: - -- `horizon_secs` -- `interval_secs` - -The `horizon_secs` parameter determines the time period over which the algorithm will execute, while -the `interval_secs` parameter sets the time between individual order executions. These parameters -determine how a primary order is split into a series of spawned orders. - -```python -from decimal import Decimal -from nautilus_trader.model.data import BarType -from nautilus_trader.test_kit.providers import TestInstrumentProvider -from nautilus_trader.examples.strategies.ema_cross_twap import EMACrossTWAP, EMACrossTWAPConfig - -# Configure your strategy -config = EMACrossTWAPConfig( - instrument_id=TestInstrumentProvider.ethusdt_binance().id, - bar_type=BarType.from_str("ETHUSDT.BINANCE-250-TICK-LAST-INTERNAL"), - trade_size=Decimal("0.05"), - fast_ema_period=10, - slow_ema_period=20, - twap_horizon_secs=10.0, # execution algorithm parameter (total horizon in seconds) - twap_interval_secs=2.5, # execution algorithm parameter (seconds between orders) -) - -# Instantiate your strategy -strategy = EMACrossTWAP(config=config) -``` - -Alternatively, you can specify these parameters dynamically per order, determining them based on -actual market conditions. In this case, the strategy configuration parameters could be provided to -an execution model which determines the horizon and interval. - -:::info -There is no limit to the number of execution algorithm parameters you can create. The parameters -must be a dictionary with string keys and primitive values (values that can be serialized -over the wire, such as ints, floats, and strings). -::: - -### Writing execution algorithms - -To build a custom execution algorithm, define a class that inherits from `ExecAlgorithm`. - -An execution algorithm is a type of `Actor`, so it's capable of the following: - -- Request and subscribe to data. -- Access the `Cache`. -- Set time alerts and/or timers using a `Clock`. - -Additionally it can: - -- Access the central `Portfolio`. -- Spawn secondary orders from a received primary (original) order. - -Once an execution algorithm is registered, and the system is running, it will receive orders off the -messages bus which are addressed to its `ExecAlgorithmId` via the `exec_algorithm_id` order parameter. -The order may also carry the `exec_algorithm_params` being a `dict[str, Any]`. - -:::warning -Because of the flexibility of the `exec_algorithm_params` dictionary, it's important to thoroughly -validate all of the key value pairs for correct operation of the algorithm (for starters that the -dictionary is not `None` and all necessary parameters actually exist). -::: - -Received orders will arrive via the following `on_order(...)` method. These received orders are -known as "primary" (original) orders when being handled by an execution algorithm. - -```python -from nautilus_trader.model.orders.base import Order - -def on_order(self, order: Order) -> None: - # Handle the order here -``` - -When the algorithm is ready to spawn a secondary order, it can use one of the following methods: - -- `spawn_market(...)` (spawns a `MARKET` order) -- `spawn_market_to_limit(...)` (spawns a `MARKET_TO_LIMIT` order) -- `spawn_limit(...)` (spawns a `LIMIT` order) - -:::note -Additional order types will be implemented in future versions, as the need arises. -::: - -Each of these methods takes the primary (original) `Order` as the first argument. The primary order -quantity will be reduced by the `quantity` passed in (becoming the spawned orders quantity). - -:::warning -The spawned quantity must not exceed the primary order's `leaves_qty` (remaining unfilled quantity). -::: - -:::note -If a spawned order is denied or rejected before acceptance, the deducted quantity is automatically -restored to the primary order. Once accepted by the venue, the reduction is considered committed. -::: - -Once the desired number of secondary orders have been spawned, and the execution routine is over, -the intention is that the algorithm will then finally send the primary (original) order. - -### Spawned orders - -All secondary orders spawned from an execution algorithm will carry a `exec_spawn_id` which is -the `ClientOrderId` of the primary (original) order, and whose `client_order_id` -derives from this original identifier with the following convention: - -- `exec_spawn_id` (primary order `client_order_id` value) -- `spawn_sequence` (the sequence number for the spawned order) - -``` -{exec_spawn_id}-E{spawn_sequence} -``` - -e.g. `O-20230404-001-000-E1` (for the first spawned order) - -:::note -The "primary" and "secondary" / "spawn" terminology was specifically chosen to avoid conflict -or confusion with the "parent" and "child" contingent orders terminology (an execution algorithm may also deal with contingent orders). -::: - -### Managing execution algorithm orders - -The `Cache` provides several methods to aid in managing (keeping track of) the activity of -an execution algorithm. Calling the below method will return all execution algorithm orders -for the given query filters. - -```python -def orders_for_exec_algorithm( - self, - exec_algorithm_id: ExecAlgorithmId, - venue: Venue | None = None, - instrument_id: InstrumentId | None = None, - strategy_id: StrategyId | None = None, - side: OrderSide = OrderSide.NO_ORDER_SIDE, -) -> list[Order]: -``` - -As well as more specifically querying the orders for a certain execution series/spawn. -Calling the below method will return all orders for the given `exec_spawn_id` (if found). - -```python -def orders_for_exec_spawn(self, exec_spawn_id: ClientOrderId) -> list[Order]: -``` - -:::note -This also includes the primary (original) order. -::: - -## Own order books - -Own order books are L3 order books that track only your own (user) orders organized by price level, maintained separately from the venue's public order books. - -### Purpose - -Own order books serve several purposes: - -- Monitor the state of your orders within the venue's public book in real-time. -- Validate order placement by checking available liquidity at price levels before submission. -- Help prevent self-trading by identifying price levels where your own orders already exist. -- Support advanced order management strategies that depend on queue position. -- Enable reconciliation between internal state and venue state during live trading. - -### Lifecycle - -Own order books are maintained per instrument and automatically updated as orders transition through their lifecycle. -Orders are added when submitted or accepted, updated when modified, and removed when filled, canceled, rejected, or expired. - -Only orders with prices can be represented in own order books. Market orders and other order types without explicit prices are excluded since they cannot be positioned at specific price levels. - -### Safe cancellation queries - -When querying own order books for orders to cancel, use a `status` filter that **excludes** `PENDING_CANCEL` to avoid processing orders already being cancelled. - -:::warning -Including `PENDING_CANCEL` in status filters can cause: - -- Duplicate cancel attempts on the same order. -- Inflated open order counts (orders in `PENDING_CANCEL` remain "open" until confirmed canceled). -- Order state explosion when multiple strategies attempt to cancel the same orders. - -::: - -The optional `accepted_buffer_ns` many methods expose is a time-based guard that only returns orders whose `ts_accepted` is at least that many nanoseconds in the past. Orders that have not yet been accepted by the venue still have `ts_accepted = 0`, so they are included once the buffer window elapses. To exclude those inflight orders you must pair the buffer with an explicit status filter (for example, restrict to `ACCEPTED` / `PARTIALLY_FILLED`). - -### Auditing - -During live trading, own order books can be periodically audited against the cache's order indexes to ensure consistency. -The audit mechanism verifies that closed orders are properly removed and that inflight orders (submitted but not yet accepted) remain tracked during venue latency windows. - -The audit interval can be configured using the `own_books_audit_interval_secs` parameter in live trading configurations. - -## Overfills - -An overfill occurs when the cumulative fill quantity for an order exceeds the original order quantity. -For example, an order for 100 units that receives fills totaling 110 units has an overfill of 10 units. - -### How overfills occur - -Overfills can result from two fundamentally different causes: - -- Duplicate fill events (a network/messaging issue). -- Genuine overfills at the matching engine (a real execution outcome). - -**Genuine overfills at the matching engine** - -In some cases, the matching engine actually executes more quantity than the order requested. -This is a real execution outcome, not a duplicate event: - -- **Matching engine race conditions**: In fast markets with high concurrency, an order may match - against multiple counterparties nearly simultaneously before being fully removed from the book. -- **Minimum lot size constraints**: If an order's remaining quantity falls below the venue's minimum - tradeable lot, some matching engines fill the minimum lot anyway rather than leaving an untradeable remainder. -- **DEX/AMM mechanics**: Decentralized exchanges using automated market makers may have execution - mechanics where actual fill quantities differ slightly from requested due to price impact calculations. -- **Multi-fill atomicity**: Some venues do not guarantee atomic fill quantities across partial - executions, allowing aggregate fills to exceed the original order quantity. - -**Duplicate fill events** - -Separate from genuine overfills, the same fill event may be delivered multiple times: - -- WebSocket reconnection replaying previously received events. -- The venue's internal retry or delivery guarantee mechanisms. -- API timing issues in the venue's execution reporting. - -The system handles duplicate events via `trade_id` deduplication (see below), but duplicates with -different `trade_id` values require overfill handling. - -**Race conditions with reconciliation** - -During live trading, the system maintains state through two parallel channels: - -- Real-time fill events arriving via WebSocket. -- Periodic reconciliation polling the venue for fill history. - -If the same fill arrives through both channels with different identifiers before deduplication -can occur, both may be applied to the order. This is particularly likely during: - -- System startup when reconciliation runs while WebSocket connections are establishing. -- Network instability causing reconnections mid-fill. -- High-frequency trading where fills arrive faster than reconciliation cycles. - -The likelihood of reconciliation race conditions increases when: - -- **Thresholds are reduced**: The `open_check_threshold_ms` and `inflight_check_threshold_ms` settings - (both default to 5,000 ms) define how long the engine waits before acting on discrepancies. - Reducing these below the round-trip latency to your venue increases the chance of processing - a fill via reconciliation before the real-time event arrives (or vice versa). -- **Reconciliation frequency is increased**: Setting `open_check_interval_secs` to aggressive values - (e.g., 1-2 seconds) increases how often the system polls the venue, creating more opportunities - for race conditions with real-time events. -- **Startup delay is reduced**: The `reconciliation_startup_delay_secs` setting (default 10 seconds) - provides time for WebSocket connections to stabilize before continuous reconciliation begins. - Reducing this increases the chance of duplicate fills during the startup window. - -See [Continuous reconciliation](live.md#continuous-reconciliation) for configuration details. - -### System behavior - -The `ExecutionEngine` checks for potential overfills before applying each fill event by comparing -the order's current `filled_qty` plus the incoming `last_qty` against the original `quantity`. - -The `allow_overfills` configuration option (default: `False`) controls how overfills are handled: - -| `allow_overfills` | Behavior | -|-------------------|----------------------------------------------------------------------------| -| `False` | Logs an error and rejects the fill, preserving the order's current state. | -| `True` | Logs a warning, applies the fill, and tracks the excess in `overfill_qty`. | - -When overfills are allowed, the order's `overfill_qty` field tracks the excess quantity. -The order transitions to `FILLED` status and `leaves_qty` is clamped to zero. - -### Duplicate fill detection - -The `Order` model enforces that each `trade_id` can only be applied once. Inside `Order.apply()`, -a hard check raises an error if the incoming fill's `trade_id` already exists on the order. -This is the invariant that prevents double-counting executions. - -**Core engine path (backtest and real-time event processing)** - -In the core `ExecutionEngine` (used for backtests and processing real-time fill events), before -calling `apply()`, the engine checks `Order.is_duplicate_fill()` which compares: - -- `trade_id` -- `order_side` -- `last_px` -- `last_qty` - -If all fields match an existing fill exactly, the event is skipped gracefully with a warning log. -This avoids raising an error for benign exact replays (e.g., from WebSocket reconnection). -If the `trade_id` matches but other fields differ ("noisy replay"), the 4-field check passes -but `Order.apply()` will raise an error due to the duplicate `trade_id`. The engine catches -this error, logs the exception with full context, and drops the fill - it does not crash. - -**Live reconciliation sanitizer** - -During live reconciliation, `LiveExecutionEngine` pre-filters on `trade_id` alone *before* -generating fill events. This check runs before the 4-field check described above. If a fill -report arrives with a `trade_id` that already exists on the order, it is skipped regardless -of whether the price or quantity differs. When the data does differ, a warning is logged to -alert operators to potential venue data quality issues. - -This pre-filtering ensures that "noisy duplicates" from venue replays or reconciliation races -are filtered out before they can trigger model integrity errors. If a venue legitimately needs -to correct fill data, it should use proper execution report semantics rather than resending -with the same `trade_id`. - -### Configuration - -For live trading, enable overfill tolerance in the `LiveExecEngineConfig`: - -```python -from nautilus_trader.live.config import LiveExecEngineConfig - -config = LiveExecEngineConfig( - allow_overfills=True, # Log warning instead of rejecting -) -``` - -:::tip -Enable `allow_overfills=True` when trading on venues known to emit duplicate fills or when -position reconciliation races with exchange fill events are expected. Monitor the logs for -overfill warnings to identify patterns that may require venue-specific handling. -::: - -:::warning -When `allow_overfills=False` (the default), rejected fills may cause position discrepancies -between the system and the venue. Use the [reconciliation](live.md#execution-reconciliation) -features to detect and resolve such discrepancies. -::: - -## Related guides - -- [Orders](orders.md) - Order types and management. -- [Positions](positions.md) - Position tracking from executions. -- [Strategies](strategies.md) - Order submission from strategies. diff --git a/nautilus-docs/docs/concepts/greeks.md b/nautilus-docs/docs/concepts/greeks.md deleted file mode 100644 index 707b4a8..0000000 --- a/nautilus-docs/docs/concepts/greeks.md +++ /dev/null @@ -1,321 +0,0 @@ -# Greeks - -Nautilus provides two paths for working with option Greeks -(sensitivities of option prices to changes in market variables): - -1. **Venue-provided Greeks (Rust/PyO3)** -- real-time Greeks streamed from venues - like Deribit and Bybit via the `OptionGreeks` data type and the option chain - aggregation system. -2. **Local Greeks calculator (Cython/Python)** -- the `GreeksCalculator` class that - computes Black-Scholes Greeks from cached market data, with support for portfolio - aggregation, shock scenarios, and beta weighting. - -Either path works independently or together. Venue-provided Greeks arrive -through the data subscription system and require no local computation. The local -calculator covers venues that do not stream Greeks, backtesting, and custom -adjustments (shocks, beta weighting, percent Greeks). - -## Venue-provided Greeks (Rust/PyO3) - -### OptionGreeks - -The `OptionGreeks` type represents venue-provided sensitivities for a single option -contract. It is a Rust-native type exposed to Python via PyO3. - -| Field | Type | Description | -|--------------------|------------------|-----------------------------------------------------| -| `instrument_id` | `InstrumentId` | The option contract these Greeks apply to. | -| `delta` | `float` | Rate of change of option price per unit underlying. | -| `gamma` | `float` | Rate of change of delta per unit underlying. | -| `vega` | `float` | Sensitivity to a 1% change in implied volatility. | -| `theta` | `float` | Daily time decay (dV/dt / 365.25). | -| `rho` | `float` | Sensitivity to a change in interest rate. | -| `mark_iv` | `float` or None | Mark implied volatility. | -| `bid_iv` | `float` or None | Bid implied volatility. | -| `ask_iv` | `float` or None | Ask implied volatility. | -| `underlying_price` | `float` or None | Underlying price at time of calculation. | -| `open_interest` | `float` or None | Open interest for the contract. | -| `ts_event` | `int` | UNIX timestamp (nanoseconds) of the event. | -| `ts_init` | `int` | UNIX timestamp (nanoseconds) when initialized. | - -Subscribe from an actor or strategy: - -```python -self.subscribe_option_greeks(instrument_id, client_id=ClientId("DERIBIT")) -``` - -Handle updates: - -```python -def on_option_greeks(self, greeks: OptionGreeks) -> None: - self.log.info(f"delta={greeks.delta:.4f} gamma={greeks.gamma:.6f}") -``` - -See the [Options](options.md) guide for the full subscription API including option -chain aggregation, strike range filtering, and snapshot modes. - -### Underlying Rust types - -The core Rust implementation lives in `crates/model/src/data/greeks.rs`: - -- `OptionGreekValues` -- a plain struct with `delta`, `gamma`, `vega`, `theta`, `rho` - fields. Implements `Add` and `Mul` for aggregation. -- `OptionGreeks` (in `crates/model/src/data/option_chain.rs`) -- wraps - `OptionGreekValues` with `instrument_id`, implied volatility fields, and timestamps. - Implements `Deref` so you can access Greeks fields directly. -- `HasGreeks` trait -- provides a `greeks()` method returning `OptionGreekValues`. - Implemented by both `OptionGreekValues` and `OptionGreeks`. - -### Black-Scholes functions (Rust/PyO3) - -Low-level pricing functions exposed to Python from `crates/model/src/data/greeks.rs`: - -```python -from nautilus_trader.core.nautilus_pyo3 import ( - black_scholes_greeks, - imply_vol, - imply_vol_and_greeks, - refine_vol_and_greeks, -) - -# Compute Greeks given known volatility -result = black_scholes_greeks(s=100.0, r=0.05, b=0.0, vol=0.20, is_call=True, k=100.0, t=0.25) -# result.delta, result.gamma, result.vega, result.theta, result.price, result.vol - -# Imply volatility from market price, then compute Greeks -result = imply_vol_and_greeks(s=100.0, r=0.05, b=0.0, is_call=True, k=100.0, t=0.25, price=5.0) - -# Refine volatility from a starting vol estimate (faster convergence) -result = refine_vol_and_greeks(s=100.0, r=0.05, b=0.0, is_call=True, k=100.0, t=0.25, - target_price=5.0, initial_vol=0.18) -``` - -The `BlackScholesGreeksResult` returned by these functions contains: `price`, `vol`, -`delta`, `gamma`, `vega`, `theta`, and `itm_prob`. - -**Conventions:** - -- Vega is scaled by 0.01 (sensitivity to a 1 percentage point vol change). -- Theta is scaled by 1/365.25 (daily decay). -- American-style options are priced as European for Greeks computation. - -## Local Greeks calculator (Cython/Python) - -### GreeksCalculator - -The `GreeksCalculator` class in `nautilus_trader/model/greeks.pyx` computes -Black-Scholes Greeks from cached market data. It is accessible from any actor or -strategy. - -```python -from nautilus_trader.model.greeks import GreeksCalculator - -# Typically created in on_start() -calculator = GreeksCalculator(cache=self.cache, clock=self.clock) -``` - -#### Instrument Greeks - -Compute Greeks for a single instrument (option or underlying) with quantity of 1: - -```python -greeks = calculator.instrument_greeks( - instrument_id=option_id, - flat_interest_rate=0.0425, # used if no yield curve in cache -) -# Returns GreeksData or None -``` - -The calculator: - -1. Looks up the instrument and its underlying in the cache. -2. Retrieves current prices (MID preferred, LAST as fallback). -3. Looks up yield curves from the cache (falls back to `flat_interest_rate`). -4. Implies volatility from the market price using `imply_vol_and_greeks`. -5. Returns a `GreeksData` object with all computed values. - -For non-option instruments (futures, equities), the calculator returns a `GreeksData` -with `delta=1` (or beta-weighted delta) and no gamma/vega/theta. - -**Shock scenarios** -- apply hypothetical changes to spot, volatility, or time: - -```python -greeks = calculator.instrument_greeks( - instrument_id=option_id, - spot_shock=10.0, # +10 points on underlying - vol_shock=0.02, # +2% absolute vol increase - time_to_expiry_shock=1/365, # roll forward one day -) -``` - -**Volatility update** -- refine implied vol from a cached starting point for faster -convergence: - -```python -greeks = calculator.instrument_greeks( - instrument_id=option_id, - update_vol=True, # use cached vol as starting point - cache_greeks=True, # store result for next iteration -) -``` - -**Beta-weighted Greeks** -- express delta and gamma in terms of an index: - -```python -greeks = calculator.instrument_greeks( - instrument_id=option_id, - index_instrument_id=InstrumentId.from_str("SPX.CBOE"), - beta_weights={underlying_id: 1.15}, - percent_greeks=True, -) -``` - -**Time-weighted vega** -- normalize vega across different expirations: - -```python -greeks = calculator.instrument_greeks( - instrument_id=option_id, - vega_time_weight_base=30, # normalize to 30-day vega -) -``` - -#### Portfolio Greeks - -Aggregate Greeks across all open positions matching filter criteria: - -```python -portfolio = calculator.portfolio_greeks( - underlyings=["AAPL", "MSFT"], - venue=Venue("CBOE"), - strategy_id=StrategyId("DELTA_HEDGE-001"), - flat_interest_rate=0.0425, - index_instrument_id=InstrumentId.from_str("SPX.CBOE"), - beta_weights=beta_dict, - percent_greeks=True, -) -# Returns PortfolioGreeks: pnl, price, delta, gamma, vega, theta -``` - -Filters: - -- `underlyings` -- list of symbol prefixes (e.g., `["AAPL"]` matches AAPL stock and - all AAPL options). -- `venue` -- restrict to a single venue. -- `instrument_id` -- restrict to a single instrument. -- `strategy_id` -- restrict to a single strategy. -- `side` -- filter by position side (LONG, SHORT). -- `greeks_filter` -- callable that accepts `PortfolioGreeks` per position; return - `True` to include. - -### GreeksData - -`GreeksData` is a Python custom data class (`@customdataclass`) that carries the full -context of a single instrument's Greeks computation. It extends `Data` and supports -Arrow serialization, cache storage, and catalog persistence. - -| Field | Type | Description | -|---------------------|-----------------|--------------------------------------------------------| -| `instrument_id` | `InstrumentId` | The instrument. | -| `is_call` | `bool` | True for call, False for put. | -| `strike` | `float` | Strike price. | -| `expiry` | `int` | Expiry date as YYYYMMDD integer. | -| `expiry_in_days` | `int` | Days to expiry. | -| `expiry_in_years` | `float` | Years to expiry (days / 365.25). | -| `multiplier` | `float` | Contract multiplier. | -| `quantity` | `float` | Position quantity (always 1 from `instrument_greeks`). | -| `underlying_price` | `float` | Underlying price used in calculation. | -| `interest_rate` | `float` | Interest rate used. | -| `cost_of_carry` | `float` | Cost of carry (r - dividend yield; 0 for futures). | -| `vol` | `float` | Implied volatility. | -| `pnl` | `float` | PnL relative to position entry (if position provided). | -| `price` | `float` | Model price. | -| `delta` | `float` | Delta. | -| `gamma` | `float` | Gamma. | -| `vega` | `float` | Vega (dV / 1% vol change). | -| `theta` | `float` | Theta (daily decay). | -| `itm_prob` | `float` | In-the-money probability. | - -`GreeksData` scales to portfolio level via its `to_portfolio_greeks()` method, which -multiplies all values by the contract `multiplier`. The `*` operator applies position -quantity: - -```python -position_greeks = signed_qty * instrument_greeks # returns PortfolioGreeks -``` - -### PortfolioGreeks - -`PortfolioGreeks` is the aggregated result from `portfolio_greeks()`. It supports -addition (`+`) for combining positions and scalar multiplication (`*`) for scaling: - -| Field | Type | Description | -|---------|---------|------------------------| -| `pnl` | `float` | Aggregate PnL. | -| `price` | `float` | Aggregate model value. | -| `delta` | `float` | Portfolio delta. | -| `gamma` | `float` | Portfolio gamma. | -| `vega` | `float` | Portfolio vega. | -| `theta` | `float` | Portfolio theta. | - -### YieldCurveData - -`YieldCurveData` stores an interest rate or dividend yield curve. The `GreeksCalculator` -looks up curves from the cache by currency code (for interest rates) or by underlying -instrument ID (for dividend yields). - -```python -from nautilus_trader.model.greeks_data import YieldCurveData -import numpy as np - -curve = YieldCurveData( - ts_event=0, - ts_init=0, - curve_name="USD", - tenors=np.array([0.25, 0.5, 1.0, 2.0]), - interest_rates=np.array([0.04, 0.042, 0.045, 0.048]), -) - -# Callable: interpolates rate for a given tenor -rate = curve(0.75) # quadratic interpolation -``` - -## Choosing between the two paths - -| Criterion | Venue-provided (`OptionGreeks`) | Local calculator (`GreeksCalculator`) | -|------------------------------|----------------------------------------|------------------------------------------| -| Computation | Done by the venue | Local Black-Scholes | -| Latency | Arrives with market data | Computed on demand | -| Venues | Deribit, Bybit (adapters with support) | Any venue with option instruments | -| Shock scenarios | Not supported | Spot, vol, and time shocks | -| Portfolio aggregation | Manual (iterate `OptionChainSlice`) | Built-in via `portfolio_greeks()` | -| Beta weighting | Not supported | Built-in | -| Backtest support | Via recorded `OptionGreeks` data | From cached prices at any point in time | -| Greeks available | delta, gamma, vega, theta, rho, IV, OI | delta, gamma, vega, theta, itm_prob, vol | -| Data type | `OptionGreeks` (Rust/PyO3) | `GreeksData` (Python `@customdataclass`) | - -## Greek definitions - -For reference, the Greeks that Nautilus computes: - -| Greek | Symbol | Definition | -|------------|--------|-------------------------------------------------------------------------------| -| Delta | `d` | First derivative of option price with respect to underlying price (dV/dS). | -| Gamma | `g` | Second derivative of option price with respect to underlying price (d2V/dS2). | -| Vega | `v` | Sensitivity to a 1 percentage point change in implied volatility (dV/dVol). | -| Theta | `t` | Daily time decay: change in option price per calendar day (dV/dt / 365.25). | -| Rho | `r` | Sensitivity to a change in the risk-free interest rate (dV/dr). | -| ITM prob | - | Probability that the option finishes in the money: P(ϕS_T > ϕK), where ϕ = 1 for calls and ϕ = -1 for puts. | - -## Examples - -Complete working examples are available in the repository: - -- `examples/live/bybit/bybit_option_greeks.py` -- subscribe to Bybit venue-provided Greeks. -- `examples/live/deribit/deribit_option_greeks.py` -- subscribe to Deribit venue-provided Greeks. - -## Related guides - -- [Options](options.md) - Option instruments, chain subscriptions, and strike filtering. -- [Data](data.md) - Built-in data types, custom data, and the subscription model. -- [Actors](actors.md) - Subscription and handler reference. -- [Strategies](strategies.md) - Strategy implementation and handler methods. diff --git a/nautilus-docs/docs/concepts/index.md b/nautilus-docs/docs/concepts/index.md deleted file mode 100644 index 28068fe..0000000 --- a/nautilus-docs/docs/concepts/index.md +++ /dev/null @@ -1,113 +0,0 @@ -# Concepts - -These guides explain the core components, architecture, and design of NautilusTrader. - -## Overview - -Main features and intended use cases for the platform. - -## Architecture - -The principles, structures, and designs that underpin the platform. - -## Actors - -The `Actor` is the base component for interacting with the trading system. -Covers capabilities and implementation details. - -## Strategies - -How to implement trading strategies using the `Strategy` component. - -## Instruments - -Instrument definitions for tradable assets and contracts. - -## Value types - -The immutable numeric types (`Price`, `Quantity`, `Money`) used throughout the platform, -including their arithmetic behavior, precision handling, and type-specific constraints. - -## Data - -Built-in data types for the trading domain, and how to work with custom data. - -## Options - -Option instrument types, venue-provided Greeks streaming, option chain subscriptions -with strike range filtering, and snapshot aggregation. - -## Greeks - -Option Greeks (delta, gamma, vega, theta) from two paths: venue-provided real-time -Greeks via the Rust/PyO3 `OptionGreeks` type, and the local `GreeksCalculator` for -Black-Scholes computation with shock scenarios, beta weighting, and portfolio aggregation. - -## Custom data - -How the custom data system works across Python and Rust: registration, persistence, -Arrow encoding, and runtime routing through actors and strategies. - -## Order book - -The high-performance order book, own order tracking, filtered views for net liquidity, and binary market support. - -## Execution - -Trade execution and order management across multiple strategies and venues simultaneously (per instance), -including the components involved and the flow of execution messages (commands and events). - -## Orders - -Available order types, supported execution instructions, advanced order types, and emulated orders. - -## Positions - -Position lifecycle, aggregation from order fills, PnL calculations, and position snapshotting -for netting OMS configurations. - -## Cache - -The `Cache` is the central in-memory store for all trading-related data. -Covers capabilities and best practices. - -## Message Bus - -The `MessageBus` enables decoupled messaging between components, supporting point-to-point, -publish/subscribe, and request/response patterns. - -## Portfolio - -The `Portfolio` tracks all positions across strategies and instruments, providing a unified view -of holdings, risk exposure, and performance. - -## Reports - -Execution reports, portfolio analysis, PnL accounting, and backtest post-run analysis. - -## Logging - -High-performance logging for both backtesting and live trading, implemented in Rust. - -## Backtesting - -Running simulated trading on historical data using a specific system implementation. - -## Visualization - -Interactive tearsheets for analyzing backtest results, including charts, themes, -customization options, and custom visualizations via the extensible chart registry. - -## Live trading - -Deploying backtested strategies in real-time without code changes, and the key differences -between backtesting and live trading. - -## Adapters - -Requirements and best practices for developing integration adapters for data providers and trading venues. - -:::note -The Python API reference (linked in the sidebar) is the source of truth for the platform. -If there are discrepancies between these guides and the API reference, the API reference is correct. -::: diff --git a/nautilus-docs/docs/concepts/instruments.md b/nautilus-docs/docs/concepts/instruments.md deleted file mode 100644 index 179fc34..0000000 --- a/nautilus-docs/docs/concepts/instruments.md +++ /dev/null @@ -1,541 +0,0 @@ -# Instruments - -An instrument represents the specification for any tradable asset or contract. All -instrument types are implemented as Rust structs that implement the `Instrument` -trait. In Python, these are exposed as Cython extension types (via -`nautilus_trader.model.instruments`), with parallel PyO3 representations that are -converted to Cython types at the boundary. Pure Rust systems use the Rust types -directly. The platform supports a range of asset classes and instrument classes: - -- `Equity`: Listed shares or ETFs traded on cash markets. -- `CurrencyPair`: Spot FX or crypto pair in BASE/QUOTE format traded in cash markets. -- `Commodity`: Spot commodity instrument (e.g., gold or oil) traded in cash markets. -- `IndexInstrument`: Spot index calculated from constituents; used as a reference price and not directly tradable. -- `FuturesContract`: Deliverable futures contract with defined underlying, expiry, and multiplier. -- `FuturesSpread`: Exchange-defined multi-leg futures strategy (e.g., calendar or inter-commodity) quoted as one instrument. -- `CryptoFuture`: Dated, deliverable crypto futures contract with fixed expiry, underlying crypto, and settlement currency. -- `CryptoPerpetual`: Perpetual futures contract (perpetual swap) on crypto with no expiry; can be inverse or quanto-settled. -- `PerpetualContract`: Asset-class agnostic perpetual swap for any underlying (FX, equities, commodities, indexes, crypto). -- `OptionContract`: Exchange-traded option (put or call) on an underlying with strike and expiry. -- `OptionSpread`: Exchange-defined multi-leg options strategy (e.g., vertical, calendar, straddle) quoted as one instrument. -- `CryptoOption`: Option on a crypto underlying with crypto quote/settlement; supports inverse or quanto styles. -- `BinaryOption`: Fixed-payout option that settles to 0 or 1 based on a binary outcome. -- `Cfd`: Over-the-counter Contract for Difference that tracks an underlying and is cash-settled. -- `BettingInstrument`: Sports/gaming market selection (e.g., team or runner) tradable on betting venues. -- `SyntheticInstrument`: Synthetic instrument with prices derived from component instruments using a formula. - -## Symbology - -All instruments should have a unique `InstrumentId`, which is made up of both the native symbol, and venue ID, separated by a period. -For example, on the Binance Futures crypto exchange, the Ethereum Perpetual Futures Contract has the instrument ID `ETHUSDT-PERP.BINANCE`. - -All native symbols *should* be unique for a venue (this is not always the case e.g. Binance share native symbols between spot and futures markets), -and the `{symbol.venue}` combination *must* be unique for a Nautilus system. - -:::warning -The correct instrument must be matched to a market dataset such as ticks or order book data for logically sound operation. -An incorrectly specified instrument may truncate data or otherwise produce surprising results. -::: - -## Backtesting - -Generic test instruments can be instantiated through the `TestInstrumentProvider`: - -```python -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -audusd = TestInstrumentProvider.default_fx_ccy("AUD/USD") -``` - -```python -from nautilus_trader.adapters.binance.spot.providers import BinanceSpotInstrumentProvider -from nautilus_trader.model import InstrumentId - -provider = BinanceSpotInstrumentProvider(client=binance_http_client) -await provider.load_all_async() - -btcusdt = InstrumentId.from_str("BTCUSDT.BINANCE") -instrument = provider.find(btcusdt) -``` - -Or defined directly by constructing a specific instrument type: - -```python -from nautilus_trader.model.instruments import OptionContract - -instrument = OptionContract(...) # provide all necessary parameters -``` - -```rust -use nautilus_model::instruments::CurrencyPair; -use nautilus_model::identifiers::{InstrumentId, Symbol}; -use nautilus_model::types::{Currency, Price, Quantity}; - -let instrument = CurrencyPair::new( - InstrumentId::from("EUR/USD.SIM"), - Symbol::from("EUR/USD"), - Currency::from("EUR"), - Currency::from("USD"), - 5, // price_precision - 0, // size_precision - Price::from("0.00001"), // price_increment - Quantity::from("1"), // size_increment - // ... remaining parameters -); -``` - -See the full instrument [API Reference](/docs/python-api-latest/model/instruments.html). - -## Live trading - -Live integration adapters have `InstrumentProvider` implementations that automatically -cache the latest instrument definitions for the venue. Refer to a particular instrument -by passing the matching `InstrumentId` to data and execution methods that require one. - -## Finding instruments - -Since the same actor/strategy classes can be used for both backtest and live trading, you can -get instruments in exactly the same way through the central cache: - -```python -from nautilus_trader.model import InstrumentId - -instrument_id = InstrumentId.from_str("ETHUSDT-PERP.BINANCE") -instrument = self.cache.instrument(instrument_id) -``` - -```rust -use nautilus_model::identifiers::InstrumentId; - -let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); -let instrument = cache.instrument(&instrument_id); -``` - -It's also possible to subscribe to any changes to a particular instrument: - -```python -self.subscribe_instrument(instrument_id) -``` - -Or subscribe to all instrument changes for an entire venue: - -```python -from nautilus_trader.model import Venue - -binance = Venue("BINANCE") -self.subscribe_instruments(binance) -``` - -When an update to the instrument(s) is received by the `DataEngine`, the object(s) will -be passed to the `on_instrument()` handler. Override this method with actions -to take upon receiving an instrument update: - -```python -from nautilus_trader.model.instruments import Instrument - -def on_instrument(self, instrument: Instrument) -> None: - # Take some action on an instrument update - pass -``` - -## Precision - -Precision defines the number of decimal places allowed for prices and quantities on a -given instrument. Every instrument specifies a `price_precision` and `size_precision` -that determine the valid fractional resolution for that market. - -NautilusTrader enforces precision strictly by design. This section explains the rationale -and mechanics behind this approach. - -### Why precision is enforced - -**Realistic market simulation.** Real exchanges only accept prices and sizes at specific -precisions. A crypto spot market may support prices to 2 decimal places (e.g., `50000.01`) -while a different market supports 8 (e.g., `0.00012345`). Allowing arbitrary precision -in a backtest would produce fills at price levels that could never exist in production, -leading to misleading performance metrics. - -**Venue compatibility.** Most exchanges validate price and size precision on incoming -orders and reject those that exceed the instrument's specification. Enforcing precision -at the platform level catches a common class of these issues early. Note that venues may -also enforce tick-multiple or step-size constraints beyond what the `RiskEngine` currently -validates, so precision compliance alone does not guarantee venue acceptance. - -**Deterministic calculations.** Fixed-point arithmetic with explicit precision eliminates -floating-point drift and ensures calculations are reproducible across platforms and -environments. Two systems processing the same data will always produce identical results. - -**Data integrity.** The backtesting matching engine validates that all incoming market -data (quotes, trades, bars) matches the instrument's declared precision. This catches -mismatches between instrument definitions and data sources early, preventing silent -corruption of fill prices and quantities. - -### How precision works - -Each instrument defines two precision values: - -| Field | Constrains | Example | -|-------------------|--------------------------------------|------------------| -| `price_precision` | Order prices, trigger prices, fills. | `2` → `50000.01` | -| `size_precision` | Order quantities, fill quantities. | `5` → `1.00001` | - -These precisions are paired with minimum increments: - -| Field | Purpose | -|-------------------|------------------------------------------| -| `price_increment` | Smallest valid price change (tick size). | -| `size_increment` | Smallest valid quantity change. | - -The increment's own precision must exactly match the instrument's declared precision. -For example, an instrument with `price_precision=2` and `price_increment=Price(0.01, 2)` -is valid, but a mismatch between these values will raise an error at instrument creation. - -### Where precision is enforced - -Precision is validated at multiple levels throughout the platform: - -1. **Instrument creation**: The precision of `price_increment` and `size_increment` must - match `price_precision` and `size_precision` respectively. -2. **Risk engine**: Before an order reaches the venue, the `RiskEngine` checks that the - order's price and quantity precision do not exceed the instrument's limits. Orders that - fail this check are denied. -3. **Matching engine**: During backtesting, the matching engine validates that all incoming - market data matches the instrument's precision. Mismatches raise a `RuntimeError` - immediately. - -:::warning -The `RiskEngine` does not round values automatically. If you create a `Price` with -5 decimal places on an instrument that supports 2, the order will be denied. Use -`instrument.make_price()` and `instrument.make_qty()` to round explicitly. -::: - -### Working with instrument precision - -Use the instrument's factory methods to create values with correct precision: - -```python -instrument = self.cache.instrument(instrument_id) - -price = instrument.make_price(0.90500) -quantity = instrument.make_qty(150) -``` - -These methods round the input to the instrument's declared precision, ensuring the -result will pass precision checks. Other validation rules still apply (e.g., min/max -quantity limits), and `make_qty()` will raise if the rounded value is zero. - -:::tip -Always use `instrument.make_price()` and `instrument.make_qty()` when creating order -parameters. This avoids precision mismatch errors and ensures your values have the -correct number of decimal places for the instrument. -::: - -If you encounter precision mismatch errors during backtesting, verify that: - -1. The instrument definition matches your data source's precision. -2. Data was not inadvertently rounded or truncated during loading. -3. Custom data loaders preserve the original precision metadata. - -## Limits - -Certain value limits are optional for instruments and can be `None`, these are exchange -dependent and can include: - -- `max_quantity` (maximum quantity for a single order). -- `min_quantity` (minimum quantity for a single order). -- `max_notional` (maximum value of a single order). -- `min_notional` (minimum value of a single order). -- `max_price` (maximum valid quote or order price). -- `min_price` (minimum valid quote or order price). - -:::note -Most of these limits are checked by the Nautilus `RiskEngine`, otherwise exceeding -published limits *can* result in the exchange rejecting orders. -::: - -## Margins and fees - -Margin calculations are handled by the `MarginAccount` class. This section explains how margins work and introduces key concepts you need to know. - -### When do margins apply? - -Each exchange (e.g., CME or Binance) operates with a specific account type that determines whether margin calculations are applicable. -When setting up an exchange venue, you'll specify one of these account types: - -- `AccountType.MARGIN`: Accounts that use margin calculations, which are explained below. -- `AccountType.CASH`: Simple accounts where margin calculations do not apply. -- `AccountType.BETTING`: Accounts designed for betting, which also do not involve margin calculations. - -### Vocabulary - -To understand trading on margin, let’s start with some key terms: - -**Notional Value**: The total contract value in the quote currency. It represents the full market value of your position. For example, with EUR/USD futures on CME (symbol 6E). - -- Each contract represents 125,000 EUR (EUR is base currency, USD is quote currency). -- If the current market price is 1.1000, the notional value equals 125,000 EUR × 1.1000 (price of EUR/USD) = 137,500 USD. - -**Leverage** (`leverage`): The ratio that determines how much market exposure you can control relative to your account deposit. For example, with 10× leverage, you can control 10,000 USD worth of positions with 1,000 USD in your account. - -**Initial Margin** (`margin_init`): The margin rate required to open a position. It represents the minimum amount of funds that must be available in your account to open new positions. This is only a pre-check; no funds are actually locked. - -**Maintenance Margin** (`margin_maint`): The margin rate required to keep a position open. This amount is locked in your account to maintain the position. It is always lower than the initial margin. You can view the total blocked funds (sum of maintenance margins for open positions) using the following in your strategy: - -```python -self.portfolio.balances_locked(venue) -``` - -**Maker/Taker Fees**: The fees charged by exchanges based on your order's interaction with the market: - -- Maker Fee (`maker_fee`): A fee (typically lower) charged when you "make" liquidity by placing an order that remains on the order book. For example, a limit buy order below the current price adds liquidity, and the *maker* fee applies when it fills. -- Taker Fee (`taker_fee`): A fee (typically higher) charged when you "take" liquidity by placing an order that executes immediately. For instance, a market buy order or a limit buy above the current price removes liquidity, and the *taker* fee applies. - -**Fee rate sign convention**: Nautilus uses a consistent sign convention for fee rates across all adapters and the backtesting engine: - -- **Positive fee rate** = commission (fee charged, reducing account balance). -- **Negative fee rate** = rebate (fee earned, increasing account balance). - -For example, a maker fee of `-0.00025` means you receive a 0.025% rebate for providing liquidity, while a taker fee of `0.00075` means you pay a 0.075% commission for taking liquidity. - -:::note -Different exchanges use different sign conventions in their APIs. Nautilus adapters normalize these to the convention above. If you're manually specifying fee rates for backtesting, ensure you follow this convention. -::: - -:::tip -Not all exchanges or instruments implement maker/taker fees. If absent, set both `maker_fee` and `taker_fee` to 0 for the `Instrument` (e.g., `FuturesContract`, `Equity`, `CurrencyPair`, `Commodity`, `Cfd`, `BinaryOption`, `BettingInstrument`). -::: - -### Margin calculation formula - -The `MarginAccount` class calculates margins using the following formulas: - -```python -# Initial margin calculation -margin_init = (notional_value / leverage * margin_init) + (notional_value / leverage * taker_fee) - -# Maintenance margin calculation -margin_maint = (notional_value / leverage * margin_maint) + (notional_value / leverage * taker_fee) -``` - -**Key Points**: - -- Both formulas follow the same structure but use their respective margin rates (`margin_init` and `margin_maint`). -- Each formula consists of two parts: - - **Primary margin calculation**: Based on notional value, leverage, and margin rate. - - **Fee Adjustment**: Accounts for the maker/taker fee. - -### Implementation details - -For those interested in exploring the technical implementation: - -- [nautilus_trader/accounting/accounts/margin.pyx](https://github.com/nautechsystems/nautilus_trader/blob/develop/nautilus_trader/accounting/accounts/margin.pyx) -- Key methods: `calculate_margin_init(self, ...)` and `calculate_margin_maint(self, ...)` - -## Commissions - -Trading commissions represent the fees charged by exchanges or brokers for executing trades. -While maker/taker fees are common in cryptocurrency markets, traditional exchanges like CME often -employ other fee structures, such as per-contract commissions. -NautilusTrader supports multiple commission models to accommodate diverse fee structures across different markets. - -### Built-in fee models - -The framework provides two built-in fee model implementations: - -1. `MakerTakerFeeModel`: Implements the maker/taker fee structure common in cryptocurrency exchanges, where fees are - calculated as a percentage of the trade value. -2. `FixedFeeModel`: Applies a fixed commission per trade, regardless of the trade size. - -### Creating custom fee models - -While the built-in fee models cover common scenarios, you might encounter situations requiring specific commission structures. -NautilusTrader's flexible architecture allows you to implement custom fee models by inheriting from `FeeModel`. - -For example, if you're trading futures on exchanges that charge per-contract commissions (like CME), you can implement -a custom fee model. When creating custom fee models, we inherit from the `FeeModel` base class, which is implemented -in Cython for performance reasons. This Cython implementation is reflected in the parameter naming convention, -where type information is incorporated into parameter names using underscores (like `Order_order` or `Quantity_fill_qty`). - -While these parameter names might look unusual to Python developers, they're a result of Cython's type system and help -maintain consistency with the framework's core components. Here's how you could create a per-contract commission model: - -```python -class PerContractFeeModel(FeeModel): - def __init__(self, commission: Money): - super().__init__() - self.commission = commission - - def get_commission(self, Order_order, Quantity_fill_qty, Price_fill_px, Instrument_instrument): - total_commission = Money(self.commission * Quantity_fill_qty, self.commission.currency) - return total_commission -``` - -This custom implementation calculates the total commission by multiplying a `fixed per-contract fee` by the `number -of contracts` traded. The `get_commission(...)` method receives information about the order, fill quantity, fill price -and instrument, allowing for flexible commission calculations based on these parameters. - -Our new class `PerContractFeeModel` inherits class `FeeModel`, which is implemented in Cython, -so notice the Cython-style parameter names in the method signature: - -- `Order_order`: The order object, with type prefix `Order_`. -- `Quantity_fill_qty`: The fill quantity, with type prefix `Quantity_`. -- `Price_fill_px`: The fill price, with type prefix `Price_`. -- `Instrument_instrument`: The instrument object, with type prefix `Instrument_`. - -These parameter names follow NautilusTrader's Cython naming conventions, where the prefix indicates the expected type. -While this might seem verbose compared to typical Python naming conventions, it ensures type safety and consistency -with the framework's Cython codebase. - -### Using fee models in practice - -To use any fee model in your trading system, whether built-in or custom, you specify it when setting up the venue. -Here's an example using the custom per-contract fee model: - -```python -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.objects import Money, Currency - -engine.add_venue( - venue=venue, - oms_type=OmsType.NETTING, - account_type=AccountType.MARGIN, - base_currency=USD, - fee_model=PerContractFeeModel(Money(2.50, USD)), # 2.50 USD per contract - starting_balances=[Money(1_000_000, USD)], # Starting with 1,000,000 USD balance -) -``` - -:::tip -When implementing custom fee models, ensure they accurately reflect the fee structure of your target exchange. -Even small discrepancies in commission calculations can significantly impact strategy performance metrics during backtesting. -::: - -### Additional info - -The raw instrument definition as provided by the exchange (typically from JSON serialized data) is also -included as a generic Python dictionary. This is to retain all information -which is not necessarily part of the unified Nautilus API, and is available to the user -at runtime by calling the `.info` property. - -## Synthetic instruments - -The platform supports creating customized synthetic instruments, which can generate synthetic quote -and trades. These are useful for: - -- Enabling `Actor` and `Strategy` components to subscribe to quote or trade feeds. -- Triggering emulated orders. -- Constructing bars from synthetic quotes or trades. - -Synthetic instruments cannot be traded directly, as they are constructs that only exist locally -within the platform. They serve as analytical tools, providing useful metrics based on their component -instruments. - -In the future, we plan to support order management for synthetic instruments, enabling trading of -their component instruments based on the synthetic instrument's behavior. - -:::info -The venue for a synthetic instrument is always designated as `'SYNTH'`. -::: - -### Formula - -A synthetic instrument is composed of a combination of two or more component instruments (which -can include instruments from multiple venues), as well as a "derivation formula". -Utilizing the dynamic expression engine powered by the [evalexpr](https://github.com/ISibboI/evalexpr) -Rust crate, the platform can evaluate the formula to calculate the latest synthetic price tick -from the incoming component instrument prices. - -See the `evalexpr` documentation for a full description of available features, operators and precedence. - -:::tip -Before defining a new synthetic instrument, ensure that all component instruments are already defined and exist in the cache. -::: - -### Subscribing - -The following example demonstrates the creation of a new synthetic instrument with an actor/strategy. -This synthetic instrument will represent a simple spread between Bitcoin and -Ethereum spot prices on Binance. For this example, it is assumed that spot instruments for -`BTCUSDT.BINANCE` and `ETHUSDT.BINANCE` are already present in the cache. - -```python -from nautilus_trader.model.instruments import SyntheticInstrument - -btcusdt_binance_id = InstrumentId.from_str("BTCUSDT.BINANCE") -ethusdt_binance_id = InstrumentId.from_str("ETHUSDT.BINANCE") - -# Define the synthetic instrument -synthetic = SyntheticInstrument( - symbol=Symbol("BTC-ETH:BINANCE"), - price_precision=8, - components=[ - btcusdt_binance_id, - ethusdt_binance_id, - ], - formula=f"{btcusdt_binance_id} - {ethusdt_binance_id}", - ts_event=self.clock.timestamp_ns(), - ts_init=self.clock.timestamp_ns(), -) - -# Recommended to store the synthetic instruments ID somewhere -self._synthetic_id = synthetic.id - -# Add the synthetic instrument for use by other components -self.add_synthetic(synthetic) - -# Subscribe to quotes for the synthetic instrument -self.subscribe_quote_ticks(self._synthetic_id) -``` - -:::note -The `instrument_id` for the synthetic instrument in the above example will be structured as `{symbol}.{SYNTH}`, resulting in `'BTC-ETH:BINANCE.SYNTH'`. -::: - -### Updating formulas - -It's also possible to update a synthetic instrument formulas at any time. The following example -shows how to achieve this with an actor/strategy. - -```python -# Recover the synthetic instrument from the cache (assuming `synthetic_id` was assigned) -synthetic = self.cache.synthetic(self._synthetic_id) - -# Update the formula to take the average -new_formula = "(BTCUSDT.BINANCE + ETHUSDT.BINANCE) / 2" -synthetic.change_formula(new_formula) - -# Now update the synthetic instrument -self.update_synthetic(synthetic) -``` - -### Trigger instrument IDs - -The platform allows for emulated orders to be triggered based on synthetic instrument prices. In -the following example, we build upon the previous one to submit a new emulated order. -This order will be retained in the emulator until a trigger from synthetic quotes releases it. -It will then be submitted to Binance as a MARKET order: - -```python -order = self.strategy.order_factory.limit( - instrument_id=ETHUSDT_BINANCE.id, - order_side=OrderSide.BUY, - quantity=Quantity.from_str("1.5"), - price=Price.from_str("30000.00000000"), # <-- Synthetic instrument price - emulation_trigger=TriggerType.DEFAULT, - trigger_instrument_id=self._synthetic_id, # <-- Synthetic instrument identifier -) - -self.strategy.submit_order(order) -``` - -### Error handling - -The platform validates inputs including the derivation formula for synthetic instruments. -Invalid or erroneous inputs may still lead to undefined behavior. - -See the [`SyntheticInstrument` API Reference](/docs/python-api-latest/model/instruments.html#nautilus_trader.model.instruments.synthetic.SyntheticInstrument) for input requirements and potential exceptions. - -## Related guides - -- [Data](data.md) - Market data types for instruments. -- [Orders](orders.md) - Orders reference instruments. diff --git a/nautilus-docs/docs/concepts/live.md b/nautilus-docs/docs/concepts/live.md deleted file mode 100644 index 25caf48..0000000 --- a/nautilus-docs/docs/concepts/live.md +++ /dev/null @@ -1,597 +0,0 @@ -# Live Trading - -NautilusTrader deploys backtested strategies to live markets with no code changes. - -**Live trading involves real financial risk. Before deploying to production, understand -system configuration, node operations, execution reconciliation, and the differences -between backtesting and live trading.** - -:::danger[Jupyter notebooks not recommended for live trading] -Do not run live trading nodes in Jupyter notebooks. Event loop conflicts and -operational risks make them unsuitable: - -- Jupyter runs its own asyncio event loop, which conflicts with `TradingNode`'s event loop. -- Workarounds like `nest_asyncio` are not production-grade. -- Cells can run out of order, kernels can crash, and state can disappear. -- Notebooks lack the logging, monitoring, and graceful shutdown needed for production trading. - -Use Jupyter for backtesting, analysis, and experimentation. For live trading, run nodes -as standalone Python scripts or services. -::: - -:::warning[One TradingNode per process] -Running multiple `TradingNode` instances concurrently in the same process is not supported due to global singleton state. -Add multiple strategies to a single node, or run additional nodes in separate processes for parallel execution. - -See [Processes and threads](architecture.md#processes-and-threads) for details. -::: - -:::warning[Do not block the event loop] -User code on the event loop thread (strategy callbacks, actor handlers, `on_event` methods) -must return quickly. This applies to both Python and Rust. Blocking operations like model -inference, heavy calculations, or synchronous I/O cause missed fills, stale data, and -delayed order submissions. Offload long-running work to an executor or a separate thread/process. -::: - -:::info[Platform differences] -Windows signal handling differs from Unix-like systems. If you are running on Windows, please read -the note on [Windows signal handling](#windows-signal-handling) for guidance on graceful shutdown -behavior and Ctrl+C (SIGINT) support. -::: - -## Configuration - -### `TradingNodeConfig` - -`TradingNodeConfig` inherits from `NautilusKernelConfig` and adds live-specific options: - -```python -from nautilus_trader.config import TradingNodeConfig - -config = TradingNodeConfig( - trader_id="MyTrader-001", - - # Component configurations - cache=CacheConfig(), - message_bus=MessageBusConfig(), - data_engine=LiveDataEngineConfig(), - risk_engine=LiveRiskEngineConfig(), - exec_engine=LiveExecEngineConfig(), - portfolio=PortfolioConfig(), - - # Client configurations - data_clients={ - "BINANCE": BinanceDataClientConfig(), - }, - exec_clients={ - "BINANCE": BinanceExecClientConfig(), - }, -) -``` - -#### Core configuration parameters - -| Setting | Default | Description | -|--------------------------|--------------|---------------------------------------------| -| `trader_id` | "TRADER-001" | Unique trader identifier (name-tag format). | -| `instance_id` | `None` | Optional unique instance identifier. | -| `timeout_connection` | 30.0 | Connection timeout in seconds. | -| `timeout_reconciliation` | 10.0 | Reconciliation timeout in seconds. | -| `timeout_portfolio` | 10.0 | Portfolio initialization timeout. | -| `timeout_disconnection` | 10.0 | Disconnection timeout. | -| `timeout_post_stop` | 5.0 | Post-stop cleanup timeout. | - -#### Cache database configuration - -```python -from nautilus_trader.config import CacheConfig -from nautilus_trader.config import DatabaseConfig - -cache_config = CacheConfig( - database=DatabaseConfig( - host="localhost", - port=6379, - username="nautilus", - password="pass", - timeout=2.0, - ), - encoding="msgpack", # or "json" - timestamps_as_iso8601=True, - buffer_interval_ms=100, - flush_on_start=False, -) -``` - -#### MessageBus configuration - -```python -from nautilus_trader.config import MessageBusConfig -from nautilus_trader.config import DatabaseConfig - -message_bus_config = MessageBusConfig( - database=DatabaseConfig(timeout=2), - timestamps_as_iso8601=True, - use_instance_id=False, - types_filter=[QuoteTick, TradeTick], # Filter specific message types - stream_per_topic=False, - autotrim_mins=30, # Automatic message trimming - heartbeat_interval_secs=1, -) -``` - -### Multi-venue configuration - -A node can connect to multiple venues. This example configures both -spot and futures markets for Binance: - -```python -config = TradingNodeConfig( - trader_id="MultiVenue-001", - - # Multiple data clients for different market types - data_clients={ - "BINANCE_SPOT": BinanceDataClientConfig( - account_type=BinanceAccountType.SPOT, - testnet=False, - ), - "BINANCE_FUTURES": BinanceDataClientConfig( - account_type=BinanceAccountType.USDT_FUTURES, - testnet=False, - ), - }, - - # Corresponding execution clients - exec_clients={ - "BINANCE_SPOT": BinanceExecClientConfig( - account_type=BinanceAccountType.SPOT, - testnet=False, - ), - "BINANCE_FUTURES": BinanceExecClientConfig( - account_type=BinanceAccountType.USDT_FUTURES, - testnet=False, - ), - }, -) -``` - -### ExecutionEngine configuration - -`LiveExecEngineConfig` controls order processing, execution events, and -venue reconciliation. For full details see the -[API Reference](/docs/python-api-latest/config.html#nautilus_trader.live.config.LiveExecEngineConfig). - -#### Reconciliation - -Recovers missed order and position events to keep system state consistent with the venue. - -| Setting | Default | Description | -|---------------------------------|---------|---------------------------------------------------------------------------------| -| `reconciliation` | True | Activate reconciliation at startup to align internal state with the venue. | -| `reconciliation_lookback_mins` | None | How far back (minutes) to request past events for reconciling uncached state. | -| `reconciliation_instrument_ids` | None | Include list of instrument IDs to reconcile. | -| `filtered_client_order_ids` | None | Client order IDs to skip during reconciliation (for venue-side duplicates). | - -See [Execution reconciliation](#execution-reconciliation) for details. - -#### Order filtering - -Controls which order events and reports the system processes, preventing conflicts -across trading nodes. - -| Setting | Default | Description | -|------------------------------------|---------|-------------------------------------------------------------------------------| -| `filter_unclaimed_external_orders` | False | Drop unclaimed external orders so they do not affect the strategy. | -| `filter_position_reports` | False | Drop position status reports. Useful when multiple nodes trade one account. | - -:::note[Order tagging behavior] -Reconciliation tags orders by origin: - -- **`VENUE` tag**: external orders discovered at the venue (placed outside this system). -- **`RECONCILIATION` tag**: synthetic orders generated to align position discrepancies. - -When `filter_unclaimed_external_orders` is enabled, only `VENUE`-tagged orders are filtered. -`RECONCILIATION`-tagged orders are never filtered, so position alignment always succeeds. -::: - -#### Continuous reconciliation - -A background loop starts after startup reconciliation completes. It: - -- Monitors in-flight orders for delays exceeding a configured threshold. -- Reconciles open orders with the venue at configurable intervals. -- Audits internal *own* order books against the venue's public books. - -The loop waits for startup reconciliation to finish before starting periodic checks. -The `reconciliation_startup_delay_secs` parameter adds a further delay *after* startup -reconciliation completes, giving the system time to stabilize. - -When retries are exhausted, the engine resolves the order as follows: - -**In-flight order timeout resolution** (venue does not respond after max retries): - -| Current status | Resolved to | Rationale | -|------------------|-------------|--------------------------------------------| -| `SUBMITTED` | `REJECTED` | No confirmation received from venue. | -| `PENDING_UPDATE` | `CANCELED` | Modification remains unacknowledged. | -| `PENDING_CANCEL` | `CANCELED` | Venue never confirmed the cancellation. | - -**Order consistency checks** (when cache state differs from venue state): - -| Cache status | Venue status | Resolution | Rationale | -|--------------------|--------------|-------------|---------------------------------------------------------------------| -| `SUBMITTED` | Not found | `REJECTED` | Order never confirmed by venue (e.g., lost during network error). | -| `ACCEPTED` | Not found | `REJECTED` | Order doesn't exist at venue, likely was never successfully placed. | -| `ACCEPTED` | `CANCELED` | `CANCELED` | Venue canceled the order (user action or venue-initiated). | -| `ACCEPTED` | `EXPIRED` | `EXPIRED` | Order reached GTD expiration at venue. | -| `ACCEPTED` | `REJECTED` | `REJECTED` | Venue rejected after initial acceptance (rare but possible). | -| `PARTIALLY_FILLED` | `CANCELED` | `CANCELED` | Order canceled at venue with fills preserved. | -| `PARTIALLY_FILLED` | Not found | `CANCELED` | Order doesn't exist but had fills (reconciles fill history). | - -:::note -**Reconciliation caveats:** - -- **"Not found" resolutions** only apply in full-history mode (`open_check_open_only=False`). - Open-only mode (the default) skips these checks because venue "open orders" endpoints - exclude closed orders by design, making it impossible to distinguish missing orders from - recently closed ones. -- **Recent order protection**: the engine skips reconciliation for orders whose last event - falls within the `open_check_threshold_ms` window (default 5s). This prevents false - positives from race conditions where the venue is still processing. -- **Targeted query safeguard**: before marking an order `REJECTED` or `CANCELED` when - "not found", the engine issues a single-order query to the venue. - This catches false negatives from bulk query limitations or timing delays. -- **`FILLED` orders** that are "not found" at the venue are silently ignored. Venues - commonly drop completed orders from their query results. - -::: - -#### Retry coordination and lookback behavior - -The inflight loop and open-order loop share a single retry counter -(`_recon_check_retries`), bounded by `inflight_check_retries` and -`open_check_missing_retries` respectively. The stricter limit wins, -and avoids duplicate venue queries for the same order state. - -When the open-order loop exhausts retries, the engine issues one targeted -`GenerateOrderStatusReport` probe before applying a terminal state. If the -venue returns the order, reconciliation proceeds and the retry counter resets. - -**Single-order query protection**: the engine caps single-order queries per -cycle via `max_single_order_queries_per_cycle` (default: 10). Remaining -orders are deferred to the next cycle. A configurable delay -(`single_order_query_delay_ms`, default: 100ms) spaces out consecutive -queries to avoid rate limits. This handles bulk query failures across hundreds of orders -without overwhelming the venue API. - -Orders older than `open_check_lookback_mins` rely on this targeted probe. -Keep the lookback generous for venues with short history windows. Increase -`open_check_threshold_ms` if venue timestamps lag the local clock, so -recently updated orders are not marked missing prematurely. - -| Setting | Default | Description | -|--------------------------------------|----------------|--------------------------------------------------------------------------------------------------| -| `inflight_check_interval_ms` | 2,000 ms | How often to check in-flight order status. Set to 0 to disable. | -| `inflight_check_threshold_ms` | 5,000 ms | Time before an in-flight order triggers a venue status check. Lower if colocated. | -| `inflight_check_retries` | 5 retries | Retry attempts to verify an in-flight order with the venue. | -| `open_check_interval_secs` | None | How often (seconds) to check open orders at the venue. None or 0.0 disables. Recommended: 5-10s.| -| `open_check_open_only` | True | When true, query only open orders; when false, fetch full history (resource-intensive). | -| `open_check_lookback_mins` | 60 min | Lookback window (minutes) for order status polling. Only orders modified within this window. | -| `open_check_threshold_ms` | 5,000 ms | Minimum time since last cached event before acting on venue discrepancies. | -| `open_check_missing_retries` | 5 retries | Max retries before resolving an order open in cache but not found at venue. | -| `max_single_order_queries_per_cycle` | 10 | Cap on single-order queries per cycle. Prevents rate-limit exhaustion. | -| `single_order_query_delay_ms` | 100 ms | Delay (ms) between single-order queries to avoid rate limits. | -| `reconciliation_startup_delay_secs` | 10.0 s | Delay (seconds) *after* startup reconciliation before continuous checks begin. | -| `own_books_audit_interval_secs` | None | Interval (seconds) between auditing own order books against public books. | -| `position_check_interval_secs` | None | Interval (seconds) between position consistency checks. On discrepancy, queries for missing fills. None disables. Recommended: 30-60s. | -| `position_check_lookback_mins` | 60 min | Lookback window (minutes) for querying fill reports on position discrepancy. | -| `position_check_threshold_ms` | 5,000 ms | Minimum time since last local activity before acting on position discrepancies. | -| `position_check_retries` | 3 retries | Max attempts per instrument before the engine stops retrying that discrepancy. Once exceeded, an error is logged and the discrepancy is no longer actively reconciled until it clears. | - -:::warning - -- **`open_check_lookback_mins`**: do not reduce below 60 minutes. A short window - triggers false "missing order" resolutions because orders fall outside the query range. -- **`reconciliation_startup_delay_secs`**: do not reduce below 10 seconds in production. - The delay lets the system stabilize after startup reconciliation before continuous - checks begin. - -::: - -#### Additional options - -| Setting | Default | Description | -|------------------------------------|---------|-------------------------------------------------------------------------------------------------| -| `allow_overfills` | False | Allow fills exceeding order quantity (logs warning). Useful when reconciliation races fills. | -| `generate_missing_orders` | True | Generate LIMIT orders during reconciliation to align position discrepancies (strategy `EXTERNAL`, tag `RECONCILIATION`). | -| `snapshot_orders` | False | Take order snapshots on order events. | -| `snapshot_positions` | False | Take position snapshots on position events. | -| `snapshot_positions_interval_secs` | None | Interval (seconds) between position snapshots. | -| `debug` | False | Enable debug logging for execution. | - -#### Memory management - -Periodically purges closed orders, closed positions, and account events from the -in-memory cache, keeping memory bounded during long-running or HFT sessions. - -| Setting | Default | Description | -|----------------------------------------|---------|------------------------------------------------------------------------------------| -| `purge_closed_orders_interval_mins` | None | How often (minutes) to purge closed orders from memory. Recommended: 10-15 min. | -| `purge_closed_orders_buffer_mins` | None | How long (minutes) an order must be closed before purging. Recommended: 60 min. | -| `purge_closed_positions_interval_mins` | None | How often (minutes) to purge closed positions from memory. Recommended: 10-15 min. | -| `purge_closed_positions_buffer_mins` | None | How long (minutes) a position must be closed before purging. Recommended: 60 min. | -| `purge_account_events_interval_mins` | None | How often (minutes) to purge account events from memory. Recommended: 10-15 min. | -| `purge_account_events_lookback_mins` | None | How old (minutes) an account event must be before purging. Recommended: 60 min. | -| `purge_from_database` | False | Also delete from the backing database (Redis/PostgreSQL). **Use with caution**. | - -Setting an interval enables the purge loop; leaving it unset disables scheduling and -deletion. Database records are unaffected unless `purge_from_database` is true. Each -loop delegates to the cache APIs described in -[Purging cached state](cache.md#purging-cached-state). - -#### Queue management - -| Setting | Default | Description | -|----------------------------------|---------|---------------------------------------------------------------------------------| -| `qsize` | 100,000 | Size of internal queue buffers. | -| `graceful_shutdown_on_exception` | False | Gracefully shut down on unexpected queue processing exceptions (not user code). | - -### Strategy configuration - -For a complete parameter list see the `StrategyConfig` -[API Reference](/docs/python-api-latest/config.html#nautilus_trader.trading.config.StrategyConfig). - -#### Identification - -| Setting | Default | Description | -|----------------|---------|---------------------------------------------------------------| -| `strategy_id` | None | Unique strategy identifier. | -| `order_id_tag` | None | Unique tag appended to this strategy's order IDs. | - -#### Order management - -| Setting | Default | Description | -|-----------------------------|---------|--------------------------------------------------------------------------------------------| -| `oms_type` | None | [OMS type](../concepts/execution#oms-configuration) for position ID and order processing. | -| `use_uuid_client_order_ids` | False | Use UUID4 values for client order IDs. | -| `external_order_claims` | None | Instrument IDs whose external orders this strategy claims. | -| `manage_contingent_orders` | False | Automatically manage OTO, OCO, and OUO contingent orders. | -| `manage_gtd_expiry` | False | Manage GTD expirations for orders. | - -### Windows signal handling - -:::warning -Windows: asyncio event loops do not implement `loop.add_signal_handler`. As a result, the legacy -`TradingNode` does not receive OS signals via asyncio on Windows. Use Ctrl+C (SIGINT) handling or -programmatic shutdown; SIGTERM parity is not expected on Windows. -::: - -On Windows, asyncio event loops do not implement `loop.add_signal_handler`, so Unix-style -signal integration is unavailable. `TradingNode` does not receive OS signals via asyncio -on Windows and will not stop gracefully unless you intervene. - -Recommended approaches: - -- Wrap `run` with `try/except KeyboardInterrupt` and call `node.stop()` then `node.dispose()`. - Ctrl+C raises `KeyboardInterrupt` in the main thread, giving you a clean teardown path. -- Publish a `ShutdownSystem` command programmatically (or call `shutdown_system(...)` from - an actor/component) to trigger the same shutdown path. - -The “inflight check loop task still pending” message appears because the normal graceful -shutdown path is not triggered. This is tracked as -[#2785](https://github.com/nautechsystems/nautilus_trader/issues/2785). - -The v2 `LiveNode` already handles Ctrl+C via `tokio::signal::ctrl_c()` and a Python SIGINT -bridge, so runner and tasks shut down cleanly. - -Example pattern for Windows: - -```python -try: - node.run() -except KeyboardInterrupt: - pass -finally: - try: - node.stop() - finally: - node.dispose() -``` - -## Execution reconciliation - -Execution reconciliation aligns the venue's actual order and position state with the -system's internal state built from events. Only the `LiveExecutionEngine` performs -reconciliation, since backtesting controls both sides. - -:::note[Terminology] -An **in-flight order** is one awaiting venue acknowledgement: - -- `SUBMITTED` - initial submission, awaiting accept/reject. -- `PENDING_UPDATE` - modification requested, awaiting confirmation. -- `PENDING_CANCEL` - cancellation requested, awaiting confirmation. - -These orders are monitored by the continuous reconciliation loop to detect stale or lost messages. -::: - -Two scenarios: - -- **Cached state exists**: report data generates missing events to align the state. -- **No cached state**: all orders and positions at the venue are generated from scratch. - -:::tip -Persist all execution events to the cache database. This reduces reliance on venue history -and allows full recovery even with short lookback windows. -::: - -### Reconciliation configuration - -Unless `reconciliation` is set to false, the execution engine reconciles state for each -venue at startup. The `reconciliation_lookback_mins` parameter controls how far back the -engine requests history. - -:::tip -Leave `reconciliation_lookback_mins` unset. This lets the engine request the maximum -execution history the venue provides. -::: - -:::warning -Executions before the lookback window still generate alignment events, but with some -information loss that a longer window would avoid. Some venues also filter or drop -older execution data. Persisting all events to the cache database prevents both issues. -::: - -Each strategy can claim external orders for an instrument ID generated during reconciliation -via the `external_order_claims` config parameter. This lets a strategy resume managing open -orders when no cached state exists. - -Orders generated with strategy ID `EXTERNAL` and tag `RECONCILIATION` during position -reconciliation are internal to the engine. They cannot be claimed via `external_order_claims` -and should not be managed by user strategies. - -:::tip -To detect external orders in your strategy, check `order.strategy_id.value == "EXTERNAL"`. These orders participate in portfolio calculations and position tracking like any other order. -::: - -For all live trading options, see the `LiveExecEngineConfig` [API Reference](/docs/python-api-latest/config.html#nautilus_trader.live.config.LiveExecEngineConfig). - -### Reconciliation procedure - -All adapter execution clients follow the same reconciliation procedure, calling three methods -to produce an execution mass status: - -- `generate_order_status_reports` -- `generate_fill_reports` -- `generate_position_status_reports` - -```mermaid -flowchart TD - Start[Startup Reconciliation] --> Fetch[Fetch venue reports
orders, fills, positions] - Fetch --> Dedup[Deduplicate reports
log warnings for duplicates] - Dedup --> Orders[Order Reconciliation
align order states, generate missing events] - Orders --> Fills[Fill Reconciliation
verify fills, generate missing OrderFilled events] - Fills --> Pos[Position Reconciliation
compare net positions per instrument] - Pos --> Match{Positions
match venue?} - Match -->|Yes| Done[Reconciliation complete
system ready for trading] - Match -->|No| Gen[Generate missing orders
strategy: EXTERNAL, tag: RECONCILIATION] - Gen --> Done -``` - -The system reconciles its state against these reports, which represent external reality: - -- **Duplicate check**: - - Deduplicates order reports within the batch and logs warnings. - - Logs duplicate trade IDs as warnings for investigation. -- **Order reconciliation**: - - Generates and applies events to move orders from cached state to current state. - - Infers `OrderFilled` events for missing trade reports. - - Generates external order events for unrecognized client order IDs or reports missing a client order ID. - - Verifies fill report data consistency with tolerance-based price and commission comparisons. -- **Position reconciliation**: - - Matches the net position per instrument against venue position reports using instrument precision. - - Generates external order events when order reconciliation leaves a position that differs from the venue. - - When `generate_missing_orders` is enabled (default: True), generates orders with strategy ID `EXTERNAL` and tag `RECONCILIATION` to align discrepancies. - - Falls through a price hierarchy when generating reconciliation orders: - 1. **Calculated reconciliation price** (preferred): targets the correct average position. - 2. **Market mid-price**: uses the current bid-ask midpoint. - 3. **Current position average**: uses the existing position's average price. - 4. **MARKET order** (last resort): used only when no price data exists (no positions, no market data). - - Uses LIMIT orders when a price can be determined (cases 1-3) to preserve PnL accuracy. - - Skips zero quantity differences after precision rounding. -- **Partial window adjustment**: - - When `reconciliation_lookback_mins` is set, the window may miss opening fills. - - The system adjusts fills using lifecycle analysis to reconstruct positions accurately: - - Detects zero-crossings (position qty crosses through FLAT) to identify separate lifecycles. - - Adds synthetic opening fills when the earliest lifecycle is incomplete. - - Filters out closed lifecycles when the current lifecycle matches the venue position. - - Replaces a mismatched current lifecycle with a synthetic fill reflecting the venue position. - - Synthetic fills use calculated reconciliation prices to target correct average positions. - - See [Partial window adjustment scenarios](#partial-window-adjustment-scenarios) for details. -- **Exception handling**: - - Individual adapter failures do not abort the entire reconciliation process. - - Fill reports arriving before order status reports are deferred until order state is available. - -If reconciliation fails, the system logs an error and does not start. - -### Common reconciliation scenarios - -The tables below cover startup reconciliation (mass status) and runtime checks (in-flight order checks, open-order polls, own-books audits). - -#### Startup reconciliation - -| Scenario | Description | System behavior | -|----------------------------------------|------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------| -| **Order state discrepancy** | Local state differs from venue (e.g., local `SUBMITTED`, venue `REJECTED`). | Updates local order to match venue state, emits missing events. | -| **Missed fills** | Venue filled an order but the engine missed the event. | Generates missing `OrderFilled` events. | -| **Multiple fills** | Order has partial fills, some missed by the engine. | Reconstructs complete fill history from venue reports. | -| **External orders** | Orders exist on venue but not in local cache. | Creates orders with strategy ID `EXTERNAL` and tag `VENUE`. | -| **Partially filled then canceled** | Order partially filled then canceled by venue. | Updates state to `CANCELED`, preserves fill history. | -| **Different fill data** | Venue reports different fill price/commission than cached. | Preserves cached data, logs discrepancies. | -| **Filtered orders** | Orders marked for filtering via config. | Skips based on `filtered_client_order_ids` or instrument filters. | -| **Duplicate order reports** | Multiple orders share the same identifier. | Deduplicates with warning logged. | -| **Position quantity mismatch (long)** | Internal long position differs from venue (e.g., 100 vs 150). | Generates BUY LIMIT with calculated price when `generate_missing_orders=True`. | -| **Position quantity mismatch (short)** | Internal short position differs from venue (e.g., -100 vs -150). | Generates SELL LIMIT with calculated price when `generate_missing_orders=True`. | -| **Position reduction** | Venue position smaller than internal (e.g., internal 150 long, venue 100 long). | Generates opposite-side LIMIT order with calculated price. | -| **Position side flip** | Internal position opposite of venue (e.g., internal 100 long, venue 50 short). | Generates LIMIT order to close internal and open external position. | -| **Internal reconciliation orders** | Orders with strategy ID `EXTERNAL` and tag `RECONCILIATION`. | Never filtered, regardless of `filter_unclaimed_external_orders`. | - -#### Runtime checks - -| Scenario | Description | System behavior | -|-----------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| -| **In-flight order timeout** | Order remains unconfirmed beyond threshold. | After `inflight_check_retries`, resolves to `REJECTED`. | -| **Open orders check discrepancy** | Periodic poll detects a venue state change. | Confirms status at `open_check_interval_secs` and applies transitions. | -| **Own books audit mismatch** | Own order books diverge from venue public books. | Audits at `own_books_audit_interval_secs`, logs inconsistencies. | - -### Common reconciliation issues - -- **Missing trade reports**: Some venues filter out older trades. Increase `reconciliation_lookback_mins` or cache all events locally. -- **Position mismatches**: External orders that predate the lookback window cause position drift. Flatten the account before restarting to reset state. -- **Duplicate order IDs**: Deduplicated with warnings logged. Frequent duplicates may indicate venue data integrity issues. -- **Precision differences**: Small decimal differences are handled using instrument precision. Large discrepancies may indicate missing orders. -- **Out-of-order reports**: Fill reports arriving before order status reports are deferred until order state is available. - -:::tip -For persistent issues, drop cached state or flatten accounts before restarting. -::: - -### Reconciliation invariants - -The reconciliation system maintains three invariants: - -1. **Position quantity**: the final quantity matches the venue within instrument precision. -2. **Average entry price**: the position's average entry price matches the venue's reported price within tolerance (default 0.01%). -3. **PnL integrity**: all generated fills, including synthetic fills, use calculated prices that preserve correct unrealized PnL. - -These hold even when: - -- The reconciliation window misses complete fill history. -- Fills are missing from venue reports. -- Position lifecycles span beyond the lookback window. -- Multiple zero-crossings have occurred. - -### Partial window adjustment scenarios - -When `reconciliation_lookback_mins` limits the window, the system analyzes position lifecycles -from fills and adjusts to reconstruct positions accurately. - -| Scenario | Description | System behavior | -|--------------------------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------------------| -| **Complete lifecycle** | All fills from opening to current state are captured. | No adjustment. | -| **Incomplete single lifecycle** | Window misses opening fills, no zero-crossings. | Adds synthetic opening fill with calculated price. | -| **Multiple lifecycles, current matches** | Zero-crossings detected, current lifecycle matches venue. | Filters out old lifecycles, returns current only. | -| **Multiple lifecycles, current mismatch** | Zero-crossings detected, current lifecycle differs from venue. | Replaces current lifecycle with a single synthetic fill. | -| **Flat position** | Venue reports FLAT regardless of fill history. | No adjustment. | -| **No fills** | Window contains no fill reports. | No adjustment, empty result. | - -**Key concepts:** - -- **Zero-crossing**: position quantity crosses through zero (FLAT), marking a lifecycle boundary. -- **Lifecycle**: a sequence of fills between zero-crossings representing one open-close cycle. -- **Synthetic fill**: a calculated fill report representing missing activity, priced to achieve the correct average position. -- **Tolerance**: position matching uses configurable price tolerance (default 0.0001 = 0.01%) to absorb minor calculation differences. - -## Related guides - -- [Adapters](adapters.md) - Venue connectivity. -- [Execution](execution.md) - Order execution in live environments. -- [Backtesting](backtesting.md) - Testing strategies before deployment. diff --git a/nautilus-docs/docs/concepts/logging.md b/nautilus-docs/docs/concepts/logging.md deleted file mode 100644 index 91e9b2d..0000000 --- a/nautilus-docs/docs/concepts/logging.md +++ /dev/null @@ -1,470 +0,0 @@ -# Logging - -The platform provides logging for both backtesting and live trading using a high-performance logging subsystem implemented in Rust -with a standardized facade from the `log` crate. - -The core logger operates in a separate thread and uses a multi-producer single-consumer (MPSC) channel to receive log messages. -This design ensures that the main thread remains performant, avoiding potential bottlenecks caused by log string formatting or file I/O operations. - -Logging output is configurable and supports: - -- **stdout/stderr writer** for console output -- **file writer** for persistent storage of logs - -:::info -Infrastructure such as [Vector](https://github.com/vectordotdev/vector) can be integrated to collect and aggregate events within your system. -::: - -## Architecture - -The logging subsystem captures events from multiple sources and routes them through an MPSC channel to a dedicated logging thread: - -```mermaid -flowchart TB - subgraph Sources["Log Sources"] - PY["Python Logger"] - NAUT["Nautilus Rust Components"] - LOG["External Rust Libraries
(using log crate)
rustls, etc."] - end - - subgraph Filtering["Filtering"] - LF["log_level / log_level_file
(LoggingConfig)"] - end - - subgraph Logger["Nautilus Logger"] - NL["Logger
(implements log::Log)"] - end - - subgraph Channel["MPSC Channel"] - TX["Sender (tx)"] - RX["Receiver (rx)"] - end - - subgraph Thread["Logging Thread"] - LT["Log Writer"] - end - - subgraph Output["Output"] - STDOUT["stdout/stderr"] - FILE["Log Files"] - end - - PY --> NL - NAUT --> NL - LOG --> LF --> NL - - NL --> TX --> RX --> LT - LT --> STDOUT - LT --> FILE - - subgraph Tracing["Tracing Subscriber (optional)"] - TRACE["External Rust Libraries
(using tracing crate)
hyper_util, h2, tokio, etc."] - EF["RUST_LOG
(EnvFilter)"] - FMT["fmt::Layer"] - end - - TRACE --> EF --> FMT --> STDOUT -``` - -- **Python and Nautilus components**: Log directly through the Nautilus Logger. -- **External `log` crate users**: Filtered by `log_level`/`log_level_file` in `LoggingConfig`. -- **External `tracing` crate users**: When enabled, output goes directly to stdout (separate from Nautilus logging), filtered by the `RUST_LOG` environment variable. -- **Logging thread**: All Nautilus log events are sent through an MPSC channel to a dedicated thread, ensuring the main thread isn't blocked by I/O operations. - -## Configuration - -Logging can be configured by importing the `LoggingConfig` object. -By default, log events with an 'INFO' `LogLevel` and higher are written to stdout/stderr. - -Log level (`LogLevel`) values include the following (matching standard log level conventions). - -The following log levels are supported: - -- `OFF` - Disable logging. -- `TRACE` - Most verbose; only emitted by Rust components (cannot be generated from Python). -- `DEBUG` - Detailed diagnostic information. -- `INFO` - General operational messages. -- `WARNING` - Potential issues that don't prevent operation. -- `ERROR` - Errors that may affect functionality. - -:::tip -You can set `TRACE` as a filter level to capture trace logs from Rust components, even though Python code cannot emit them directly. -::: - -See the `LoggingConfig` [API Reference](/docs/python-api-latest/config.html#nautilus_trader.common.config.LoggingConfig) for further details. - -Logging can be configured in the following ways: - -- Minimum `LogLevel` for stdout/stderr. -- Minimum `LogLevel` for log files. -- Maximum size before rotating a log file. -- Maximum number of backup log files to maintain when rotating. -- Automatic log file naming with date or timestamp components, or custom log file name. -- Directory for writing log files. -- Plain text or JSON log file formatting. -- Filtering of individual components by log level. -- ANSI colors in log lines. -- Bypass logging entirely. -- Print Rust config to stdout at initialization. -- Optionally initialize logging via the PyO3 bridge (`use_pyo3`) to capture log events emitted by Rust components. -- Truncate existing log file on startup if it already exists (`clear_log_file`) - -### Standard output logging - -Log messages are written to the console via stdout/stderr writers. The minimum log level can be configured using the `log_level` parameter. - -### File logging - -Log files are written to the current working directory by default. The naming convention and rotation behavior are configurable and follow specific patterns based on your settings. - -You can specify a custom log directory using `log_directory` and/or a custom file basename using `log_file_name`. - -**Log file formats:** - -- `None` (default) - Plain text format with `.log` extension. -- `"json"` - JSON format with `.json` extension, useful for log aggregation tools. - -For detailed information about log file naming conventions and rotation behavior, see the [Log file rotation](#log-file-rotation) and [Log file naming convention](#log-file-naming-convention) sections below. - -#### Log file rotation - -Rotation behavior depends on both the presence of a size limit and whether a custom file name is provided: - -- **Size-based rotation**: - - Enabled by specifying the `log_file_max_size` parameter (e.g., `100_000_000` for 100 MB). - - When writing a log entry would make the current file exceed this size, the file is closed and a new one is created. -- **Date-based rotation (default naming only)**: - - Applies when no `log_file_max_size` is specified and no custom `log_file_name` is provided. - - At each UTC date change (midnight), the current log file is closed and a new one is started, creating one file per UTC day. -- **No rotation**: - - When a custom `log_file_name` is provided without a `log_file_max_size`, logs continue to append to the same file. - - Note: Size-based rotation takes precedence - if both a custom name and size limit are provided, rotation still occurs. -- **Backup file management**: - - Controlled by the `log_file_max_backup_count` parameter (default: 5), limiting the total number of rotated files kept. - - When this limit is exceeded, the oldest backup files are automatically removed. - -#### Log file naming convention - -The default naming convention ensures log files are uniquely identifiable and timestamped. -The format depends on whether file rotation is enabled: - -**With file rotation enabled**: - -- **Format**: `{trader_id}_{%Y-%m-%d_%H%M%S:%3f}_{instance_id}.{log|json}` -- **Example**: `TESTER-001_2025-04-09_210721:521_d7dc12c8-7008-4042-8ac4-017c3db0fc38.log` -- **Components**: - - `{trader_id}`: The trader identifier (e.g., `TESTER-001`). - - `{%Y-%m-%d_%H%M%S:%3f}`: Full ISO 8601-compliant datetime with millisecond resolution. - - `{instance_id}`: A unique instance identifier. - - `{log|json}`: File suffix based on format setting. - -**Without size-based rotation (default naming)**: - -- **Format**: `{trader_id}_{%Y-%m-%d}_{instance_id}.{log|json}` -- **Example**: `TESTER-001_2025-04-09_d7dc12c8-7008-4042-8ac4-017c3db0fc38.log` -- **Components**: - - `{trader_id}`: The trader identifier. - - `{%Y-%m-%d}`: Date only (YYYY-MM-DD). - - `{instance_id}`: A unique instance identifier. - - `{log|json}`: File suffix based on format setting. -- **Note**: With default naming and no size limit, logs rotate daily at UTC midnight. - -**Custom naming**: - -If `log_file_name` is set (e.g., `my_custom_log`): - -- With rotation disabled: The file will be named exactly as provided (e.g., `my_custom_log.log`). -- With rotation enabled: The file will include the custom name and timestamp (e.g., `my_custom_log_2025-04-09_210721:521.log`). - -### Component log filtering - -The `log_component_levels` parameter can be used to set log levels for each component individually. -The input value should be a dictionary of component ID strings to log level strings: `dict[str, str]`. - -Below is an example of a trading node logging configuration that includes some of the options mentioned above: - -```python -from nautilus_trader.config import LoggingConfig -from nautilus_trader.config import TradingNodeConfig - -config_node = TradingNodeConfig( - trader_id="TESTER-001", - logging=LoggingConfig( - log_level="INFO", - log_level_file="DEBUG", - log_file_format="json", - log_component_levels={ "Portfolio": "INFO" }, - ), - ... # Omitted -) -``` - -For backtesting, the `BacktestEngineConfig` class can be used instead of `TradingNodeConfig`, as the same options are available. - -### Environment variable configuration - -The `NAUTILUS_LOG` environment variable provides an alternative way to configure logging using a semicolon-separated spec string. This is useful for Rust-only binaries or when you want to override logging settings without modifying code. - -```bash -export NAUTILUS_LOG="stdout=Info;fileout=Debug;RiskEngine=Error;is_colored" -``` - -**Supported keys:** - -| Key | Type | Description | -|-----------------------|-----------|--------------------------------------------------| -| `stdout` | Log level | Maximum level for stdout output. | -| `fileout` | Log level | Maximum level for file output. | -| `is_colored` | Flag | Enable ANSI colors (default: true). | -| `print_config` | Flag | Print config to stdout at startup. | -| `log_components_only` | Flag | Only log components with explicit filters. | -| `` | Log level | Component-specific level (exact match). | -| `` | Log level | Module-specific level (prefix match, Rust only). | - -Flags are enabled by their presence in the spec string (no value needed). Log levels are case-insensitive: `Off`, `Trace`, `Debug`, `Info`, `Warning` (or `Warn`), `Error`. - -:::note -For Rust-only binaries, setting `NAUTILUS_LOG` enables lazy initialization of the logging subsystem on first use, without requiring explicit `init_logging()` calls. -::: - -### Components-only logging - -When focusing on a subset of noisy systems, enable `log_components_only` to log messages only from components explicitly listed in `log_component_levels`. All other components are suppressed regardless of the global `log_level` or file level. - -Example (Python configuration): - -```python -logging = LoggingConfig( - log_level="INFO", - log_component_levels={ - "RiskEngine": "DEBUG", - "Portfolio": "INFO", - }, - log_components_only=True, -) -``` - -If configuring via the environment using the Rust spec string, include `log_components_only` alongside component filters, for example: - -```bash -export NAUTILUS_LOG="stdout=Info;log_components_only;RiskEngine=Debug;Portfolio=Info" -``` - -### Module path filtering (Rust only) - -When using the `NAUTILUS_LOG` environment variable, you can filter by Rust module paths in addition to component names. Keys containing `::` are treated as module path filters with prefix matching, while keys without `::` are component filters with exact matching. - -```bash -# Filter all adapters to Warn, but allow Debug for OKX specifically -export NAUTILUS_LOG="stdout=Info;nautilus_okx=Warn;nautilus_okx::websocket=Debug" -``` - -The longest matching prefix takes precedence. In the example above, `nautilus_okx::websocket::handler` would use the `Debug` level (longer prefix), while `nautilus_okx::data` would use `Warn`. - -:::tip -Rust log macros automatically capture the module path when no explicit component is provided. This enables module-level filtering to work with standard logging calls. -::: - -:::note -Module path filtering is only available via the `NAUTILUS_LOG` environment variable. The Python `log_component_levels` configuration uses component name matching only. -::: - -:::warning -If `log_components_only=True` (or `log_components_only` is present in the spec string) and `log_component_levels` is empty, no log messages will be emitted to stdout/stderr or files. Add at least one component filter or disable components-only logging. -::: - -### Log colors - -ANSI color codes improve log readability in terminals. -In environments that do not support ANSI color rendering (such as some cloud environments or text editors), -these color codes may not be appropriate as they can appear as raw text. - -To accommodate for such scenarios, the `LoggingConfig.log_colors` option can be set to `false`. -Disabling `log_colors` will prevent the addition of ANSI color codes to the log messages, -which avoids raw escape codes in environments without color support. - -## Using a logger directly - -It's possible to use `Logger` objects directly, and these can be initialized anywhere (very similar to the Python built-in `logging` API). - -If you ***aren't*** using an object which already initializes a `NautilusKernel` (and logging) such as `BacktestEngine` or `TradingNode`, -then you can activate logging in the following way: - -```python -from nautilus_trader.common.component import init_logging -from nautilus_trader.common.component import Logger - -log_guard = init_logging() -logger = Logger("MyLogger") -``` - -See the [`init_logging` API Reference](/docs/python-api-latest/common.html) for further details. - -:::warning -Only one logging subsystem can be initialized per process with an `init_logging` call. Multiple `LogGuard` instances (up to 255) can exist concurrently, and the logging thread will remain active until all guards are dropped. -::: - -## LogGuard: managing log lifecycle - -The `LogGuard` ensures that the logging subsystem remains active and operational throughout the lifecycle of a process. -It prevents premature shutdown of the logging subsystem when running multiple engines in the same process. - -### Reference counting implementation - -The logging system uses reference counting to track active `LogGuard` instances: - -- **Counter increments**: When a new `LogGuard` is created, an atomic counter is incremented. -- **Counter decrements**: When a `LogGuard` is dropped, the counter is decremented. -- **Logging thread termination**: When the counter reaches zero (last `LogGuard` dropped), the logging thread is properly joined to ensure all pending log messages are written before the process terminates. -- **Maximum guards**: The system supports up to 255 concurrent `LogGuard` instances. Attempting to create more raises a `RuntimeError`. - -This mechanism ensures that: - -1. `LogGuard` keeps the logging thread alive and flushes on drop; abrupt termination (crashes, kill signals) can still lose buffered logs. -2. The logging thread remains active as long as any `LogGuard` exists. -3. On graceful shutdown, all buffered logs are properly flushed to their destinations. - -### Why use LogGuard? - -Without a `LogGuard`, any attempt to run sequential engines in the same process may result in errors such as: - -``` -Error sending log event: [INFO] ... -``` - -This occurs because the logging subsystem's underlying channel and Rust `Logger` are closed when the first engine is disposed. -As a result, subsequent engines lose access to the logging subsystem, leading to these errors. - -By using a `LogGuard`, you can ensure consistent logging behavior across multiple backtests or engine runs in the same process. -The `LogGuard` retains the resources of the logging subsystem and ensures that logs continue to function correctly, -even as engines are disposed and initialized. - -:::note -Using `LogGuard` is required to maintain consistent logging behavior throughout a process with multiple engines. -::: - -## Running multiple engines - -The following example demonstrates how to use a `LogGuard` when running multiple engines sequentially in the same process: - -```python -log_guard = None # Initialize LogGuard reference - -for i in range(number_of_backtests): - engine = setup_engine(...) - - # Assign reference to LogGuard - if log_guard is None: - log_guard = engine.get_log_guard() - - # Add actors and execute the engine - actors = setup_actors(...) - engine.add_actors(actors) - engine.run() - engine.dispose() # Dispose safely -``` - -### Steps - -- **Initialize LogGuard once**: The `LogGuard` is obtained from the first engine (`engine.get_log_guard()`) and is retained throughout the process. This ensures that the logging subsystem remains active. -- **Dispose engines safely**: Each engine is safely disposed of after its backtest completes. The `LogGuard` remains valid after `engine.dispose()` - only the engine is cleaned up, not the logging subsystem. -- **Reuse LogGuard**: The same `LogGuard` instance is reused for subsequent engines, preventing the logging subsystem from shutting down prematurely. - -### Considerations - -- **Multiple LogGuards per process**: The system supports up to 255 concurrent `LogGuard` instances per process. Each guard increments a reference counter when created and decrements it when dropped. -- **Thread safety**: The logging subsystem, including `LogGuard`, is thread-safe, ensuring consistent behavior even in multi-threaded environments. -- **Automatic cleanup**: When the last `LogGuard` is dropped (reference count reaches zero), the logging thread is properly joined to ensure all pending logs are written before the process terminates. - -## Tracing subscriber for external Rust libraries - -External Rust crates that use the `tracing` crate can have their log output displayed by enabling -the tracing subscriber. This is useful for debugging external dependencies or when integrating -custom Rust components (such as feature extractors or adapters) compiled as separate PyO3 extensions. - -### Enabling the subscriber - -Enable the tracing subscriber by setting `use_tracing=True` in `LoggingConfig`: - -```python -from nautilus_trader.config import LoggingConfig -from nautilus_trader.config import TradingNodeConfig - -config_node = TradingNodeConfig( - trader_id="TESTER-001", - logging=LoggingConfig( - log_level="INFO", - use_tracing=True, - ), - ... # Omitted -) -``` - -Alternatively, call `init_tracing()` directly: - -```python -from nautilus_trader.core import nautilus_pyo3 - -nautilus_pyo3.init_tracing() -``` - -### Filtering with RUST_LOG - -The `RUST_LOG` environment variable controls which tracing events are displayed: - -```bash -# Show debug logs from your crate, warn and above from hyper -RUST_LOG=my_feature_extractor=debug,hyper=warn python my_script.py -``` - -If `RUST_LOG` is not set, the default filter level is `warn`. - -### How it works - -The tracing subscriber uses a `tracing-subscriber` fmt layer with a custom formatter to output -directly to stdout. This is separate from the Nautilus logging infrastructure - tracing output -uses a Nautilus-aligned format with nanosecond timestamps. - -Example tracing output: - -``` -2026-01-24T05:51:42.809619000Z [DEBUG] hyper_util::client::legacy::connect::http: connecting to 104.18.5.240:443 -2026-01-24T05:51:42.810543000Z [DEBUG] hyper_util::client::legacy::pool: pooling idle connection for ("https", api.example.com) -``` - -**Differences from Nautilus logging:** - -- Tracing output goes directly to stdout, not through the Nautilus logging thread. -- Tracing events are not written to Nautilus log files. -- Filtering is controlled exclusively by `RUST_LOG`, independent of `LoggingConfig`. - -For external libraries that use the `log` crate (such as `rustls`), their events go through -the Nautilus logger and are filtered by `log_level`/`log_level_file` in `LoggingConfig`. - -:::tip -`RUST_LOG` only affects crates using `tracing`. For crates using `log`, configure verbosity -via `LoggingConfig` or the `NAUTILUS_LOG` environment variable (e.g., `NAUTILUS_LOG=stdout=Debug`). -::: - -:::note -The tracing subscriber can only be initialized once per process. When using `use_tracing=True` in -`LoggingConfig`, subsequent kernel creations safely skip re-initialization. Direct calls to -`init_tracing()` when already initialized will raise an error. -::: - -## Platform-specific considerations - -### Windows shutdown behavior - -On Windows, non-deterministic garbage collection during interpreter shutdown can occasionally -prevent the logging thread from joining properly. When the last `LogGuard` is dropped, the -logging subsystem signals the background thread to close and joins it to ensure all pending -messages are written. If Python's garbage collector delays dropping the guard until after -interpreter shutdown has begun, this join may not complete, resulting in truncated logs. - -This issue is tracked in GitHub [issue #3027](https://github.com/nautechsystems/nautilus_trader/issues/3027). -A more deterministic shutdown mechanism is under consideration. - -## Related guides - -- [Architecture](architecture.md) - System architecture including logging infrastructure. diff --git a/nautilus-docs/docs/concepts/message_bus.md b/nautilus-docs/docs/concepts/message_bus.md deleted file mode 100644 index e94f182..0000000 --- a/nautilus-docs/docs/concepts/message_bus.md +++ /dev/null @@ -1,473 +0,0 @@ -# Message Bus - -The `MessageBus` enables communication between system components through message passing. -This design creates a loosely coupled architecture where components interact without -direct dependencies. - -The *messaging patterns* include: - -- Point-to-Point -- Publish/Subscribe -- Request/Response - -Messages exchanged via the `MessageBus` fall into three categories: - -- Data -- Events -- Commands - -## Data and signal publishing - -While the `MessageBus` is a lower-level component that users typically interact with indirectly, -`Actor` and `Strategy` classes provide convenient methods built on top of it: - -```python -def publish_data(self, data_type: DataType, data: Data) -> None: -def publish_signal(self, name: str, value, ts_event: int = 0) -> None: -``` - -These methods allow you to publish custom data and signals efficiently without needing to work directly with the `MessageBus` interface. - -## Direct access - -For advanced users or specialized use cases, direct access to the message bus is available within `Actor` and `Strategy` -classes through the `self.msgbus` reference, which provides the full message bus interface. - -To publish a custom message directly, you can specify a topic as a `str` and any Python `object` as the message payload, for example: - -```python -self.msgbus.publish("MyTopic", "MyMessage") -``` - -## Messaging styles - -NautilusTrader is an **event-driven** framework where components communicate by sending and receiving messages. -Understanding the different messaging styles helps when building trading systems. - -This guide explains the three primary messaging patterns available in NautilusTrader: - -| **Messaging Style** | **Purpose** | **Best For** | -|:---------------------------------------------|:--------------------------------------------|:------------------------------------------------------| -| **MessageBus - Publish/Subscribe to topics** | Low-level, direct access to the message bus | Custom events, system-level communication | -| **Actor-Based - Publish/Subscribe Data** | Structured trading data exchange | Trading metrics, indicators, data needing persistence | -| **Actor-Based - Publish/Subscribe Signal** | Lightweight notifications | Simple alerts, flags, status updates | - -Each approach serves different purposes. This section helps you decide which pattern to use. - -### MessageBus publish/subscribe to topics - -#### Concept - -The `MessageBus` is the central hub for all messages in NautilusTrader. It enables a **publish/subscribe** pattern -where components can publish events to **named topics**, and other components can subscribe to receive those messages. -This decouples components, allowing them to interact indirectly via the message bus. - -#### Key benefits and use cases - -The message bus approach is ideal when you need: - -- **Cross-component communication** within the system. -- **Flexibility** to define any topic and send any type of payload (any Python object). -- **Decoupling** between publishers and subscribers who don't need to know about each other. -- **Global Reach** where messages can be received by multiple subscribers. -- Working with events that don't fit within the predefined `Actor` model. -- Advanced scenarios requiring full control over messaging. - -#### Considerations - -- You must track topic names manually (typos could result in missed messages). -- You must define handlers manually. - -#### Quick overview code - -```python -from nautilus_trader.core.message import Event - -# Define a custom event -class Each10thBarEvent(Event): - TOPIC = "each_10th_bar" # Topic name - def __init__(self, bar): - self.bar = bar - -# Subscribe in a component (in Strategy) -self.msgbus.subscribe(Each10thBarEvent.TOPIC, self.on_each_10th_bar) - -# Publish an event (in Strategy) -event = Each10thBarEvent(bar) -self.msgbus.publish(Each10thBarEvent.TOPIC, event) - -# Handler (in Strategy) -def on_each_10th_bar(self, event: Each10thBarEvent): - self.log.info(f"Received 10th bar: {event.bar}") -``` - -#### Full example - -[MessageBus Example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/example_09_messaging_with_msgbus) - -### Actor-based publish/subscribe data - -#### Concept - -This approach provides a way to exchange trading specific data between `Actor`s in the system. -(note: each `Strategy` inherits from `Actor`). It inherits from `Data`, which ensures proper timestamping -and ordering of events - crucial for correct backtest processing. - -#### Key benefits and use cases - -The Data publish/subscribe approach works well when you need: - -- **Exchange of structured trading data** like market data, indicators, custom metrics, or option greeks. -- **Proper event ordering** via built-in timestamps (`ts_event`, `ts_init`) crucial for backtest accuracy. -- **Data persistence and serialization** through the `@customdataclass` decorator, integrating with NautilusTrader's data catalog system. -- **Standardized trading data exchange** between system components. - -#### Considerations - -- Requires defining a class that inherits from `Data` or uses `@customdataclass`. - -#### Inheriting from `Data` vs. using `@customdataclass` - -**Inheriting from `Data` class:** - -- Defines abstract properties `ts_event` and `ts_init` that must be implemented by the subclass. These ensure proper data ordering in backtests based on timestamps. - -**The `@customdataclass` decorator:** - -- Adds `ts_event` and `ts_init` attributes if they are not already present. -- Provides serialization functions: `to_dict()`, `from_dict()`, `to_bytes()`, `to_arrow()`, etc. -- Enables data persistence and external communication. - -#### Quick overview code - -```python -from nautilus_trader.core.data import Data -from nautilus_trader.model.custom import customdataclass - -@customdataclass -class GreeksData(Data): - delta: float - gamma: float - -# Publish data (in Actor / Strategy) -data = GreeksData(delta=0.75, gamma=0.1, ts_event=1_630_000_000_000_000_000, ts_init=1_630_000_000_000_000_000) -self.publish_data(GreeksData, data) - -# Subscribe to receiving data (in Actor / Strategy) -self.subscribe_data(GreeksData) - -# Handler (this is static callback function with fixed name) -def on_data(self, data: Data): - if isinstance(data, GreeksData): - self.log.info(f"Delta: {data.delta}, Gamma: {data.gamma}") -``` - -#### Full example - -[Actor-Based Data Example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/example_10_messaging_with_actor_data) - -### Actor-based publish/subscribe signal - -#### Concept - -**Signals** are a lightweight way to publish and subscribe to simple notifications within the actor framework. -This is the simplest messaging approach, requiring no custom class definitions. - -#### Key benefits and use cases - -The Signal messaging approach works well when you need: - -- **Simple, lightweight notifications/alerts** like "RiskThresholdExceeded" or "TrendUp". -- **Quick, on-the-fly messaging** without defining custom classes. -- **Broadcasting alerts or flags** as primitive data (`int`, `float`, or `str`). -- **Easy API integration** with straightforward methods (`publish_signal`, `subscribe_signal`). -- **Multiple subscriber communication** where all subscribers receive signals when published. -- **Minimal setup overhead** with no class definitions required. - -#### Considerations - -- Each signal can contain only **single value** of type: `int`, `float`, and `str`. That means no support for complex data structures or other Python types. -- In the `on_signal` handler, you can only differentiate between signals using `signal.value`, as the signal name is not accessible in the handler. - -#### Quick overview code - -```python -# Define signal constants for better organization (optional but recommended) -import types -from nautilus_trader.core.datetime import unix_nanos_to_dt -from nautilus_trader.common.enums import LogColor - -signals = types.SimpleNamespace() -signals.NEW_HIGHEST_PRICE = "NewHighestPriceReached" -signals.NEW_LOWEST_PRICE = "NewLowestPriceReached" - -# Subscribe to signals (in Actor/Strategy) -self.subscribe_signal(signals.NEW_HIGHEST_PRICE) -self.subscribe_signal(signals.NEW_LOWEST_PRICE) - -# Publish a signal (in Actor/Strategy) -self.publish_signal( - name=signals.NEW_HIGHEST_PRICE, - value=signals.NEW_HIGHEST_PRICE, # value can be the same as name for simplicity - ts_event=bar.ts_event, # timestamp from triggering event -) - -# Handler (this is static callback function with fixed name) -def on_signal(self, signal): - # IMPORTANT: We match against signal.value, not signal.name - match signal.value: - case signals.NEW_HIGHEST_PRICE: - self.log.info( - f"New highest price was reached. | " - f"Signal value: {signal.value} | " - f"Signal time: {unix_nanos_to_dt(signal.ts_event)}", - color=LogColor.GREEN - ) - case signals.NEW_LOWEST_PRICE: - self.log.info( - f"New lowest price was reached. | " - f"Signal value: {signal.value} | " - f"Signal time: {unix_nanos_to_dt(signal.ts_event)}", - color=LogColor.RED - ) -``` - -#### Full example - -[Actor-Based Signal Example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/example_11_messaging_with_actor_signals) - -### Summary and decision guide - -Here's a quick reference to help you decide which messaging style to use: - -#### Decision guide: Which style to choose? - -| **Use Case** | **Recommended Approach** | **Setup required** | -|:--------------------------------------------|:--------------------------------------------------------------------------------|:-------------------| -| Custom events or system-level communication | `MessageBus` + Pub/Sub to topic | Topic + Handler management | -| Structured trading data | `Actor` + Pub/Sub Data + optional `@customdataclass` if serialization is needed | New class definition inheriting from `Data` (handler `on_data` is predefined) | -| Simple alerts/notifications | `Actor` + Pub/Sub Signal | Signal name only | - -## External publishing - -The `MessageBus` can be *backed* with any database or message broker technology which has an -integration written for it, this then enables external publishing of messages. - -:::info -Redis is currently supported for all serializable messages which are published externally. -The minimum supported Redis version is 6.2 (required for [streams](https://redis.io/docs/latest/develop/data-types/streams/) functionality). -::: - -Under the hood, when a backing database (or any other compatible technology) is configured, -all outgoing messages are first serialized, then transmitted via a Multiple-Producer Single-Consumer (MPSC) channel to a separate thread (implemented in Rust). -In this separate thread, the message is written to its final destination, which is presently Redis streams. - -Offloading I/O to a separate thread keeps the main thread unblocked. - -### Serialization - -Nautilus supports serialization for: - -- All Nautilus built-in types (serialized as dictionaries `dict[str, Any]` containing serializable primitives). -- Python primitive types (`str`, `int`, `float`, `bool`, `bytes`). - -You can add serialization support for custom types by registering them through the `serialization` subpackage. - -```python -def register_serializable_type( - cls, - to_dict: Callable[[Any], dict[str, Any]], - from_dict: Callable[[dict[str, Any]], Any], -): - ... -``` - -- `cls`: The type to register. -- `to_dict`: The delegate to instantiate a dict of primitive types from the object. -- `from_dict`: The delegate to instantiate the object from a dict of primitive types. - -## Configuration - -The message bus external backing technology can be configured by importing the `MessageBusConfig` object and passing this to -your `TradingNodeConfig`. Each of these config options will be described below. - -```python -... # Other config omitted -message_bus=MessageBusConfig( - database=DatabaseConfig(), - encoding="json", - timestamps_as_iso8601=True, - buffer_interval_ms=100, - autotrim_mins=30, - use_trader_prefix=True, - use_trader_id=True, - use_instance_id=False, - streams_prefix="streams", - types_filter=[QuoteTick, TradeTick], -) -... -``` - -### Database config - -A `DatabaseConfig` must be provided, for a default Redis setup on the local -loopback you can pass a `DatabaseConfig()`, which will use defaults to match. - -### Encoding - -Two encodings are currently supported by the built-in `Serializer` used by the `MessageBus`: - -- JSON (`json`) -- MessagePack (`msgpack`) - -Use the `encoding` config option to control the message writing encoding. - -:::tip -The `msgpack` encoding is used by default as it offers the most optimal serialization and memory performance. -We recommend using `json` encoding for human readability when performance is not a primary concern. -::: - -### Timestamp formatting - -By default timestamps are formatted as UNIX epoch nanosecond integers. Alternatively you can -configure ISO 8601 string formatting by setting the `timestamps_as_iso8601` to `True`. - -### Message stream keys - -Message stream keys are essential for identifying individual trader nodes and organizing messages within streams. -They can be tailored to meet your specific requirements and use cases. In the context of message bus streams, a trader key is typically structured as follows: - -``` -trader:{trader_id}:{instance_id}:{streams_prefix} -``` - -The following options are available for configuring message stream keys: - -#### Trader prefix - -If the key should begin with the `trader` string. - -#### Trader ID - -If the key should include the trader ID for the node. - -#### Instance ID - -Each trader node is assigned a unique 'instance ID,' which is a UUIDv4. This instance ID helps distinguish individual traders when messages -are distributed across multiple streams. You can include the instance ID in the trader key by setting the `use_instance_id` configuration option to `True`. -This is particularly useful when you need to track and identify traders across various streams in a multi-node trading system. - -#### Streams prefix - -The `streams_prefix` string enables you to group all streams for a single trader instance or organize -messages for multiple instances. Configure this by passing a string to the `streams_prefix` configuration -option, ensuring other prefixes are set to false. - -#### Stream per topic - -Indicates whether the producer will write a separate stream for each topic. This is particularly -useful for Redis backings, which do not support wildcard topics when listening to streams. -If set to False, all messages will be written to the same stream. - -:::info -Redis does not support wildcard stream topics. For better compatibility with Redis, it is recommended to set this option to False. -::: - -### Types filtering - -When messages are published on the message bus, they are serialized and written to a stream if a backing -for the message bus is configured and enabled. To prevent flooding the stream with data like high-frequency -quotes, you may filter out certain types of messages from external publication. - -To enable this filtering mechanism, pass a list of `type` objects to the `types_filter` parameter in the message bus configuration, -specifying which types of messages should be excluded from external publication. - -```python -from nautilus_trader.config import MessageBusConfig -from nautilus_trader.model.data import QuoteTick -from nautilus_trader.model.data import TradeTick - -# Create a MessageBusConfig instance with types filtering -message_bus = MessageBusConfig( - types_filter=[QuoteTick, TradeTick] -) -``` - -### Stream auto-trimming - -The `autotrim_mins` configuration parameter allows you to specify the lookback window in minutes for automatic stream trimming in your message streams. -Automatic stream trimming helps manage the size of your message streams by removing older messages, ensuring that the streams remain manageable in terms of storage and performance. - -:::info -The current Redis implementation will maintain the `autotrim_mins` as a maximum width (plus roughly a minute, as streams are trimmed no more than once per minute). -Rather than a maximum lookback window based on the current wall clock time. -::: - -## External streams - -The message bus within a `TradingNode` (node) is referred to as the "internal message bus". -A producer node is one which publishes messages onto an external stream (see [external publishing](#external-publishing)). -The consumer node listens to external streams to receive and publish deserialized message payloads on its internal message bus. - -```mermaid -flowchart TB - producer[Producer Node] - stream[Stream] - consumer1[Consumer Node 1] - consumer2[Consumer Node 2] - - producer --> stream - stream --> consumer1 - stream --> consumer2 -``` - -:::tip -Set the `LiveDataEngineConfig.external_clients` with the list of `client_id`s intended to represent the external streaming clients. -The `DataEngine` will filter out subscription commands for these clients, ensuring that the external streaming provides the necessary data for any subscriptions to these clients. -::: - -### Example configuration - -The following example details a streaming setup where a producer node publishes Binance data externally, -and a downstream consumer node publishes these data messages onto its internal message bus. - -#### Producer node - -We configure the `MessageBus` of the producer node to publish to a `"binance"` stream. -The settings `use_trader_id`, `use_trader_prefix`, and `use_instance_id` are all set to `False` -to ensure a simple and predictable stream key that the consumer nodes can register for. - -```python -message_bus=MessageBusConfig( - database=DatabaseConfig(timeout=2), - use_trader_id=False, - use_trader_prefix=False, - use_instance_id=False, - streams_prefix="binance", # <--- - stream_per_topic=False, - autotrim_mins=30, -), -``` - -#### Consumer node - -We configure the `MessageBus` of the consumer node to receive messages from the same `"binance"` stream. -The node will listen to the external stream keys to publish these messages onto its internal message bus. -Additionally, we declare the client ID `"BINANCE_EXT"` as an external client. This ensures that the -`DataEngine` does not attempt to send data commands to this client ID, as we expect these messages to be -published onto the internal message bus from the external stream, to which the node has subscribed to the relevant topics. - -```python -data_engine=LiveDataEngineConfig( - external_clients=[ClientId("BINANCE_EXT")], -), -message_bus=MessageBusConfig( - database=DatabaseConfig(timeout=2), - external_streams=["binance"], # <--- -), -``` - -## Related guides - -- [Actors](actors.md) - Actors use the message bus for event handling. -- [Architecture](architecture.md) - Message bus role in system architecture. diff --git a/nautilus-docs/docs/concepts/options.md b/nautilus-docs/docs/concepts/options.md deleted file mode 100644 index 514bc9b..0000000 --- a/nautilus-docs/docs/concepts/options.md +++ /dev/null @@ -1,291 +0,0 @@ -# Options - -Nautilus provides first-class support for options trading across traditional -and crypto markets. This includes option-specific instrument types, venue-provided -Greeks streaming, option chain aggregation, and a local Black-Scholes Greeks calculator -for risk management. - -## Option instrument types - -The platform defines several option instrument types: - -| Instrument | Description | -|------------------|----------------------------------------------------------------------------------------| -| `OptionContract` | Exchange-traded option (put or call) on an underlying with strike and expiry. | -| `OptionSpread` | Exchange-defined multi-leg options strategy (vertical, calendar, straddle) as one line. | -| `CryptoOption` | Option on a crypto underlying with crypto quote/settlement; inverse or quanto styles. | -| `BinaryOption` | Fixed-payout option that settles to 0 or 1 based on a binary outcome. | - -Greeks-relevant metadata varies by instrument type: - -- `OptionContract`, `CryptoOption` -- full Greeks inputs: `strike_price`, - `option_kind` (CALL/PUT), `expiration_utc`, `underlying`, `multiplier`. -- `OptionSpread` -- a combination of up to 4 option legs, each weighted by a - ratio. Has `underlying`, `expiration_utc`, and `strategy_type` (vertical, - calendar, straddle, etc.). Per-leg `strike_price` and `option_kind` live on - each leg's `OptionContract`, not on the spread itself. Greeks are computed - per leg and aggregated. Spreads are commonly used for orders (the exchange - executes as a single order), while the individual legs appear as positions. -- `BinaryOption` -- has `expiration_utc` and `outcome`/`description`, but no - `strike_price`, `option_kind`, or `underlying`. - -## Subscribing to Greeks - -Venues like Deribit and Bybit publish real-time Greeks alongside their options markets. -Nautilus provides two subscription levels: - -- **Per-instrument Greeks** -- subscribe to individual option contracts. -- **Option chain slices** -- subscribe to an aggregated view of an entire option series. - -### Per-instrument Greeks - -Subscribe to venue-provided Greeks for a single option contract from an actor or strategy: - -```python -from nautilus_trader.model.identifiers import ClientId - -client_id = ClientId("DERIBIT") -self.subscribe_option_greeks(instrument_id, client_id=client_id) -``` - -Handle incoming updates by implementing the `on_option_greeks` handler: - -```python -def on_option_greeks(self, greeks) -> None: - self.log.info( - f"{greeks.instrument_id}: " - f"delta={greeks.delta:.4f} gamma={greeks.gamma:.6f} " - f"vega={greeks.vega:.4f} theta={greeks.theta:.4f} " - f"mark_iv={greeks.mark_iv} underlying={greeks.underlying_price}" - ) -``` - -To stop receiving updates: - -```python -self.unsubscribe_option_greeks(instrument_id, client_id=client_id) -``` - -### Option chain subscriptions - -An option chain subscription aggregates quotes and Greeks across all strikes in an -option series into periodic `OptionChainSlice` snapshots. The `DataEngine` creates -one `OptionChainManager` per series and owns the full lifecycle: routing incoming -data through the manager, publishing snapshots, and managing wire subscriptions. - -```python -from nautilus_trader.core import nautilus_pyo3 - -series_id = nautilus_pyo3.OptionSeriesId(...) # identifies the series (venue, underlying, expiry) - -# Subscribe to 5 strikes above and below ATM, snapshot every 1000ms -strike_range = nautilus_pyo3.StrikeRange.atm_relative(strikes_above=5, strikes_below=5) -self.subscribe_option_chain( - series_id, - strike_range=strike_range, - snapshot_interval_ms=1000, -) -``` - -Handle snapshots by implementing the `on_option_chain` handler: - -```python -def on_option_chain(self, chain) -> None: - for strike in chain.strikes(): - call = chain.get_call(strike) - put = chain.get_put(strike) - if call and call.greeks: - self.log.info(f"Call {strike}: delta={call.greeks.delta:.4f}") -``` - -### Strike range filtering - -`StrikeRange` controls which strikes are active in a chain subscription: - -| Variant | Description | Example | -|----------------|------------------------------------------------------|-----------------------------------------------| -| `Fixed` | Subscribe to an explicit set of strikes. | `nautilus_pyo3.StrikeRange.fixed([...])` | -| `AtmRelative` | N strikes above and N below the current ATM strike. | `nautilus_pyo3.StrikeRange.atm_relative(5, 5)` | -| `AtmPercent` | All strikes within a percentage band around ATM. | `nautilus_pyo3.StrikeRange.atm_percent(0.10)` | - -For ATM-based variants, subscriptions are deferred until the ATM price is determined. -ATM is derived from the forward price embedded in venue-provided `OptionGreeks` updates -(the `underlying_price` field). It can also be seeded from an initial forward price -fetched via HTTP, allowing instant bootstrap before live WebSocket ticks arrive. As ATM -shifts, the active strike set rebalances automatically. - -### Snapshot vs. raw mode - -The `snapshot_interval_ms` parameter controls publishing behavior: - -- **Snapshot mode** (`snapshot_interval_ms=1000`): Quotes and Greeks accumulate in a - buffer and publish as an `OptionChainSlice` on a timer. Suitable for periodic - portfolio rebalancing or UI display. -- **Raw mode** (`snapshot_interval_ms=None`): Each quote or Greeks update publishes - a slice immediately. Suitable for latency-sensitive strategies that react to - individual updates. - -## Option chain architecture - -The option chain system is event-driven and built around per-series isolation. The -`DataEngine` creates one `OptionChainManager` (a PyO3 wrapper around the Rust -`OptionChainAggregator` and `AtmTracker`) per subscribed option series. The engine -owns the lifecycle: subscription routing, timer management, and message bus publishing. -The manager handles only aggregation state and ATM tracking. - -```mermaid -flowchart TD - subgraph DataEngine - DE[DataEngine] - TMR[SnapshotTimer] - end - - subgraph "OptionChainManager (per series)" - MGR[Manager / PyO3] - AGG[OptionChainAggregator] - ATM[AtmTracker] - end - - DC[DataClient] -- QuoteTick --> DE - DC -- OptionGreeks --> DE - DE -- "handle_quote()" --> MGR - DE -- "handle_greeks()" --> MGR - MGR --> AGG - MGR --> ATM - ATM -- "forward price" --> AGG - TMR -- "timer tick" --> DE - DE -- "snapshot()" --> MGR - MGR -- "OptionChainSlice" --> DE - DE -- publish --> MB((MessageBus)) - MB -- "on_option_chain" --> S[Actor / Strategy] - DE -- "sub/unsub" --> DC -``` - -### Component responsibilities - -#### DataEngine - -Holds one `OptionChainManager` per active `OptionSeriesId`. On -`SubscribeOptionChain`, it resolves instruments from the cache, creates the -manager, subscribes active instruments to the data client, and sets up the -snapshot timer. On each timer tick, it calls `manager.check_rebalance()` and -`manager.snapshot()`, forwarding any subscription changes directly to the data -client. On `UnsubscribeOptionChain` or when all instruments expire, it tears -down the manager, cancels the timer, and unsubscribes wire-level feeds. - -#### OptionChainManager (PyO3) - -A thin PyO3 wrapper around `OptionChainAggregator` and `AtmTracker`. It does -not interact with the message bus, clock, or data clients. The `DataEngine` -feeds it market data through `handle_quote()` and `handle_greeks()`, and -retrieves snapshots via `snapshot()`. Both `handle_*` methods return a boolean -indicating whether ATM bootstrap occurred (first ATM price arrived), which the -engine uses to trigger subscription of the real active instrument set. - -#### OptionChainAggregator - -Accumulates quotes and Greeks into call/put buffers using keep-latest semantics. -Instruments that did not update since the last snapshot are still included. Greeks -that arrive before any quote for an instrument are held in a `pending_greeks` -buffer and attached when the first quote arrives. On each `snapshot()` call, the -aggregator produces an immutable `OptionChainSlice`. - -#### AtmTracker - -Derives the ATM price reactively from the `underlying_price` field in incoming -`OptionGreeks` events (the venue-provided forward price for that expiry). It can -be pre-seeded from an HTTP forward price response for instant bootstrap without -waiting for WebSocket ticks. - -### Bootstrap and rebalancing - -For ATM-based strike ranges (`AtmRelative`, `AtmPercent`), the active instrument -set cannot be determined until the ATM price is known. There are two bootstrap -paths: - -**Instant bootstrap (forward price available):** - -1. `DataEngine` receives `SubscribeOptionChain`, resolves all instruments for the - series from the cache, and requests forward prices from the data client. -2. When the forward price response arrives, the engine creates the manager with - the ATM price pre-seeded. The manager computes the active strike set during - construction. -3. The engine subscribes the active instruments immediately. - -**Deferred bootstrap (no forward price):** - -1. Same as above, but no matching forward price is found in the response. -2. The engine creates the manager with no initial ATM price. The active set is - empty and no wire subscriptions are made for the chain. -3. Bootstrap depends on relevant Greeks data already flowing from other - subscriptions (e.g., per-instrument `subscribe_option_greeks` calls). When - the engine feeds an `OptionGreeks` event with `underlying_price` through - `handle_greeks()`, the manager bootstraps and returns `True`. The engine - then subscribes the now-active instrument set. - -Once bootstrapped, the aggregator monitors ATM drift. On each snapshot timer tick, -the engine calls `check_rebalance()` which returns any instruments to add or -remove. A hysteresis threshold and cooldown period prevent thrashing near strike -boundaries. - -## OptionGreeks data type - -`OptionGreeks` carries venue-provided sensitivities and implied volatility for a -single option contract: - -| Field | Type | Description | -|--------------------|------------------|-------------------------------------------------------| -| `instrument_id` | `InstrumentId` | The option contract these Greeks apply to. | -| `delta` | `float` | Rate of change of option price per unit underlying. | -| `gamma` | `float` | Rate of change of delta per unit underlying. | -| `vega` | `float` | Sensitivity to a 1% change in implied volatility. | -| `theta` | `float` | Daily time decay (dV/dt / 365.25). | -| `rho` | `float` | Sensitivity to a change in interest rate. | -| `mark_iv` | `float` or None | Mark implied volatility. | -| `bid_iv` | `float` or None | Bid implied volatility. | -| `ask_iv` | `float` or None | Ask implied volatility. | -| `underlying_price` | `float` or None | Underlying price at time of calculation. | -| `open_interest` | `float` or None | Open interest for the contract. | -| `ts_event` | `int` | UNIX timestamp (nanoseconds) of the event. | -| `ts_init` | `int` | UNIX timestamp (nanoseconds) when initialized. | - -## OptionChainSlice data type - -`OptionChainSlice` is a point-in-time snapshot of an entire option series. - -Properties: - -| Property | Type | Description | -|--------------|----------------------|------------------------------------------| -| `series_id` | `OptionSeriesId` | The option series identifier. | -| `atm_strike` | `Price` or None | Current ATM strike (if determined). | -| `ts_event` | `int` | UNIX timestamp (nanoseconds). | -| `ts_init` | `int` | UNIX timestamp (nanoseconds). | - -Call and put data are accessed through methods, not as direct properties. -Each `OptionStrikeData` returned by these methods contains a `quote` (`QuoteTick`) -and an optional `greeks` (`OptionGreeks`) for that strike. - -Methods: - -- `strikes()` -- all unique strike prices in the chain. -- `strike_count()`, `call_count()`, `put_count()` -- counts. -- `get_call(strike)`, `get_put(strike)` -- full `OptionStrikeData`. -- `get_call_greeks(strike)`, `get_put_greeks(strike)` -- Greeks only. -- `get_call_quote(strike)`, `get_put_quote(strike)` -- quote only. -- `is_empty()` -- true if the chain has no data. - -## Adapter support - -The following adapters currently support option Greeks subscriptions: - -| Adapter | Per-instrument Greeks | Option chains | -|---------|:---------------------:|:-------------:| -| Deribit | ✓ | ✓ | -| Bybit | ✓ | ✓ | - -## See also - -- [Greeks](greeks.md) -- local Greeks calculation and portfolio risk management. -- [Data](data.md) -- built-in data types and the subscription model. -- [Actors](actors.md) -- subscription and handler reference table. diff --git a/nautilus-docs/docs/concepts/order_book.md b/nautilus-docs/docs/concepts/order_book.md deleted file mode 100644 index 8240996..0000000 --- a/nautilus-docs/docs/concepts/order_book.md +++ /dev/null @@ -1,246 +0,0 @@ -# Order Book - -NautilusTrader provides a high-performance order book implemented in Rust, capable of -maintaining full book state from L1 through L3 data. The `OrderBook` is the primary -component for tracking public market depth, while the `OwnOrderBook` tracks your own -orders separately, enabling filtered views that show true available liquidity. - -:::note -This guide documents the Rust API. These types are also available from Python via -PyO3 bindings (`nautilus_pyo3.OrderBook`, `nautilus_pyo3.OwnOrderBook`). The legacy -Cython `OrderBook` (`nautilus_trader.model.book.OrderBook`) returned by -`cache.order_book()` has a similar but not identical interface. Refer to the -API reference for differences. -::: - -## Book types - -`OrderBook` instances are maintained per instrument for both backtesting and live trading: - -- `L3_MBO`: **Market by order** data. Tracks every order at every price level, keyed by order ID. -- `L2_MBP`: **Market by price** data. Aggregates orders by price level (one entry per price). -- `L1_MBP`: **Top-of-book** data, also known as best bid and offer (BBO). Captures only the - best prices. - -:::note -Top-of-book data such as `QuoteTick`, `TradeTick` and `Bar` can also maintain `L1_MBP` books. -::: - -## Subscribing to book data - -Strategies and actors subscribe to order book updates through the following methods. -Subscriptions and handlers are part of the Python strategy/actor layer: - -```python -# L3/L2 incremental deltas -self.subscribe_order_book_deltas(instrument_id) - -# Aggregated depth snapshots (up to 10 levels) -self.subscribe_order_book_depth(instrument_id) - -# Full book snapshots at a timed interval -self.subscribe_order_book_at_interval(instrument_id, interval_ms=1000) -``` - -Each subscription type delivers data to the corresponding handler: - -```python -def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: - ... - -def on_order_book_depth(self, depth: OrderBookDepth10) -> None: - ... - -def on_order_book(self, order_book: OrderBook) -> None: - ... -``` - -## Accessing the book - -The `OrderBook` exposes top-of-book accessors: - -```rust -let best_bid: Option = book.best_bid_price(); -let best_ask: Option = book.best_ask_price(); -let spread: Option = book.spread(); -let midpoint: Option = book.midpoint(); -``` - -## Analysis methods - -The `OrderBook` supports market depth analysis and execution simulation: - -```rust -// Average fill price for a given quantity -let avg_px = book.get_avg_px_for_quantity(quantity, OrderSide::Buy); - -// Average price and quantity for a target exposure (notional) -let (price, qty, exposure) = - book.get_avg_px_qty_for_exposure(target_exposure, OrderSide::Buy); - -// Cumulative quantity available at or better than a price -let qty = book.get_quantity_for_price(price, OrderSide::Buy); - -// Quantity at a specific price level only -let qty = book.get_quantity_at_level(price, OrderSide::Buy, 2); - -// Simulate fills against the book -let fills: Vec<(Price, Quantity)> = book.simulate_fills(&order); - -// All crossed levels regardless of order quantity -let levels = book.get_all_crossed_levels(OrderSide::Buy, price, 2); -``` - -## Integrity checks - -The `book_check_integrity` function validates that the book state is consistent -with its type: - -- **L1_MBP**: No more than one level per side. -- **L2_MBP**: No more than one order per price level. -- **L3_MBO**: No structural constraints (any number of orders at any level). -- **All types**: Best bid must not exceed best ask (crossed book). Locked markets - (bid == ask) are considered valid. - -These checks run internally during delta application. The instrument ID of incoming -deltas is also validated against the book's instrument ID, returning -`BookIntegrityError::InstrumentMismatch` on mismatch. - -## Pretty printing - -Both `OrderBook` and `OwnOrderBook` provide a `pprint` method that renders the book -as a human-readable table: - -```rust -book.pprint(5, None); -book.pprint(5, Some(Decimal::new(1, 2))); // group_size = 0.01 -``` - -The `group_size` parameter buckets price levels into coarser groups for instruments -with fine tick sizes. The output is a formatted table with bids on the left, prices -in the center, and asks on the right. - -## Own order book - -The `OwnOrderBook` tracks your own working orders separately from the public book. -Market making and other strategies use it to find the true available liquidity -at each price level (public size minus your own orders). - -The cache maintains own order books automatically as orders are submitted, accepted, -and filled. - -### Order lifecycle - -The `OwnOrderBook` tracks orders through their lifecycle. Orders are added when -submitted and updated as events arrive (accepted, partially filled, etc.). -Each `OwnBookOrder` carries: - -- `status`: Current order status (SUBMITTED, ACCEPTED, PARTIALLY_FILLED, etc.). -- `ts_accepted`: Timestamp when the order was accepted by the venue. -- `ts_submitted`: Timestamp when the order was submitted. - -These fields are used by the filtering logic to selectively include or exclude -orders from filtered views (see [Status and time filtering](#status-and-time-filtering)). - -### Auditing - -The `audit_open_orders` method reconciles the book against a set of known open -order IDs. Any orders in the book not in the provided set are removed and logged -as audit errors. The cache calls this periodically to keep the own book in sync -with the execution system. - -### Querying - -```rust -// Check if a specific order is tracked -let in_book = own_book.is_order_in_book(&client_order_id); - -// Get all tracked order IDs per side -let bid_ids = own_book.bid_client_order_ids(); -let ask_ids = own_book.ask_client_order_ids(); - -// Aggregated quantities per price level -let bid_qty = own_book.bid_quantity(None, None, None, None, None); -let ask_qty = own_book.ask_quantity(None, None, None, None, None); - -// Pretty print -own_book.pprint(5, None); -``` - -### Filtered views - -Subtract your own orders from the public book to see net available liquidity: - -```rust -// Filtered maps of price -> quantity (own orders subtracted) -let net_bids = book.bids_filtered_as_map(Some(10), Some(&own_book), None, None, None); -let net_asks = book.asks_filtered_as_map(Some(10), Some(&own_book), None, None, None); - -// Full filtered OrderBook with all analysis methods available -let filtered = book.filtered_view(Some(&own_book), Some(10), None, None, None); -let avg_px = filtered.get_avg_px_for_quantity(quantity, OrderSide::Buy); -``` - -The `filtered_view` method returns a new `OrderBook` with your own sizes subtracted, -giving access to the full set of analysis methods (`spread`, `midpoint`, -`get_avg_px_for_quantity`, etc.) on the net book. - -### Status and time filtering - -Filtered views support optional status and time-based filtering for own orders: - -```rust -let status = Some(AHashSet::from([OrderStatus::Accepted])); - -// Only subtract ACCEPTED orders (ignore SUBMITTED, PENDING_CANCEL, etc.) -let filtered = book.filtered_view(Some(&own_book), None, status, None, None); -``` - -The `accepted_buffer_ns` parameter provides a grace period: when set, only orders -where `ts_accepted + buffer <= now` are included. This excludes recently accepted -orders that may not yet appear in the public book feed. The buffer applies to the -`ts_accepted` field regardless of order status. Combine with a status filter to -also exclude non-accepted orders. - -```rust -// Only subtract orders accepted at least 500ms ago -let filtered = book.filtered_view( - Some(&own_book), - None, - None, - Some(500_000_000), - Some(clock.timestamp_ns()), -); -``` - -## Binary markets - -For binary/prediction markets (e.g., Polymarket), instruments have two complementary -sides (YES and NO) where prices sum to 1.0. A bid on the NO side at 0.40 is -economically equivalent to an ask on the YES side at 0.60. - -The `OwnOrderBook::combined_with_opposite` method handles this transformation, -merging your orders from both sides into a single view: - -```rust -let yes_own = own_yes_book - .cloned() - .unwrap_or_else(|| OwnOrderBook::new(yes_instrument_id)); - -let no_own = own_no_book - .cloned() - .unwrap_or_else(|| OwnOrderBook::new(no_instrument_id)); - -// Merge NO-side orders with parity price transform (1 - price) -let combined = yes_own.combined_with_opposite(&no_own).unwrap(); - -// Filter the public YES book using the combined own book -let filtered = book.filtered_view(Some(&combined), None, None, None, None); -``` - -The transformation works as follows: - -- NO asks at price P become bids at price 1 - P in the combined book. -- NO bids at price P become asks at price 1 - P in the combined book. - -This gives a complete picture of your own liquidity across both sides of the market. diff --git a/nautilus-docs/docs/concepts/orders.md b/nautilus-docs/docs/concepts/orders.md deleted file mode 100644 index aecdc1e..0000000 --- a/nautilus-docs/docs/concepts/orders.md +++ /dev/null @@ -1,838 +0,0 @@ -# Orders - -NautilusTrader supports a broad set of order types and execution instructions, exposing as much -of a trading venue's functionality as possible. Traders can define instructions and contingencies -for order execution and management across any trading strategy. - -## Overview - -All order types are derived from two fundamentals: *Market* and *Limit* orders. In terms of liquidity, they are opposites. -*Market* orders consume liquidity by executing immediately at the best available price, whereas *Limit* -orders provide liquidity by resting in the order book at a specified price until matched. - -The order types available for the platform are (using the `OrderType` enum values): - -- `MARKET` -- `LIMIT` -- `STOP_MARKET` -- `STOP_LIMIT` -- `MARKET_TO_LIMIT` -- `MARKET_IF_TOUCHED` -- `LIMIT_IF_TOUCHED` -- `TRAILING_STOP_MARKET` -- `TRAILING_STOP_LIMIT` - -:::info -NautilusTrader provides a unified API for many order types and execution instructions, but not all venues support every option. -If an order includes an instruction or option the target venue does not support, the system does not submit it. -Instead, it logs a clear, explanatory error. -::: - -### Terminology - -- An order is **aggressive** if its type is `MARKET` or if it executes as a *marketable* order (i.e., takes liquidity). -- An order is **passive** if it is not marketable (i.e., provides liquidity). -- An order is **active local** if it remains within the local system boundary in one of the following three non-terminal statuses: - - `INITIALIZED` - - `EMULATED` - - `RELEASED` -- An order is **in-flight** when at one of the following statuses: - - `SUBMITTED` - - `PENDING_UPDATE` - - `PENDING_CANCEL` -- An order is **open** when at one of the following (non-terminal) statuses: - - `ACCEPTED` - - `TRIGGERED` - - `PENDING_UPDATE` - - `PENDING_CANCEL` - - `PARTIALLY_FILLED` -- An order is **closed** when at one of the following (terminal) statuses: - - `DENIED` - - `REJECTED` - - `CANCELED` - - `EXPIRED` - - `FILLED` - -### Order state flow - -The following diagram illustrates the order lifecycle and primary state transitions: - -```mermaid -flowchart TB - subgraph local ["Active Local"] - Initialized - Emulated - Released - end - - subgraph flight ["In-Flight"] - Submitted - PendingUpdate - PendingCancel - end - - subgraph open ["Open (on venue)"] - Accepted - Triggered - PartiallyFilled - end - - subgraph closed ["Closed (terminal)"] - Denied - Rejected - Canceled - Expired - Filled - end - - Initialized -->|"Emulation trigger"| Emulated - Initialized -->|"Submit"| Submitted - Initialized -->|"System denied"| Denied - Emulated -->|"Triggered locally"| Released - Released --> Submitted - - Submitted -->|"Venue ACK"| Accepted - Submitted --> Rejected - - Accepted -->|"Stop hit"| Triggered - Accepted --> PartiallyFilled - Triggered --> PartiallyFilled - PartiallyFilled -->|"More fills"| PartiallyFilled - - Accepted --> PendingUpdate - Accepted --> PendingCancel - PartiallyFilled --> PendingUpdate - PartiallyFilled --> PendingCancel - PendingUpdate --> Accepted - PendingCancel --> Canceled - - Accepted --> Filled - Triggered --> Filled - PartiallyFilled --> Filled - PartiallyFilled --> Canceled - Accepted --> Expired -``` - -### Order status definitions - -| Status | Description | -|--------------------|-------------------------------------------------------------------------------------------| -| `INITIALIZED` | Order is instantiated within the Nautilus system. | -| `DENIED` | Order was denied by Nautilus for being invalid, unprocessable, or exceeding a risk limit. | -| `EMULATED` | Order is being emulated by the `OrderEmulator` component. | -| `RELEASED` | Order was released from the `OrderEmulator` component. | -| `SUBMITTED` | Order was submitted to the venue (awaiting acknowledgement). | -| `ACCEPTED` | Order was acknowledged by the venue as received and valid (may now be working). | -| `REJECTED` | Order was rejected by the trading venue. | -| `CANCELED` | Order was canceled (terminal). | -| `EXPIRED` | Order reached its GTD expiration (terminal). | -| `TRIGGERED` | Order's STOP price was triggered on the venue. | -| `PENDING_UPDATE` | Order is pending a modification request on the venue. | -| `PENDING_CANCEL` | Order is pending a cancellation request on the venue. | -| `PARTIALLY_FILLED` | Order has been partially filled on the venue. | -| `FILLED` | Order has been completely filled (terminal). | - -## Execution instructions - -Certain venues allow a trader to specify conditions and restrictions on -how an order will be processed and executed. The following is a brief -summary of the different execution instructions available. - -### Time in force - -The order's time in force specifies how long the order will remain open or active before any -remaining quantity is canceled. - -- `GTC` **(Good Till Cancel)**: The order remains active until canceled by the trader or the venue. -- `IOC` **(Immediate or Cancel / Fill and Kill)**: The order executes immediately, with any unfilled portion canceled. -- `FOK` **(Fill or Kill)**: The order executes immediately in full or not at all. -- `GTD` **(Good Till Date)**: The order remains active until a specified expiration date and time. -- `DAY` **(Good for session/day)**: The order remains active until the end of the current trading session. -- `AT_THE_OPEN` **(OPG)**: The order is only active at the open of the trading session. -- `AT_THE_CLOSE`: The order is only active at the close of the trading session. - -### Expire time - -This instruction is to be used in conjunction with the `GTD` time in force to specify the time -at which the order will expire and be removed from the venue's order book (or order management system). - -### Post-only - -An order which is marked as `post_only` will only ever participate in providing liquidity to the -limit order book, and never initiating a trade which takes liquidity as an aggressor. This option is -important for market makers, or traders seeking to restrict the order to a liquidity *maker* fee tier. - -### Reduce-only - -An order which is set as `reduce_only` will only ever reduce an existing position on an instrument and -never open a new position (if already flat). The exact behavior of this instruction can vary between venues. - -However, the behavior in the Nautilus `SimulatedExchange` is typical of a real venue. - -- Order will be canceled if the associated position is closed (becomes flat). -- Order quantity will be reduced as the associated position's size decreases. - -### Display quantity - -The `display_qty` specifies the portion of a *Limit* order which is displayed on the limit order book. -These are also known as iceberg orders as there is a visible portion to be displayed, with more quantity which is hidden. -Specifying a display quantity of zero is also equivalent to setting an order as `hidden`. - -### Trigger type - -Also known as [trigger method](https://www.interactivebrokers.com/en/software/tws/usersguidebook/configuretws/Modify%20the%20Stop%20Trigger%20Method.htm) -which is applicable to conditional trigger orders, specifying the method of triggering the stop price. - -- `DEFAULT`: The default trigger type for the venue (typically `LAST_PRICE` or `BID_ASK`). -- `LAST_PRICE`: The trigger price will be based on the last traded price. -- `BID_ASK`: The trigger price will be based on the bid for buy orders and ask for sell orders. -- `DOUBLE_LAST`: The trigger price will be based on the last two consecutive last prices. -- `DOUBLE_BID_ASK`: The trigger price will be based on the last two consecutive bid or ask prices as applicable. -- `LAST_OR_BID_ASK`: The trigger price will be based on either the last price or bid/ask. -- `MID_POINT`: The trigger price will be based on the mid-point between the bid and ask. -- `MARK_PRICE`: The trigger price will be based on the venue's mark price for the instrument. -- `INDEX_PRICE`: The trigger price will be based on the venue's index price for the instrument. - -### Trigger offset type - -Applicable to conditional trailing-stop trigger orders, specifies the method of triggering modification -of the stop price based on the offset from the *market* (bid, ask or last price as applicable). - -- `DEFAULT`: The default offset type for the venue (typically `PRICE`). -- `PRICE`: The offset is based on a price difference. -- `BASIS_POINTS`: The offset is based on a price percentage difference expressed in basis points (100bp = 1%). -- `TICKS`: The offset is based on a number of ticks. -- `PRICE_TIER`: The offset is based on a venue-specific price tier. - -### Contingent orders - -More advanced relationships can be specified between orders. -For example, child orders can be assigned to trigger only when the parent is activated or filled, or orders can be -linked so that one cancels or reduces the quantity of another. See the [Advanced Orders](#advanced-orders) section for more details. - -## Order factory - -The easiest way to create new orders is by using the built-in `OrderFactory`, which is -automatically attached to every `Strategy` class. This factory will take care -of lower level details - such as ensuring the correct trader ID and strategy ID are assigned, generation -of a necessary initialization ID and timestamp, and abstracts away parameters which don't necessarily -apply to the order type being created, or are only needed to specify more advanced execution instructions. - -This leaves the factory with simpler order creation methods to work with, all the -examples use an `OrderFactory` from within a `Strategy` context. - -See the [`OrderFactory` API Reference](/docs/python-api-latest/common.html#nautilus_trader.common.factories.OrderFactory) for further details. - -## Order types - -The following describes the order types which are available for the platform with a code example. -Any optional parameters will be clearly marked with a comment which includes the default value. - -### Market - -A *Market* order is an instruction by the trader to immediately trade -the given quantity at the best price available. You can also specify several -time in force options, and indicate whether this order is only intended to reduce -a position. - -In the following example we create a *Market* order on the Interactive Brokers [IdealPro](https://ibkr.info/node/1708) Forex ECN -to BUY 100,000 AUD using USD: - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import MarketOrder - -order: MarketOrder = self.order_factory.market( - instrument_id=InstrumentId.from_str("AUD/USD.IDEALPRO"), - order_side=OrderSide.BUY, - quantity=Quantity.from_int(100_000), - time_in_force=TimeInForce.IOC, # <-- optional (default GTC) - reduce_only=False, # <-- optional (default False) - tags=["ENTRY"], # <-- optional (default None) -) -``` - -See the [`MarketOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.market.MarketOrder) for further details. - -### Limit - -A *Limit* order is placed on the limit order book at a specific price, and will only -execute at that price (or better). - -In the following example we create a *Limit* order on the Binance Futures Crypto exchange to SELL 20 ETHUSDT-PERP Perpetual Futures -contracts at a limit price of 5000 USDT, as a market maker. - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import LimitOrder - -order: LimitOrder = self.order_factory.limit( - instrument_id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), - order_side=OrderSide.SELL, - quantity=Quantity.from_int(20), - price=Price.from_str("5_000.00"), - time_in_force=TimeInForce.GTC, # <-- optional (default GTC) - expire_time=None, # <-- optional (default None) - post_only=True, # <-- optional (default False) - reduce_only=False, # <-- optional (default False) - display_qty=None, # <-- optional (default None which indicates full display) - tags=None, # <-- optional (default None) -) -``` - -See the [`LimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.limit.LimitOrder) for further details. - -### Stop-Market - -A *Stop-Market* order is a conditional order which once triggered, will immediately -place a *Market* order. This order type is often used as a stop-loss to limit losses, either -as a SELL order against LONG positions, or as a BUY order against SHORT positions. - -In the following example we create a *Stop-Market* order on the Binance Spot/Margin exchange -to SELL 1 BTC at a trigger price of 100,000 USDT, active until further notice: - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import StopMarketOrder - -order: StopMarketOrder = self.order_factory.stop_market( - instrument_id=InstrumentId.from_str("BTCUSDT.BINANCE"), - order_side=OrderSide.SELL, - quantity=Quantity.from_int(1), - trigger_price=Price.from_int(100_000), - trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) - time_in_force=TimeInForce.GTC, # <-- optional (default GTC) - expire_time=None, # <-- optional (default None) - reduce_only=False, # <-- optional (default False) - tags=None, # <-- optional (default None) -) -``` - -See the [`StopMarketOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.stop_market.StopMarketOrder) for further details. - -### Stop-Limit - -A *Stop-Limit* order is a conditional order which once triggered will immediately place -a *Limit* order at the specified price. - -In the following example we create a *Stop-Limit* order on the Currenex FX ECN to BUY 50,000 GBP at a limit price of 1.3000 USD -once the market hits the trigger price of 1.30010 USD, active until midday 6th June, 2022 (UTC): - -```python -import pandas as pd -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import StopLimitOrder - -order: StopLimitOrder = self.order_factory.stop_limit( - instrument_id=InstrumentId.from_str("GBP/USD.CURRENEX"), - order_side=OrderSide.BUY, - quantity=Quantity.from_int(50_000), - price=Price.from_str("1.30000"), - trigger_price=Price.from_str("1.30010"), - trigger_type=TriggerType.BID_ASK, # <-- optional (default DEFAULT) - time_in_force=TimeInForce.GTD, # <-- optional (default GTC) - expire_time=pd.Timestamp("2022-06-06T12:00"), - post_only=True, # <-- optional (default False) - reduce_only=False, # <-- optional (default False) - tags=None, # <-- optional (default None) -) -``` - -See the [`StopLimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.stop_limit.StopLimitOrder) for further details. - -### Market-To-Limit - -A *Market-To-Limit* order submits as a market order at the current best price. -If the order partially fills, the system cancels the remainder and resubmits it as a *Limit* order at the executed price. - -In the following example we create a *Market-To-Limit* order on the Interactive Brokers [IdealPro](https://ibkr.info/node/1708) Forex ECN -to BUY 200,000 USD using JPY: - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import MarketToLimitOrder - -order: MarketToLimitOrder = self.order_factory.market_to_limit( - instrument_id=InstrumentId.from_str("USD/JPY.IDEALPRO"), - order_side=OrderSide.BUY, - quantity=Quantity.from_int(200_000), - time_in_force=TimeInForce.GTC, # <-- optional (default GTC) - reduce_only=False, # <-- optional (default False) - display_qty=None, # <-- optional (default None which indicates full display) - tags=None, # <-- optional (default None) -) -``` - -See the [`MarketToLimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.market_to_limit.MarketToLimitOrder) for further details. - -### Market-If-Touched - -A *Market-If-Touched* order is a conditional order which once triggered will immediately -place a *Market* order. This order type is often used to enter a new position on a stop price, -or to take profits for an existing position, either as a SELL order against LONG positions, -or as a BUY order against SHORT positions. - -In the following example we create a *Market-If-Touched* order on the Binance Futures exchange -to SELL 10 ETHUSDT-PERP Perpetual Futures contracts at a trigger price of 10,000 USDT, active until further notice: - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import MarketIfTouchedOrder - -order: MarketIfTouchedOrder = self.order_factory.market_if_touched( - instrument_id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), - order_side=OrderSide.SELL, - quantity=Quantity.from_int(10), - trigger_price=Price.from_str("10_000.00"), - trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) - time_in_force=TimeInForce.GTC, # <-- optional (default GTC) - expire_time=None, # <-- optional (default None) - reduce_only=False, # <-- optional (default False) - tags=["ENTRY"], # <-- optional (default None) -) -``` - -See the [`MarketIfTouchedOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.market_if_touched.MarketIfTouchedOrder) for further details. - -### Limit-If-Touched - -A *Limit-If-Touched* order is a conditional order which once triggered will immediately place -a *Limit* order at the specified price. - -In the following example we create a *Limit-If-Touched* order to BUY 5 BTCUSDT-PERP Perpetual Futures contracts on the -Binance Futures exchange at a limit price of 30,100 USDT (once the market hits the trigger price of 30,150 USDT), -active until midday 6th June, 2022 (UTC): - -```python -import pandas as pd -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import LimitIfTouchedOrder - -order: LimitIfTouchedOrder = self.order_factory.limit_if_touched( - instrument_id=InstrumentId.from_str("BTCUSDT-PERP.BINANCE"), - order_side=OrderSide.BUY, - quantity=Quantity.from_int(5), - price=Price.from_str("30_100"), - trigger_price=Price.from_str("30_150"), - trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) - time_in_force=TimeInForce.GTD, # <-- optional (default GTC) - expire_time=pd.Timestamp("2022-06-06T12:00"), - post_only=True, # <-- optional (default False) - reduce_only=False, # <-- optional (default False) - tags=["TAKE_PROFIT"], # <-- optional (default None) -) -``` - -See the [`LimitIfTouchedOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.limit_if_touched.LimitIfTouchedOrder) for further details. - -### Trailing-Stop-Market - -A *Trailing-Stop-Market* order is a conditional order which trails a stop trigger price -a fixed offset away from the defined market price. Once triggered a *Market* order will -immediately be placed. - -In the following example we create a *Trailing-Stop-Market* order on the Binance Futures exchange to SELL 10 ETHUSD-PERP COIN_M margined -Perpetual Futures Contracts activating at a price of 5,000 USD, then trailing at an offset of 1% (in basis points) away from the current last traded price: - -```python -import pandas as pd -from decimal import Decimal -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model.enums import TrailingOffsetType -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import TrailingStopMarketOrder - -order: TrailingStopMarketOrder = self.order_factory.trailing_stop_market( - instrument_id=InstrumentId.from_str("ETHUSD-PERP.BINANCE"), - order_side=OrderSide.SELL, - quantity=Quantity.from_int(10), - activation_price=Price.from_str("5_000"), - trigger_type=TriggerType.LAST_PRICE, # <-- optional (default DEFAULT) - trailing_offset=Decimal(100), - trailing_offset_type=TrailingOffsetType.BASIS_POINTS, - time_in_force=TimeInForce.GTC, # <-- optional (default GTC) - expire_time=None, # <-- optional (default None) - reduce_only=True, # <-- optional (default False) - tags=["TRAILING_STOP-1"], # <-- optional (default None) -) -``` - -See the [`TrailingStopMarketOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.trailing_stop_market.TrailingStopMarketOrder) for further details. - -### Trailing-Stop-Limit - -A *Trailing-Stop-Limit* order is a conditional order which trails a stop trigger price -a fixed offset away from the defined market price. Once triggered a *Limit* order will -immediately be placed at the defined price (which is also updated as the market moves until triggered). - -In the following example we create a *Trailing-Stop-Limit* order on the Currenex FX ECN to BUY 1,250,000 AUD using USD -at a limit price of 0.71000 USD, activating at 0.72000 USD then trailing at a stop offset of 0.00100 USD -away from the current ask price, active until further notice: - -```python -import pandas as pd -from decimal import Decimal -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model.enums import TrailingOffsetType -from nautilus_trader.model import InstrumentId -from nautilus_trader.model import Price -from nautilus_trader.model import Quantity -from nautilus_trader.model.orders import TrailingStopLimitOrder - -order: TrailingStopLimitOrder = self.order_factory.trailing_stop_limit( - instrument_id=InstrumentId.from_str("AUD/USD.CURRENEX"), - order_side=OrderSide.BUY, - quantity=Quantity.from_int(1_250_000), - price=Price.from_str("0.71000"), - activation_price=Price.from_str("0.72000"), - trigger_type=TriggerType.BID_ASK, # <-- optional (default DEFAULT) - limit_offset=Decimal("0.00050"), - trailing_offset=Decimal("0.00100"), - trailing_offset_type=TrailingOffsetType.PRICE, - time_in_force=TimeInForce.GTC, # <-- optional (default GTC) - expire_time=None, # <-- optional (default None) - reduce_only=True, # <-- optional (default False) - tags=["TRAILING_STOP"], # <-- optional (default None) -) -``` - -See the [`TrailingStopLimitOrder` API Reference](/docs/python-api-latest/model/orders.html#nautilus_trader.model.orders.trailing_stop_limit.TrailingStopLimitOrder) for further details. - -## Advanced orders - -The following guide should be read in conjunction with the specific documentation from the broker or venue -involving these order types, lists/groups and execution instructions (such as for Interactive Brokers). - -### Order lists - -Combinations of contingent orders, or larger order bulks can be grouped together into a list with a common -`order_list_id`. The orders contained in this list may or may not have a contingent relationship with -each other, as this is specific to how the orders themselves are constructed, and the -specific venue they are being routed to. - -### Contingency types - -- **OTO (One-Triggers-Other)** – a parent order that, once executed, automatically places one or more child orders. - - *Full-trigger model*: child order(s) are released **only after the parent is completely filled**. Common at most retail equity/option brokers (e.g. Schwab, Fidelity, TD Ameritrade) and many spot-crypto venues (Binance, Coinbase). - - *Partial-trigger model*: child order(s) are released **pro-rata to each partial fill**. Used by professional-grade platforms such as Interactive Brokers, most futures/FX OMSs, and Kraken Pro. - -- **OCO (One-Cancels-Other)** – two (or more) linked live orders where executing one cancels the remainder. - -- **OUO (One-Updates-Other)** – two (or more) linked live orders where executing one reduces the open quantity of the remainder. - -:::info -These contingency types relate to ContingencyType FIX tag <1385> . -::: - -#### One-Triggers-Other (OTO) - -An OTO order involves two parts: - -1. **Parent order** – submitted to the matching engine immediately. -2. **Child order(s)** – held *off-book* until the trigger condition is met. - -##### Trigger models - -| Trigger model | When are child orders released? | -|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| **Full trigger** | When the parent order’s cumulative quantity equals its original quantity (i.e., it is *fully* filled). | -| **Partial trigger** | Immediately upon each partial execution of the parent; the child’s quantity matches the executed amount and is increased as further fills occur. | - -:::info -The default backtest venue for NautilusTrader uses a *partial-trigger model* for OTO orders. -To opt-in to a *full-trigger mode*, set `oto_trigger_mode="FULL"` for the venue (e.g. via `BacktestVenueConfig`). -::: - -**Working with partial-trigger in production:** - -If your strategy requires full-trigger semantics but the venue or backtest engine uses partial-trigger: - -1. Submit the parent order without contingent children. -2. Subscribe to `OrderFilled` events for the parent order. -3. Only submit child orders (stop-loss, take-profit) after confirming the parent is fully filled. -4. Use `order.is_closed` and `order.filled_qty == order.quantity` to verify complete fill. - -> **Why the distinction matters** -> *Full trigger* leaves a risk window: any partially filled position is live without its protective exit until the remaining quantity fills. -> *Partial trigger* mitigates that risk by ensuring every executed lot instantly has its linked stop/limit, at the cost of creating more order traffic and updates. - -An OTO order can use any supported asset type on the venue (e.g. stock entry with option hedge, futures entry with OCO bracket, crypto spot entry with TP/SL). - -| Venue / Adapter ID | Asset classes | Trigger rule for child | Practical notes | -|----------------------------------------------|---------------------------|---------------------------------------------|-------------------------------------------------------------------| -| Binance / Binance Futures (`BINANCE`) | Spot, perpetual futures | **Partial or full** – fires on first fill. | OTOCO/TP-SL children appear instantly; monitor margin usage. | -| Bybit Spot (`BYBIT`) | Spot | **Full** – child placed after completion. | TP-SL preset activates only once the limit order is fully filled. | -| Bybit Perps (`BYBIT`) | Perpetual futures | **Partial and full** – configurable. | “Partial-position” mode sizes TP-SL as fills arrive. | -| Kraken Futures (`KRAKEN`) | Futures & perps | **Partial and full** – automatic. | Child quantity matches every partial execution. | -| OKX (`OKX`) | Spot, futures, options | **Full** – attached stop waits for fill. | Position-level TP-SL can be added separately. | -| Interactive Brokers (`INTERACTIVE_BROKERS`) | Stocks, options, FX, fut | **Configurable** – OCA can pro-rate. | `OcaType 2/3` reduces remaining child quantities. | -| dYdX v4 (`DYDX`) | Perpetual futures (DEX) | On-chain condition (size exact). | TP-SL triggers by oracle price; partial fill not applicable. | -| Polymarket (`POLYMARKET`) | Prediction market (DEX) | N/A. | Advanced contingency handled entirely at the strategy layer. | -| Betfair (`BETFAIR`) | Sports betting | N/A. | Advanced contingency handled entirely at the strategy layer. | - -#### One-Cancels-Other (OCO) - -An OCO order is a set of linked orders where the execution of **any** order (full *or partial*) triggers a best-efforts cancellation of the others. -Both orders are live simultaneously; once one starts filling, the venue attempts to cancel the unexecuted portion of the remainder. - -#### One-Updates-Other (OUO) - -An OUO order is a set of linked orders where execution of one order causes an immediate *reduction* of open quantity in the other order(s). -Both orders are live concurrently, and each partial execution proportionally updates the remaining quantity of its peer order on a best-effort basis. - -### Contingent order validation - -When working with contingent orders (OTO, OCO, OUO), be aware of the following validation rules and error scenarios: - -**Order list requirements:** - -- All orders in a contingent group must share the same `order_list_id`. -- Parent orders must be submitted before or simultaneously with their children. -- Child orders reference their parent via `parent_order_id`. - -**Modification rules:** - -- Parent orders can typically be modified while pending, but modifications may cascade to children. -- Child orders can be modified independently on most venues, but check venue-specific behavior. -- Canceling a parent order will cancel all associated child orders. - -**Common error scenarios:** - -| Scenario | System behavior | -|----------|-----------------| -| Child references non-existent parent | Order denied with `INVALID_ORDER` error | -| Parent canceled before children trigger | Children automatically canceled | -| OCO sibling filled before cancel propagates | Partial fill honored, remaining quantity canceled | -| Insufficient margin for bracket | Entry may execute, children rejected separately | - -:::warning -Always handle `OrderDenied` and `OrderRejected` events in your strategy, especially for contingent orders where -partial failures can leave positions unprotected. -::: - -### Bracket orders - -Bracket orders are an advanced order type that allows traders to set both take-profit and stop-loss -levels for a position simultaneously. This involves placing a parent order (entry order) and two child -orders: a take-profit `LIMIT` order and a stop-loss `STOP_MARKET` order. When the parent order executes, -the system places the child orders. The take-profit closes the position if the market moves favorably, and the stop-loss limits losses if it moves unfavorably. - -Bracket orders can be easily created using the [OrderFactory](/docs/python-api-latest/common.html#nautilus_trader.common.factories.OrderFactory), -which supports various order types, parameters, and instructions. - -:::warning -You should be aware of the margin requirements of positions, as bracketing a position will consume -more order margin. -::: - -## Emulated orders - -### Introduction - -Emulation lets you use order types even when your trading venue does not natively support them. - -Nautilus locally mimics the behavior of these order types (such as `STOP_LIMIT` or `TRAILING_STOP` orders) -while using only `MARKET` and `LIMIT` orders for actual execution on the venue. - -When you create an emulated order, Nautilus continuously tracks a specific type of market price (specified by the -`emulation_trigger` parameter) and based on the order type and conditions you've set, will automatically submit -the appropriate fundamental order (`MARKET` / `LIMIT`) when the triggering condition is met. - -For example, if you create an emulated `STOP_LIMIT` order, Nautilus will monitor the market price until your `stop` -price is reached, and then automatically submits a `LIMIT` order to the venue. - -To perform emulation, Nautilus needs to know which **type of market price** it should monitor. -By default, it uses bid and ask prices (quotes), which is why you'll often see `emulation_trigger=TriggerType.DEFAULT` -in examples (this is equivalent to using `TriggerType.BID_ASK`). However, Nautilus supports various other price types, -that can guide the emulation process. - -### Submitting an order for emulation - -The only requirement to emulate an order is to pass a `TriggerType` to the `emulation_trigger` -parameter of an `Order` constructor, or `OrderFactory` creation method. The following -emulation trigger types are currently supported: - -- `NO_TRIGGER`: disables local emulation completely and order is fully submitted to the venue. -- `DEFAULT`: which is the same as `BID_ASK`. -- `BID_ASK`: emulated using quotes to trigger. -- `LAST_PRICE`: emulated using trades to trigger. - -The choice of trigger type determines how the order emulation will behave: - -- For `STOP` orders, the trigger price will be compared against the specified trigger type. -- For `TRAILING_STOP` orders, the trailing offset will be updated based on the specified trigger type. -- For `LIMIT` orders being emulated, the limit price will be compared against the specified trigger type to determine when to release the order as a `MARKET` order. - -Here are all the available values you can set into `emulation_trigger` parameter and their purposes: - -| Trigger Type | Description | Common use cases | -|:------------------|:-----------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------| -| `NO_TRIGGER` | Disables emulation completely. The order is sent directly to the venue without any local processing. | When you want to use the venue's native order handling, or for simple order types that don't need emulation. | -| `DEFAULT` | Same as `BID_ASK`. This is the standard choice for most emulated orders. | General-purpose emulation when you want to work with the "default" type of market prices. | -| `BID_ASK` | Uses the best bid and ask prices (quotes) to guide emulation. | Stop orders, trailing stops, and other orders that should react to the current market spread. | -| `LAST_PRICE` | Uses the price of the most recent trade to guide emulation. | Orders that should trigger based on actual executed trades rather than quotes. | -| `DOUBLE_LAST` | Uses two consecutive last trade prices to confirm the trigger condition. | When you want additional confirmation of price movement before triggering. | -| `DOUBLE_BID_ASK` | Uses two consecutive bid/ask price updates to confirm the trigger condition. | When you want extra confirmation of quote movements before triggering. | -| `LAST_OR_BID_ASK` | Triggers on either last trade price or bid/ask prices. | When you want to be more responsive to any type of price movement. | -| `MID_POINT` | Uses the middle point between the best bid and ask prices. | Orders that should trigger based on the theoretical fair price. | -| `MARK_PRICE` | Uses the mark price (common in derivatives markets) for triggering. | Particularly useful for futures and perpetual contracts. | -| `INDEX_PRICE` | Uses an underlying index price for triggering. | When trading derivatives that track an index. | - -### Technical details - -The platform makes it possible to emulate most order types locally, regardless -of whether the type is supported on a trading venue. The logic and code paths for -order emulation are exactly the same for all [environment contexts](architecture.md#environment-contexts) -and use a common `OrderEmulator` component. - -:::note -There is no limitation on the number of emulated orders you can have per running instance. -::: - -### Lifecycle - -An emulated order will progress through the following stages: - -1. Submitted by a `Strategy` through the `submit_order` method. -2. Sent to the `RiskEngine` for pre-trade risk checks (it may be denied at this point). -3. Sent to the `OrderEmulator` where it is *held* / emulated. -4. Once triggered, emulated order is transformed into a `MARKET` or `LIMIT` order and released (submitted to the venue). -5. Released order undergoes final risk checks before venue submission. - -:::note -Emulated orders are subject to the same risk controls as *regular* orders, and can be -modified and canceled by a trading strategy in the normal way. They will also be included -when canceling all orders. -::: - -:::info -An emulated order will retain its original client order ID throughout its entire life cycle, making it easy to query -through the cache. -::: - -#### Held emulated orders - -The following will occur for an emulated order now *held* by the `OrderEmulator` component: - -- The original `SubmitOrder` command will be cached. -- The emulated order will be processed inside a local `MatchingCore` component. -- The `OrderEmulator` will subscribe to any needed market data (if not already) to update the matching core. -- The emulated order can be modified (by the trader) and updated (by the market) until *released* or canceled. - -#### Released emulated orders - -Once data arrival triggers / matches an emulated order locally, the following -*release* actions will occur: - -- The order will be transformed to either a `MARKET` or `LIMIT` order (see below table) through an additional `OrderInitialized` event. -- The orders `emulation_trigger` will be set to `NONE` (it will no longer be treated as an emulated order by any component). -- The order attached to the original `SubmitOrder` command will be sent back to the `RiskEngine` for additional checks since any modification/updates. -- If not denied, then the command will continue to the `ExecutionEngine` and on to the trading venue via an `ExecutionClient` as normal. - -### Order types which can be emulated - -The following table lists which order types are possible to emulate, and -which order type they transform to when being released for submission to the -trading venue. - -| Order type for emulation | Can emulate | Released type | -|:-------------------------|:------------|:--------------| -| `MARKET` | | n/a | -| `MARKET_TO_LIMIT` | | n/a | -| `LIMIT` | ✓ | `MARKET` | -| `STOP_MARKET` | ✓ | `MARKET` | -| `STOP_LIMIT` | ✓ | `LIMIT` | -| `MARKET_IF_TOUCHED` | ✓ | `MARKET` | -| `LIMIT_IF_TOUCHED` | ✓ | `LIMIT` | -| `TRAILING_STOP_MARKET` | ✓ | `MARKET` | -| `TRAILING_STOP_LIMIT` | ✓ | `LIMIT` | - -### Querying - -When writing trading strategies, it may be necessary to know the state of emulated orders in the system. -There are several ways to query emulation status: - -#### Through the Cache - -The following `Cache` methods are available: - -- `self.cache.orders_emulated(...)`: Returns all currently emulated orders. -- `self.cache.is_order_emulated(...)`: Checks if a specific order is emulated. -- `self.cache.orders_emulated_count(...)`: Returns the count of emulated orders. - -See the full [API reference](/docs/python-api-latest/cache.html) for additional details. - -#### Direct order queries - -You can query order objects directly using: - -- `order.is_emulated` - -If either of these return `False`, then the order has been *released* from the -`OrderEmulator`, and so is no longer considered an emulated order (or was never an emulated order). - -:::warning -Do not hold a local reference to an emulated order. The order object transforms -when the emulated order is *released*. Use the `Cache` instead. -::: - -### Persistence and recovery - -If a running system either crashes or shuts down with active emulated orders, then -they will be reloaded inside the `OrderEmulator` from any configured cache database. -This preserves order state across system restarts and recoveries. - -### Best practices - -When working with emulated orders, consider the following best practices: - -1. Always use the `Cache` for querying or tracking emulated orders rather than storing local references -2. Be aware that emulated orders transform to different types when released -3. Remember that emulated orders undergo risk checks both at submission and release - -:::note -Order emulation allows you to use advanced order types even on venues that don't natively support them, -making your trading strategies more portable across different venues. -::: - -## Related guides - -- [Execution](execution.md) - Order execution and fill handling. -- [Positions](positions.md) - Positions created from order fills. -- [Strategies](strategies.md) - Order management from strategies. diff --git a/nautilus-docs/docs/concepts/overview.md b/nautilus-docs/docs/concepts/overview.md deleted file mode 100644 index d60e92f..0000000 --- a/nautilus-docs/docs/concepts/overview.md +++ /dev/null @@ -1,255 +0,0 @@ -# Overview - -## Introduction - -NautilusTrader is an open-source, production-grade, Rust-native engine for multi-asset, -multi-venue trading systems. - -The system spans research, deterministic simulation, and live execution within a single -event-driven architecture, with Python serving as the control plane for strategy logic, -configuration, and orchestration. - -This separation provides the performance and safety of a compiled trading engine with -the flexibility of Python for system composition and strategy development. -Trading systems can also be written entirely in Rust for mission-critical workloads. - -The same execution semantics and deterministic time model operate in both research and -live systems. Strategies deploy from research to production with no code changes, -providing research-to-live parity and reducing the divergence that typically introduces -deployment risk. - -NautilusTrader is asset-class-agnostic. Any venue with a REST API or WebSocket feed can be -integrated through modular adapters. Current integrations span crypto exchanges (CEX and -DEX), traditional markets (FX, equities, futures, options), and betting exchanges. - -## Features - -- **Fast**: Rust core with asynchronous networking using [tokio](https://crates.io/crates/tokio). -- **Reliable**: Type- and thread-safety backed by Rust, with optional Redis-backed state persistence. -- **Portable**: Runs on Linux, macOS, and Windows. Deploy using Docker. -- **Flexible**: Modular adapters integrate any REST API or WebSocket feed. -- **Advanced**: Time in force `IOC`, `FOK`, `GTC`, `GTD`, `DAY`, `AT_THE_OPEN`, `AT_THE_CLOSE`, advanced order types and conditional triggers. Execution instructions `post-only`, `reduce-only`, and icebergs. Contingency orders including `OCO`, `OUO`, `OTO`. -- **Customizable**: User-defined components, or assemble entire systems from scratch using the [cache](cache.md) and [message bus](message_bus.md). -- **Backtesting**: Multiple venues, instruments, and strategies simultaneously using historical quote tick, trade tick, bar, order book, and custom data with nanosecond resolution. -- **Live**: Identical strategy implementations between research and live deployment. -- **Multi-venue**: Run market-making and cross-venue strategies across multiple venues simultaneously. -- **AI training**: Engine fast enough to train AI trading agents (RL/ES). - -## Why NautilusTrader? - -Trading strategy research typically happens in Python using vectorized approaches, while -production trading systems are built separately using event-driven architectures in -compiled languages. - -NautilusTrader removes this separation. - -A Rust-native core provides a deterministic event-driven runtime for both research and live -execution, while Python serves as the control plane. The same architecture, execution -semantics, and time model operate across both environments, allowing strategies to move -from research to production without reimplementation. - -Python bindings are provided via [PyO3](https://pyo3.rs), with an ongoing migration from -Cython. No Rust toolchain is required at install time. - -## Use cases - -There are three main use cases for this software package: - -- Backtest trading systems on historical data (`backtest`). -- Simulate trading systems with real-time data and virtual execution (`sandbox`). -- Deploy trading systems live on real or paper accounts (`live`). - -The codebase provides a framework for building the software layer of systems that achieve the above. -The default `backtest` and `live` system implementations live in their respectively named subpackages. -A `sandbox` environment can be built using the sandbox adapter. - -:::note - -- All examples will use these default system implementations. -- We consider trading strategies to be subcomponents of end-to-end trading systems, these systems -include the application and infrastructure layers. - -::: - -## Distributed - -The platform integrates into larger distributed systems. -Nearly all configuration and domain objects serialize using JSON, MessagePack, or Apache Arrow -(Feather) for communication over the network. - -## Common core - -The common system core is used by all node [environment contexts](architecture.md#environment-contexts) (`backtest`, `sandbox`, and `live`). -User-defined `Actor`, `Strategy` and `ExecAlgorithm` components are managed consistently across these environment contexts. - -## Backtesting - -Feed data to a `BacktestEngine` either directly or through a higher-level `BacktestNode` and -`ParquetDataCatalog`, then run the data through the system with nanosecond resolution. - -## Live trading - -A `TradingNode` ingests data and events from multiple data and execution clients, supporting both -demo/paper trading accounts and real accounts. Running asynchronously on a single -[event loop](https://docs.python.org/3/library/asyncio-eventloop.html) provides high performance, -with the option to use the [uvloop](https://github.com/MagicStack/uvloop) implementation -(available for Linux and macOS) for additional throughput. - -## Domain model - -The platform features a trading domain model that includes various value types such as -`Price` and `Quantity`, as well as more complex entities such as `Order` and `Position` objects, -which are used to aggregate multiple events to determine state. - -## Timestamps - -All timestamps use nanosecond precision in UTC. - -Timestamp strings follow ISO 8601 (RFC 3339) format with either 9 digits (nanoseconds) or 3 digits (milliseconds) of decimal precision, -(but mostly nanoseconds) always maintaining all digits including trailing zeros. -These can be seen in log messages, and debug/display outputs for objects. - -A timestamp string consists of: - -- Full date component always present: `YYYY-MM-DD`. -- `T` separator between date and time components. -- Always nanosecond precision (9 decimal places) or millisecond precision (3 decimal places) for certain cases such as GTD expiry times. -- Always UTC timezone designated by `Z` suffix. - -Example: `2024-01-05T15:30:45.123456789Z` - -For the complete specification, refer to [RFC 3339: Date and Time on the Internet](https://datatracker.ietf.org/doc/html/rfc3339). - -## UUIDs - -The platform uses Universally Unique Identifiers (UUID) version 4 (RFC 4122) for unique identifiers. -Our high-performance implementation uses the `uuid` crate for correctness validation when parsing from strings, -ensuring input UUIDs comply with the specification. - -A valid UUID v4 consists of: - -- 32 hexadecimal digits displayed in 5 groups. -- Groups separated by hyphens: `8-4-4-4-12` format. -- Version 4 designation (indicated by the third group starting with "4"). -- RFC 4122 variant designation (indicated by the fourth group starting with "8", "9", "a", or "b"). - -Example: `2d89666b-1a1e-4a75-b193-4eb3b454c757` - -For the complete specification, refer to [RFC 4122: A Universally Unique Identifier (UUID) URN Namespace](https://datatracker.ietf.org/doc/html/rfc4122). - -## Data types - -The following market data types can be requested historically, and also subscribed to as live streams when available from a venue / data provider, and implemented in an integrations adapter. - -- `OrderBookDelta` (L1/L2/L3) -- `OrderBookDeltas` (container type) -- `OrderBookDepth10` (fixed depth of 10 levels per side) -- `QuoteTick` -- `TradeTick` -- `Bar` -- `Instrument` -- `InstrumentStatus` -- `InstrumentClose` - -The following `PriceType` options can be used for bar aggregations: - -- `BID` -- `ASK` -- `MID` -- `LAST` - -## Bar aggregations - -The following `BarAggregation` methods are available: - -- `MILLISECOND` -- `SECOND` -- `MINUTE` -- `HOUR` -- `DAY` -- `WEEK` -- `MONTH` -- `YEAR` -- `TICK` -- `VOLUME` -- `VALUE` (a.k.a Dollar bars) -- `RENKO` (price-based bricks) -- `TICK_IMBALANCE` -- `TICK_RUNS` -- `VOLUME_IMBALANCE` -- `VOLUME_RUNS` -- `VALUE_IMBALANCE` -- `VALUE_RUNS` - -Currently implemented aggregations: - -- `MILLISECOND` -- `SECOND` -- `MINUTE` -- `HOUR` -- `DAY` -- `WEEK` -- `MONTH` -- `YEAR` -- `TICK` -- `VOLUME` -- `VALUE` -- `RENKO` - -Aggregations listed above that are not repeated in the implemented list are planned but not yet available. - -The price types and bar aggregations can be combined with step sizes >= 1 in any way through a `BarSpecification`. -This allows alternative bars to be aggregated for live trading. - -## Account types - -The following account types are available for both live and backtest environments: - -- `Cash` single-currency (base currency) -- `Cash` multi-currency -- `Margin` single-currency (base currency) -- `Margin` multi-currency -- `Betting` single-currency - -## Order types - -The following order types are available (when possible on a venue): - -- `MARKET` -- `LIMIT` -- `STOP_MARKET` -- `STOP_LIMIT` -- `MARKET_TO_LIMIT` -- `MARKET_IF_TOUCHED` -- `LIMIT_IF_TOUCHED` -- `TRAILING_STOP_MARKET` -- `TRAILING_STOP_LIMIT` - -## Value types - -The following value types are backed by either 128-bit or 64-bit raw integer values, depending on the -[precision mode](../getting_started/installation.md#precision-mode) used during compilation. - -- `Price` -- `Quantity` -- `Money` - -### High-precision mode (128-bit) - -When the `high-precision` feature flag is **enabled** (default), values use the specification: - -| Type | Raw backing | Max precision | Min value | Max value | -|:-------------|:------------|:--------------|:--------------------|:-------------------| -| `Price` | `i128` | 16 | -17,014,118,346,046 | 17,014,118,346,046 | -| `Money` | `i128` | 16 | -17,014,118,346,046 | 17,014,118,346,046 | -| `Quantity` | `u128` | 16 | 0 | 34,028,236,692,093 | - -### Standard-precision mode (64-bit) - -When the `high-precision` feature flag is **disabled**, values use the specification: - -| Type | Raw backing | Max precision | Min value | Max value | -|:-------------|:------------|:--------------|:--------------------|:-------------------| -| `Price` | `i64` | 9 | -9,223,372,036 | 9,223,372,036 | -| `Money` | `i64` | 9 | -9,223,372,036 | 9,223,372,036 | -| `Quantity` | `u64` | 9 | 0 | 18,446,744,073 | diff --git a/nautilus-docs/docs/concepts/portfolio.md b/nautilus-docs/docs/concepts/portfolio.md deleted file mode 100644 index 16177ff..0000000 --- a/nautilus-docs/docs/concepts/portfolio.md +++ /dev/null @@ -1,169 +0,0 @@ -# Portfolio - -The Portfolio is the central hub for managing and tracking all positions across active strategies for the trading node or backtest. -It consolidates position data from multiple instruments, providing a unified view of your holdings, risk exposure, and overall performance. - -## Currency conversion - -The Portfolio supports automatic currency conversion for PnL and exposure calculations, -allowing you to view results in your preferred currency. This is particularly useful when -trading across multiple instruments with different settlement currencies or managing multiple -accounts with different base currencies. - -### Supported conversions - -Currency conversion is available for the following portfolio queries: - -- `realized_pnl()` / `realized_pnls()` - Convert realized PnL to target currency. -- `unrealized_pnl()` / `unrealized_pnls()` - Convert unrealized PnL to target currency. -- `total_pnl()` / `total_pnls()` - Convert total PnL to target currency. -- `net_exposure()` / `net_exposures()` - Convert net exposure to target currency. - -All methods accept an optional `target_currency` parameter to specify the desired output -currency. - -### Single account behavior - -When querying a single account without specifying `target_currency`, the Portfolio -automatically converts values to that account's base currency: - -```python -# Returns exposure in the account's base currency (e.g., USD) -exposure = portfolio.net_exposures(venue=BINANCE, account_id=account_id) -``` - -### Multi-account behavior - -When querying multiple accounts simultaneously, behavior depends on whether you query -all instruments (`net_exposures()`) or a single instrument (`net_exposure()`): - -**For `net_exposures()` (all instruments):** - -- **Same base currency**: Automatically converts to the common base currency. -- **Different base currencies**: Returns a dict with multiple currencies, each converted - to its account's base currency. Provide `target_currency` for single-currency results. - -**For `net_exposure()` (single instrument across accounts):** - -- **Different base currencies**: Returns `None` unless you provide `target_currency`. - -```python -# Scenario 1: Multiple accounts, all with USD base currency -exposures = portfolio.net_exposures(venue=BINANCE) -# Returns {USD: Money(...)} - -# Scenario 2: Multiple accounts with different base currencies (USD and EUR) -exposures = portfolio.net_exposures(venue=BINANCE) -# Returns {USD: Money(...), EUR: Money(...)} - -# Force single currency across accounts -exposures = portfolio.net_exposures(venue=BINANCE, target_currency=USD) -# Returns {USD: Money(...)} -``` - -### Conversion failures - -When `target_currency` is provided and currency conversion fails, behavior depends on -the method type: - -- **Single-value methods** (`realized_pnl`, `unrealized_pnl`, `total_pnl`, `net_exposure`): - Return `None` and log an error to prevent incorrect values. -- **Dict-returning methods** (`realized_pnls`, `unrealized_pnls`, `total_pnls`, `net_exposures`): - Omit instruments that fail conversion but return results for successful conversions. - -:::warning -Exchange rate data must be available when using `target_currency` for cross-currency -aggregation. -::: - -### Conversion price types - -When converting exposures to a target currency, the Portfolio uses different price types -depending on the position composition: - -- **All long positions**: Uses `BID` prices (conservative for long exposure). -- **All short positions**: Uses `ASK` prices (conservative for short exposure). -- **Mixed positions**: Uses `MID` prices (neutral when both long and short exist). - -This ensures conversions reflect realistic market conditions where you would liquidate -long positions at bid and cover short positions at ask. For mixed positions, mid-pricing -provides a neutral valuation. - -If `use_mark_xrates` is enabled in the portfolio configuration, `MARK` prices replace -`MID` prices for mixed positions and general conversions. - -## Portfolio statistics - -There are a variety of [built-in portfolio statistics](https://github.com/nautechsystems/nautilus_trader/tree/develop/crates/analysis/src/statistics) -which analyse a trading portfolio's performance for both backtests and live trading. - -The statistics are generally categorized as follows. - -- PnLs based statistics (per currency) -- Returns based statistics -- Positions based statistics -- Orders based statistics - -You can also call a trader's `PortfolioAnalyzer` and calculate statistics at any arbitrary -time, including *during* a backtest, or live trading session. - -## Custom statistics - -Custom portfolio statistics can be defined by inheriting from the `PortfolioStatistic` -base class, and implementing any of the `calculate_` methods. - -For example, the following is the implementation for the built-in `WinRate` statistic: - -```python -import pandas as pd -from typing import Any -from nautilus_trader.analysis.statistic import PortfolioStatistic - - -class WinRate(PortfolioStatistic): - """ - Calculates the win rate from a realized PnLs series. - """ - - def calculate_from_realized_pnls(self, realized_pnls: pd.Series) -> Any | None: - # Preconditions - if realized_pnls is None or realized_pnls.empty: - return 0.0 - - # Calculate statistic - winners = [x for x in realized_pnls if x > 0.0] - losers = [x for x in realized_pnls if x <= 0.0] - - return len(winners) / float(max(1, (len(winners) + len(losers)))) -``` - -These statistics can then be registered with a traders `PortfolioAnalyzer`. - -```python -stat = WinRate() - -# Register with the portfolio analyzer -engine.portfolio.analyzer.register_statistic(stat) -``` - -See the [`PortfolioAnalyzer` API Reference](/docs/python-api-latest/analysis.html#nautilus_trader.analysis.analyzer.PortfolioAnalyzer) for all available methods. - -:::tip -Your statistic should handle degenerate inputs such as `None`, empty series, or insufficient data. -Return `None` for unknown/incalculable values, or a reasonable default like `0.0` when semantically appropriate (e.g., win rate with no trades). -::: - -## Backtest analysis - -Following a backtest run, the engine passes realized PnLs, returns, positions, and orders data to each registered -statistic. Any output is then displayed in the tear sheet under the `Portfolio Performance` heading, grouped as: - -- Realized PnL statistics (per currency) -- Returns statistics (for the entire portfolio) -- General statistics derived from position and order data (for the entire portfolio) - -## Related guides - -- [Positions](positions.md) - Position tracking within portfolios. -- [Reports](reports.md) - Generate portfolio analysis reports. -- [Visualization](visualization.md) - Visualize portfolio performance. diff --git a/nautilus-docs/docs/concepts/positions.md b/nautilus-docs/docs/concepts/positions.md deleted file mode 100644 index 61b92dd..0000000 --- a/nautilus-docs/docs/concepts/positions.md +++ /dev/null @@ -1,460 +0,0 @@ -# Positions - -This guide explains how positions work in NautilusTrader, including their lifecycle, aggregation -from order fills, profit and loss calculations, and the important concept of position snapshotting -for netting OMS configurations. - -## Overview - -A position represents an open exposure to a particular instrument in the market. Positions are -fundamental to tracking trading performance and risk, as they aggregate all fills for a particular -instrument and continuously calculate metrics like unrealized PnL, average entry price, and total -exposure. - -The system automatically creates positions when orders fill and tracks them -from open to close. The platform supports both netting -and hedging position management styles through its OMS (Order Management System) configuration. - -## Position lifecycle - -### Creation - -The system opens a position on the first fill: - -- **NETTING OMS**: Opens on first fill for an instrument (one position per instrument). -- **HEDGING OMS**: Opens on first fill for a new `position_id` (multiple positions per instrument). - -A position tracks: - -- Opening order and fill details. -- Entry side (`LONG` or `SHORT`). -- Initial quantity and average price. -- Timestamps for initialization and opening. - -:::tip -You can access positions through the Cache using `self.cache.position(position_id)` or -`self.cache.positions(instrument_id=instrument_id)` from within your actors/strategies. -::: - -### Updates - -As additional fills occur, the position: - -- Aggregates quantities from buy and sell fills. -- Recalculates average entry and exit prices. -- Updates peak quantity (maximum exposure reached). -- Tracks all associated order IDs and trade IDs. -- Accumulates commissions by currency. - -### Closure - -A position closes when the net quantity becomes zero (`FLAT`). At closure: - -- The closing order ID is recorded. -- Duration is calculated from open to close. -- Final realized PnL is computed. -- In `NETTING` OMS, when the position later reopens, the engine snapshots the closed state to preserve historical PnL (see [Position snapshotting](#position-snapshotting)). - -## Order fill aggregation - -Positions aggregate order fills to maintain an accurate view of market exposure. The aggregation -process handles both sides of trading activity: - -### Buy fills - -When a BUY order fills: - -- Increases long exposure or reduces short exposure. -- Updates average entry price for opening trades. -- Updates average exit price for closing trades. -- Calculates realized PnL for any closed portion. - -### Sell fills - -When a SELL order fills: - -- Increases short exposure or reduces long exposure. -- Updates average entry price for opening trades. -- Updates average exit price for closing trades. -- Calculates realized PnL for any closed portion. - -### Net position calculation - -The position maintains a `signed_qty` field representing the net exposure: - -- Positive values indicate `LONG` positions. -- Negative values indicate `SHORT` positions. -- Zero indicates a `FLAT` (closed) position. - -```python -# Example: Position aggregation -# Initial BUY 100 units at $50 -signed_qty = +100 # LONG position - -# Subsequent SELL 150 units at $55 -signed_qty = -50 # Now SHORT position - -# Final BUY 50 units at $52 -signed_qty = 0 # Position FLAT (closed) -``` - -## Position adjustments - -Position adjustments track quantity or PnL changes that occur outside of normal order fills, -ensuring the position quantity accurately reflects the true net asset position. The system -generates `PositionAdjusted` events for these scenarios. - -### Base currency commissions - -When trading spot currency pairs (e.g., BTC/USDT) or FX spot, commissions paid in the base -currency directly affect the net quantity received or delivered: - -- **Opening fills**: Commission is deducted from the traded quantity. A buy of 1.0 BTC with - 0.001 BTC commission results in a net long position of 0.999 BTC. -- **Closing fills**: Commission is applied to `signed_qty` because it affects actual inventory. - Selling a 0.999 BTC LONG position with 0.000999 BTC commission leaves you SHORT 0.000999 BTC, - not FLAT, because you gave up 0.999999 BTC total. -- **Flips**: Commission affects the final position size on both sides of the flip. - -:::note -Base currency commissions only apply to spot currency pairs and FX spot instruments where the -commission currency matches `instrument.base_currency`. For other instruments, commissions are -tracked separately and do not affect position quantity. -::: - -### Funding payments - -Funding adjustments track periodic payments for perpetual futures without affecting position -quantity. These are logged with `quantity_change = None` and can include PnL impacts. - -### Adjustment tracking - -All adjustments are preserved in the position event history: - -- `position.adjustments` returns the list of all `PositionAdjusted` events. -- Each adjustment includes type (`COMMISSION` or `FUNDING`), quantity change, and timestamps. -- The adjustment history is cleared when positions close and reopen. When events are purged, - commission adjustments tied to the removed fills are regenerated while non-commission adjustments - (for example funding) are preserved. - -## OMS types and position management - -NautilusTrader supports two primary OMS types that fundamentally affect how positions are tracked -and managed. An `OmsType.UNSPECIFIED` option also exists, which defaults to the component's -context. For full details, see the [Execution guide](execution.md#order-management-system-oms). - -### `NETTING` - -In `NETTING` mode, all fills for an instrument are aggregated into a single position: - -- One position per instrument ID. -- All fills contribute to the same position. -- Position flips from `LONG` to `SHORT` (or vice versa) as net quantity changes. -- Historical snapshots preserve closed position states. - -### `HEDGING` - -In `HEDGING` mode, multiple positions can exist for the same instrument: - -- Multiple simultaneous `LONG` and `SHORT` positions. -- Each position has a unique position ID. -- Positions are tracked independently. -- No automatic netting across positions. -- Closed positions remain in cache history but do not reopen; new fills create new positions. - -:::warning -When using `HEDGING` mode, be aware of increased margin requirements as each position -consumes margin independently. Some venues may not support true hedging mode and will -net positions automatically. -::: - -### Strategy vs venue OMS - -The platform allows different OMS configurations for strategies and venues: - -| Strategy OMS | Venue OMS | Behavior | -|--------------|-----------|-------------------------------------------------------------| -| `NETTING` | `NETTING` | Single position per instrument at both strategy and venue. | -| `HEDGING` | `HEDGING` | Multiple positions supported at both levels. | -| `NETTING` | `HEDGING` | Venue tracks multiple, Nautilus maintains single position. | -| `HEDGING` | `NETTING` | Venue tracks single, Nautilus maintains virtual positions. | - -:::tip -For most trading scenarios, keeping strategy and venue OMS types aligned simplifies -position management. Override configurations are primarily useful for prop trading -desks or when interfacing with legacy systems. See the [Live guide](live.md) -for venue-specific OMS configuration. -::: - -## Position snapshotting - -Position snapshotting is an important feature for `NETTING` OMS configurations that preserves -the state of closed positions for accurate PnL tracking and reporting. - -### Why snapshotting matters - -In a `NETTING` system, when a position closes (becomes `FLAT`) and then reopens with a new trade, -the position object is reset to track the new exposure. Without snapshotting, the historical -realized PnL from the previous position cycle would be lost. - -### How it works - -When a `NETTING` position closes and then receives a new fill for the same instrument, the execution -engine snapshots the closed position state before resetting it, preserving: - -- Final quantities and prices. -- Realized PnL. -- All fill events. -- Commission totals. - -This snapshot is stored in the cache indexed by position ID. The position then resets for the new -cycle while previous snapshots remain accessible. The Portfolio aggregates PnL across all snapshots -for accurate totals. - -:::note -This historical snapshot mechanism differs from optional position state snapshots (`snapshot_positions`), -which periodically record open-position state for telemetry. See the [Live guide](live.md) for -`snapshot_positions` and `snapshot_positions_interval_secs` settings. -::: - -### Example scenario - -```python -# NETTING OMS Example -# Cycle 1: Open LONG position -BUY 100 units at $50 # Position opens -SELL 100 units at $55 # Position closes, PnL = $500 -# Snapshot taken preserving $500 realized PnL - -# Cycle 2: Open SHORT position -SELL 50 units at $54 # Position reopens (SHORT) -BUY 50 units at $52 # Position closes, PnL = $100 -# Snapshot taken preserving $100 realized PnL - -# Total realized PnL = $500 + $100 = $600 (from snapshots) -``` - -Without snapshotting, only the most recent cycle's PnL would be available, leading to -incorrect reporting and analysis. - -## PnL calculations - -NautilusTrader provides PnL calculations that account for instrument -specifications and market conventions. - -### Realized PnL - -Calculated when positions are partially or fully closed: - -```python -# For standard instruments -realized_pnl = (exit_price - entry_price) * closed_quantity * multiplier - -# For inverse instruments (side-aware) -# LONG: realized_pnl = closed_quantity * multiplier * (1/entry_price - 1/exit_price) -# SHORT: realized_pnl = closed_quantity * multiplier * (1/exit_price - 1/entry_price) -``` - -The engine automatically applies the correct formula based on position side. - -### Unrealized PnL - -Calculated using current market prices for open positions. The `price` parameter accepts any -reference price (bid, ask, mid, last, or mark): - -```python -position.unrealized_pnl(last_price) # Using last traded price -position.unrealized_pnl(bid_price) # Conservative for LONG positions -position.unrealized_pnl(ask_price) # Conservative for SHORT positions -``` - -Returns `Money(0, settlement_currency)` for `FLAT` positions regardless of the price provided. - -### Total PnL - -Combines realized and unrealized components: - -```python -total_pnl = position.total_pnl(current_price) -# Returns realized_pnl + unrealized_pnl -``` - -### Currency considerations - -- PnL is calculated in the instrument's settlement currency. -- For Forex, this is typically the quote currency. -- For inverse contracts, PnL may be in the base currency. -- Portfolio aggregates realized PnL per instrument in settlement currency. -- Multi-currency totals require conversion outside the Position class. - -## Commissions and costs - -Positions track all trading costs: - -- Commissions are accumulated by currency. -- Each fill's commission is added to the running total. -- Multiple commission currencies are supported. -- Realized PnL includes commissions only when denominated in the settlement currency. -- Other commissions are tracked separately and may require conversion. - -```python -commissions = position.commissions() -# Returns list[Money] with aggregated commission totals per currency - -notional = position.notional_value(current_price) -# Returns Money in quote currency (standard) or base currency (inverse) -``` - -**Limitations:** - -- Panics if inverse instrument has no `base_currency` set. -- Does not handle quanto contracts (returns quote currency instead of settlement currency). -- For quanto instruments, use `instrument.calculate_notional_value()` instead. - -## Position properties and state - -### Identifiers - -- `id`: Unique position identifier. -- `instrument_id`: The traded instrument. -- `account_id`: Account where position is held. -- `trader_id`: The trader who owns the position. -- `strategy_id`: The strategy managing the position. -- `opening_order_id`: Client order ID that opened the position. -- `closing_order_id`: Client order ID that closed the position. - -### Position state - -- `side`: Current position side (`LONG`, `SHORT`, or `FLAT`). -- `entry`: Direction of the currently open position (`Buy` for `LONG`, `Sell` for `SHORT`). Updates when position flips direction. -- `quantity`: Current absolute position size. -- `signed_qty`: Signed position size (positive for `LONG`, negative for `SHORT`). -- `peak_qty`: Maximum quantity reached during position lifetime. -- `is_open`: Whether position is currently open. -- `is_closed`: Whether position is closed (`FLAT`). -- `is_long`: Whether position side is `LONG`. -- `is_short`: Whether position side is `SHORT`. - -### Pricing and valuation - -- `avg_px_open`: Average entry price. -- `avg_px_close`: Average exit price when closing. -- `realized_pnl`: Realized profit/loss. -- `realized_return`: Realized return as decimal (e.g., 0.05 for 5%). -- `quote_currency`: Quote currency of the instrument. -- `base_currency`: Base currency if applicable. -- `settlement_currency`: Currency for PnL settlement. - -### Instrument specifications - -- `multiplier`: Contract multiplier. -- `price_precision`: Decimal precision for prices. -- `size_precision`: Decimal precision for quantities. -- `is_inverse`: Whether instrument is inverse. - -### Timestamps - -- `ts_init`: When position was initialized. -- `ts_opened`: When position was opened. -- `ts_last`: Last update timestamp. -- `ts_closed`: When position was closed. -- `duration_ns`: Duration from open to close in nanoseconds. - -### Associated data - -- `symbol`: The instrument's ticker symbol. -- `venue`: The trading venue. -- `client_order_ids`: All client order IDs associated with position. -- `venue_order_ids`: All venue order IDs associated with position. -- `trade_ids`: All trade/fill IDs from venue. -- `events`: All order fill events applied to position. -- `event_count`: Total number of fill events applied. -- `last_event`: Most recent fill event. -- `last_trade_id`: Most recent trade ID. - -:::info -For complete type information and detailed property documentation, see the Position -[API Reference](/docs/python-api-latest/model/position.html#nautilus_trader.model.position.Position). -::: - -## Events and tracking - -Positions maintain a complete history of events: - -- All order fill events are stored chronologically. -- Associated client order IDs are tracked. -- Trade IDs from the venue are preserved. -- Event count indicates total fills applied. - -This historical data enables: - -- Detailed position analysis. -- Trade reconciliation. -- Performance attribution. -- Audit trails. - -:::tip -Use `position.events` to access the full history of fills for reconciliation. -The `position.trade_ids` property helps match against broker statements. -See the [Execution guide](execution.md) for reconciliation best practices. -::: - -## Numerical precision - -Position calculations use 64-bit floating-point (`f64`) arithmetic for PnL and average price computations. -While fixed-point types (`Price`, `Quantity`, `Money`) preserve exact precision at configured decimal places, -internal calculations convert to `f64` for performance and overflow safety. - -### Design rationale - -The platform uses `f64` for position calculations to balance performance and accuracy: - -- Floating-point operations are significantly faster than arbitrary-precision arithmetic. -- Raw integer multiplication can overflow even with 128-bit integers. -- Each calculation starts from precise fixed-point values, avoiding cumulative error. -- IEEE-754 double precision provides ~15 decimal digits of accuracy. - -### Validated precision characteristics - -Testing confirms `f64` arithmetic maintains accuracy for typical trading scenarios: - -- Standard amounts: No precision loss for amounts ≥ 0.01 in standard currencies. -- High-precision instruments: 9-decimal crypto prices preserved within 1e-6 tolerance. -- Sequential fills: 100 fills show no drift (commission accuracy to 1e-10). -- Extreme prices: Handles range from 0.00001 to 99,999.99999 without overflow. -- Round-trip trades: Opening and closing at same price produces exact PnL (commissions only). - -For implementation details, see `test_position_pnl_precision_*` tests in `crates/model/src/position.rs`. - -:::note -For regulatory compliance or audit trails requiring exact decimal arithmetic, consider using `Decimal` -types from external libraries. Very small amounts below `f64` epsilon (~1e-15) may round to zero. -This does not affect realistic trading scenarios with standard currency precisions (typically 2-9 decimals). -::: - -## Integration with other components - -Positions interact with several key components: - -- **Portfolio**: Aggregates positions across instruments and strategies. -- **ExecutionEngine**: Creates and updates positions from fills. -- **Cache**: Stores position state and snapshots. -- **RiskEngine**: Monitors position limits and exposure. - -:::note -Positions are not created for spread instruments. While contingent orders can still trigger for spreads, -they operate without position linkage. The engine handles spread instruments separately from regular positions. -::: - -## Summary - -Positions are central to tracking trading activity and performance. Understanding how positions -aggregate fills, calculate PnL, and handle different OMS configurations matters when building -trading strategies. Position snapshotting provides accurate historical tracking in `NETTING` -mode, and the event history supports detailed analysis and reconciliation. - -## Related guides - -- [Orders](orders.md) - Orders that create and modify positions. -- [Execution](execution.md) - Fill handling that updates positions. -- [Portfolio](portfolio.md) - Portfolio-level position aggregation. diff --git a/nautilus-docs/docs/concepts/reports.md b/nautilus-docs/docs/concepts/reports.md deleted file mode 100644 index 1f599ff..0000000 --- a/nautilus-docs/docs/concepts/reports.md +++ /dev/null @@ -1,435 +0,0 @@ -# Reports - -This guide explains the portfolio analysis and reporting capabilities provided by the `ReportProvider` -class, and how these reports are used for PnL accounting and backtest post-run analysis. - -## Overview - -The `ReportProvider` class in NautilusTrader generates structured analytical reports from -trading data, transforming raw orders, fills, positions, and account states into pandas DataFrames -for analysis and visualization. These reports help you evaluate strategy performance, -analyze execution quality, and verify PnL accounting. - -Reports can be generated using two approaches: - -- **Trader helper methods** (recommended): Convenient methods like `trader.generate_orders_report()`. -- **ReportProvider directly**: For more control over data selection and filtering. - -Reports provide consistent analytics across both backtesting and live trading environments, -enabling reliable performance evaluation and strategy comparison. - -## Available reports - -The `ReportProvider` class offers several static methods to generate reports from trading data. -Each report returns a pandas DataFrame with specific columns and indexing for easy analysis. - -### Orders report - -Generates a full view of all orders: - -```python -# Using Trader helper method (recommended) -orders_report = trader.generate_orders_report() - -# Or using ReportProvider directly -from nautilus_trader.analysis import ReportProvider - -orders = cache.orders() -orders_report = ReportProvider.generate_orders_report(orders) -``` - -**Returns `pd.DataFrame`. Key columns include:** - -| Column | Description | -|--------------------|---------------------------------------------------------| -| `client_order_id` | Index - unique order identifier. | -| `instrument_id` | Trading instrument. | -| `strategy_id` | Strategy that created the order. | -| `trader_id` | Trader identifier. | -| `account_id` | Account identifier (if assigned). | -| `venue_order_id` | Venue-assigned order ID (if accepted). | -| `side` | BUY or SELL. | -| `type` | MARKET, LIMIT, etc. | -| `status` | Current order status. | -| `quantity` | Original order quantity (string). | -| `filled_qty` | Amount filled (string). | -| `price` | Limit price (order-type dependent). | -| `avg_px` | Average fill price (if filled). | -| `time_in_force` | Time-in-force instruction. | -| `ts_init` | Order initialization timestamp (Unix nanoseconds). | -| `ts_last` | Last update timestamp (Unix nanoseconds). | - -Additional columns vary by order type (e.g., `trigger_price` for stop orders, `expire_time` for -GTD orders). See `Order.to_dict()` for the complete field list. - -### Order fills report - -Provides a summary of filled orders (one row per order): - -```python -# Using Trader helper method (recommended) -fills_report = trader.generate_order_fills_report() - -# Or using ReportProvider directly -orders = cache.orders() -fills_report = ReportProvider.generate_order_fills_report(orders) -``` - -This report includes only orders with `filled_qty > 0` and contains the same columns as the -orders report, but filtered to executed orders only. Note that `ts_init` and `ts_last` are -converted to datetime objects in this report for easier analysis. - -### Fills report - -Details individual fill events (one row per fill): - -```python -# Using Trader helper method (recommended) -fills_report = trader.generate_fills_report() - -# Or using ReportProvider directly -orders = cache.orders() -fills_report = ReportProvider.generate_fills_report(orders) -``` - -**Returns `pd.DataFrame`. Key columns include:** - -| Column | Description | -|--------------------|------------------------------------------| -| `client_order_id` | Index - order identifier. | -| `trade_id` | Unique trade/fill identifier. | -| `venue_order_id` | Venue-assigned order ID. | -| `instrument_id` | Trading instrument. | -| `strategy_id` | Strategy that created the order. | -| `account_id` | Account identifier. | -| `position_id` | Associated position ID (if applicable). | -| `order_side` | BUY or SELL. | -| `order_type` | Order type (MARKET, LIMIT, etc.). | -| `last_px` | Fill execution price (string). | -| `last_qty` | Fill execution quantity (string). | -| `currency` | Currency of the fill. | -| `liquidity_side` | MAKER or TAKER. | -| `commission` | Commission amount and currency. | -| `ts_event` | Fill timestamp (datetime). | -| `ts_init` | Initialization timestamp (datetime). | - -See `OrderFilled.to_dict()` for the complete field list. - -### Positions report - -Position analysis including snapshots: - -```python -# Using Trader helper method (recommended) -# Automatically includes snapshots for NETTING OMS -positions_report = trader.generate_positions_report() - -# Or using ReportProvider directly -positions = cache.positions() -snapshots = cache.position_snapshots() # For NETTING OMS -positions_report = ReportProvider.generate_positions_report( - positions=positions, - snapshots=snapshots -) -``` - -**Returns `pd.DataFrame`. Key columns include:** - -| Column | Description | -|--------------------|------------------------------------------| -| `position_id` | Index - unique position identifier. | -| `instrument_id` | Trading instrument. | -| `strategy_id` | Strategy that managed the position. | -| `trader_id` | Trader identifier. | -| `account_id` | Account identifier. | -| `opening_order_id` | Order ID that opened the position. | -| `closing_order_id` | Order ID that closed the position. | -| `entry` | Entry side (BUY or SELL). | -| `side` | Position side (LONG, SHORT, or FLAT). | -| `quantity` | Current position size. | -| `peak_qty` | Maximum size reached. | -| `avg_px_open` | Average entry price. | -| `avg_px_close` | Average exit price (if closed). | -| `commissions` | List of commissions paid. | -| `realized_pnl` | Realized profit/loss. | -| `realized_return` | Return percentage. | -| `ts_init` | Position initialization timestamp. | -| `ts_opened` | Opening timestamp (datetime). | -| `ts_last` | Last update timestamp. | -| `ts_closed` | Closing timestamp (datetime or NA). | -| `duration_ns` | Position duration in nanoseconds. | -| `is_snapshot` | Whether this is a historical snapshot. | - -### Account report - -Tracks account balance and margin changes over time: - -```python -# Using Trader helper method (recommended) -# Requires venue parameter -from nautilus_trader.model.identifiers import Venue -venue = Venue("BINANCE") -account_report = trader.generate_account_report(venue) - -# Or using ReportProvider directly -account = cache.account(account_id) -account_report = ReportProvider.generate_account_report(account) -``` - -**Returns `pd.DataFrame`. Columns include:** - -| Column | Description | -|-----------------|--------------------------------------------| -| `ts_event` | Index - timestamp of account state change. | -| `account_id` | Account identifier. | -| `account_type` | Type of account (e.g., SPOT, MARGIN). | -| `base_currency` | Base currency for the account. | -| `total` | Total balance amount (string). | -| `free` | Available balance (string). | -| `locked` | Balance locked in orders (string). | -| `currency` | Currency of the balance. | -| `reported` | Whether balance was reported by venue. | -| `margins` | Margin information (list, if applicable). | -| `info` | Additional venue-specific information. | - -Each row represents a balance entry; accounts with multiple currencies produce multiple rows -per account state event. - -## PnL accounting considerations - -Accurate PnL accounting requires careful consideration of several factors: - -### Position-based PnL - -- **Realized PnL**: Calculated when positions are partially or fully closed. -- **Unrealized PnL**: Marked-to-market using current prices. -- **Commission impact**: Only included when in settlement currency. - -:::warning -PnL calculations depend on the OMS type. In `NETTING` OMS, position snapshots -preserve historical PnL when positions reopen. Always include snapshots in -reports for accurate total PnL calculation. In `HEDGING` OMS, snapshots are -not used since each position has a unique ID and is never reopened. -::: - -### Multi-currency accounting - -When dealing with multiple currencies: - -- Each position tracks PnL in its settlement currency. -- Portfolio aggregation requires currency conversion. -- Commission currencies may differ from settlement currency. - -```python -# Accessing PnL across positions -for position in positions: - realized = position.realized_pnl # In settlement currency - unrealized = position.unrealized_pnl(last_price) - - # Handle multi-currency aggregation (illustrative) - # Note: Currency conversion requires user-provided exchange rates - if position.settlement_currency != base_currency: - # Apply conversion rate from your data source - # rate = get_exchange_rate(position.settlement_currency, base_currency) - # realized_converted = realized.as_double() * rate - pass -``` - -### Snapshot considerations - -For `NETTING` OMS: - -```python -from nautilus_trader.model.objects import Money - -# Include snapshots for complete PnL (per currency) -pnl_by_currency = {} - -# Add PnL from current positions -for position in cache.positions(instrument_id=instrument_id): - if position.realized_pnl: - currency = position.realized_pnl.currency - if currency not in pnl_by_currency: - pnl_by_currency[currency] = 0.0 - pnl_by_currency[currency] += position.realized_pnl.as_double() - -# Add PnL from historical snapshots -for snapshot in cache.position_snapshots(instrument_id=instrument_id): - if snapshot.realized_pnl: - currency = snapshot.realized_pnl.currency - if currency not in pnl_by_currency: - pnl_by_currency[currency] = 0.0 - pnl_by_currency[currency] += snapshot.realized_pnl.as_double() - -# Create Money objects for each currency -total_pnls = [Money(amount, currency) for currency, amount in pnl_by_currency.items()] -``` - -## Backtest post-run analysis - -After a backtest completes, analysis is available through various reports -and the portfolio analyzer. - -### Accessing backtest results - -```python -# After backtest run -engine.run(start=start_time, end=end_time) - -# Generate reports using Trader helper methods -orders_report = engine.trader.generate_orders_report() -positions_report = engine.trader.generate_positions_report() -fills_report = engine.trader.generate_fills_report() - -# Or access data directly for custom analysis -orders = engine.cache.orders() -positions = engine.cache.positions() -snapshots = engine.cache.position_snapshots() -``` - -### Portfolio statistics - -The portfolio analyzer provides performance metrics: - -```python -# Access portfolio analyzer -portfolio = engine.portfolio - -# Get different categories of statistics -stats_pnls = portfolio.analyzer.get_performance_stats_pnls() -stats_returns = portfolio.analyzer.get_performance_stats_returns() -stats_general = portfolio.analyzer.get_performance_stats_general() -``` - -:::info -For detailed information about available statistics and creating custom metrics, -see the [Portfolio guide](portfolio.md#portfolio-statistics). The Portfolio guide covers: - -- Built-in statistics categories (PnLs, returns, positions, orders based). -- Creating custom statistics with `PortfolioStatistic`. -- Registering and using custom metrics. - -::: - -### Visualization - -NautilusTrader provides interactive tearsheets and plots via Plotly: - -```python -from nautilus_trader.analysis import create_tearsheet - -# After backtest run -engine.run() - -# Generate interactive HTML tearsheet -create_tearsheet(engine, output_path="tearsheet.html") -``` - -This creates an interactive HTML report with: - -- Equity curve -- Drawdown analysis -- Monthly returns heatmap -- Performance statistics table -- Returns distribution - -For more control, generate individual plots: - -```python -from nautilus_trader.analysis import create_equity_curve - -returns = engine.portfolio.analyzer.returns() -fig = create_equity_curve(returns, title="My Strategy Equity") -fig.show() # Display in browser -fig.write_image("equity.png") # Export to PNG (requires kaleido) -``` - -Install visualization dependencies: - -```bash -uv pip install "nautilus_trader[visualization]" -``` - -## Report generation patterns - -### Live trading - -During live trading, generate reports periodically: - -```python -import pandas as pd - -class ReportingActor(Actor): - def on_start(self): - # Schedule periodic reporting - self.clock.set_timer( - name="generate_reports", - interval=pd.Timedelta(minutes=30), - callback=self.generate_reports - ) - - def generate_reports(self, event): - # Generate and log reports - positions_report = self.trader.generate_positions_report() - - # Save or transmit report - positions_report.to_csv(f"positions_{event.ts_event}.csv") -``` - -### Performance analysis - -For backtest analysis: - -```python -import pandas as pd - -# Run the backtest -engine.run(start=start_time, end=end_time) - -# Collect results -positions_closed = engine.cache.positions_closed() -stats_pnls = engine.portfolio.analyzer.get_performance_stats_pnls() -stats_returns = engine.portfolio.analyzer.get_performance_stats_returns() -stats_general = engine.portfolio.analyzer.get_performance_stats_general() - -# Create summary dictionary -results = { - "total_positions": len(positions_closed), - "pnl_total": stats_pnls.get("PnL (total)"), - "sharpe_ratio": stats_returns.get("Sharpe Ratio (252 days)"), - "profit_factor": stats_general.get("Profit Factor"), - "win_rate": stats_general.get("Win Rate"), -} - -# Display results -results_df = pd.DataFrame([results]) -print(results_df.T) # Transpose for vertical display -``` - -:::info -Reports are generated from in-memory data structures. For large-scale analysis -or long-running systems, consider persisting reports to a database for efficient -querying. See the [Cache guide](cache.md) for persistence options. -::: - -## Integration with other components - -The `ReportProvider` works with several system components: - -- **Cache**: Source of all trading data (orders, positions, accounts) for reports. -- **Portfolio**: Uses reports for performance analysis and metrics calculation. -- **BacktestEngine**: Uses reports for post-run analysis and visualization. -- **Position snapshots**: Required for accurate PnL reporting in `NETTING` OMS. - -## Summary - -The `ReportProvider` generates reports from orders, fills, positions, and account -states as structured DataFrames for analysis and visualization. For accurate total -PnL in `NETTING` OMS, include position snapshots when generating reports. - -## Related guides - -- [Visualization](visualization.md) - Interactive tearsheets and charts from backtest results. -- [Portfolio](portfolio.md) - Portfolio statistics and performance metrics. -- [Backtesting](backtesting.md) - Running backtests that generate reports. -- [Cache](cache.md) - Cache system that stores data for reports. diff --git a/nautilus-docs/docs/concepts/strategies.md b/nautilus-docs/docs/concepts/strategies.md deleted file mode 100644 index b5520c6..0000000 --- a/nautilus-docs/docs/concepts/strategies.md +++ /dev/null @@ -1,692 +0,0 @@ -# Strategies - -A strategy inherits the `Strategy` class and implements -the methods its logic requires. - -**Capabilities**: - -- All `Actor` capabilities. -- Order management. - -**Relationship with actors**: -The `Strategy` class inherits from `Actor`, which means strategies have access to all actor functionality -plus order management capabilities. - -:::tip -We recommend reviewing the [Actors](actors.md) guide before diving into strategy development. -::: - -Strategies can be added to Nautilus systems in any [environment contexts](architecture.md#environment-contexts) and will start sending commands and receiving -events based on their logic as soon as the system starts. - -With these building blocks of data ingest, event handling, and order management (discussed below), -you can build any type of strategy including directional, momentum, re-balancing, -pairs, market making, etc. - -See the [`Strategy` API Reference](/docs/python-api-latest/trading.html) for all available methods. - -There are two main parts of a Nautilus trading strategy: - -- The strategy implementation itself, defined by inheriting the `Strategy` class. -- The *optional* strategy configuration, defined by inheriting the `StrategyConfig` class. - -:::tip -Once a strategy is defined, the same source code can be used for backtesting and live trading. -::: - -The main capabilities of a strategy include: - -- Historical data requests. -- Live data feed subscriptions. -- Setting time alerts or timers. -- Cache access. -- Portfolio access. -- Creating and managing orders and positions. - -## Strategy implementation - -A trading strategy inherits from `Strategy`, so you must define a constructor. -At minimum, initialize the base class: - -```python -from nautilus_trader.trading.strategy import Strategy - -class MyStrategy(Strategy): - def __init__(self) -> None: - super().__init__() # <-- the superclass must be called to initialize the strategy -``` - -From here, you can implement handlers as necessary to perform actions based on state transitions -and events. - -:::warning -Do not call components such as `clock` and `logger` in the `__init__` constructor (which is prior to registration). -This is because the systems clock and logging subsystem have not yet been initialized. -::: - -### Handlers - -Handlers are methods on the `Strategy` class that perform actions based on events or state changes. -These methods use the `on_*` prefix. Implement any or all of them as your strategy requires. - -Multiple handlers exist for similar event types to give you control over granularity. -Respond to a specific event with a dedicated handler, or use a generic handler for a range -of related events (using typical switch statement logic). -The system calls handlers in sequence from most specific to most general. - -#### Stateful actions - -Lifecycle state changes trigger these handlers. Recommendations: - -- Use the `on_start` method to initialize your strategy (e.g., fetch instruments, subscribe to data). -- Use the `on_stop` method for cleanup tasks (e.g., cancel open orders, close open positions, unsubscribe from data). - -```python -def on_start(self) -> None: -def on_stop(self) -> None: -def on_resume(self) -> None: -def on_reset(self) -> None: -def on_dispose(self) -> None: -def on_degrade(self) -> None: -def on_fault(self) -> None: -def on_save(self) -> dict[str, bytes]: # Returns user-defined dictionary of state to be saved -def on_load(self, state: dict[str, bytes]) -> None: -``` - -#### Data handling - -These handlers receive data updates, including built-in market data and custom user-defined data. - -```python -from nautilus_trader.core import Data -from nautilus_trader.model import OrderBook -from nautilus_trader.model import Bar -from nautilus_trader.model import QuoteTick -from nautilus_trader.model import TradeTick -from nautilus_trader.model import OrderBookDeltas -from nautilus_trader.model import InstrumentClose -from nautilus_trader.model import InstrumentStatus -from nautilus_trader.model import OptionChainSlice -from nautilus_trader.model import OptionGreeks -from nautilus_trader.model.instruments import Instrument - -def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: -def on_order_book(self, order_book: OrderBook) -> None: -def on_quote_tick(self, tick: QuoteTick) -> None: -def on_trade_tick(self, tick: TradeTick) -> None: -def on_bar(self, bar: Bar) -> None: -def on_instrument(self, instrument: Instrument) -> None: -def on_instrument_status(self, data: InstrumentStatus) -> None: -def on_instrument_close(self, data: InstrumentClose) -> None: -def on_option_greeks(self, greeks: OptionGreeks) -> None: -def on_option_chain(self, chain: OptionChainSlice) -> None: -def on_historical_data(self, data: Data) -> None: -def on_data(self, data: Data) -> None: # Custom data passed to this handler -def on_signal(self, signal: Data) -> None: # Custom signals passed to this handler -``` - -#### Order management - -These handlers receive events related to orders. -`OrderEvent` type messages are passed to handlers in the following sequence: - -1. Specific handler (e.g., `on_order_accepted`, `on_order_rejected`, etc.) -2. `on_order_event(...)` -3. `on_event(...)` - -```python -from nautilus_trader.model.events import OrderAccepted -from nautilus_trader.model.events import OrderCanceled -from nautilus_trader.model.events import OrderCancelRejected -from nautilus_trader.model.events import OrderDenied -from nautilus_trader.model.events import OrderEmulated -from nautilus_trader.model.events import OrderEvent -from nautilus_trader.model.events import OrderExpired -from nautilus_trader.model.events import OrderFilled -from nautilus_trader.model.events import OrderInitialized -from nautilus_trader.model.events import OrderModifyRejected -from nautilus_trader.model.events import OrderPendingCancel -from nautilus_trader.model.events import OrderPendingUpdate -from nautilus_trader.model.events import OrderRejected -from nautilus_trader.model.events import OrderReleased -from nautilus_trader.model.events import OrderSubmitted -from nautilus_trader.model.events import OrderTriggered -from nautilus_trader.model.events import OrderUpdated - -def on_order_initialized(self, event: OrderInitialized) -> None: -def on_order_denied(self, event: OrderDenied) -> None: -def on_order_emulated(self, event: OrderEmulated) -> None: -def on_order_released(self, event: OrderReleased) -> None: -def on_order_submitted(self, event: OrderSubmitted) -> None: -def on_order_rejected(self, event: OrderRejected) -> None: -def on_order_accepted(self, event: OrderAccepted) -> None: -def on_order_canceled(self, event: OrderCanceled) -> None: -def on_order_expired(self, event: OrderExpired) -> None: -def on_order_triggered(self, event: OrderTriggered) -> None: -def on_order_pending_update(self, event: OrderPendingUpdate) -> None: -def on_order_pending_cancel(self, event: OrderPendingCancel) -> None: -def on_order_modify_rejected(self, event: OrderModifyRejected) -> None: -def on_order_cancel_rejected(self, event: OrderCancelRejected) -> None: -def on_order_updated(self, event: OrderUpdated) -> None: -def on_order_filled(self, event: OrderFilled) -> None: -def on_order_event(self, event: OrderEvent) -> None: # All order event messages are eventually passed to this handler -``` - -#### Position management - -These handlers receive events related to positions. -`PositionEvent` type messages are passed to handlers in the following sequence: - -1. Specific handler (e.g., `on_position_opened`, `on_position_changed`, etc.) -2. `on_position_event(...)` -3. `on_event(...)` - -```python -from nautilus_trader.model.events import PositionChanged -from nautilus_trader.model.events import PositionClosed -from nautilus_trader.model.events import PositionEvent -from nautilus_trader.model.events import PositionOpened - -def on_position_opened(self, event: PositionOpened) -> None: -def on_position_changed(self, event: PositionChanged) -> None: -def on_position_closed(self, event: PositionClosed) -> None: -def on_position_event(self, event: PositionEvent) -> None: # All position event messages are eventually passed to this handler -``` - -#### Generic event handling - -This handler will eventually receive all event messages which arrive at the strategy, including those for -which no other specific handler exists. - -```python -from nautilus_trader.core.message import Event - -def on_event(self, event: Event) -> None: -``` - -#### Handler example - -The following example shows a typical `on_start` handler method implementation (taken from the example EMA cross strategy). -Here we can see the following: - -- Indicators being registered to receive bar updates. -- Historical data being requested (to hydrate the indicators). -- Live data being subscribed to. - -```python -def on_start(self) -> None: - """ - Actions to be performed on strategy start. - """ - self.instrument = self.cache.instrument(self.instrument_id) - if self.instrument is None: - self.log.error(f"Could not find instrument for {self.instrument_id}") - self.stop() # Transitions strategy to STOPPED state - return - - # Register the indicators for updating - self.register_indicator_for_bars(self.bar_type, self.fast_ema) - self.register_indicator_for_bars(self.bar_type, self.slow_ema) - - # Get historical data - self.request_bars(self.bar_type) - - # Subscribe to live data - self.subscribe_bars(self.bar_type) - self.subscribe_quote_ticks(self.instrument_id) -``` - -### Clock and timers - -Strategies have access to a `Clock` which provides a number of methods for creating -different timestamps, as well as setting time alerts or timers to trigger `TimeEvent`s. - -See the [`Clock` API Reference](/docs/python-api-latest/common.html) for all available methods. - -#### Current timestamps - -While there are multiple ways to obtain current timestamps, here are two commonly used methods as examples: - -To get the current UTC timestamp as a tz-aware `pd.Timestamp`: - -```python -import pandas as pd - - -now: pd.Timestamp = self.clock.utc_now() -``` - -To get the current UTC timestamp as nanoseconds since the UNIX epoch: - -```python -unix_nanos: int = self.clock.timestamp_ns() -``` - -#### Time alerts - -Time alerts can be set which will result in a `TimeEvent` being dispatched to the `on_event` handler at the -specified alert time. In a live context, this might be slightly delayed by a few microseconds. - -This example sets a time alert to trigger one minute from the current time: - -```python -import pandas as pd - -# Fire a TimeEvent one minute from now -self.clock.set_time_alert( - name="MyTimeAlert1", - alert_time=self.clock.utc_now() + pd.Timedelta(minutes=1), -) -``` - -#### Timers - -Continuous timers can be set up which will generate a `TimeEvent` at regular intervals until the timer expires -or is canceled. - -This example sets a timer to fire once per minute, starting immediately: - -```python -import pandas as pd - -# Fire a TimeEvent every minute -self.clock.set_timer( - name="MyTimer1", - interval=pd.Timedelta(minutes=1), -) -``` - -### Cache access - -The trader's central `Cache` stores data and execution objects (orders, positions, etc). -Many methods are available with filtering. Here are some basic use cases. - -#### Fetching data - -The following example fetches data from the cache (assuming some instrument ID attribute is assigned). -These methods return `None` if the requested data is not available. - -```python -last_quote = self.cache.quote_tick(self.instrument_id) -last_trade = self.cache.trade_tick(self.instrument_id) -last_bar = self.cache.bar(bar_type) -``` - -#### Fetching execution objects - -The following example shows how individual order and position objects can be fetched from the cache: - -```python -order = self.cache.order(client_order_id) -position = self.cache.position(position_id) -``` - -See the [`Cache` API Reference](/docs/python-api-latest/cache.html) for all available methods. - -### Portfolio access - -The trader's central `Portfolio` provides account and positional information. -The following shows a general outline of available methods. - -#### Account and positional information - -```python -import decimal - -from nautilus_trader.accounting.accounts.base import Account -from nautilus_trader.model import Venue -from nautilus_trader.model import Currency -from nautilus_trader.model import Money -from nautilus_trader.model import InstrumentId - -def account(self, venue: Venue) -> Account - -def balances_locked(self, venue: Venue) -> dict[Currency, Money] -def margins_init(self, venue: Venue) -> dict[Currency, Money] -def margins_maint(self, venue: Venue) -> dict[Currency, Money] -def unrealized_pnls(self, venue: Venue) -> dict[Currency, Money] -def realized_pnls(self, venue: Venue) -> dict[Currency, Money] -def net_exposures(self, venue: Venue) -> dict[Currency, Money] - -def unrealized_pnl(self, instrument_id: InstrumentId) -> Money -def realized_pnl(self, instrument_id: InstrumentId) -> Money -def net_exposure(self, instrument_id: InstrumentId) -> Money -def net_position(self, instrument_id: InstrumentId) -> decimal.Decimal - -def is_net_long(self, instrument_id: InstrumentId) -> bool -def is_net_short(self, instrument_id: InstrumentId) -> bool -def is_flat(self, instrument_id: InstrumentId) -> bool -def is_completely_flat(self) -> bool -``` - -See the [`Portfolio` API Reference](/docs/python-api-latest/portfolio.html) for all available methods. - -#### Reports and analysis - -The `Portfolio` also exposes a `PortfolioAnalyzer`, which accepts a flexible amount of data -(to accommodate different lookback windows). The analyzer tracks and generates performance -metrics and statistics. - -See the [`PortfolioAnalyzer` API Reference](/docs/python-api-latest/analysis.html) and [Portfolio statistics](portfolio.md#portfolio-statistics) guide. - -### Trading commands - -The following trading commands are available for order management. -See also the [Execution](../concepts/execution.md) guide for the full flow through the system. - -#### Submitting orders - -An `OrderFactory` is provided on the base class for every `Strategy` as a convenience, reducing -the amount of boilerplate required to create different `Order` objects (although these objects -can still be initialized directly with the `Order.__init__(...)` constructor if the trader prefers). - -The component a `SubmitOrder` or `SubmitOrderList` command will flow to for execution depends on the following: - -- If an `emulation_trigger` is specified, the command will *firstly* be sent to the `OrderEmulator`. -- If an `exec_algorithm_id` is specified (with no `emulation_trigger`), the command will *firstly* be sent to the relevant `ExecAlgorithm`. -- Otherwise, the command will *firstly* be sent to the `RiskEngine`. - -This example submits a `LIMIT` BUY order for emulation (see [Emulated Orders](orders.md#emulated-orders)): - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TriggerType -from nautilus_trader.model.orders import LimitOrder - - -def buy(self) -> None: - """ - Users simple buy method (example). - """ - order: LimitOrder = self.order_factory.limit( - instrument_id=self.instrument_id, - order_side=OrderSide.BUY, - quantity=self.instrument.make_qty(self.trade_size), - price=self.instrument.make_price(5000.00), - emulation_trigger=TriggerType.LAST_PRICE, - ) - - self.submit_order(order) -``` - -:::info -You can specify both order emulation and an execution algorithm. In this case, the order is -first sent to the `OrderEmulator`, and upon release is then routed to the `ExecAlgorithm`. -::: - -This example submits a `MARKET` BUY order to a TWAP execution algorithm: - -```python -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.enums import TimeInForce -from nautilus_trader.model import ExecAlgorithmId - - -def buy(self) -> None: - """ - Users simple buy method (example). - """ - order: MarketOrder = self.order_factory.market( - instrument_id=self.instrument_id, - order_side=OrderSide.BUY, - quantity=self.instrument.make_qty(self.trade_size), - time_in_force=TimeInForce.FOK, - exec_algorithm_id=ExecAlgorithmId("TWAP"), - exec_algorithm_params={"horizon_secs": 20, "interval_secs": 2.5}, - ) - - self.submit_order(order) -``` - -#### Canceling orders - -Orders can be canceled individually, as a batch, or all orders for an instrument (with an optional side filter). - -If the order is already *closed* or already pending cancel, then a warning will be logged. - -If the order is currently *open* then the status will become `PENDING_CANCEL`. - -The component a `CancelOrder`, `CancelAllOrders` or `BatchCancelOrders` command will flow to for execution depends on the following: - -- If the order is currently emulated, the command will *firstly* be sent to the `OrderEmulator`. -- If an `exec_algorithm_id` is specified (with no `emulation_trigger`), and the order is still active within the local system, the command will *firstly* be sent to the relevant `ExecAlgorithm`. -- Otherwise, the order will *firstly* be sent to the `ExecutionEngine`. - -:::info -Any managed GTD timer will also be canceled after the command has left the strategy. -::: - -The following shows how to cancel an individual order: - -```python -self.cancel_order(order) -``` - -The following shows how to cancel a batch of orders: - -```python -from nautilus_trader.model.orders import Order - - -my_order_list: list[Order] = [order1, order2, order3] -self.cancel_orders(my_order_list) -``` - -The following shows how to cancel all orders: - -```python -self.cancel_all_orders() -``` - -#### Modifying orders - -Orders can be modified individually when emulated, or *open* on a venue (if supported). - -If the order is already *closed* or already pending cancel, then a warning will be logged. -If the order is currently *open* then the status will become `PENDING_UPDATE`. - -:::warning -At least one value must differ from the original order for the command to be valid. -::: - -The component a `ModifyOrder` command will flow to for execution depends on the following: - -- If the order is currently emulated, the command will *firstly* be sent to the `OrderEmulator`. -- Otherwise, the order will *firstly* be sent to the `RiskEngine`. - -:::info -Once an order is under the control of an execution algorithm, it cannot be directly modified by a strategy (only canceled). -::: - -The following shows how to modify the size of `LIMIT` BUY order currently *open* on a venue: - -```python -from nautilus_trader.model import Quantity - - -new_quantity: Quantity = Quantity.from_int(5) -self.modify_order(order, new_quantity) -``` - -:::info -The price and trigger price can also be modified (when emulated or supported by a venue). -::: - -#### Market exit - -The `market_exit()` method provides a graceful way to exit all positions and cancel all orders -for a strategy. The strategy remains running after the exit completes, allowing you to re-enter -positions later if desired. - -```python -self.market_exit() -``` - -The market exit process: - -1. Cancels all open and in-flight orders for the strategy. -2. Closes all open positions with market orders. -3. Periodically checks (at `market_exit_interval_ms`) until all orders resolve and positions close. -4. Calls `post_market_exit()` once flat, or after `market_exit_max_attempts` is reached. - -Two hooks are available for custom logic: - -- `on_market_exit()` - Called when the exit process begins. -- `post_market_exit()` - Called when the exit process completes. - -```python -class MyStrategy(Strategy): - def on_market_exit(self) -> None: - self.log.info("Beginning market exit...") - - def post_market_exit(self) -> None: - self.log.info("Market exit complete") -``` - -During a market exit, non-reduce-only orders are automatically denied. For order lists, -if any order in the list is non-reduce-only, the entire list is denied to preserve list -semantics (e.g., bracket orders with interdependencies). - -To check if an exit is in progress (e.g., to skip order submission logic), use `is_exiting()`: - -```python -def on_quote_tick(self, tick: QuoteTick) -> None: - if self.is_exiting(): - return # Skip order logic during exit - # ... normal order logic -``` - -To automatically perform a market exit when the strategy is stopped, set `manage_stop=True`: - -```python -config = StrategyConfig(manage_stop=True) -``` - -With this option, calling `stop()` will first perform a market exit, then stop the strategy -once flat. - -Configuration options in `StrategyConfig`: - -- `manage_stop` (default: False) - If True, `stop()` performs a market exit before stopping. -- `market_exit_interval_ms` (default: 100) - Interval between exit completion checks. -- `market_exit_max_attempts` (default: 100) - Maximum checks before completing the exit. -- `market_exit_time_in_force` (default: None/GTC) - Time in force for closing market orders. -- `market_exit_reduce_only` (default: True) - If closing market orders should be reduce only. - -## Strategy configuration - -A separate configuration class gives full flexibility over where and how a strategy -is instantiated. Configurations serialize over the wire, enabling distributed backtesting -and remote live trading. - -This is opt-in. You can skip configuration and pass parameters directly to your -strategy constructor. If you want distributed backtests or remote live trading, -define a configuration. - -Here is an example configuration: - -```python -from decimal import Decimal -from nautilus_trader.config import StrategyConfig -from nautilus_trader.model import Bar, BarType -from nautilus_trader.model import InstrumentId -from nautilus_trader.trading.strategy import Strategy - - -# Configuration definition -class MyStrategyConfig(StrategyConfig): - instrument_id: InstrumentId # example value: "ETHUSDT-PERP.BINANCE" - bar_type: BarType # example value: "ETHUSDT-PERP.BINANCE-15-MINUTE[LAST]-EXTERNAL" - fast_ema_period: int = 10 - slow_ema_period: int = 20 - trade_size: Decimal - order_id_tag: str - - -# Strategy definition -class MyStrategy(Strategy): - def __init__(self, config: MyStrategyConfig) -> None: - # Always initialize the parent Strategy class - # After this, configuration is stored and available via `self.config` - super().__init__(config) - - # Custom state variables - self.time_started = None - self.count_of_processed_bars: int = 0 - - def on_start(self) -> None: - self.time_started = self.clock.utc_now() # Remember time, when strategy started - self.subscribe_bars(self.config.bar_type) # See how configuration data are exposed via `self.config` - - def on_bar(self, bar: Bar): - self.count_of_processed_bars += 1 # Update count of processed bars - - -# Instantiate configuration with specific values. By setting: -# - InstrumentId - we parameterize the instrument the strategy will trade. -# - BarType - we parameterize bar-data, that strategy will trade. -config = MyStrategyConfig( - instrument_id=InstrumentId.from_str("ETHUSDT-PERP.BINANCE"), - bar_type=BarType.from_str("ETHUSDT-PERP.BINANCE-15-MINUTE[LAST]-EXTERNAL"), - trade_size=Decimal(1), - order_id_tag="001", -) - -# Pass configuration to our trading strategy. -strategy = MyStrategy(config=config) -``` - -Access configuration values through `self.config`. -This provides clear separation between: - -- Configuration data (accessed via `self.config`): - - Contains initial settings, that define how the strategy works. - - Example: `self.config.trade_size`, `self.config.instrument_id` - -- Strategy state variables (as direct attributes): - - Track any custom state of the strategy. - - Example: `self.time_started`, `self.count_of_processed_bars` - -This separation makes code easier to understand and maintain. - -:::note -Even though it often makes sense to define a strategy which will trade a single -instrument. The number of instruments a single strategy can work with is only limited by machine resources. -::: - -### Managed GTD expiry - -It's possible for the strategy to manage expiry for orders with a time in force of GTD (*Good 'till Date*). -This may be desirable if the exchange/broker does not support this time in force option, or for any -reason you prefer the strategy to manage this. - -To use this option, pass `manage_gtd_expiry=True` to your `StrategyConfig`. When an order is submitted with -a time in force of GTD, the strategy will automatically start an internal time alert. -Once the internal GTD time alert is reached, the order will be canceled (if not already *closed*). - -Some venues (such as Binance Futures) support the GTD time in force, so to avoid conflicts when using -`managed_gtd_expiry` you should set `use_gtd=False` for your execution client config. - -### Multiple strategies - -If you intend running multiple instances of the same strategy, with different -configurations (such as trading different instruments), then you will need to define -a unique `order_id_tag` for each of these strategies (as shown above). - -:::note -The platform has built-in safety measures: if two strategies share a duplicated strategy ID, -a `RuntimeError` is raised during registration indicating the strategy ID is already registered. -::: - -The reason for this is that the system must be able to identify which strategy -various commands and events belong to. A strategy ID is made up of the -strategy class name, and the strategies `order_id_tag` separated by a hyphen. For -example the above config would result in a strategy ID of `MyStrategy-001`. - -See the [`StrategyId` API Reference](/docs/python-api-latest/model/identifiers.html) for further details. - -## Related guides - -- [Actors](actors.md) - Base class that strategies extend. -- [Orders](orders.md) - Order types and management from strategies. -- [Backtesting](backtesting.md) - Test strategies with historical data. diff --git a/nautilus-docs/docs/concepts/value_types.md b/nautilus-docs/docs/concepts/value_types.md deleted file mode 100644 index 306eaae..0000000 --- a/nautilus-docs/docs/concepts/value_types.md +++ /dev/null @@ -1,303 +0,0 @@ -# Value Types - -NautilusTrader provides specialized value types for representing core trading concepts: -`Price`, `Quantity`, and `Money`. These types use fixed-point arithmetic internally -for performant, deterministic calculations across different platforms -and environments. - -## Overview - -| Type | Purpose | Signed | Currency | -|------------|------------------------------------------|--------|----------| -| `Quantity` | Trade sizes, order amounts, positions. | No | - | -| `Price` | Market prices, quotes, price levels. | Yes | - | -| `Money` | Monetary amounts, P&L, account balances. | Yes | Yes | - -## Immutability - -All value types are **immutable**. Once a value is constructed, it cannot be changed. -Operations do not mutate the original object. - -```python -from nautilus_trader.model.objects import Quantity - -qty1 = Quantity(100, precision=0) -qty2 = Quantity(50, precision=0) - -# This creates a NEW Quantity; qty1 and qty2 are unchanged -result = qty1 + qty2 - -print(qty1) # 100 -print(qty2) # 50 -print(result) # 150 -``` - -This design provides several benefits: - -- **Thread safety**: Immutable values can be safely shared across threads without synchronization. -- **Predictability**: Values never change unexpectedly, making debugging easier. -- **Hashability**: Immutable types can be used as dictionary keys and in sets. - -## Arithmetic operations - -Value types support standard arithmetic operators (`+`, `-`, `*`, `/`, `%`, `//`) -and unary operators (`-`, `+`, `abs`). The return type depends on the operator -and the operand types. - -### Same-type binary operations - -Addition and subtraction of the same value type return that type, preserving -domain meaning (a price plus a price is still a price): - -| Operation | Result | -|-----------------------|------------| -| `Quantity + Quantity` | `Quantity` | -| `Quantity - Quantity` | `Quantity` | -| `Price + Price` | `Price` | -| `Price - Price` | `Price` | -| `Money + Money` | `Money` | -| `Money - Money` | `Money` | - -```python -from nautilus_trader.model.objects import Price - -price1 = Price(100.50, precision=2) -price2 = Price(0.25, precision=2) - -result = price1 + price2 # Returns Price(100.75, precision=2) -print(type(result)) # -``` - -Multiplication, division, floor division, and modulo between two values of the -same type return `Decimal`: - -| Operation | Result | -|-----------------------|-----------| -| `Price * Price` | `Decimal` | -| `Price / Price` | `Decimal` | -| `Price // Price` | `Decimal` | -| `Price % Price` | `Decimal` | - -The same pattern applies to `Quantity` and `Money`. - -These operations do not return the original type because the result has different -dimensional meaning. Multiplying a price by a price produces "price squared", not -a price. Dividing a quantity by a quantity produces a dimensionless ratio, not a -quantity. Returning `Decimal` makes the unit change explicit and prevents -misinterpretation of the result as a value with the original unit. - -### Unary operations - -Unary operators preserve the value type where the result is valid for that type: - -| Operation | `Price` | `Quantity` | `Money` | -|--------------|-----------|------------|-----------| -| `-x` (neg) | `Price` | `Decimal` | `Money` | -| `+x` (pos) | `Price` | `Quantity` | `Money` | -| `abs(x)` | `Price` | `Quantity` | `Money` | -| `int(x)` | `int` | `int` | `int` | -| `float(x)` | `float` | `float` | `float` | -| `round(x)` | `Decimal` | `Decimal` | `Decimal` | - -`Quantity.__neg__` returns `Decimal` rather than `Quantity` because `Quantity` is -unsigned and cannot represent a negative value. - -```python -from nautilus_trader.model.objects import Price, Quantity, Money -from nautilus_trader.model.currencies import USD - -price = Price(100.50, precision=2) -print(-price) # -100.50 -print(type(-price)) # - -money = Money(-50.00, USD) -print(abs(money)) # 50.00 USD -print(type(abs(money))) # - -qty = Quantity(10, precision=0) -print(+qty) # 10 -print(type(+qty)) # -``` - -### Mixed-type operations - -When operating with other numeric types, the result type follows Python's -[numeric tower](https://docs.python.org/3/library/numbers.html) conventions. The general -principle is that operations widen to the more general type: `float` operations return -`float`, while `int` and `Decimal` operations return `Decimal` for precision preservation. - -This applies to all six binary operators (`+`, `-`, `*`, `/`, `//`, `%`) and works -in both directions (`value op scalar` and `scalar op value`): - -| Left operand | Right operand | Result type | -|--------------|---------------|-------------| -| Value type | `int` | `Decimal` | -| Value type | `float` | `float` | -| Value type | `Decimal` | `Decimal` | -| `int` | Value type | `Decimal` | -| `float` | Value type | `float` | -| `Decimal` | Value type | `Decimal` | - -```python -from decimal import Decimal -from nautilus_trader.model.objects import Quantity - -qty = Quantity(100, precision=0) - -# Quantity + int → Decimal -result1 = qty + 50 -print(type(result1)) # - -# Quantity + float → float -result2 = qty + 50.5 -print(type(result2)) # - -# Quantity + Decimal → Decimal -result3 = qty + Decimal("50") -print(type(result3)) # -``` - -## Precision handling - -Each value type stores a precision field indicating the number of decimal places. -Precision is set at construction and is immutable. There is no "unspecified" precision. - -### Fixed-point representation - -Value types are stored internally as integers scaled to a global fixed precision -(e.g., 10^16 in high-precision mode), not floating-point numbers. The `precision` -field tracks the number of decimal places used at construction, controlling display -formatting and serialization, but the underlying raw value always uses the global scale. - -```python -from nautilus_trader.model.objects import Price - -p1 = Price(1.23, precision=2) # displays as "1.23" -p2 = Price(1.230, precision=3) # displays as "1.230" - -p1 == p2 # True: same underlying value -str(p1) # "1.23" -str(p2) # "1.230" -``` - -**Precision controls display, not identity.** Two prices with the same decimal value but -different precisions are equal. The `precision` field determines string formatting and -how many decimal places are shown, but equality is based on the underlying numeric value. - -**Market data serialization uses precision metadata.** When market data types (quotes, -trades, order book deltas) are written to Parquet or Arrow format, precision is stored in -the file metadata so that values can be correctly decoded. All market data values within -a single file must share the same precision. - -:::note -If a venue changes an instrument's tick size (and thus its precision), data files written -before and after the change will have different precision metadata and should not be -consolidated into a single file. -::: - -For how instrument-level precision constrains valid prices and quantities, see the -[Precision](instruments.md#precision) section of the Instruments guide. - -### Arithmetic precision - -When performing arithmetic between values with different precisions, the result -uses the maximum precision of the operands. - -```python -from nautilus_trader.model.objects import Price - -price1 = Price(100.5, precision=1) # 1 decimal place -price2 = Price(0.125, precision=3) # 3 decimal places - -result = price1 + price2 -print(result) # 100.625 -print(result.precision) # 3 (max of 1 and 3) -``` - -## Type-specific constraints - -### Quantity - -`Quantity` represents non-negative amounts. Attempting to create a negative quantity -or subtract a larger quantity from a smaller one raises an error: - -```python -from nautilus_trader.model.objects import Quantity - -# This raises ValueError: Quantity cannot be negative -qty = Quantity(-100, precision=0) - -# This also raises ValueError -qty1 = Quantity(50, precision=0) -qty2 = Quantity(100, precision=0) -result = qty1 - qty2 # Would be -50, which is invalid -``` - -### Money - -`Money` values include a currency. Addition and subtraction between `Money` values -require matching currencies: - -```python -from nautilus_trader.model.objects import Money -from nautilus_trader.model.currencies import USD, EUR - -usd_amount = Money(100.00, USD) -eur_amount = Money(50.00, EUR) - -# This works - same currency -result = usd_amount + Money(25.00, USD) - -# This raises ValueError - currency mismatch -result = usd_amount + eur_amount -``` - -## Common patterns - -### Accumulating values - -Since value types are immutable, accumulate by reassigning: - -```python -from nautilus_trader.model.objects import Money -from nautilus_trader.model.currencies import USD - -total = Money(0.00, USD) -amounts = [Money(100.00, USD), Money(50.00, USD), Money(25.00, USD)] - -for amount in amounts: - total = total + amount # Reassign to new Money instance - -print(total) # 175.00 USD -``` - -### Converting to other types - -Value types provide conversion methods: - -```python -from nautilus_trader.model.objects import Price - -price = Price(123.456, precision=3) - -# Convert to Decimal (preserves precision) -decimal_value = price.as_decimal() - -# Convert to float -float_value = price.as_double() - -# Convert to string -string_value = str(price) # "123.456" -``` - -### Creating from strings - -Parse value types from string representations: - -```python -from nautilus_trader.model.objects import Quantity, Price, Money - -qty = Quantity.from_str("100.5") -price = Price.from_str("99.95") -money = Money.from_str("1000.00 USD") -``` diff --git a/nautilus-docs/docs/concepts/visualization.md b/nautilus-docs/docs/concepts/visualization.md deleted file mode 100644 index 2212b5c..0000000 --- a/nautilus-docs/docs/concepts/visualization.md +++ /dev/null @@ -1,571 +0,0 @@ -# Visualization - -NautilusTrader provides interactive HTML tearsheets for analyzing backtest results through -an extensible visualization system built on Plotly. You can generate reports with minimal -code and add custom charts and themes. - -## Overview - -The visualization system has three parts: - -1. **Chart Registry** - Decoupled chart definitions that can be extended with custom visualizations. -2. **Theme System** - Consistent styling with built-in and custom themes. -3. **Configuration** - Declarative specification of what to render and how to display it. - -All visualization outputs are self-contained HTML files that can be viewed in any modern -browser, shared with stakeholders, or archived for future reference. - -:::note -The visualization system requires `plotly>=6.3.1`. Install it with: - -```bash -uv pip install "nautilus_trader[visualization]" -``` - -or - -```bash -uv pip install "plotly>=6.3.1" -``` - -::: - -## Tearsheets - -A tearsheet is a performance report that combines multiple charts and -statistics into a single interactive visualization. Tearsheets are generated after -completing a backtest run and provide immediate visual feedback on strategy performance. - -### Quick start - -Generate a tearsheet with default settings: - -```python -from nautilus_trader.analysis import create_tearsheet -from nautilus_trader.backtest.engine import BacktestEngine - -# After running your backtest -engine.run() - -# Generate tearsheet -create_tearsheet( - engine=engine, - output_path="backtest_results.html", -) -``` - -This produces an HTML file with all default charts, using the light theme and automatic -layout. Open `backtest_results.html` in your browser to view the interactive tearsheet. - -### Customization - -Control which charts appear and how they're styled: - -```python -from nautilus_trader.analysis import TearsheetConfig -from nautilus_trader.analysis import TearsheetDrawdownChart -from nautilus_trader.analysis import TearsheetEquityChart -from nautilus_trader.analysis import TearsheetRunInfoChart -from nautilus_trader.analysis import TearsheetStatsTableChart - -config = TearsheetConfig( - charts=[ - TearsheetRunInfoChart(), - TearsheetStatsTableChart(), - TearsheetEquityChart(), - TearsheetDrawdownChart(), - ], - theme="nautilus_dark", - height=2000, -) - -create_tearsheet( - engine=engine, - output_path="custom_tearsheet.html", - config=config, -) -``` - -### Currency filtering - -For multi-currency backtests, filter statistics to a specific currency: - -```python -from nautilus_trader.model.currencies import USD - -create_tearsheet( - engine=engine, - output_path="usd_only.html", - currency=USD, # Currency object, shows only USD statistics -) -``` - -When `currency` is `None` (default), statistics for all currencies are displayed separately in the tearsheet. - -## Available charts - -The tearsheet can include any combination of the following built-in charts: - -| Chart Name | Type | Description | -|--------------------|--------------|----------------------------------------------------------| -| `run_info` | Table | Run metadata and account balances. | -| `stats_table` | Table | Performance statistics (PnL, returns, general metrics). | -| `equity` | Line | Cumulative returns over time with optional benchmark. | -| `drawdown` | Area | Drawdown percentage from peak equity. | -| `monthly_returns` | Heatmap | Monthly return percentages organized by year. | -| `distribution` | Histogram | Distribution of individual return values. | -| `rolling_sharpe` | Line | 60-day rolling Sharpe ratio. | -| `yearly_returns` | Bar | Annual return percentages. | -| `bars_with_fills` | Candlestick | Price bars (OHLC) with order fills overlaid as markers. | - -All charts are registered in the chart registry and are configured via chart objects in -`TearsheetConfig.charts` (each chart object maps to a built-in chart name). - -### Run information table - -The `run_info` chart displays key metadata about the backtest run: - -- Run ID, start time, finish time -- Backtest period (start/end dates) -- Total iterations processed -- Event, order, and position counts -- Account starting and ending balances (per currency) - -This table appears in the top-left position by default. - -### Performance statistics table - -The `stats_table` chart displays performance metrics organized into sections: - -- **PnL Statistics** (per currency): Total PnL, win rate, profit factor, etc. -- **Returns Statistics**: Sharpe ratio, Sortino ratio, max drawdown, etc. -- **General Statistics**: Total trades, average trade duration, etc. - -This table appears in the top-right position by default. - -### Equity curve - -The `equity` chart plots cumulative returns over the backtest period. When `benchmark_returns` -is provided to `create_tearsheet()`, the benchmark is overlaid for comparison. - -```python -import pandas as pd - -# Load benchmark returns (e.g., from a market index) -# Index should be datetime, aligned with strategy returns timeframe -benchmark_returns = pd.read_csv("sp500_returns.csv", index_col=0, parse_dates=True)["return"] - -create_tearsheet( - engine=engine, - output_path="with_benchmark.html", - benchmark_returns=benchmark_returns, - benchmark_name="S&P 500", -) -``` - -The benchmark series is plotted as-is; ensure the index aligns with your strategy's return dates for accurate comparison. - -## Themes - -Themes control the visual styling of charts including colors, fonts, and backgrounds. -NautilusTrader provides four built-in themes: - -| Theme Name | Description | Use Case | -|-----------------|------------------------------------------------|-------------------------------| -| `plotly_white` | Clean light theme with dark gray headers. | Default, professional reports.| -| `plotly_dark` | Dark background with standard Plotly colors. | Low-light environments. | -| `nautilus` | Light theme with NautilusTrader brand colors. | Official light mode. | -| `nautilus_dark` | Dark theme with teal/cyan signature colors. | Official dark mode. | - -### Selecting a theme - -Specify the theme in `TearsheetConfig`: - -```python -config = TearsheetConfig(theme="nautilus_dark") -create_tearsheet(engine=engine, config=config) -``` - -### Custom themes - -Register a custom theme for consistent branding across all visualizations: - -```python -from nautilus_trader.analysis import register_theme - -register_theme( - name="corporate", - template="plotly_white", # Base Plotly template - colors={ - "primary": "#003366", # Navy blue - "positive": "#2e8b57", # Sea green - "negative": "#c41e3a", # Cardinal red - "neutral": "#808080", # Gray - "background": "#ffffff", # White - "grid": "#e5e5e5", # Light gray - # Optional table colors (defaults will be provided if omitted) - "table_section": "#e5e5e5", - "table_row_odd": "#f8f8f8", - "table_row_even": "#ffffff", - "table_text": "#000000", - } -) - -# Use the custom theme -config = TearsheetConfig(theme="corporate") -``` - -The theme system automatically provides sensible defaults for `table_*` colors based on -the `background` and `grid` colors, ensuring backward compatibility with themes registered -before table-specific colors were introduced. - -## Configuration - -The `TearsheetConfig` class provides declarative control over tearsheet generation: - -```python -from nautilus_trader.analysis import GridLayout -from nautilus_trader.analysis import TearsheetConfig -from nautilus_trader.analysis import TearsheetDrawdownChart -from nautilus_trader.analysis import TearsheetEquityChart -from nautilus_trader.analysis import TearsheetStatsTableChart - -config = TearsheetConfig( - charts=[ - TearsheetEquityChart(), - TearsheetDrawdownChart(), - TearsheetStatsTableChart(), - ], - theme="nautilus_dark", - title="Q4 2024 Strategy Performance", - height=1800, - include_benchmark=True, - benchmark_name="SPY", - layout=GridLayout( - rows=2, - cols=2, - heights=[0.60, 0.40], - vertical_spacing=0.08, - horizontal_spacing=0.12, - ), -) -``` - -### Configuration parameters - -| Parameter | Type | Default | Description | -|---------------------|-------------------------------|-----------------------------------|-----------------------------------------------| -| `charts` | `list[TearsheetChart]` | All built-in charts | List of chart objects to include (in order). | -| `theme` | `str` | `"plotly_white"` | Theme name for styling. | -| `layout` | `GridLayout` | `None` (auto-calculated) | Custom subplot grid layout. | -| `title` | `str` | Auto-generated with strategy/time | Tearsheet title. | -| `include_benchmark` | `bool` | `True` | Show benchmark when provided. | -| `benchmark_name` | `str` | `"Benchmark"` | Display name for benchmark. | -| `height` | `int` | `1500` | Total height in pixels. | -| `show_logo` | `bool` | `True` | Display NautilusTrader logo (reserved for future use).| - -When `layout` is `None`, the grid dimensions and row heights are automatically calculated -based on the number of charts. For 8 charts (the default), a 4×2 grid is used with -heights `[0.50, 0.22, 0.16, 0.12]` to give more space to the top row tables. - -## Custom charts - -The registry pattern lets you add custom charts. Charts are functions that -render traces onto a Plotly figure object. - -### Registering a custom chart - -```python -from nautilus_trader.analysis.tearsheet import register_chart -import plotly.graph_objects as go - -def my_custom_chart(returns, output_path=None, title="Custom Chart", theme="plotly_white"): - """ - Create a custom visualization. - - This function signature matches the built-in chart functions for consistency. - """ - from nautilus_trader.analysis.themes import get_theme - - theme_config = get_theme(theme) - - # Create your visualization - fig = go.Figure() - fig.add_trace(go.Scatter( - x=returns.index, - y=returns.cumsum(), - mode="lines", - name="Custom Metric", - line={"color": theme_config["colors"]["primary"]}, - )) - - fig.update_layout( - title=title, - template=theme_config["template"], - xaxis_title="Date", - yaxis_title="Value", - ) - - if output_path: - fig.write_html(output_path) - - return fig - -# Register the chart for standalone use (via `get_chart()` / `list_charts()`) -register_chart("my_custom", my_custom_chart) -``` - -### Tearsheet integration - -For full tearsheet integration with proper grid placement, use the lower-level registration. - -:::warning -The `_register_tearsheet_chart` function is internal API and may change between releases. -For most use cases, prefer `register_chart` for standalone charts or contribute new built-in -charts upstream. -::: - -```python -from nautilus_trader.analysis import TearsheetConfig -from nautilus_trader.analysis import TearsheetCustomChart -from nautilus_trader.analysis import TearsheetEquityChart -from nautilus_trader.analysis import TearsheetStatsTableChart -from nautilus_trader.analysis.tearsheet import _register_tearsheet_chart - -def _render_my_metric(fig, row, col, returns, theme_config, **kwargs): - """ - Render custom metric directly onto a subplot. - - Parameters - ---------- - fig : go.Figure - The figure to add traces to. - row : int - Subplot row position. - col : int - Subplot column position. - returns : pd.Series - Strategy returns from analyzer. - theme_config : dict - Theme configuration dictionary. - **kwargs : dict - Additional parameters (stats_pnls, stats_returns, benchmark_returns, etc.). - """ - metric_values = returns.rolling(30).std() * 100 # Example metric - - fig.add_trace( - go.Scatter( - x=returns.index, - y=metric_values, - mode="lines", - name="30-Day Volatility", - line={"color": theme_config["colors"]["neutral"]}, - ), - row=row, - col=col, - ) - - fig.update_xaxes(title_text="Date", row=row, col=col) - fig.update_yaxes(title_text="Volatility (%)", row=row, col=col) - -# Register for tearsheet use -_register_tearsheet_chart( - name="volatility", - subplot_type="scatter", - title="Rolling Volatility (30-day)", - renderer=_render_my_metric, -) - -# Now "volatility" can be used in TearsheetConfig.charts: -config = TearsheetConfig( - charts=[ - TearsheetStatsTableChart(), - TearsheetEquityChart(), - TearsheetCustomChart(chart="volatility"), - ], -) -``` - -The renderer function receives all necessary data (returns, statistics, theme configuration) -and renders directly onto the specified subplot position. - -## Offline analysis - -For situations where you have precomputed statistics but not a `BacktestEngine` instance, -use the lower-level API: - -```python -from nautilus_trader.analysis.tearsheet import create_tearsheet_from_stats - -# Load precomputed data (structure matches PortfolioAnalyzer output) -stats_pnls = {"USD": {"PnL (total)": 1500.0, "Win Rate": 0.55, ...}} # Per-currency -stats_returns = {"Sharpe Ratio (252 days)": 1.2, "Max Drawdown": -0.15, ...} -stats_general = {"Avg Winner": 100.0, "Avg Loser": -50.0, ...} -returns = pd.Series(...) # Daily returns with datetime index - -create_tearsheet_from_stats( - stats_pnls=stats_pnls, - stats_returns=stats_returns, - stats_general=stats_general, - returns=returns, - output_path="offline_analysis.html", -) -``` - -The dictionary keys should match those returned by `PortfolioAnalyzer.get_performance_stats_*()`. - -This approach is useful for: - -- Analyzing results from multiple backtest runs stored separately. -- Comparing strategies using precomputed metrics. -- Integrating with external analysis pipelines. - -## Best practices - -### Chart selection - -- Use default charts for exploratory analysis to see all available metrics. -- Customize charts when you know which metrics matter for your strategy. -- Remove irrelevant charts to reduce visual clutter and file size. - -### Theme usage - -- Use `plotly_white` for professional reports and presentations. -- Use `nautilus_dark` for official materials or low-light viewing. -- Create custom themes to match internal guidelines or personal preferences. - -### Performance considerations - -- Tearsheet HTML files contain all data inline and can be several megabytes for long backtests. -- Consider generating separate tearsheets for different analysis timeframes. -- For very large datasets, use the individual chart functions instead of full tearsheets. - -### Custom statistics integration - -Custom charts work best when paired with [custom statistics](reports.md) registered in the -`PortfolioAnalyzer`. This ensures your visualizations display metrics computed consistently -with the rest of the system: - -```python -from nautilus_trader.analysis.statistic import PortfolioStatistic - -class MyCustomStatistic(PortfolioStatistic): - """Custom metric for specialized strategy analysis.""" - - def calculate_from_returns(self, returns): - # Your calculation logic - return custom_metric_value - -# Register with analyzer -analyzer.register_statistic(MyCustomStatistic()) - -# Now available in stats_returns for custom charts -``` - -## API levels - -The visualization system provides two API levels: - -### High-level API - -Recommended for most use cases: - -```python -create_tearsheet(engine=engine, config=config) -``` - -Automatically extracts data from the `BacktestEngine`, generates all configured charts, -and produces a complete HTML tearsheet. - -### Low-level API - -For advanced customization or offline analysis: - -```python -create_tearsheet_from_stats( - stats_pnls=stats_pnls, - stats_returns=stats_returns, - stats_general=stats_general, - returns=returns, - run_info=run_info, - account_info=account_info, - config=config, -) -``` - -Provides fine-grained control over data inputs and allows analysis of precomputed statistics. - -### Standalone chart functions - -Individual chart functions can be used independently to generate single-purpose HTML visualizations -or Plotly figures for custom analysis workflows. - -#### Price bars with fills - -The `create_bars_with_fills` function generates a candlestick chart with order fills overlaid, -useful for visually analyzing strategy execution within price action. It can be used standalone -or included in tearsheets: - -```python -from nautilus_trader.analysis import create_bars_with_fills -from nautilus_trader.analysis import create_tearsheet -from nautilus_trader.analysis import TearsheetBarsWithFillsChart -from nautilus_trader.analysis import TearsheetConfig -from nautilus_trader.analysis import TearsheetEquityChart -from nautilus_trader.analysis import TearsheetStatsTableChart -from nautilus_trader.model.data import BarType - -# Standalone usage -bar_type = BarType.from_str("ESM4.XCME-1-MINUTE-LAST-EXTERNAL") -fig = create_bars_with_fills( - engine=engine, - bar_type=bar_type, - title="ES Futures - Entry/Exit Analysis", -) -fig.show() # Display in Jupyter -fig.write_html("bars_with_fills.html") # Or save to file - -# Include in tearsheet -config = TearsheetConfig( - charts=[ - TearsheetStatsTableChart(), - TearsheetEquityChart(), - TearsheetBarsWithFillsChart( - bar_type="ESM4.XCME-1-MINUTE-LAST-EXTERNAL", - title="Bars with Fills", - ), - ], -) -create_tearsheet(engine=engine, config=config) - -# Multiple bars-with-fills charts in one tearsheet -config = TearsheetConfig( - charts=[ - TearsheetStatsTableChart(), - TearsheetEquityChart(), - TearsheetBarsWithFillsChart( - bar_type=f"{instrument.id}-5-MINUTE-MID-INTERNAL", - title=f"Bars with Order Fills - {instrument.id}", - ), - TearsheetBarsWithFillsChart( - bar_type=f"{other_instrument.id}-5-MINUTE-MID-INTERNAL", - title=f"Bars with Order Fills - {other_instrument.id}", - ), - ], -) -create_tearsheet(engine=engine, config=config) -``` - -The visualization shows candlesticks for OHLC price action with triangle markers representing order -fills (green up-triangles for buys, red down-triangles for sells). Charts that need extra -configuration (like `bar_type`) take those parameters directly on the chart object -(e.g. `TearsheetBarsWithFillsChart(bar_type=...)`). - -Other individual chart functions include `create_equity_curve`, `create_drawdown_chart`, -`create_monthly_returns_heatmap`, and more. See the API reference for the complete list. - -## Related guides - -- [Backtesting](backtesting.md) - Learn how to run backtests that generate tearsheets. -- [Reports](reports.md) - Understand the underlying statistics displayed in tearsheets. -- [Portfolio](portfolio.md) - Explore portfolio tracking and performance metrics. diff --git a/nautilus-docs/docs/dev_templates/criterion_template.rs b/nautilus-docs/docs/dev_templates/criterion_template.rs deleted file mode 100644 index b6ac217..0000000 --- a/nautilus-docs/docs/dev_templates/criterion_template.rs +++ /dev/null @@ -1,52 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. -// https://nautechsystems.io -// -// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ------------------------------------------------------------------------------------------------- - -//! Criterion benchmark template. -//! -//! Copy this file into `crates//benches/` and adjust the names and -//! imports. Compile with -//! -//! ```bash -//! cargo bench -p --bench -//! ``` - -use std::hint::black_box; - -use criterion::{criterion_group, criterion_main, Criterion}; - -// ----------------------------------------------------------------------------- -// Replace `my_function` and set-up code with your real workload. -// ----------------------------------------------------------------------------- - -fn my_function(input: &[u8]) -> usize { - input.iter().map(|b| *b as usize).sum() -} - -fn prepare_data() -> Vec { - (0u8..=255).collect() -} - -fn bench_my_function(c: &mut Criterion) { - let data = prepare_data(); - - c.bench_function("my_function", |b| { - b.iter(|| { - let _ = black_box(my_function(&data)); - }); - }); -} - -criterion_group!(benches, bench_my_function); -criterion_main!(benches); diff --git a/nautilus-docs/docs/dev_templates/iai_template.rs b/nautilus-docs/docs/dev_templates/iai_template.rs deleted file mode 100644 index 741a856..0000000 --- a/nautilus-docs/docs/dev_templates/iai_template.rs +++ /dev/null @@ -1,33 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. -// https://nautechsystems.io -// -// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ------------------------------------------------------------------------------------------------- - -//! iai benchmark template. -//! -//! Copy this file into `crates//benches/` and adjust names and -//! imports. - -use std::hint::black_box; - -// ----------------------------------------------------------------------------- -// Replace `fast_add` with the real function you want to measure. -// ----------------------------------------------------------------------------- - -fn fast_add() -> i32 { - let a = black_box(1); - let b = black_box(2); - a + b -} - -iai::main!(fast_add); diff --git a/nautilus-docs/docs/developer_guide/adapters.md b/nautilus-docs/docs/developer_guide/adapters.md deleted file mode 100644 index c4e8e82..0000000 --- a/nautilus-docs/docs/developer_guide/adapters.md +++ /dev/null @@ -1,2131 +0,0 @@ -# Adapters - -## Introduction - -This developer guide provides specifications for how to build an integration adapter for the NautilusTrader platform. - -Adapters connect to trading venues and data providers, translating their native APIs into the platform’s unified interface and normalized domain model. - -## Structure of an adapter - -NautilusTrader adapters follow a layered architecture pattern with: - -- **Rust core** for networking clients and performance-sensitive operations. -- **Python layer** for integrating Rust clients into the platform's data and execution engines. - -### Rust core (`crates/adapters/your_adapter/`) - -The Rust layer handles: - -- **HTTP client**: Raw API communication, request signing, rate limiting. -- **WebSocket client**: Low-latency streaming connections, message parsing. -- **Parsing**: Fast conversion of venue data to Nautilus domain models. -- **Python bindings**: PyO3 exports to make Rust functionality available to Python. - -Typical Rust structure: - -``` -crates/adapters/your_adapter/ -├── src/ -│ ├── common/ # Shared types and utilities -│ │ ├── consts.rs # Venue constants / broker IDs -│ │ ├── credential.rs # API key storage and signing helpers -│ │ ├── enums.rs # Venue enums mirrored in REST/WS payloads -│ │ ├── error.rs # Adapter-level error aggregation (when applicable) -│ │ ├── models.rs # Shared model types -│ │ ├── parse.rs # Shared parsing helpers -│ │ ├── retry.rs # Retry classification (when applicable) -│ │ ├── urls.rs # Environment & product aware base-url resolvers -│ │ └── testing.rs # Fixtures reused across unit tests -│ ├── http/ # HTTP client implementation -│ │ ├── client.rs # HTTP client with authentication -│ │ ├── error.rs # HTTP-specific error types -│ │ ├── models.rs # Structs for REST payloads -│ │ ├── parse.rs # Response parsing functions -│ │ └── query.rs # Request and query builders -│ ├── websocket/ # WebSocket implementation -│ │ ├── client.rs # WebSocket client -│ │ ├── dispatch.rs # Execution event dispatch and order routing -│ │ ├── enums.rs # WebSocket-specific enums -│ │ ├── error.rs # WebSocket-specific error types -│ │ ├── handler.rs # Feed handler (I/O boundary) -│ │ ├── messages.rs # Frame and message enums -│ │ ├── parse.rs # Message parsing functions -│ │ └── subscription.rs # Subscription topic helpers (optional) -│ ├── python/ # PyO3 Python bindings -│ │ ├── enums.rs # Python-exposed enums -│ │ ├── http.rs # Python HTTP client bindings -│ │ ├── urls.rs # Python URL helpers -│ │ ├── websocket.rs # Python WebSocket client bindings -│ │ └── mod.rs # Module exports -│ ├── config.rs # Configuration structures -│ ├── data.rs # Data client implementation -│ ├── execution.rs # Execution client implementation -│ ├── factories.rs # Factory functions -│ └── lib.rs # Library entry point -├── tests/ # Integration tests with mock servers -│ ├── data_client.rs # Data client integration tests -│ ├── exec_client.rs # Execution client integration tests -│ ├── http.rs # HTTP client integration tests -│ └── websocket.rs # WebSocket client integration tests -└── test_data/ # Canonical venue payloads -``` - -### Python layer (`nautilus_trader/adapters/your_adapter`) - -The Python layer provides the integration interface through these components: - -1. **Instrument Provider**: Supplies instrument definitions via `InstrumentProvider`. -2. **Data Client**: Handles market data feeds and historical data requests via `LiveDataClient` and `LiveMarketDataClient`. -3. **Execution Client**: Manages order execution via `LiveExecutionClient`. -4. **Factories**: Converts venue-specific data to Nautilus domain models. -5. **Configuration**: User-facing configuration classes for client settings. - -Typical Python structure: - -``` -nautilus_trader/adapters/your_adapter/ -├── config.py # Configuration classes -├── constants.py # Adapter constants -├── data.py # LiveDataClient/LiveMarketDataClient -├── execution.py # LiveExecutionClient -├── factories.py # Instrument factories -├── providers.py # InstrumentProvider -└── __init__.py # Package initialization -``` - -## Adapter implementation sequence - -Follow this dependency-driven order when building an adapter. Each phase -builds on the previous one. Implement the Rust core before any Python layer. - -### Phase 1: Rust core infrastructure - -Build the low-level networking and parsing foundation. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 1.1 | HTTP error types | Define HTTP-specific error enum with retryable/non-retryable variants (`http/error.rs`). | -| 1.2 | HTTP client | Implement credentials, request signing, rate limiting, and retry logic. | -| 1.3 | HTTP API models | Define request/response structs for REST endpoints (`http/models.rs`, `http/query.rs`). | -| 1.4 | HTTP parsing | Convert venue responses to Nautilus domain models (`http/parse.rs`, `common/parse.rs`). | -| 1.5 | WebSocket error types | Define WebSocket-specific error enum (`websocket/error.rs`). | -| 1.6 | WebSocket client | Implement connection lifecycle, authentication, heartbeat, and reconnection. | -| 1.7 | WebSocket messages | Define streaming payload types (`websocket/messages.rs`). | -| 1.8 | WebSocket parsing | Convert stream messages to Nautilus domain models (`websocket/parse.rs`). | -| 1.9 | Python bindings | Expose Rust functionality via PyO3 (`python/mod.rs`). | - -**Milestone**: Rust crate compiles, unit tests pass, HTTP/WebSocket clients can authenticate and stream/request raw data. - -### Phase 2: Instrument definitions - -Instruments are the foundation: both data and execution clients depend on them. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 2.1 | Instrument parsing | Parse venue instrument definitions into Nautilus types (spot, perpetual, future, option). | -| 2.2 | Instrument provider | Implement `InstrumentProvider` to load, filter, and cache instruments. | -| 2.3 | Symbol mapping | Handle venue-specific symbol formats and Nautilus `InstrumentId` conversion. | - -**Milestone**: `InstrumentProvider.load_all_async()` returns valid Nautilus instruments. - -### Phase 3: Market data - -Build data subscriptions and historical data requests. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 3.1 | Public WebSocket streams | Subscribe to order books, trades, tickers, and other public channels. | -| 3.2 | Historical data requests | Fetch historical bars, trades, and order book snapshots via HTTP. | -| 3.3 | Data client (Python) | Implement `LiveDataClient` or `LiveMarketDataClient` wiring Rust clients to the data engine. | - -**Milestone**: Data client connects, subscribes to instruments, and emits market data to the platform. - -### Phase 4: Order execution - -Build order management and account state. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 4.1 | Private WebSocket streams | Subscribe to order updates, fills, positions, and account balance changes. | -| 4.2 | Basic order submission | Implement market and limit orders via HTTP or WebSocket. | -| 4.3 | Order modification/cancel | Implement order amendment and cancellation. | -| 4.4 | Execution client (Python) | Implement `LiveExecutionClient` wiring Rust clients to the execution engine. | -| 4.5 | Execution reconciliation | Generate order, fill, and position status reports for startup reconciliation. | - -**Milestone**: Execution client submits orders, receives fills, and reconciles state on connect. - -### Phase 5: Advanced features - -Extend coverage based on venue capabilities. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 5.1 | Advanced order types | Conditional orders, stop-loss, take-profit, trailing stops, iceberg, etc. | -| 5.2 | Batch operations | Batch order submission, batch cancellation, mass cancel. | -| 5.3 | Venue-specific features | Options chains, funding rates, liquidations, or other venue-specific data. | - -### Phase 6: Configuration and factories - -Wire everything together for production usage. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 6.1 | Configuration classes | Create `LiveDataClientConfig` and `LiveExecClientConfig` subclasses. | -| 6.2 | Factory functions | Implement factory functions to instantiate clients from configuration. | -| 6.3 | Environment variables | Support credential resolution from environment variables. | - -### Phase 7: Testing and documentation - -Validate the integration and document usage. - -| Step | Component | Description | -|------|----------------------------|----------------------------------------------------------------------------------------------| -| 7.1 | Rust unit tests | Test parsers, signing helpers, and business logic in `#[cfg(test)]` blocks. | -| 7.2 | Rust integration tests | Test HTTP/WebSocket clients against mock Axum servers in `tests/`. | -| 7.3 | Python integration tests | Test data/execution clients in `tests/integration_tests/adapters//`. | -| 7.4 | Example scripts | Provide runnable examples demonstrating data subscription and order execution. | - -See the [Testing](#testing) section for detailed test organization guidelines. - ---- - -## Rust adapter patterns - -### Common code (`common/`) - -Group venue constants, credential helpers, enums, and reusable parsers under `src/common`. -Adapters such as OKX keep submodules like `consts`, `credential`, `enums`, and `urls` alongside a `testing` module -for fixtures, providing a single place for cross-cutting pieces. -When an adapter has multiple environments or product categories, add a dedicated `common::urls` helper so -REST/WebSocket base URLs stay in sync with the Python layer. - -### URL resolution - -Define URL constants and resolution functions in `common/urls.rs`: - -```rust -const VENUE_WS_URL: &str = "wss://stream.venue.com/ws"; -const VENUE_TESTNET_WS_URL: &str = "wss://testnet-stream.venue.com/ws"; - -pub const fn get_ws_base_url(testnet: bool) -> &'static str { - if testnet { VENUE_TESTNET_WS_URL } else { VENUE_WS_URL } -} -``` - -Config structs should provide override fields (`base_url_http`, `base_url_ws`, etc.) that fall back -to these defaults when unset. - -### Configurations (`config.rs`) - -Expose typed config structs in `src/config.rs` so Python callers toggle venue-specific behaviour -(see how OKX wires demo URLs, retries, and channel flags). -Keep defaults minimal and delegate URL selection to helpers in `common::urls`. - -All config structs (data and execution) must implement `Default`. This enables the -`..Default::default()` pattern in examples and tests, keeping only the fields that differ -from defaults visible: - -```rust -let exec_config = VenueExecClientConfig { - trader_id, - account_id, - environment: VenueEnvironment::Testnet, - ..Default::default() -}; -``` - -Default values should use sensible production defaults: credentials as `None` (resolved -from environment at runtime), mainnet URLs, standard timeouts. For `trader_id` and -`account_id`, use placeholder values like `TraderId::from("TRADER-001")` and -`AccountId::from("VENUE-001")`. - -### Error taxonomy (`common/error.rs`) - -For adapters with multiple client types, define an adapter-level error enum in `common/error.rs` that -aggregates component errors: - -```rust -#[derive(Debug, thiserror::Error)] -pub enum VenueError { - #[error("HTTP error: {0}")] - Http(#[from] VenueHttpError), - - #[error("WebSocket error: {0}")] - WebSocket(#[from] VenueWsError), - - #[error("Build error: {0}")] - Build(#[from] VenueBuildError), -} -``` - -This enables unified error handling at the adapter boundary while preserving component-specific -error details for debugging. - -### Retry classification (`common/retry.rs`) - -When an adapter needs sophisticated retry logic, define a retry classification module in `common/retry.rs` -that distinguishes between retryable, non-retryable, and fatal errors: - -```rust -#[derive(Debug, thiserror::Error)] -pub enum VenueError { - #[error("Retryable error: {source}")] - Retryable { - #[source] - source: VenueRetryableError, - retry_after: Option, - }, - - #[error("Non-retryable error: {source}")] - NonRetryable { - #[source] - source: VenueNonRetryableError, - }, - - #[error("Fatal error: {source}")] - Fatal { - #[source] - source: VenueFatalError, - }, -} -``` - -Include helper methods like `from_http_status()`, `from_rate_limit_headers()`, `is_retryable()`, -`is_fatal()`, and `retry_after()` to enable consistent error classification across the adapter. -See BitMEX and Bybit adapters for reference implementations. - -### Python exports (`python/mod.rs`) - -Mirror the Rust surface area through PyO3 modules by re-exporting clients, enums, and helper functions. -When new functionality lands in Rust, add it to `python/mod.rs` so the Python layer stays in sync -(the OKX adapter is a good reference). - -### Python bindings (`python/`) - -Expose Rust functionality to Python through PyO3. -Mark venue-specific structs that need Python access with `#[pyclass]` and implement `#[pymethods]` blocks with -`#[getter]` attributes for field access. - -For async methods in the HTTP client, use `pyo3_async_runtimes::tokio::future_into_py` to convert Rust futures -into Python awaitables. -When returning lists of custom types, map each item with `Py::new(py, item)` before constructing the Python list. -Register all exported classes and enums in `python/mod.rs` using `m.add_class::()` so they're available -to Python code. - -Follow the pattern established in other adapters: prefixing Python-facing methods with `py_*` in Rust while using -`#[pyo3(name = "method_name")]` to expose them without the prefix. - -When delivering instruments from WebSocket to Python, use `instrument_any_to_pyobject()` which returns PyO3 types -for caching. -For the reverse direction (Python→Rust), use `pyobject_to_instrument_any()` in `cache_instrument()` methods. -Never call `.into_py_any()` directly on `InstrumentAny` as it doesn't implement the required trait. - -### Type qualification - -Adapter-specific types (enums, structs) and Nautilus domain types should not be fully qualified. -Import them at the module level and use short names (e.g., `OKXContractType` instead of -`crate::common::enums::OKXContractType`, `InstrumentId` instead of `nautilus_model::identifiers::InstrumentId`). -This keeps code concise and readable. -Only fully qualify types from `anyhow` and `tokio` to avoid ambiguity with similarly-named types from other crates. - -### String interning - -Use `ustr::Ustr` for any non-unique strings the platform stores repeatedly (venues, symbols, instrument IDs) to -minimise allocations and comparisons. - -### Instrument cache standardization - -All clients that cache instruments must implement three methods with standardized names: `cache_instruments()` -(plural, bulk replace), `cache_instrument()` (singular, upsert), and `get_instrument()` (retrieve by symbol). -WebSocket clients store instruments in `Arc>` on the outer client for -thread-safe access across clones. - -### Testing helpers (`common/testing.rs`) - -Store shared fixtures and payload loaders in `src/common/testing.rs` for use across HTTP and WebSocket unit tests. -This keeps `#[cfg(test)]` helpers out of production modules and encourages reuse. - -### Factory module (`factories.rs`) - -Complex adapters may define a `factories.rs` module for converting venue data to Nautilus types. -This centralizes transformation logic that would otherwise be scattered across HTTP and WebSocket -parsers: - -```rust -// factories.rs -pub fn create_instrument( - venue_instrument: &VenueInstrument, - ts_init: UnixNanos, -) -> anyhow::Result { - match venue_instrument.instrument_type { - InstrumentType::Perpetual => parse_perpetual(venue_instrument, ts_init), - InstrumentType::Future => parse_future(venue_instrument, ts_init), - InstrumentType::Option => parse_option(venue_instrument, ts_init), - } -} -``` - -Use this pattern when the same venue data structures are parsed in multiple places (HTTP responses, -WebSocket updates, historical data). - -### Connection lifecycle (`connect`) - -Both data and execution clients follow a strict initialization order during `connect()` to prevent -race conditions with reconciliation and strategy startup. The platform waits for all clients to -signal connected before running reconciliation or starting strategies, so all initialization must -complete within `connect()`. - -#### Data client - -1. **Fetch instruments via REST** - call `bootstrap_instruments()` or equivalent. -2. **Cache locally** - populate the client's internal instrument map and HTTP client cache. -3. **Emit to data engine** - send each instrument as `DataEvent::Instrument` via `data_sender`. - These events are queued during startup and processed before reconciliation runs. -4. **Cache to WebSocket** - call `ws.cache_instruments()` so the handler can parse messages. -5. **Connect WebSocket** - establish the streaming connection. - -```rust -async fn connect(&mut self) -> anyhow::Result<()> { - let instruments = self.bootstrap_instruments().await?; - ws.cache_instruments(instruments); - ws.connect().await?; - ws.wait_until_active(10.0).await?; - // ... -} -``` - -#### Execution client - -1. **Initialize instruments** - fetch via REST if not already cached. Cache to HTTP, WebSocket, - and broadcaster clients. -2. **Connect WebSocket** - establish the private streaming connection. -3. **Subscribe to channels** - orders, executions, positions, wallet/margin. -4. **Start WebSocket stream handler** - begin processing incoming messages. -5. **Fetch account state** - request account state via REST and emit via `emitter.send_account_state()`. -6. **Await account registered** - poll the cache until the account is registered (30s timeout). - This ensures the portfolio can process orders during reconciliation. -7. **Signal connected** - call `self.core.set_connected()`. - -```rust -async fn connect(&mut self) -> anyhow::Result<()> { - self.ensure_instruments_initialized_async().await?; - - self.ws_client.connect().await?; - self.ws_client.wait_until_active(10.0).await?; - // ... subscribe channels, start stream ... - - self.refresh_account_state().await?; - self.await_account_registered(30.0).await?; - - self.core.set_connected(); - Ok(()) -} -``` - -The `await_account_registered` method polls `self.core.cache().account(&account_id)` at 10ms -intervals until the account appears or the timeout expires. - -## HTTP client patterns - -Adapters use a two-layer HTTP client architecture: a raw client for low-level API operations and a domain -client for high-level logic. The split also enables efficient cloning for Python bindings. - -### Client structure - -The architecture consists of two complementary clients: - -1. **Raw client** (`MyRawHttpClient`) - Low-level API methods matching venue endpoints. -2. **Domain client** (`MyHttpClient`) - High-level methods using Nautilus domain types. - -```rust -use std::sync::Arc; -use nautilus_network::http::HttpClient; - -// Raw HTTP client - low-level API methods matching venue endpoints -pub struct MyRawHttpClient { - base_url: String, - client: HttpClient, // Use nautilus_network::http::HttpClient, not reqwest directly - credential: Option, - retry_manager: RetryManager, - cancellation_token: CancellationToken, -} - -// Domain HTTP client - wraps raw client with Arc, provides high-level API -pub struct MyHttpClient { - pub(crate) inner: Arc, - // Additional domain-specific state (e.g., instrument cache) - instruments: DashMap, -} -``` - -**Key points**: - -- **Raw client** (`MyRawHttpClient`) contains low-level HTTP methods named to match venue endpoints - (e.g., `get_instruments`, `get_balance`, `place_order`). These methods take venue-specific query - objects and return venue-specific response types. -- **Domain client** (`MyHttpClient`) wraps the raw client in an `Arc` for efficient cloning (required - for Python bindings). It provides high-level methods that accept Nautilus domain types - (e.g., `InstrumentId`, `ClientOrderId`) and return domain objects. It may also cache instruments - or other venue metadata. -- Use `nautilus_network::http::HttpClient` instead of `reqwest::Client` directly for rate limiting, - retry logic, and consistent error handling. -- Both clients are exposed to Python, but the domain client is the primary interface. - -### Parser functions - -Parser functions convert venue-specific data structures into Nautilus domain objects. Place them in -`common/parse.rs` for cross-cutting conversions (instruments, trades, bars) or `http/parse.rs` for -REST-specific transformations. Each parser takes venue data plus context (account IDs, timestamps, -instrument references) and returns a Nautilus domain type wrapped in `Result`. - -**Standard patterns:** - -- Handle string-to-numeric conversions with proper error context using `.parse::()` and `anyhow::Context`. -- Check for empty strings before parsing optional fields - venues often return `""` instead of omitting fields. -- Map venue enums to Nautilus enums explicitly with `match` statements rather than implementing automatic conversions that could hide mapping errors. -- Accept instrument references when precision or other metadata is required for constructing Nautilus types (quantities, prices). -- Use descriptive function names: `parse_position_status_report`, `parse_order_status_report`, `parse_trade_tick`. - -Place parsing helpers (`parse_price_with_precision`, `parse_timestamp`) in the same module as private functions when they're reused across multiple parsers. - -### Method naming and organization - -The raw client mirrors venue endpoints with venue-specific parameter and response types. The domain -client wraps it and exposes high-level methods that accept Nautilus domain types. - -**Naming conventions:** - -- **Raw client methods**: Named to match venue endpoints as closely as possible (e.g., `get_instruments`, `get_balance`, `place_order`). These methods are internal to the raw client and take venue-specific types (builders, JSON values). -- **Domain client methods**: Named based on operation semantics (e.g., `request_instruments`, `submit_order`, `cancel_order`). These are the methods exposed to Python and take Nautilus domain objects (InstrumentId, ClientOrderId, OrderSide, etc.). - -**Domain method flow:** - -Domain methods follow a three-step pattern: build venue-specific parameters from Nautilus types, call the corresponding raw client method, then parse the response. For endpoints returning domain objects (positions, orders, trades), call parser functions from `common/parse`. For endpoints returning raw venue data (fee rates, balances), extract the result directly from the response envelope. Methods prefixed with `request_*` indicate they return domain data, while methods like `submit_*`, `cancel_*`, or `modify_*` perform actions and return acknowledgments. - -The domain client wraps the raw client in an `Arc` for efficient cloning required by Python bindings. - -### Query parameter builders - -Use the `derive_builder` crate with proper defaults and ergonomic Option handling: - -```rust -use derive_builder::Builder; - -#[derive(Clone, Debug, Deserialize, Serialize, Builder)] -#[serde(rename_all = "camelCase")] -#[builder(setter(into, strip_option), default)] -pub struct InstrumentsInfoParams { - pub category: ProductType, - #[serde(skip_serializing_if = "Option::is_none")] - pub symbol: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub limit: Option, -} - -impl Default for InstrumentsInfoParams { - fn default() -> Self { - Self { - category: ProductType::Linear, - symbol: None, - limit: None, - } - } -} -``` - -**Key attributes:** - -- `#[builder(setter(into, strip_option), default)]` - enables clean API: `.symbol("BTCUSDT")` instead of `.symbol(Some("BTCUSDT".to_string()))`. -- `#[serde(skip_serializing_if = "Option::is_none")]` - omits optional fields from query strings. -- Always implement `Default` for builder parameters. - -### Request signing and authentication - -Keep signing logic in a `Credential` struct under `common/credential.rs`: - -- Store API keys using `Ustr` for efficient comparison, secrets in `Box<[u8]>` with `#[zeroize]`. -- Implement `sign()` and `sign_bytes()` methods that compute HMAC-SHA256 signatures. -- Pass the credential to the raw HTTP client; the domain client delegates signing through the inner client. - -For WebSocket authentication, the handler constructs login messages using the same `Credential::sign()` method with a WebSocket-specific timestamp format. - -### Credential module structure - -Each adapter's `common/credential.rs` must provide two things: - -1. **`credential_env_vars()` free function**: returns environment variable names as a tuple. -2. **`Credential::resolve()` method**: resolves credentials from config values or environment - variables using `resolve_env_var_pair` from `nautilus_core::env`. - -Config structs are DTOs and must not contain credential resolution logic. All resolution -belongs in `credential.rs`. - -**Standard layout:** - -```rust -use nautilus_core::env::resolve_env_var_pair; - -/// Returns the environment variable names for API credentials. -pub fn credential_env_vars(is_testnet: bool) -> (&'static str, &'static str) { - if is_testnet { - ("{VENUE}_TESTNET_API_KEY", "{VENUE}_TESTNET_API_SECRET") - } else { - ("{VENUE}_API_KEY", "{VENUE}_API_SECRET") - } -} - -impl Credential { - /// Resolves credentials from provided values or environment variables. - pub fn resolve( - api_key: Option, - api_secret: Option, - is_testnet: bool, - ) -> Option { - let (key_var, secret_var) = credential_env_vars(is_testnet); - let (k, s) = resolve_env_var_pair(api_key, api_secret, key_var, secret_var)?; - Some(Self::new(k, s)) - } -} -``` - -### Environment variable conventions - -Adapters load API credentials from environment variables when not provided directly, avoiding -hardcoded secrets. - -**Naming conventions:** - -| Environment | API Key Variable | API Secret Variable | -|--------------|---------------------------|---------------------| -| Mainnet/Live | `{VENUE}_API_KEY` | `{VENUE}_API_SECRET` | -| Testnet | `{VENUE}_TESTNET_API_KEY` | `{VENUE}_TESTNET_API_SECRET` | -| Demo | `{VENUE}_DEMO_API_KEY` | `{VENUE}_DEMO_API_SECRET` | - -Some venues require additional credentials: - -- OKX: `OKX_API_PASSPHRASE` - -**Key principles:** - -- Environment variable names must be centralized in `credential_env_vars()`, never - duplicated as string literals across files. -- Environment variable resolution should happen in core Rust code, not Python bindings. -- Use `get_or_env_var_opt` for optional credentials (returns `None` if missing). -- Use `get_or_env_var` when credentials are required (returns error if missing). -- Invalid credentials (e.g. malformed keys) must fail fast with an error, never silently - degrade to unauthenticated mode. - -### Error handling and retry logic - -Use the `RetryManager` from `nautilus_network` for consistent retry behavior. - -### Rate limiting - -Configure rate limiting through `HttpClient` using `LazyLock` static variables. - -**Naming conventions:** - -- REST quotas: `{VENUE}_REST_QUOTA` (e.g., `OKX_REST_QUOTA`, `BYBIT_REST_QUOTA`) -- WebSocket quotas: `{VENUE}_WS_{OPERATION}_QUOTA` (e.g., `OKX_WS_CONNECTION_QUOTA`, `OKX_WS_ORDER_QUOTA`) -- Rate limit keys: `{VENUE}_RATE_LIMIT_KEY_{OPERATION}` (e.g., `OKX_RATE_LIMIT_KEY_SUBSCRIPTION`, `OKX_RATE_LIMIT_KEY_ORDER`) - -**Standard rate limit keys for WebSocket:** - -| Key | Operations | -|---------------------------------|----------------------------------| -| `*_RATE_LIMIT_KEY_SUBSCRIPTION` | Subscribe, unsubscribe, login. | -| `*_RATE_LIMIT_KEY_ORDER` | Place orders (regular and algo). | -| `*_RATE_LIMIT_KEY_CANCEL` | Cancel orders, mass cancel. | -| `*_RATE_LIMIT_KEY_AMEND` | Amend/modify orders. | - -**Example:** - -```rust -pub static OKX_REST_QUOTA: LazyLock = - LazyLock::new(|| Quota::per_second(NonZeroU32::new(250).unwrap())); - -pub static OKX_WS_SUBSCRIPTION_QUOTA: LazyLock = - LazyLock::new(|| Quota::per_hour(NonZeroU32::new(480).unwrap())); - -pub const OKX_RATE_LIMIT_KEY_ORDER: &str = "order"; -``` - -Pass rate limit keys when sending WebSocket messages to enforce per-operation quotas: - -```rust -self.send_with_retry(payload, Some(vec![OKX_RATE_LIMIT_KEY_ORDER.to_string()])).await -``` - -## WebSocket client patterns - -WebSocket clients handle real-time streaming data. They manage connection state, authentication, -subscriptions, and reconnection logic. - -### Client structure - -WebSocket adapters use a **two-layer architecture** to separate Python-accessible state from high-performance async I/O: - -#### Connection state tracking - -Track connection state using `Arc>` to provide lock-free, race-free visibility across all clones: - -```rust -use arc_swap::ArcSwap; - -pub struct MyWebSocketClient { - connection_mode: Arc>, // Shared connection mode (lock-free) - signal: Arc, // Cancellation signal for graceful shutdown - // ... -} -``` - -**Pattern breakdown:** - -- **Outer `Arc`**: Shared across all clones (Python bindings clone clients before async operations). -- **`ArcSwap`**: Enables atomic pointer replacement via `.store()` without replacing the outer Arc. -- **Inner `Arc`**: The actual connection state from `WebSocketClient::connection_mode_atomic()`. - -Initialize with a placeholder atomic (`ConnectionMode::Closed`), then in `connect()` call -`.store(client.connection_mode_atomic())` to atomically swap to the real client's state. -All clones see updates instantly through lock-free `.load()` calls in `is_active()`. - -The underlying `WebSocketClient` sends a `RECONNECTED` sentinel message when reconnection completes, triggering resubscription logic in the handler. - -**Outer client** (`{Venue}WebSocketClient`): - -- Orchestrates connection lifecycle, authentication, subscriptions. -- Maintains state for Python access using `Arc>`. -- Tracks subscription state for reconnection logic. -- Stores instruments cache for replay on reconnect. -- Sends commands to handler via `cmd_tx` channel. -- Receives venue events via `out_rx` channel. - -**Inner handler** (`{Venue}WsFeedHandler`): - -- Runs in dedicated Tokio task as stateless I/O boundary. -- Owns `WebSocketClient` exclusively (no `RwLock` needed). -- Processes commands from `cmd_rx` → serializes to JSON → sends via WebSocket. -- Receives raw WebSocket messages → deserializes into `{Venue}WsFrame` → converts to `{Venue}WsMessage` → emits via `out_tx`. -- Owns pending request state using `AHashMap` (single-threaded, no locking). -- Uses `VecDeque<{Venue}WsMessage>` to buffer multi-message yields from a single frame parse. - -Some venues expose separate WebSocket endpoints for market data and order management -(different URLs, authentication flows, or message protocols). In this case, split into -two client+handler pairs under `websocket/data/` and `websocket/orders/` subdirectories, -each following the same two-layer pattern. Name them `{Venue}MdWebSocketClient` / -`{Venue}MdWsFeedHandler` and `{Venue}OrdersWebSocketClient` / `{Venue}OrdersWsFeedHandler`. - -**Communication pattern:** - -```mermaid -flowchart LR - subgraph client["Client (orchestrator)"] - cmd_tx["cmd_tx
├ Subscribe { args }
├ PlaceOrder { params }
└ MassCancel { id }"] - out_rx["out_rx
← {Venue}WsMessage
← Authenticated
← ChannelData"] - end - - subgraph handler["Handler (I/O boundary)"] - cmd_rx[cmd_rx] - out_tx[out_tx] - ws[WebSocket] - end - - cmd_tx --> cmd_rx - cmd_rx -->|"serialize"| ws - ws -->|"parse → transform"| out_tx - out_tx --> out_rx -``` - -**Key principles:** - -- **No shared locks on hot path**: Handler owns `WebSocketClient`, client sends commands via lock-free mpsc channel. -- **Command pattern for all sends**: Subscriptions, orders, cancellations all route through `HandlerCommand` enum. -- **Event pattern for state**: Handler emits `{Venue}WsMessage` events (including `Authenticated`), client maintains state from events. -- **Pending state ownership**: Handler owns `AHashMap` for matching responses (no `Arc` between layers). -- **Message buffering**: Handler uses `VecDeque<{Venue}WsMessage>` for frames that produce multiple output messages. The `next()` method drains the queue before polling channels. -- **Python constraint**: Client uses `Arc` only for state Python might query; handler uses `AHashMap` for internal matching. - -### Authentication - -Authentication state is managed through events: - -- Handler processes `Login` response → **returns** `{Venue}WsMessage::Authenticated` immediately. -- Client receives event → updates local auth state → proceeds with subscriptions. -- `AuthTracker` may be shared via `Arc` for state queries, but handler returns events directly (no blocking). - -**Note**: The `Authenticated` message is consumed in the client's spawn loop for reconnection -flow coordination and is not forwarded to downstream consumers (data/execution clients). -Downstream consumers can query authentication state via `AuthTracker` if needed. The execution -client's `Authenticated` handler only logs at debug level with no important logic depending -on this event. - -### Subscription management - -#### Shared `SubscriptionState` pattern - -The `SubscriptionState` struct from `nautilus_network::websocket` is shared between client and handler using `Arc>` internally for thread-safe access: - -- **`SubscriptionState` is shared via `Arc`**: Both client and handler receive `.clone()` of the same instance (shallow clone of Arc pointers). -- **Responsibility split**: Client tracks user intent (`mark_subscribe`, `mark_unsubscribe`), handler tracks server confirmations (`confirm_subscribe`, `confirm_unsubscribe`, `mark_failure`). -- **Why both need it**: Single source of truth with lock-free concurrent access, no synchronization overhead. - -#### Subscription lifecycle - -A **subscription** represents any topic in one of two states: - -| State | Description | -|---------------|-------------| -| **Pending** | Subscription request sent to venue, awaiting acknowledgment. | -| **Confirmed** | Venue acknowledged subscription and is actively streaming data. | - -State transitions follow this lifecycle: - -| Trigger | Method Called | From State | To State | Notes | -|-------------------|----------------------|------------|-----------|-------| -| User subscribes | `mark_subscribe()` | — | Pending | Topic added to pending set. | -| Venue confirms | `confirm()` | Pending | Confirmed | Moved from pending to confirmed. | -| Venue rejects | `mark_failure()` | Pending | Pending | Stays pending for retry on reconnect. | -| User unsubscribes | `mark_unsubscribe()` | Confirmed | Pending | Temporarily pending until ack. | -| Unsubscribe ack | `clear_pending()` | Pending | Removed | Topic fully removed. | - -**Key principles**: - -- `subscription_count()` reports **only confirmed subscriptions**, not pending ones. -- Failed subscriptions remain pending and are automatically retried on reconnect. -- Both confirmed and pending subscriptions are restored after reconnection. -- Unsubscribe operations must check the `op` field in acknowledgments to avoid re-confirming topics. - -#### Topic format patterns - -Adapters use venue-specific delimiters to structure subscription topics: - -| Adapter | Delimiter | Example | Pattern | -|------------|-----------|------------------------|------------------------------| -| **BitMEX** | `:` | `trade:XBTUSD` | `{channel}:{symbol}` | -| **OKX** | `:` | `trades:BTC-USDT-SWAP` | `{channel}:{symbol}` | -| **Bybit** | `.` | `orderbook.50.BTCUSDT` | `{channel}.{depth}.{symbol}` | - -Parse topics using `split_once()` with the appropriate delimiter to extract channel and symbol components. - -### Reconnection logic - -On reconnection, restore authentication and subscriptions: - -1. **Track subscriptions**: Preserve original subscription arguments in collections (e.g., `Arc`) to avoid parsing topics back to arguments. - -2. **Reconnection flow**: - - Receive `{Venue}WsMessage::Reconnected` from handler. - - If authenticated: Re-authenticate and wait for confirmation. - - Restore all tracked subscriptions via handler commands. - -**Preserving subscription arguments:** - -Store original subscription arguments in a separate collection to enable deterministic reconnection -replay without parsing topics back into arguments: - -```rust -pub struct MyWebSocketClient { - subscription_state: Arc, - subscription_args: Arc>, // topic → original args - // ... -} - -impl MyWebSocketClient { - async fn subscribe(&self, args: SubscriptionArgs) -> Result<(), Error> { - let topic = args.to_topic(); - self.subscription_state.mark_subscribe(&topic); - self.subscription_args.insert(topic.clone(), args.clone()); - self.send_cmd(HandlerCommand::Subscribe(args)).await - } - - async fn unsubscribe(&self, topic: &str) -> Result<(), Error> { - self.subscription_state.mark_unsubscribe(topic); - self.subscription_args.remove(topic); - self.send_cmd(HandlerCommand::Unsubscribe(topic.to_string())).await - } - - async fn restore_subscriptions(&self) { - for entry in self.subscription_args.iter() { - let _ = self.send_cmd(HandlerCommand::Subscribe(entry.value().clone())).await; - } - } -} -``` - -This avoids complex topic parsing and ensures subscriptions are replayed exactly as originally -requested. - -### Ping/Pong handling - -Support both WebSocket control frame pings and application-level text pings: - -- **Control frame pings**: Handled automatically by `WebSocketClient` via the `PingHandler` callback. -- **Text pings**: Some venues (e.g., OKX) use `"ping"`/`"pong"` text messages. Configure `heartbeat_msg: Some(TEXT_PING.to_string())` in `WebSocketConfig` and respond to incoming `TEXT_PING` with `TEXT_PONG` in the handler. - -The handler should check for ping messages early in the message processing loop and respond immediately to maintain connection health. - -### Disconnection lifecycle (`close`) - -The `close()` method follows a three-step shutdown sequence: signal, command, await. - -```rust -impl MyWebSocketClient { - pub async fn close(&mut self) -> Result<(), MyWsError> { - tracing::debug!("Starting close process"); - - // 1. Send disconnect command so handler can clean up gracefully - if let Err(e) = self.cmd_tx.read().await.send(HandlerCommand::Disconnect) { - tracing::warn!("Failed to send disconnect command to handler: {e}"); - } - - // 2. Set stop signal so handler loop exits after processing disconnect - self.signal.store(true, Ordering::Release); - - // 3. Await task handle with timeout, abort if stuck - if let Some(task_handle) = self.task_handle.take() { - match Arc::try_unwrap(task_handle) { - Ok(handle) => { - let abort_handle = handle.abort_handle(); - match tokio::time::timeout(Duration::from_secs(2), handle).await { - Ok(Ok(())) => tracing::debug!("Handler task completed"), - Ok(Err(e)) => tracing::error!("Handler task error: {e:?}"), - Err(_) => { - tracing::warn!("Timeout waiting for handler task, aborting"); - abort_handle.abort(); - } - } - } - Err(arc_handle) => { - tracing::debug!("Cannot unwrap task handle, aborting"); - arc_handle.abort(); - } - } - } - - Ok(()) - } -} -``` - -**Key points:** - -- Send `Disconnect` before setting the stop signal so the handler processes it before exiting. -- Return `Result<(), {Venue}WsError>` so callers can handle failures. -- Use `Ordering::Release` on the signal store so the handler sees the write. -- Extract `abort_handle` before awaiting so it remains available after timeout. -- When `Arc::try_unwrap` fails (other clones exist), abort directly. - -### Stream consumption (`stream`) - -The outer client exposes a `stream()` method that hands ownership of `out_rx` to the -caller as an async stream. Data and execution clients call this once to drive their -message processing loop: - -```rust -impl MyWebSocketClient { - pub fn stream(&mut self) -> impl Stream + 'static { - let rx = self - .out_rx - .take() - .expect("Stream receiver already taken or not connected"); - let mut rx = Arc::try_unwrap(rx) - .expect("Cannot take ownership - other references exist"); - async_stream::stream! { - while let Some(msg) = rx.recv().await { - yield msg; - } - } - } -} -``` - -The data/execution client consumes the stream in a `tokio::select!` loop with a -cancellation token or stop signal, matching on `{Venue}WsMessage` variants and calling -parse functions to produce Nautilus domain types. - -### Subscription topic helpers (`subscription.rs`) - -When a venue's subscription topics have complex structure (multiple parameter types, -instrument type / family / ID variants, candle width encoding), extract topic building -and parsing into `websocket/subscription.rs`. This keeps `client.rs` focused on -connection lifecycle and `handler.rs` focused on I/O. - -For venues with simple `{channel}:{symbol}` topics, inline helpers in the client are -sufficient and a separate module is not needed. - -### Handler configuration constants - -Define handler-specific tuning constants for consistent behavior: - -| Constant | Purpose | Typical value | -|----------------------------|--------------------------------------------------|---------------| -| `DEFAULT_HEARTBEAT_SECS` | Interval for sending keep-alive messages. | 15-30 | -| `WEBSOCKET_AUTH_WINDOW_MS` | Maximum age for authentication timestamps. | 5000-30000 | -| `BATCH_PROCESSING_LIMIT` | Maximum messages processed per event loop cycle. | 100-1000 | - -Place these in `websocket/handler.rs` or `common/consts.rs` depending on scope. - -### Message routing - -The handler uses two message enums to separate wire deserialization from emitted events. -The data and execution client layers convert emitted events into Nautilus domain types. - -Define two enums: - -1. **`{Venue}WsFrame`**: Serde-deserialized wire frames. Contains every JSON shape the venue - can send (login responses, subscription acks, channel data, order responses, errors, pings). - Typically `pub(super)` since only the handler uses it. - -2. **`{Venue}WsMessage`**: Handler output events emitted on `out_tx`. Contains the subset of - wire data the client needs plus synthetic control variants (`Reconnected`, `Authenticated`, - `SendFailed`) that have no wire representation. This is the `pub` type consumers match on. - -The handler deserializes raw text into `{Venue}WsFrame`, handles control frames internally -(subscription acks, login, pings), and converts relevant frames into `{Venue}WsMessage` events -sent via `out_tx`. The client receives from `out_rx` and routes to data/execution callbacks, -which convert venue types to Nautilus domain types using parse functions. - -#### Message type naming convention - -Types prefixed with the venue name (e.g., `OKX`, `Bitmex`) contain raw exchange-specific types. -Types prefixed with `Nautilus` contain normalized domain types ready for the trading system. - -**Wire frame enum (serde-deserialized, handler-internal):** - -```rust -pub(super) enum MyWsFrame { - Login { event, code, msg, conn_id }, - Subscription { event, arg, conn_id, code, msg }, - OrderResponse { id, op, code, msg, data }, - BookData { arg, action, data: Vec }, - Data { arg, data: Value }, - Error { code, msg }, - Ping, - Reconnected, -} -``` - -**Handler output enum (emitted to client):** - -```rust -pub enum MyWsMessage { - BookData { arg, action, data: Vec }, - ChannelData { channel, inst_id, data: Value }, - Orders(Vec), - OrderResponse { id, op, code, msg, data }, - SendFailed { request_id, client_order_id, op, error }, - Instruments(Vec), - Error(MyWebSocketError), - Reconnected, - Authenticated, -} -``` - -The frame enum includes every wire shape (login acks, subscription acks, pings) for -deserialization. The output enum drops shapes the handler consumes internally and adds synthetic -variants (`Authenticated`, `SendFailed`) that originate in handler logic, not on the wire. - -Include `OrderResponse` for venue acknowledgements (place, cancel, amend) and `SendFailed` for -WebSocket send failures after retries are exhausted. The execution client dispatch layer converts -these into Nautilus rejection events (`OrderRejected`, `OrderCancelRejected`, etc.). - -**Conversion in data/exec client:** - -The data client's message loop matches on `{Venue}WsMessage` variants and calls parse functions -to produce Nautilus domain types (`Data`, `OrderBookDeltas`, etc.). The execution client's -dispatch layer handles `OrderResponse`, `SendFailed`, and `Orders` variants. This keeps -the handler focused on I/O and deserialization while the client layers own domain conversion. - -The execution dispatch converts order and fill messages using a two-tier routing contract: - -1. The handler emits venue-specific order types (e.g., `Orders(Vec)`). -2. The client dispatch layer tracks which orders were submitted through this client. -3. **Tracked order**: convert venue types to order events (`OrderAccepted`, `OrderCanceled`, - `OrderFilled`, etc.) and synthesize any missing lifecycle events (e.g., `OrderAccepted` - before a fast fill). -4. **External/unknown order**: convert to reports (`OrderStatusReport` or `FillReport`) for - downstream reconciliation. - -#### `WsDispatchState` - -Execution dispatch state lives in a `WsDispatchState` struct defined in `websocket/dispatch.rs`. -It tracks which lifecycle events have already been emitted to prevent duplicates across -reconnections and fast-fill races: - -```rust -#[derive(Debug, Default)] -pub struct WsDispatchState { - pub order_identities: DashMap, - pub emitted_accepted: DashSet, - pub triggered_orders: DashSet, - pub filled_orders: DashSet, - clearing: AtomicBool, -} -``` - -| Field | Purpose | -|---------------------|-----------------------------------------------------------------| -| `order_identities` | Maps client order ID to identity metadata set at submission. | -| `emitted_accepted` | Prevents duplicate `OrderAccepted` events. | -| `triggered_orders` | Tracks conditional orders that have triggered. | -| `filled_orders` | Prevents duplicate `OrderFilled` events on reconnect replay. | -| `clearing` | Guards concurrent eviction when sets reach capacity. | - -Each `DashSet` is bounded by a `DEDUP_CAPACITY` constant (typically 10,000). When a set -reaches capacity, `evict_if_full()` clears it atomically using a compare-exchange on the -`clearing` flag to prevent concurrent clears. - -The `dispatch_ws_message()` free function in the same module routes `{Venue}WsMessage` -variants to the appropriate order event builders, using `WsDispatchState` for dedup -and `OrderIdentity` for tracked-vs-external classification. - -### Error handling - -#### Client-side error propagation - -Channel send failures (client → handler) should propagate loudly as `Result<(), Error>`: - -```rust -impl MyWebSocketClient { - async fn send_cmd(&self, cmd: HandlerCommand) -> Result<(), Error> { - self.cmd_tx.read().await.send(cmd) - .map_err(|e| Error::ClientError(format!("Handler not available: {e}"))) - } - - pub async fn submit_order(...) -> Result<(), Error> { - let cmd = HandlerCommand::PlaceOrder { ... }; - self.send_cmd(cmd).await // Propagates channel failures - } -} -``` - -#### Handler-side retry logic - -WebSocket send failures (handler → network) should be retried by the handler using `RetryManager`: - -```rust -pub struct MyWsFeedHandler { - inner: Option, - retry_manager: RetryManager, - // ... -} - -impl MyWsFeedHandler { - async fn send_with_retry(&self, payload: String, rate_limit_keys: Option>) -> Result<(), MyWsError> { - if let Some(client) = &self.inner { - self.retry_manager.execute_with_retry( - "websocket_send", - || async { - client.send_text(payload.clone(), rate_limit_keys.clone()) - .await - .map_err(|e| MyWsError::ClientError(format!("Send failed: {e}"))) - }, - should_retry_error, - create_timeout_error, - ).await - } else { - Err(MyWsError::ClientError("No active WebSocket client".to_string())) - } - } - - async fn handle_place_order(...) -> anyhow::Result<()> { - let payload = serde_json::to_string(&request)?; - - match self.send_with_retry(payload, Some(vec![RATE_LIMIT_KEY])).await { - Ok(()) => Ok(()), - Err(e) => { - // Emit SendFailed so the exec client dispatch can produce OrderRejected - let _ = self.out_tx.send(MyWsMessage::SendFailed { - request_id: request_id.clone(), - client_order_id: Some(client_order_id), - op: Some(MyWsOperation::Order), - error: e.to_string(), - }); - Err(anyhow::anyhow!("Failed to send order: {e}")) - } - } - } -} - -fn should_retry_error(error: &MyWsError) -> bool { - match error { - MyWsError::NetworkError(_) | MyWsError::Timeout(_) => true, - MyWsError::AuthenticationError(_) | MyWsError::ParseError(_) => false, - } -} -``` - -**Key principles:** - -- Client propagates channel failures immediately (handler unavailable). -- Handler retries transient WebSocket failures (network issues, timeouts). -- Handler emits `SendFailed` when retries are exhausted; the exec client dispatch converts - these into Nautilus rejection events (`OrderRejected`, `OrderCancelRejected`). -- Use `RetryManager` from `nautilus_network::retry` for consistent backoff. - -### Naming conventions - -Adapters follow standardized naming conventions for consistency across all venue integrations. - -#### Channel naming: `raw` → `msg` → `out` - -WebSocket message channels follow a two-stage transformation pipeline within the handler: - -| Stage | Type | Description | Example | -|-------|------|-------------|---------| -| `raw` | Raw WebSocket frames | Bytes/text from the network layer. | `raw_rx: UnboundedReceiver` | -| `out` | Venue-specific messages | Parsed venue message types. | `out_tx: UnboundedSender` | - -The handler deserializes raw frames into venue-specific types and emits them on `out_tx`. -The data and execution client layers then convert venue types into Nautilus domain types. - -**Example flow:** - -```rust -// Client creates output channel for venue messages -let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel(); // Venue messages (MyWsMessage) - -// Handler receives raw frames, outputs venue messages -let handler = MyWsFeedHandler::new( - cmd_rx, - raw_rx, // Input: Message (raw WebSocket frames) - out_tx, // Output: MyWsMessage - // ... -); -``` - -Channel names reflect the data transformation stage, not the destination. Use `raw_*` for raw -WebSocket frames (`Message`) and `out_*` for venue-specific message types. - -### Backpressure strategy - -WebSocket channels on latency-sensitive paths are intentionally **unbounded**. The platform -prioritizes latency and prefers an explicit crash (OOM) over delaying or dropping data. - -:::note -Do not add bounded channels, buffering limits, or backpressure unless the latency requirement changes. -::: - -#### Field naming: `inner` and command channels - -Structs holding references to lower-level components follow these conventions: - -| Field | Type | Description | -|---------------|-----------------------------------------------------|-------------| -| `inner` | `Option` | Network-level WebSocket client (handler only, exclusively owned). | -| `cmd_tx` | `Arc>>` | Command channel to handler (client side). | -| `cmd_rx` | `UnboundedReceiver` | Command channel from client (handler side). | -| `out_tx` | `UnboundedSender<{Venue}WsMessage>` | Output channel to client (handler side). | -| `out_rx` | `Option>>` | Output channel from handler (client side). | -| `task_handle` | `Option>>` | Handler task handle. | - -**Example:** - -```rust -// Client struct -pub struct MyWebSocketClient { - cmd_tx: Arc>>, - out_rx: Option>>, - task_handle: Option>>, - connection_mode: Arc>, // Lock-free connection state - // ... -} - -impl MyWebSocketClient { - async fn send_cmd(&self, cmd: HandlerCommand) -> Result<(), Error> { - self.cmd_tx.read().await.send(cmd) - .map_err(|e| Error::ClientError(format!("Handler not available: {e}"))) - } -} - -// Handler struct -pub(super) struct MyWsFeedHandler { - inner: Option, // Exclusively owned - no RwLock - cmd_rx: UnboundedReceiver, - raw_rx: UnboundedReceiver, - out_tx: UnboundedSender, - pending_requests: AHashMap, // Single-threaded - no locks - pending_messages: VecDeque, // Multi-message buffer - // ... -} -``` - -The handler exclusively owns `WebSocketClient` without locks. The client sends commands via -`cmd_tx` (wrapped in `RwLock` to allow reconnection channel replacement) and receives events -via `out_rx`. Use a `send_cmd()` helper to standardize command sending. - -#### Type naming: `{Venue}Ws{TypeSuffix}` - -All WebSocket-related types follow a standardized naming pattern: `{Venue}Ws{TypeSuffix}` - -- `{Venue}`: Capitalized venue name (e.g., `OKX`, `Bybit`, `Bitmex`, `Hyperliquid`). -- `Ws`: Abbreviated "WebSocket" (not fully spelled out). -- `{TypeSuffix}`: Full type descriptor (e.g., `Message`, `Error`, `Request`, `Response`). - -**Examples:** - -```rust -// Correct - abbreviated Ws, full type suffix -pub enum OKXWsMessage { ... } -pub enum BybitWsError { ... } -pub struct HyperliquidWsRequest { ... } -``` - -**Standard type suffixes:** - -- `Message`: WebSocket message enums. -- `Error`: WebSocket error types. -- `Request`: Request message types. -- `Response`: Response message types. - -**Tokio channel qualification:** - -Always fully qualify tokio channel types as `tokio::sync::mpsc::` to avoid ambiguity with -similarly-named types from other crates. Never import `mpsc` directly at module level. - -```rust -// Correct -let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); -``` - -### Split WebSocket architectures - -Some venues expose multiple WebSocket endpoints with distinct protocols or encodings. -When a venue requires separate connections for market data and order management, split -the `websocket/` module into submodules that mirror the connection boundaries: - -``` -src/ -├── websocket/ -│ ├── mod.rs # Re-exports from submodules -│ ├── streams/ # Market data pub/sub connection -│ │ ├── client.rs # Streams client -│ │ ├── handler.rs # Streams feed handler -│ │ ├── messages.rs # Streams message types -│ │ └── mod.rs -│ └── trading/ # Order management + user data (authenticated WS API) -│ ├── client.rs # Trading client -│ ├── handler.rs # Trading handler -│ ├── messages.rs # Trading message types -│ ├── user_data.rs # User data stream venue types (execution reports, etc.) -│ ├── parse.rs # Parse functions for user data -> Nautilus types -│ ├── error.rs # Trading error types -│ └── mod.rs -``` - -Each submodule follows the same two-layer client/handler pattern described above. The -parent `websocket/mod.rs` re-exports the public client types. - -The `trading/` module handles both order operations (place, cancel, modify) and the -user data stream (execution reports, account updates). When the venue's authenticated -WebSocket API supports `session.logon` and inline user data subscriptions, both -concerns share a single authenticated connection. This avoids a separate `execution/` -module and the deprecated REST listenKey lifecycle. - -For venues where user data events arrive on a separate stream connection (e.g., -futures APIs that return a listenKey for a dedicated stream URL), the `streams/` -handler dispatches both market data and user data events from the combined connection. - -#### Naming conventions for split architectures - -Type names include the submodule qualifier to avoid ambiguity: - -| Submodule | Command type | Message type | -|--------------|--------------------------------------|-------------------------------------| -| `streams/` | `{Venue}WsStreamsCommand` | `{Venue}WsMessage` (venue types) | -| `trading/` | `{Venue}WsTradingCommand` | `{Venue}WsTradingMessage` | - -The `{Venue}Ws` prefix follows the standard type naming convention. The qualifier -(`Streams`, `Trading`) distinguishes types that would otherwise collide across -submodules. - -#### When to split - -Split the WebSocket module when the venue has: - -- Different endpoints with different protocols (e.g., SBE binary for market data, JSON - for trading) -- A dedicated order management WebSocket API (`ws-api` style) alongside pub/sub streams -- User data delivered inline on the authenticated trading connection rather than via a - separate listenKey stream - -Do not split when a single connection handles all message types through channel-based -multiplexing (the common pattern for OKX, Bybit, and similar venues). - -## Modeling venue payloads - -Use the following conventions when mirroring upstream schemas in Rust. - -### REST models (`http::models` and `http::query`) - -- Put request and response representations in `src/http/models.rs` and derive `serde::Deserialize` (add `serde::Serialize` when the adapter sends data back). -- Mirror upstream payload names with blanket casing attributes such as `#[serde(rename_all = "camelCase")]` or `#[serde(rename_all = "snake_case")]`; only add per-field renames when the upstream key would be an invalid Rust identifier or collide with a keyword (for example `#[serde(rename = "type")] pub order_type: String`). -- Keep helper structs for query parameters in `src/http/query.rs`, deriving `serde::Serialize` to remain type-safe and reusing constants from `common::consts` instead of duplicating literals. - -### WebSocket messages (`websocket::messages`) - -- Define streaming payload types in `src/websocket/messages.rs`, giving each venue topic a struct or enum that mirrors the upstream JSON. -- Apply the same naming guidance as REST models: rely on blanket casing renames and keep field names aligned with the venue unless syntax forces a change; consider serde helpers such as `#[serde(tag = "op")]` or `#[serde(flatten)]` and document the choice. -- Note any intentional deviations from the upstream schema in code comments and module docs so other contributors can follow the mapping quickly. - ---- - -## Testing - -Adapters should ship two layers of coverage: the Rust crate that talks to the venue and the Python glue that exposes it to the wider platform. -Keep the suites deterministic and colocated with the production code they protect. - -**Key principle:** The `tests/` directory is reserved for integration tests that require external infrastructure (mock Axum servers, simulated network conditions). -Unit tests for parsing, serialization, and business logic belong in `#[cfg(test)]` blocks within source modules. - -### Rust testing - -#### Layout - -``` -crates/adapters/your_adapter/ -├── src/ -│ ├── http/ -│ │ ├── client.rs # HTTP client + unit tests -│ │ └── parse.rs # REST payload parsers + unit tests -│ └── websocket/ -│ ├── client.rs # WebSocket client + unit tests -│ └── parse.rs # Streaming parsers + unit tests -├── tests/ # Integration tests (mock servers) -│ ├── data_client.rs # Data client integration tests -│ ├── exec_client.rs # Execution client integration tests -│ ├── http.rs # HTTP client integration tests -│ └── websocket.rs # WebSocket client integration tests -└── test_data/ # Canonical venue payloads used by the suites - ├── http_{method}_{endpoint}.json # Full venue responses with retCode/result/time - └── ws_{message_type}.json # WebSocket message samples -``` - -#### Test file organization - -| File | Purpose | -|----------------------|---------------------------------------------------------------------------------------------------------------------------| -| `tests/data_client.rs` | Integration tests for the data client. Validates data subscriptions, historical data requests, and market data parsing. | -| `tests/exec_client.rs` | Integration tests for the execution client. Validates order submission, modification, cancellation, and execution reports. | -| `tests/http.rs` | Low-level HTTP client tests. Validates request signing, error handling, and response parsing against mock Axum servers. | -| `tests/websocket.rs` | WebSocket client tests. Validates connection lifecycle, authentication, subscriptions, and message routing. | - -**Guidelines:** - -- Place unit tests next to the module they exercise (`#[cfg(test)]` blocks). Use `src/common/testing.rs` (or an equivalent helper module) for shared fixtures so production files stay tidy. -- Keep Axum-based integration suites under `crates/adapters//tests/`, mirroring the public APIs (HTTP client, WebSocket client, data client, execution client). -- Data and execution client tests (`data_client.rs`, `exec_client.rs`) should focus on higher-level behavior: subscription workflows, order lifecycle, and domain model transformations. HTTP and WebSocket tests (`http.rs`, `websocket.rs`) focus on transport-level concerns. -- Store upstream payload samples (snapshots, REST replies) under `test_data/` and reference them from both unit and integration tests. Name test data files consistently: `http_get_{endpoint_name}.json` for REST responses, `ws_{message_type}.json` for WebSocket messages. Include complete venue response envelopes (status codes, timestamps, result wrappers) rather than just the data payload. Provide multiple realistic examples in each file - for instance, position data should include long, short, and flat positions to exercise all parser branches. -- **Test data sourcing**: Test data must be obtained from either official API documentation examples or directly from the live API via network calls. Never fabricate or generate test data manually, as this risks missing edge cases (e.g., negative precision values, scientific notation, unexpected field types) that only appear in real venue responses. - -#### Unit tests - -Unit tests belong in `#[cfg(test)]` blocks within source modules, not in the `tests/` directory. - -**What to test (in source modules):** - -- Deserialization of venue JSON payloads into Rust structs. -- Parsing functions that convert venue types to Nautilus domain models. -- Request signing and authentication helpers. -- Enum conversions and mapping logic. -- Price, quantity, and precision calculations. - -**What NOT to test:** - -- Standard library behavior (Vec operations, HashMap lookups, string parsing). -- Third-party crate functionality (chrono date arithmetic, serde attributes). -- Test helper code itself (fixture loaders, mock builders). - -Tests should exercise production code paths. If a test only verifies that `Vec::extend()` works or that chrono can parse a date string, it provides no value. - -##### WebSocket unit test coverage - -WebSocket unit tests exercise three areas: message deserialization, parse dispatch, and handler -logic. Each area lives in a `#[cfg(test)]` block within the module it tests. - -**Message types (`messages.rs`):** - -- Deserialize every message variant from fixture JSON files in `test_data/`. -- Round-trip tests: serialize a constructed struct, deserialize the output, and assert equality. - Round-trip tests catch field renames, missing `skip_serializing_if` attributes, and precision - loss that deserialization-only tests miss. -- Cover edge cases in venue payloads: null optional fields, empty arrays, zero quantities. - -**Parse functions (`parse.rs`):** - -- Exercise the fast-path byte scanner for each type tag or discriminant value. -- Exercise the slow-path fallback (fields not at expected byte positions). -- Verify unknown type tags produce a descriptive error, not a panic. - -**Handler logic (`handler.rs`):** - -- Verify the handler filters internal messages (heartbeats, subscription acks, pong frames) - and does not forward them to consumers. -- Verify reconnect signals trigger re-authentication and emit the `Reconnected` variant. -- Verify multi-message buffering: when a single raw frame produces multiple output messages, - all messages appear in the correct order from `next()`. -- Verify pending-order cleanup on error and success responses. - -#### Integration tests - -Integration tests belong in the `tests/` directory and exercise the public API against mock infrastructure. - -**What to test (in tests/ directory):** - -- HTTP client requests against mock Axum servers. -- WebSocket connection lifecycle, authentication, and message routing. -- Data client subscription workflows and historical data requests. -- Execution client order submission, modification, and cancellation flows. -- Error handling and retry behavior with simulated failures. - -At a minimum, review existing adapter test suites for reference patterns and verify every adapter -proves the same core behaviours. - -##### HTTP client integration coverage - -- **Happy paths** – fetch a representative public resource (e.g., instruments or mark price) and verify the - response is converted into Nautilus domain models. -- **Credential guard** – call a private endpoint without credentials and assert a structured error; repeat with - credentials to prove success. -- **Rate limiting / retry mapping** – surface venue-specific rate-limit responses and assert the adapter produces - the correct `OkxError`/`BitmexHttpError` variant so the retry policy can react. -- **Query builders** – exercise builders for paginated/time-bounded endpoints (historical trades, candles) and - assert the emitted query string matches the venue specification (`after`, `before`, `limit`, etc.). -- **Error translation** – verify non-2xx upstream responses map to adapter error enums with the original code/message attached. - -##### WebSocket client integration coverage - -- **Login handshake** – confirm a successful login flips the internal auth state and test failure cases where the - server returns a non-zero code; the client should surface an error and avoid marking itself as authenticated. -- **Ping/Pong** – prove both text-based and control-frame pings trigger immediate pong responses. -- **Subscription lifecycle** – assert subscription requests/acks are emitted for public and private channels, and that - unsubscribe calls remove entries from the cached subscription sets. -- **Reconnect behaviour** – simulate a disconnect and verify the client re-authenticates, restores public channels, - and skips private channels that were explicitly unsubscribed pre-disconnect. -- **Message routing** – feed representative data/ack/error payloads through the socket and assert they arrive on the - public stream as the correct `{Venue}WsMessage` variant. -- **Quota tagging** – (optional but recommended) validate that order/cancel/amend operations are tagged with the - appropriate quota label so rate limiting can be enforced independently of subscription traffic. - -**CI robustness:** - -- Never use bare `tokio::time::sleep()` with arbitrary durations. Tests become flaky under CI load and slower than necessary. -- Use the `wait_until_async` test helper to poll for conditions with timeout. Tests return - immediately when the condition is met and fail deterministically on timeout rather than - relying on arbitrary sleep durations. -- Prefer event-driven assertions with shared state (for example, collect `subscription_events`, track pending/confirmed topics, wait for `connection_count` transitions). -- Use adapter-specific helpers to gate on explicit signals such as "auth confirmed" or "reconnection finished" so suites remain deterministic under load. - -##### Data and execution client integration testing - -Data (`tests/data_client.rs`) and execution (`tests/exec_client.rs`) client integration tests verify the full message flow from WebSocket through parsing to event emission. - -**Test infrastructure:** - -| Component | Purpose | -|------------------------------|------------------------------------------------------------------------------------| -| Mock Axum server | Serves HTTP endpoints (instruments, fee rates, positions) and WebSocket channels. | -| `TestServerState` | Tracks connections, subscriptions, and authentication state for assertions. | -| Thread-local event channels | `set_data_event_sender()` / `set_exec_event_sender()` for capturing emitted events.| -| `wait_until_async` | Polls conditions with timeout for deterministic async assertions. | - -**Data client coverage:** - -| Test scenario | Validates | -|------------------------------|----------------------------------------------------------------| -| Connect/disconnect | Connection lifecycle, WebSocket establishment, clean shutdown. | -| Subscribe trades | Trade tick events emitted to data channel. | -| Subscribe quotes | Quote events from ticker (LINEAR) or orderbook (SPOT). | -| Subscribe book deltas | OrderBookDeltas events from orderbook snapshots/updates. | -| Subscribe mark/index prices | Filtered by subscription state (only emit when subscribed). | -| Reset state | Subscription tracking cleared, connection terminated. | -| Instruments on connect | Instrument events emitted during connection setup. | - -**Execution client coverage:** - -| Test scenario | Validates | -|------------------------------|----------------------------------------------------------------| -| Connect/disconnect | Auth handshake, private + trade WS connections, subscriptions. | -| Demo mode | Only private WS connects (trade WS skipped for HTTP fallback). | -| Order submission | Order accepted/rejected events, venue ID correlation. | -| Order modification/cancel | Update and cancel acknowledgment events. | -| Position/wallet updates | PositionStatusReport and AccountState events. | - -**Key patterns:** - -- Each `#[tokio::test]` runs on a fresh thread, ensuring thread-local channel isolation. -- Use `wait_until_async` for subscription/connection state instead of arbitrary sleeps. -- Drain instrument events before subscription tests to isolate assertions. -- Verify subscription state in `TestServerState` before asserting on emitted events. - -### Python testing - -#### Layout - -``` -tests/integration_tests/adapters/your_adapter/ -├── conftest.py # Shared fixtures (mock clients, test instruments) -├── test_data.py # Data client integration tests -├── test_execution.py # Execution client integration tests -├── test_providers.py # Instrument provider tests -├── test_factories.py # Factory and configuration tests -└── __init__.py # Package initialization -``` - -#### Test file organization - -| File | Purpose | -|---------------------|--------------------------------------------------------------------------------------------------------------------| -| `test_data.py` | Tests for `LiveDataClient` and `LiveMarketDataClient`. Validates subscriptions, data parsing, and message handling. | -| `test_execution.py` | Tests for `LiveExecutionClient`. Validates order submission, modification, cancellation, and execution reports. | -| `test_providers.py` | Tests for `InstrumentProvider`. Validates instrument loading, filtering, and caching behavior. | -| `test_factories.py` | Tests for factory functions. Validates client instantiation and configuration wiring. | - -**Guidelines:** - -- Exercise the adapter's Python surface (instrument providers, data/execution clients, factories) inside `tests/integration_tests/adapters//`. -- Mock the PyO3 boundary (`nautilus_pyo3` shims, stubbed Rust clients) so tests stay fast while verifying that configuration, factory wiring, and error handling match the exported Rust API. -- Mirror the Rust integration coverage: when the Rust suite adds a new behaviour (e.g., reconnection replay, error propagation), assert the Python layer performs the same sequence (connect/disconnect, submit/amend/cancel translations, venue ID hand-off, failure handling). BitMEX's Python tests provide the target level of detail. - ---- - -## Documentation - -All adapter documentation (module-level docs, doc comments, and inline comments) should follow the -[Documentation Style Guide](docs.md). - -### Rust documentation requirements - -Every Rust module, struct, and public method must have documentation comments. -Use third-person declarative voice (e.g., "Returns the account ID" not "Return the account ID"). - -- **Modules**: Use `//!` doc comments at the top of each file (after the license header) to describe the module's purpose. -- **Structs**: Use `///` doc comments above struct definitions. Keep descriptions concise; one sentence is often sufficient. -- **Public methods**: Every `pub fn` and `pub async fn` must have a `///` doc comment describing what the method does. - Do not document individual parameters in a separate `# Arguments` section. The type signatures and names should be self-explanatory. - Parameters may be mentioned in the description when behavior is complex or non-obvious. - -**What NOT to document**: - -- Private methods and fields (unless complex logic warrants it). -- Individual parameters/arguments (use descriptive names instead). -- Implementation details that are obvious from the code. -- Files in the `python/` module (PyO3 bindings). Documentation conventions are TBD (*may* use numpydoc specification). - ---- - -## Python adapter layer - -Step-by-step guide to building the Python layer of an adapter using the provided template. - -### Method ordering convention - -When implementing adapter classes, group methods by category in this order: - -1. **Connection handlers**: `_connect`, `_disconnect` -2. **Subscribe handlers**: `_subscribe`, `_subscribe_*` -3. **Unsubscribe handlers**: `_unsubscribe`, `_unsubscribe_*` -4. **Request handlers**: `_request`, `_request_*` - -This keeps related functionality together rather than interleaving subscribe/unsubscribe pairs. - -### InstrumentProvider - -The `InstrumentProvider` loads instrument definitions from the venue: all instruments, specific -instruments by ID, or a filtered subset. - -```python -from nautilus_trader.common.providers import InstrumentProvider -from nautilus_trader.model import InstrumentId - - -class TemplateInstrumentProvider(InstrumentProvider): - """Example `InstrumentProvider` showing the minimal overrides required for a complete integration.""" - - async def load_all_async(self, filters: dict | None = None) -> None: - raise NotImplementedError("implement `load_all_async` in your adapter subclass") - - async def load_ids_async(self, instrument_ids: list[InstrumentId], filters: dict | None = None) -> None: - raise NotImplementedError("implement `load_ids_async` in your adapter subclass") - - async def load_async(self, instrument_id: InstrumentId, filters: dict | None = None) -> None: - raise NotImplementedError("implement `load_async` in your adapter subclass") -``` - -| Method | Description | -|------------------|----------------------------------------------------------------| -| `load_all_async` | Loads all instruments asynchronously, optionally with filters. | -| `load_ids_async` | Loads specific instruments by their IDs. | -| `load_async` | Loads a single instrument by its ID. | - -### DataClient - -The `LiveDataClient` handles data feeds that are not market data: news feeds, custom data streams, -or other non-market sources. - -```python -from nautilus_trader.data.messages import RequestData -from nautilus_trader.data.messages import SubscribeData -from nautilus_trader.data.messages import UnsubscribeData -from nautilus_trader.live.data_client import LiveDataClient -from nautilus_trader.model import DataType - - -class TemplateLiveDataClient(LiveDataClient): - """Example `LiveDataClient` showing the overridable abstract methods.""" - - async def _connect(self) -> None: - raise NotImplementedError("implement `_connect` in your adapter subclass") - - async def _disconnect(self) -> None: - raise NotImplementedError("implement `_disconnect` in your adapter subclass") - - async def _subscribe(self, command: SubscribeData) -> None: - raise NotImplementedError("implement `_subscribe` in your adapter subclass") - - async def _unsubscribe(self, command: UnsubscribeData) -> None: - raise NotImplementedError("implement `_unsubscribe` in your adapter subclass") - - async def _request(self, request: RequestData) -> None: - raise NotImplementedError("implement `_request` in your adapter subclass") -``` - -| Method | Description | -|----------------|------------------------------------------------| -| `_connect` | Establishes a connection to the data provider. | -| `_disconnect` | Closes the connection to the data provider. | -| `_subscribe` | Subscribes to a specific data type. | -| `_unsubscribe` | Unsubscribes from a specific data type. | -| `_request` | Requests data from the provider. | - -### MarketDataClient - -The `MarketDataClient` handles market-specific data: order books, top-of-book quotes and trades, -instrument status updates, and historical data requests. - -```python -from nautilus_trader.data.messages import RequestBars -from nautilus_trader.data.messages import RequestData -from nautilus_trader.data.messages import RequestInstrument -from nautilus_trader.data.messages import RequestInstruments -from nautilus_trader.data.messages import RequestOrderBookDeltas -from nautilus_trader.data.messages import RequestOrderBookDepth -from nautilus_trader.data.messages import RequestOrderBookSnapshot -from nautilus_trader.data.messages import RequestQuoteTicks -from nautilus_trader.data.messages import RequestTradeTicks -from nautilus_trader.data.messages import SubscribeBars -from nautilus_trader.data.messages import SubscribeData -from nautilus_trader.data.messages import SubscribeFundingRates -from nautilus_trader.data.messages import SubscribeIndexPrices -from nautilus_trader.data.messages import SubscribeInstrument -from nautilus_trader.data.messages import SubscribeInstrumentClose -from nautilus_trader.data.messages import SubscribeInstruments -from nautilus_trader.data.messages import SubscribeInstrumentStatus -from nautilus_trader.data.messages import SubscribeMarkPrices -from nautilus_trader.data.messages import SubscribeOrderBook -from nautilus_trader.data.messages import SubscribeQuoteTicks -from nautilus_trader.data.messages import SubscribeTradeTicks -from nautilus_trader.data.messages import UnsubscribeBars -from nautilus_trader.data.messages import UnsubscribeData -from nautilus_trader.data.messages import UnsubscribeFundingRates -from nautilus_trader.data.messages import UnsubscribeIndexPrices -from nautilus_trader.data.messages import UnsubscribeInstrument -from nautilus_trader.data.messages import UnsubscribeInstrumentClose -from nautilus_trader.data.messages import UnsubscribeInstruments -from nautilus_trader.data.messages import UnsubscribeInstrumentStatus -from nautilus_trader.data.messages import UnsubscribeMarkPrices -from nautilus_trader.data.messages import UnsubscribeOrderBook -from nautilus_trader.data.messages import UnsubscribeQuoteTicks -from nautilus_trader.data.messages import UnsubscribeTradeTicks -from nautilus_trader.live.data_client import LiveMarketDataClient - - -class TemplateLiveMarketDataClient(LiveMarketDataClient): - """Example `LiveMarketDataClient` showing the overridable abstract methods.""" - - async def _connect(self) -> None: - raise NotImplementedError("implement `_connect` in your adapter subclass") - - async def _disconnect(self) -> None: - raise NotImplementedError("implement `_disconnect` in your adapter subclass") - - async def _subscribe(self, command: SubscribeData) -> None: - raise NotImplementedError("implement `_subscribe` in your adapter subclass") - - async def _subscribe_instruments(self, command: SubscribeInstruments) -> None: - raise NotImplementedError("implement `_subscribe_instruments` in your adapter subclass") - - async def _subscribe_instrument(self, command: SubscribeInstrument) -> None: - raise NotImplementedError("implement `_subscribe_instrument` in your adapter subclass") - - async def _subscribe_order_book_deltas(self, command: SubscribeOrderBook) -> None: - raise NotImplementedError("implement `_subscribe_order_book_deltas` in your adapter subclass") - - async def _subscribe_order_book_depth(self, command: SubscribeOrderBook) -> None: - raise NotImplementedError("implement `_subscribe_order_book_depth` in your adapter subclass") - - async def _subscribe_quote_ticks(self, command: SubscribeQuoteTicks) -> None: - raise NotImplementedError("implement `_subscribe_quote_ticks` in your adapter subclass") - - async def _subscribe_trade_ticks(self, command: SubscribeTradeTicks) -> None: - raise NotImplementedError("implement `_subscribe_trade_ticks` in your adapter subclass") - - async def _subscribe_mark_prices(self, command: SubscribeMarkPrices) -> None: - raise NotImplementedError("implement `_subscribe_mark_prices` in your adapter subclass") - - async def _subscribe_index_prices(self, command: SubscribeIndexPrices) -> None: - raise NotImplementedError("implement `_subscribe_index_prices` in your adapter subclass") - - async def _subscribe_bars(self, command: SubscribeBars) -> None: - raise NotImplementedError("implement `_subscribe_bars` in your adapter subclass") - - async def _subscribe_funding_rates(self, command: SubscribeFundingRates) -> None: - raise NotImplementedError("implement `_subscribe_funding_rates` in your adapter subclass") - - async def _subscribe_instrument_status(self, command: SubscribeInstrumentStatus) -> None: - raise NotImplementedError("implement `_subscribe_instrument_status` in your adapter subclass") - - async def _subscribe_instrument_close(self, command: SubscribeInstrumentClose) -> None: - raise NotImplementedError("implement `_subscribe_instrument_close` in your adapter subclass") - - async def _subscribe_option_greeks(self, command: SubscribeOptionGreeks) -> None: - raise NotImplementedError("implement `_subscribe_option_greeks` in your adapter subclass") - - async def _unsubscribe(self, command: UnsubscribeData) -> None: - raise NotImplementedError("implement `_unsubscribe` in your adapter subclass") - - async def _unsubscribe_instruments(self, command: UnsubscribeInstruments) -> None: - raise NotImplementedError("implement `_unsubscribe_instruments` in your adapter subclass") - - async def _unsubscribe_instrument(self, command: UnsubscribeInstrument) -> None: - raise NotImplementedError("implement `_unsubscribe_instrument` in your adapter subclass") - - async def _unsubscribe_order_book_deltas(self, command: UnsubscribeOrderBook) -> None: - raise NotImplementedError("implement `_unsubscribe_order_book_deltas` in your adapter subclass") - - async def _unsubscribe_order_book_depth(self, command: UnsubscribeOrderBook) -> None: - raise NotImplementedError("implement `_unsubscribe_order_book_depth` in your adapter subclass") - - async def _unsubscribe_quote_ticks(self, command: UnsubscribeQuoteTicks) -> None: - raise NotImplementedError("implement `_unsubscribe_quote_ticks` in your adapter subclass") - - async def _unsubscribe_trade_ticks(self, command: UnsubscribeTradeTicks) -> None: - raise NotImplementedError("implement `_unsubscribe_trade_ticks` in your adapter subclass") - - async def _unsubscribe_mark_prices(self, command: UnsubscribeMarkPrices) -> None: - raise NotImplementedError("implement `_unsubscribe_mark_prices` in your adapter subclass") - - async def _unsubscribe_index_prices(self, command: UnsubscribeIndexPrices) -> None: - raise NotImplementedError("implement `_unsubscribe_index_prices` in your adapter subclass") - - async def _unsubscribe_bars(self, command: UnsubscribeBars) -> None: - raise NotImplementedError("implement `_unsubscribe_bars` in your adapter subclass") - - async def _unsubscribe_funding_rates(self, command: UnsubscribeFundingRates) -> None: - raise NotImplementedError("implement `_unsubscribe_funding_rates` in your adapter subclass") - - async def _unsubscribe_instrument_status(self, command: UnsubscribeInstrumentStatus) -> None: - raise NotImplementedError("implement `_unsubscribe_instrument_status` in your adapter subclass") - - async def _unsubscribe_instrument_close(self, command: UnsubscribeInstrumentClose) -> None: - raise NotImplementedError("implement `_unsubscribe_instrument_close` in your adapter subclass") - - async def _unsubscribe_option_greeks(self, command: UnsubscribeOptionGreeks) -> None: - raise NotImplementedError("implement `_unsubscribe_option_greeks` in your adapter subclass") - - async def _request(self, request: RequestData) -> None: - raise NotImplementedError("implement `_request` in your adapter subclass") - - async def _request_instrument(self, request: RequestInstrument) -> None: - raise NotImplementedError("implement `_request_instrument` in your adapter subclass") - - async def _request_instruments(self, request: RequestInstruments) -> None: - raise NotImplementedError("implement `_request_instruments` in your adapter subclass") - - async def _request_order_book_deltas(self, request: RequestOrderBookDeltas) -> None: - raise NotImplementedError("implement `_request_order_book_deltas` in your adapter subclass") - - async def _request_order_book_depth(self, request: RequestOrderBookDepth) -> None: - raise NotImplementedError("implement `_request_order_book_depth` in your adapter subclass") - - async def _request_order_book_snapshot(self, request: RequestOrderBookSnapshot) -> None: - raise NotImplementedError("implement `_request_order_book_snapshot` in your adapter subclass") - - async def _request_quote_ticks(self, request: RequestQuoteTicks) -> None: - raise NotImplementedError("implement `_request_quote_ticks` in your adapter subclass") - - async def _request_trade_ticks(self, request: RequestTradeTicks) -> None: - raise NotImplementedError("implement `_request_trade_ticks` in your adapter subclass") - - async def _request_bars(self, request: RequestBars) -> None: - raise NotImplementedError("implement `_request_bars` in your adapter subclass") - -``` - -| Method | Description | -|------------------------------------|---------------------------------------------------------| -| `_connect` | Establishes a connection to the venue APIs. | -| `_disconnect` | Closes the connection to the venue APIs. | -| `_subscribe` | Subscribes to generic data (base for custom types). | -| `_subscribe_instruments` | Subscribes to market data for multiple instruments. | -| `_subscribe_instrument` | Subscribes to market data for a single instrument. | -| `_subscribe_order_book_deltas` | Subscribes to order book delta updates. | -| `_subscribe_order_book_depth` | Subscribes to order book depth updates. | -| `_subscribe_quote_ticks` | Subscribes to top-of-book quote updates. | -| `_subscribe_trade_ticks` | Subscribes to trade tick updates. | -| `_subscribe_mark_prices` | Subscribes to mark price updates. | -| `_subscribe_index_prices` | Subscribes to index price updates. | -| `_subscribe_bars` | Subscribes to bar/candlestick updates. | -| `_subscribe_funding_rates` | Subscribes to funding rate updates. | -| `_subscribe_instrument_status` | Subscribes to instrument status updates. | -| `_subscribe_instrument_close` | Subscribes to instrument close price updates. | -| `_subscribe_option_greeks` | Subscribes to option greeks updates. | -| `_unsubscribe` | Unsubscribes from generic data (base for custom types). | -| `_unsubscribe_instruments` | Unsubscribes from market data for multiple instruments. | -| `_unsubscribe_instrument` | Unsubscribes from market data for a single instrument. | -| `_unsubscribe_order_book_deltas` | Unsubscribes from order book delta updates. | -| `_unsubscribe_order_book_depth` | Unsubscribes from order book depth updates. | -| `_unsubscribe_quote_ticks` | Unsubscribes from quote tick updates. | -| `_unsubscribe_trade_ticks` | Unsubscribes from trade tick updates. | -| `_unsubscribe_mark_prices` | Unsubscribes from mark price updates. | -| `_unsubscribe_index_prices` | Unsubscribes from index price updates. | -| `_unsubscribe_bars` | Unsubscribes from bar updates. | -| `_unsubscribe_funding_rates` | Unsubscribes from funding rate updates. | -| `_unsubscribe_instrument_status` | Unsubscribes from instrument status updates. | -| `_unsubscribe_instrument_close` | Unsubscribes from instrument close price updates. | -| `_unsubscribe_option_greeks` | Unsubscribes from option greeks updates. | -| `_request` | Requests generic data (base for custom types). | -| `_request_instrument` | Requests historical data for a single instrument. | -| `_request_instruments` | Requests historical data for multiple instruments. | -| `_request_order_book_snapshot` | Requests an order book snapshot. | -| `_request_order_book_depth` | Requests order book depth. | -| `_request_order_book_deltas` | Requests historical order book deltas. | -| `_request_quote_ticks` | Requests historical quote tick data. | -| `_request_trade_ticks` | Requests historical trade tick data. | -| `_request_bars` | Requests historical bar data. | -| `_request_funding_rates` | Requests historical funding rate data. | - -#### Order book delta flag requirements - -When implementing `_subscribe_order_book_deltas` or streaming order book -data, adapters **must** set `RecordFlag` flags correctly on each -`OrderBookDelta`. See also [Delta flags and event boundaries](../concepts/data.md#delta-flags-and-event-boundaries). - -- **`F_LAST`**: Set on the last delta of every logical event group. The - `DataEngine` uses this flag as the flush signal when `buffer_deltas` is - enabled. Without it, deltas accumulate indefinitely and are never - published to subscribers. - -- **`F_SNAPSHOT`**: Set on all deltas that belong to a snapshot sequence - (a `Clear` action followed by `Add` actions reconstructing the book). - -- **Empty book snapshots**: When emitting a snapshot for an empty book, - the `Clear` delta must have `F_SNAPSHOT | F_LAST`. Otherwise buffered - consumers never receive it. - -- **Incremental updates**: Each venue update message ends with a delta - that has `F_LAST` set. If the venue batches multiple updates into one - message, terminate each logical group with `F_LAST`. - -```python -from nautilus_trader.model.enums import RecordFlag - -# Incremental update (single event) -delta = OrderBookDelta( - instrument_id=instrument_id, - action=BookAction.UPDATE, - order=order, - flags=RecordFlag.F_LAST, # Last (and only) delta in this event - sequence=sequence, - ts_event=ts_event, - ts_init=ts_init, -) - -# Snapshot sequence -clear_delta = OrderBookDelta( - instrument_id=instrument_id, - action=BookAction.CLEAR, - order=NULL_ORDER, - flags=RecordFlag.F_SNAPSHOT, # Not the last delta - ... -) - -last_add_delta = OrderBookDelta( - instrument_id=instrument_id, - action=BookAction.ADD, - order=last_order, - flags=RecordFlag.F_SNAPSHOT | RecordFlag.F_LAST, # End of snapshot - ... -) -``` - -:::warning -A missing `F_LAST` is a silent bug: no error is raised, but subscribers -never receive the data when buffering is enabled. -::: - -### ExecutionClient - -The `ExecutionClient` manages order submission, modification, and cancellation against the venue -trading system. - -```python -from nautilus_trader.execution.messages import BatchCancelOrders -from nautilus_trader.execution.messages import CancelAllOrders -from nautilus_trader.execution.messages import CancelOrder -from nautilus_trader.execution.messages import GenerateFillReports -from nautilus_trader.execution.messages import GenerateOrderStatusReport -from nautilus_trader.execution.messages import GenerateOrderStatusReports -from nautilus_trader.execution.messages import GeneratePositionStatusReports -from nautilus_trader.execution.messages import ModifyOrder -from nautilus_trader.execution.messages import SubmitOrder -from nautilus_trader.execution.messages import SubmitOrderList -from nautilus_trader.execution.reports import ExecutionMassStatus -from nautilus_trader.execution.reports import FillReport -from nautilus_trader.execution.reports import OrderStatusReport -from nautilus_trader.execution.reports import PositionStatusReport -from nautilus_trader.live.execution_client import LiveExecutionClient - - -class TemplateLiveExecutionClient(LiveExecutionClient): - """Example `LiveExecutionClient` outlining the required overrides.""" - - async def _connect(self) -> None: - raise NotImplementedError("implement `_connect` in your adapter subclass") - - async def _disconnect(self) -> None: - raise NotImplementedError("implement `_disconnect` in your adapter subclass") - - async def _submit_order(self, command: SubmitOrder) -> None: - raise NotImplementedError("implement `_submit_order` in your adapter subclass") - - async def _submit_order_list(self, command: SubmitOrderList) -> None: - raise NotImplementedError("implement `_submit_order_list` in your adapter subclass") - - async def _modify_order(self, command: ModifyOrder) -> None: - raise NotImplementedError("implement `_modify_order` in your adapter subclass") - - async def _cancel_order(self, command: CancelOrder) -> None: - raise NotImplementedError("implement `_cancel_order` in your adapter subclass") - - async def _cancel_all_orders(self, command: CancelAllOrders) -> None: - raise NotImplementedError("implement `_cancel_all_orders` in your adapter subclass") - - async def _batch_cancel_orders(self, command: BatchCancelOrders) -> None: - raise NotImplementedError("implement `_batch_cancel_orders` in your adapter subclass") - - async def generate_order_status_report( - self, - command: GenerateOrderStatusReport, - ) -> OrderStatusReport | None: - raise NotImplementedError("method `generate_order_status_report` must be implemented in the subclass") - - async def generate_order_status_reports( - self, - command: GenerateOrderStatusReports, - ) -> list[OrderStatusReport]: - raise NotImplementedError("method `generate_order_status_reports` must be implemented in the subclass") - - async def generate_fill_reports( - self, - command: GenerateFillReports, - ) -> list[FillReport]: - raise NotImplementedError("method `generate_fill_reports` must be implemented in the subclass") - - async def generate_position_status_reports( - self, - command: GeneratePositionStatusReports, - ) -> list[PositionStatusReport]: - raise NotImplementedError("method `generate_position_status_reports` must be implemented in the subclass") - - async def generate_mass_status( - self, - lookback_mins: int | None = None, - ) -> ExecutionMassStatus | None: - raise NotImplementedError("method `generate_mass_status` must be implemented in the subclass") -``` - -| Method | Description | -|------------------------------------|-----------------------------------------------------------| -| `_connect` | Establishes a connection to the venue APIs. | -| `_disconnect` | Closes the connection to the venue APIs. | -| `_submit_order` | Submits a new order to the venue. | -| `_submit_order_list` | Submits a list of orders to the venue. | -| `_modify_order` | Modifies an existing order on the venue. | -| `_cancel_order` | Cancels a specific order on the venue. | -| `_cancel_all_orders` | Cancels all orders for an instrument on the venue. | -| `_batch_cancel_orders` | Cancels a batch of orders for an instrument on the venue. | -| `generate_order_status_report` | Generates a report for a specific order on the venue. | -| `generate_order_status_reports` | Generates reports for all orders on the venue. | -| `generate_fill_reports` | Generates reports for filled orders on the venue. | -| `generate_position_status_reports` | Generates reports for position status on the venue. | -| `generate_mass_status` | Generates execution mass status reports. | - -### Configuration - -Configuration classes hold adapter-specific settings like API keys and connection details. - -```python -from nautilus_trader.config import LiveDataClientConfig -from nautilus_trader.config import LiveExecClientConfig - - -class TemplateDataClientConfig(LiveDataClientConfig): - """Configuration for `TemplateDataClient` instances.""" - - api_key: str - api_secret: str - base_url: str - - -class TemplateExecClientConfig(LiveExecClientConfig): - """Configuration for `TemplateExecClient` instances.""" - - api_key: str - api_secret: str - base_url: str -``` - -**Key attributes**: - -- `api_key`: The API key for authenticating with the data provider. -- `api_secret`: The API secret for authenticating with the data provider. -- `base_url`: The base URL for connecting to the data provider's API. - -## Common test scenarios - -Exercise adapters across every venue behaviour they claim to support. Incorporate these scenarios -into the Rust and Python suites. - -### Product coverage - -Test each supported product family. - -- Spot instruments -- Derivatives (perpetuals, futures, swaps) -- Options and structured products - -### Order flow - -- Cover each supported order type (limit, market, stop, conditional, etc.) under every venue time-in-force option, expiries, and rejection handling. -- Submit buy and sell market orders and assert balance, position, and average-price updates align with venue responses. -- Submit representative buy and sell limit orders, verifying acknowledgements, execution reports, full and partial fills, and cancel flows. - -### State management - -- Start sessions with existing open orders to verify the adapter reconciles state on connect before - issuing new commands. -- Seed preloaded positions and confirm position snapshots, valuation, and PnL agree with the venue prior to trading. - ---- - -## Data testing spec - -See the full [Data Testing Spec](spec_data_testing.md) for the `DataTester` test matrix. - ---- - -## Execution testing spec - -See the full [Execution Testing Spec](spec_exec_testing.md) for the `ExecTester` test matrix. diff --git a/nautilus-docs/docs/developer_guide/benchmarking.md b/nautilus-docs/docs/developer_guide/benchmarking.md deleted file mode 100644 index e450a47..0000000 --- a/nautilus-docs/docs/developer_guide/benchmarking.md +++ /dev/null @@ -1,180 +0,0 @@ -# Benchmarking - -This guide explains how NautilusTrader measures Rust performance, when to -use each tool and the conventions you should follow when adding new benches. - ---- - -## Tooling overview - -NautilusTrader relies on **two complementary benchmarking frameworks**: - -| Framework | What is it? | What it measures | When to prefer it | -|-----------|-------------|------------------|-------------------| -| [**Criterion**](https://docs.rs/criterion/latest/criterion/) | Statistical benchmark harness that produces detailed HTML reports and performs outlier detection. | Wall-clock run time with confidence intervals. | End-to-end scenarios, anything slower than ≈100 ns, visual comparisons. | -| [**iai**](https://docs.rs/iai/latest/iai/) | Deterministic micro-benchmark harness that counts retired CPU instructions via hardware counters. | Exact instruction counts (noise-free). | Ultra-fast functions, CI gating via instruction diff. | - -Most hot code paths benefit from **both** kinds of measurements. - -:::note -iai is deterministic (immune to system noise) but results are machine-specific. Use it for regression detection within CI, not for cross-machine comparisons. -::: - ---- - -## Directory layout - -Each crate keeps its performance tests in a local `benches/` folder: - -```text -crates// -└── benches/ - ├── foo_criterion.rs # Criterion group(s) - └── foo_iai.rs # iai micro benches -``` - -`Cargo.toml` must list every benchmark explicitly so `cargo bench` discovers -them: - -```toml -[[bench]] -name = "foo_criterion" -path = "benches/foo_criterion.rs" -harness = false - -[[bench]] -name = "foo_iai" -path = "benches/foo_iai.rs" -harness = false -``` - ---- - -## Writing Criterion benchmarks - -1. Perform **all expensive set-up outside** the timing loop (`b.iter`). -2. Wrap inputs/outputs in `black_box` to prevent the optimizer from removing - work. -3. Group related cases with `benchmark_group!` and set `throughput` or - `sample_size` when the defaults aren’t ideal. - -```rust -use std::hint::black_box; - -use criterion::{Criterion, criterion_group, criterion_main}; - -fn bench_my_algo(c: &mut Criterion) { - let data = prepare_data(); // Heavy set-up done once - - c.bench_function("my_algo", |b| { - b.iter(|| my_algo(black_box(&data))); - }); -} - -criterion_group!(benches, bench_my_algo); -criterion_main!(benches); -``` - ---- - -## Writing iai benchmarks - -`iai` requires functions that take **no parameters** and return a value (which -can be ignored). Keep them as small as possible so the measured instruction -count is meaningful. - -```rust -use std::hint::black_box; - -fn bench_add() -> i64 { - let a = black_box(123); - let b = black_box(456); - a + b -} - -iai::main!(bench_add); -``` - ---- - -## Running benches locally - -- **Single crate**: `cargo bench -p nautilus-core`. -- **Single benchmark module**: `cargo bench -p nautilus-core --bench time`. -- **CI performance benches**: `make cargo-ci-benches` (runs the crates included - in the CI performance workflow one at a time to avoid the mixed-panic-strategy - linker issue). - -Criterion writes HTML reports to `target/criterion/`; open `target/criterion/report/index.html` in your browser. - -### Generating a flamegraph - -`cargo-flamegraph` lets you see a sampled call-stack profile of a single -benchmark. On Linux it uses `perf`, and on macOS it uses `DTrace`. - -1. Install `cargo-flamegraph` once per machine (it installs a `cargo flamegraph` - subcommand automatically). - - ```bash - cargo install flamegraph - ``` - -2. Run a specific bench with the symbol-rich `bench` profile. - - ```bash - # example: the matching benchmark in nautilus-common - cargo flamegraph --bench matching -p nautilus-common --profile bench - ``` - -3. Open the generated `flamegraph.svg` in your browser and zoom into hot paths. - -#### Linux - -On Linux, `perf` must be available. On Debian/Ubuntu, you can install it with: - -```bash -sudo apt install linux-tools-common linux-tools-$(uname -r) -``` - -If you see an error mentioning `perf_event_paranoid` you need to relax the -kernel’s perf restrictions for the current session (root required): - -```bash -sudo sh -c 'echo 1 > /proc/sys/kernel/perf_event_paranoid' -``` - -A value of `1` is typically enough; set it back to `2` (default) or make -the change permanent via `/etc/sysctl.conf` if desired. - -#### macOS - -On macOS, `DTrace` requires root permissions, so you must run `cargo flamegraph` -with `sudo`. - -:::warning -Running with `sudo` creates files in `target/` owned by root, causing permission errors with subsequent `cargo` commands. You may need to remove root-owned files manually or run `sudo cargo clean`. -::: - -```bash -sudo cargo flamegraph --bench matching -p nautilus-common --profile bench -``` - -Because `[profile.bench]` keeps full debug symbols the SVG will show readable -function names without bloating production binaries (which still use -`panic = "abort"` and are built via `[profile.release]`). - -> **Note** Benchmark binaries are compiled with the custom `[profile.bench]` -> defined in the workspace `Cargo.toml`. That profile inherits from -> `release-debugging`, preserving full optimisation *and* debug symbols so that -> tools like `cargo flamegraph` or `perf` produce human-readable stack traces. - ---- - -## Templates - -Ready-to-copy starter files live in [`docs/dev_templates/`](../dev_templates/): - -- **Criterion**: [`criterion_template.rs`](../dev_templates/criterion_template.rs) -- **iai**: [`iai_template.rs`](../dev_templates/iai_template.rs) - -Copy the template into `benches/`, adjust imports and names, and start measuring! diff --git a/nautilus-docs/docs/developer_guide/coding_standards.md b/nautilus-docs/docs/developer_guide/coding_standards.md deleted file mode 100644 index 58456d4..0000000 --- a/nautilus-docs/docs/developer_guide/coding_standards.md +++ /dev/null @@ -1,141 +0,0 @@ -# Coding Standards - -## Code Style - -The current codebase can be used as a guide for formatting conventions. -Additional guidelines are provided below. - -### Universal formatting rules - -The following applies to **all** source files (Rust, Python, Cython, shell, etc.): - -- Use **spaces only**, never hard tab characters. -- Lines should generally stay below **100 characters**; wrap thoughtfully when necessary. -- Prefer American English spelling (`color`, `serialize`, `behavior`). - -### Shell script portability - -Shell scripts in this repository use **bash** (not POSIX sh) and must be portable across **Linux** and **macOS**. User-facing scripts (e.g., `scripts/cli/install.sh`) must also work on **Windows** via Git Bash or WSL. - -**Shebang**: Always use `#!/usr/bin/env bash` for portability. - -**Common pitfalls**: GNU and BSD utilities differ between Linux and macOS: - -| Command | Linux (GNU) | macOS (BSD) | Portable solution | -|----------------------|-------------------|-------------------|------------------------------------------| -| `sed -i` | `sed -i 's/…'` | `sed -i '' 's/…'` | Use backup extension: `sed -i.bak 's/…'` | -| `stat` (file size) | `stat -c%s file` | `stat -f%z file` | Detect with `stat --version` | -| `sha256sum` | `sha256sum file` | N/A | Use `shasum -a 256` or detect | -| `readlink -f` | Works | N/A | Avoid, or use `realpath` | -| `grep -P` (PCRE) | Works | N/A | Use `-E` (extended regex) instead | -| `date` (nanoseconds) | `date +%N` | N/A | Use `$RANDOM` for cache-busting | - -**Bash version**: macOS ships with bash 3.2; avoid bash 4+ features in user-facing scripts: - -| Feature | Bash version | Alternative | -|-----------------------------------|--------------|----------------------------------| -| Associative arrays (`declare -A`) | 4.0+ | Use files or simple arrays | -| `readarray` / `mapfile` | 4.0+ | Use `while read` loops | -| `${var,,}` / `${var^^}` (case) | 4.0+ | Use `tr '[:upper:]' '[:lower:]'` | - -**CI scripts** (`scripts/ci/*`) run on Linux runners, so bash 4+ and GNU tools are acceptable there. - -### Comment conventions - -1. Generally leave **one blank line above** every comment block or docstring so it is visually separated from code. -2. Use *sentence case* – capitalize the first letter, keep the rest lowercase unless proper nouns or acronyms. -3. Do not use double spaces after periods. -4. **Single-line comments** *must not* end with a period *unless* the line ends with a URL or inline Markdown link – in those cases leave the punctuation exactly as the link requires. -5. **Multi-line comments** should separate sentences with commas (not period-per-line). The final line *should* end with a period. -6. Keep comments concise; favor clarity and only explain the non-obvious – *less is more*. -7. Avoid emoji symbols in text. - -### Doc comment mood - -**Rust** doc comments should be written in the **indicative mood** – e.g. *"Returns a cached client."* - -This convention aligns with the prevailing style of the Rust ecosystem and makes generated -documentation feel natural to end-users. - -### Terminology and phrasing - -1. **Error messages**: Avoid using ", got" in error messages. Use more descriptive alternatives like ", was", ", received", or ", found" depending on context. - - ❌ `"Expected string, got {type(value)}"` - - ✅ `"Expected string, was {type(value)}"` - -2. **Spelling**: Use "hardcoded" (single word) rather than "hard-coded" or "hard coded" – this is the more modern and accepted spelling. - -3. **Error variable naming**: Use single-letter `e` for caught errors/exceptions: - - Rust: `Err(e)` not `Err(err)` or `Err(error)`, and `|e|` not `|err|` in closures - - Python: `except SomeError as e:` not `as err:` or `as error:` - -### Naming conventions - -1. **Internal fields**: Abbreviations are acceptable for private/internal fields (e.g., `_price_prec`, `_size_prec`) to keep hot-path code concise. - -2. **User-facing API**: Use full, descriptive names for public properties, function parameters, return types, and metric names/labels (e.g., `price_precision`, `size_precision`). This prevents abbreviated terminology from leaking into dashboards or alerts. - -3. **Error messages and logs**: Use full words for clarity (e.g., "price precision" not "price prec"). The user should never see abbreviated terminology. - -### Formatting - -1. For longer lines of code, and when passing more than a couple of arguments, you should take a new line which aligns at the next logical indent (rather than attempting a hanging 'vanity' alignment off an opening parenthesis). This practice conserves space to the right, keeps important code more central in view, and survives function/method name changes. - -2. The closing parenthesis should be located on a new line, aligned at the logical indent. - -3. Multiple hanging parameters or arguments should end with a trailing comma: - -```python -long_method_with_many_params( - some_arg1, - some_arg2, - some_arg3, # <-- trailing comma -) -``` - -## Commit messages - -Here are some guidelines for the style of your commit messages: - -1. Limit subject titles to 60 characters or fewer. Capitalize subject line and do not end with period. - -2. Use 'imperative voice', i.e. the message should describe what the commit will do if applied. - -3. Optional: Use the body to explain change. Separate from subject with a blank line. Keep under 100 character width. You can use bullet points with or without terminating periods. - -4. Optional: Provide # references to relevant issues or tickets. - -5. Optional: Provide any hyperlinks which are informative. - -### Gitlint (optional) - -Gitlint is available to help enforce commit message standards automatically. It checks that commit messages follow the guidelines above (character limits, formatting, etc.). This is **opt-in** and not enforced in CI. - -**Benefits**: Encourages concise yet expressive commit messages, helps develop clear explanations of changes. - -**Installation**: First install gitlint to run it locally: - -```bash -uv pip install gitlint -``` - -To enable gitlint as an automatic commit-msg hook: - -```bash -pre-commit install --hook-type commit-msg -``` - -**Manual usage**: Check your last commit message: - -```bash -gitlint -``` - -Configuration is in `.gitlint` at the repository root: - -- **60-character title limit**: Ensures clear rendering on GitHub and encourages brevity while remaining descriptive. -- **79-character body width**: Aligns with Python's PEP 8 conventions and the traditional limit for git tooling. - -:::note -Gitlint may be enforced in CI in the future, so adopting these practices early eases the transition. -::: diff --git a/nautilus-docs/docs/developer_guide/docs.md b/nautilus-docs/docs/developer_guide/docs.md deleted file mode 100644 index 5f3d83c..0000000 --- a/nautilus-docs/docs/developer_guide/docs.md +++ /dev/null @@ -1,134 +0,0 @@ -# Docs Style - -This guide outlines the style conventions and best practices for writing documentation for NautilusTrader. - -## General principles - -- We favor simplicity over complexity, less is more. -- We favor concise yet readable prose and documentation. -- We value standardization in conventions, style, patterns, etc. -- Documentation should be accessible to users of varying technical backgrounds. - -## Language and tone - -- Use active voice when possible ("Configure the adapter" vs "The adapter should be configured"). -- Write in present tense for describing current functionality. -- Use future tense only for planned features. -- Avoid unnecessary jargon; define technical terms on first use. -- Be direct and concise; avoid filler words like "basically", "simply", "just". -- Use parallel structure in lists; keep grammatical patterns consistent across items. - -## Markdown tables - -### Column alignment and spacing - -- Use symmetrical column widths based on the space dictated by the widest content in each column. -- Align column separators (`|`) vertically for better readability. -- Use consistent spacing around cell content. - -### Notes and descriptions - -- All notes and descriptions should have terminating periods. -- Keep notes concise but informative. -- Use sentence case (capitalize only the first letter and proper nouns). - -### Example - -```markdown -| Order Type | Spot | Margin | USDT Futures | Coin Futures | Notes | -|------------------------|------|--------|--------------|--------------|-------------------------| -| `MARKET` | ✓ | ✓ | ✓ | ✓ | | -| `STOP_MARKET` | - | ✓ | ✓ | ✓ | Not supported for Spot. | -| `MARKET_IF_TOUCHED` | - | - | ✓ | ✓ | Futures only. | -``` - -### Support indicators - -- Use `✓` for supported features. -- Use `-` for unsupported features (not `✗` or other symbols). -- When adding notes for unsupported features, emphasize with italics: `*Not supported*`. -- Leave cells empty when no content is needed. - -## Code references - -- Use backticks for inline code, method names, class names, and configuration options. -- Use code blocks for multi-line examples. -- When referencing code locations, use `file_path::function_name` or `file_path::ClassName` rather than line numbers, which become stale as code changes. - -## Headings - -We follow modern documentation conventions that prioritize readability and accessibility: - -- Use title case for the main page heading (# Level 1 only). -- Use sentence case for all subheadings (## Level 2 and below). -- Always capitalize proper nouns regardless of heading level (product names, technologies, companies, acronyms). -- Use proper heading hierarchy (don't skip levels). - -This convention aligns with industry standards used by major technology companies including Google Developer Documentation, Microsoft Docs, and Anthropic's documentation. -It improves readability, reduces cognitive load, and is more accessible for international users and screen readers. - -### Examples - -```markdown -# NautilusTrader Developer Guide - -## Getting started with Python -## Using the Binance adapter -## REST API implementation -## WebSocket data streaming -## Testing with pytest -``` - -## Lists - -- Use hyphens (`-`) for unordered list bullets; avoid `*` or `+` to keep the Markdown style consistent across the project. -- Use numbered lists only when order matters. -- Maintain consistent indentation for nested lists. -- End list items with periods when they are complete sentences. - -## Links and references - -- Use descriptive link text (avoid "click here" or "this link"). -- Reference external documentation when appropriate. -- Keep all internal links relative and accurate. - -## Technical terminology - -- Base capability matrices on the Nautilus domain model, not exchange-specific terminology. -- Mention exchange-specific terms in parentheses or notes when necessary for clarity. -- Use consistent terminology throughout the documentation. - -## Examples and code samples - -- Provide practical, working examples. -- Include necessary imports and context. -- Use realistic variable names and values. -- Add comments to explain non-obvious parts of examples. - -## Admonitions - -Use admonition blocks to highlight important information: - -| Admonition | Purpose | -|--------------|---------------------------------------------------------------| -| `:::note` | Supplementary context that clarifies but isn't essential. | -| `:::info` | Important information the reader should be aware of. | -| `:::tip` | Helpful suggestions or best practices. | -| `:::warning` | Potential pitfalls or important caveats. | -| `:::danger` | Critical issues that could cause data loss or system failure. | - -Avoid overusing admonitions; too many diminish their impact. - -## Line length and wrapping - -- Wrap lines at no more than ~100-120 characters for better readability and diff reviews. -- Break long sentences at natural points (after commas, conjunctions, or phrases). -- Avoid orphaned words on new lines when possible. -- Code blocks and URLs can exceed the line limit when necessary. - -## API documentation - -- Document parameters and return types clearly. -- Include usage examples for complex APIs. -- Explain any side effects or important behavior. -- Keep parameter descriptions concise but complete. diff --git a/nautilus-docs/docs/developer_guide/environment_setup.md b/nautilus-docs/docs/developer_guide/environment_setup.md deleted file mode 100644 index ce9709f..0000000 --- a/nautilus-docs/docs/developer_guide/environment_setup.md +++ /dev/null @@ -1,399 +0,0 @@ -# Environment Setup - -For development we recommend using the PyCharm *Professional* edition IDE, as it interprets Cython syntax. Alternatively, you could use Visual Studio Code with a Cython extension. - -[uv](https://docs.astral.sh/uv) is the preferred tool for handling all Python virtual environments and dependencies. - -[pre-commit](https://pre-commit.com/) is used to automatically run various checks, auto-formatters and linting tools at commit. - -NautilusTrader uses increasingly more [Rust](https://www.rust-lang.org), so Rust should be installed on your system as well -([installation guide](https://www.rust-lang.org/tools/install)). - -[Cap'n Proto](https://capnproto.org/) is required for serialization schema compilation. The required -version is specified in the `capnp-version` file in the repository root. Ubuntu's default package -is typically too old, so you may need to install from source (see below). - -:::info -NautilusTrader *must* compile and run on **Linux, macOS, and Windows**. Please keep portability in -mind (use `std::path::Path`, avoid Bash-isms in shell scripts, etc.). -::: - -## Setup - -The following steps are for UNIX-like systems, and only need to be completed once. - -1. Follow the [installation guide](../getting_started/installation.md) to set up the project with a modification to the final command to install development and test dependencies: - -```bash -uv sync --active --all-groups --all-extras -``` - -or - -```bash -make install -``` - -If you're developing and iterating frequently, then compiling in debug mode is often sufficient and *significantly* faster than a fully optimized build. -To install in debug mode, use: - -```bash -make install-debug -``` - -2. Set up the pre-commit hook which will then run automatically at commit: - -```bash -pre-commit install -``` - -Before opening a pull-request run the formatting and lint suite locally so that CI passes on the -first attempt: - -```bash -make format -make pre-commit -``` - -Make sure the Rust compiler reports **zero errors** – broken builds slow everyone down. - -3. **Required for Rust/PyO3 (Linux and macOS)**: When using Python installed via `uv` on Linux or macOS, set the following environment variables: - -```bash -# Add to your shell configuration (e.g., ~/.zshrc or ~/.bashrc) - -# Linux only: Set the library path for the Python interpreter -export LD_LIBRARY_PATH="$(python -c 'import sys; print(sys.base_prefix)')/lib:$LD_LIBRARY_PATH" - -# Set the Python executable path for PyO3 -export PYO3_PYTHON=$(pwd)/.venv/bin/python - -# Set the Python home path (required for Rust tests) -export PYTHONHOME=$(python -c "import sys; print(sys.base_prefix)") -``` - -:::note -The `LD_LIBRARY_PATH` export is Linux-specific and not needed on macOS or Windows. - -- `PYO3_PYTHON` tells PyO3 which Python interpreter to use, reducing unnecessary recompilation. -- `PYTHONHOME` is required when running `make cargo-test` with a `uv`-installed Python. - Without it, tests that depend on PyO3 may fail to locate the Python runtime. - -::: - -To verify your environment is configured correctly: - -```bash -python -c "import sys; print('Python:', sys.executable, sys.version)" -echo "PYO3_PYTHON: $PYO3_PYTHON" -echo "PYTHONHOME: $PYTHONHOME" -``` - -## Builds - -Following any changes to `.rs`, `.pyx` or `.pxd` files, you can re-compile by running: - -```bash -uv run --no-sync python build.py -``` - -or - -```bash -make build -``` - -If you're developing and iterating frequently, then compiling in debug mode is often sufficient and *significantly* faster than a fully optimized build. -To compile in debug mode, use: - -```bash -make build-debug -``` - -## Cap'n Proto - -[Cap'n Proto](https://capnproto.org/) is required for serialization schema compilation. -The required version is defined in the `capnp-version` file in the repository root. - -The recommended way to install the correct version on **Linux** or **macOS** is: - -```bash -./scripts/install-capnp.sh -``` - -This script ensures the pinned version is installed. Alternatively, you can install manually: - -On **macOS**, install via Homebrew: - -```bash -brew install capnp -``` - -Verify the installed version matches `capnp-version`. If Homebrew provides an older version, -install from source using the Linux instructions below. - -On **Ubuntu/Linux**, the default package is typically too old. Install from source: - -```bash -CAPNP_VERSION=$(cat capnp-version) -cd ~ -wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz -tar xzf capnproto-c++-${CAPNP_VERSION}.tar.gz -cd capnproto-c++-${CAPNP_VERSION} -./configure -make -j$(nproc) -sudo make install -sudo ldconfig -``` - -Verify installation: - -```bash -capnp --version -``` - -On **Windows**, install via Chocolatey: - -```bash -choco install capnproto -``` - -Verify the installed version matches `capnp-version`. If Chocolatey provides an older version, -see the [Cap'n Proto installation guide](https://capnproto.org/install.html) for alternative -installation methods. - -## Faster builds - -The cranelift backends reduces build time significantly for dev, testing and IDE checks. However, cranelift is available on the nightly toolchain and needs extra configuration. Install the nightly toolchain - -``` -rustup install nightly -rustup override set nightly -rustup component add rust-analyzer # install nightly lsp -rustup override set stable # reset to stable -``` - -Activate the nightly feature and use "cranelift" backend for dev and testing profiles in workspace `Cargo.toml`. You can apply the below patch using `git apply `. You can remove it using `git apply -R ` before pushing changes. - -:::warning -Do not commit these changes. The cranelift patch is for local development only and will break CI if pushed. -::: - -``` -diff --git a/Cargo.toml b/Cargo.toml -index 62b78cd8d0..beb0800211 100644 ---- a/Cargo.toml -+++ b/Cargo.toml -@@ -1,3 +1,6 @@ -+# This line needs to come before anything else in Cargo.toml -+cargo-features = ["codegen-backend"] -+ - [workspace] - resolver = "2" - members = [ -@@ -140,6 +143,7 @@ lto = false - panic = "unwind" - incremental = true - codegen-units = 256 -+codegen-backend = "cranelift" - - [profile.test] - opt-level = 0 -@@ -150,11 +154,13 @@ strip = false - lto = false - incremental = true - codegen-units = 256 -+codegen-backend = "cranelift" - - [profile.nextest] - inherits = "test" - debug = false # Improves compile times - strip = "debuginfo" # Improves compile times -+codegen-backend = "cranelift" - - [profile.release] - opt-level = 3 -``` - -Pass `RUSTUP_TOOLCHAIN=nightly` when running `make build-debug` like commands and include it in all [rust analyzer settings](#rust-analyzer-settings) for faster builds and IDE checks. - -## Services - -You can use `docker-compose.yml` file located in `.docker` directory -to bootstrap the Nautilus working environment. This will start the following services: - -```bash -docker-compose up -d -``` - -If you only want specific services running (like `postgres` for example), you can start them with command: - -```bash -docker-compose up -d postgres -``` - -Used services are: - -- `postgres`: Postgres database with root user `POSTGRES_USER` which defaults to `postgres`, `POSTGRES_PASSWORD` which defaults to `pass` and `POSTGRES_DB` which defaults to `postgres`. -- `redis`: Redis server. -- `pgadmin`: PgAdmin4 for database management and administration. - -:::info -Please use this as development environment only. For production, use a proper and more secure setup. -::: - -After the services has been started, you must log in with `psql` cli to create `nautilus` Postgres database. -To do that you can run, and type `POSTGRES_PASSWORD` from docker service setup - -```bash -psql -h localhost -p 5432 -U postgres -``` - -After you have logged in as `postgres` administrator, run `CREATE DATABASE` command with target db name (we use `nautilus`): - -``` -psql (16.2, server 15.2 (Debian 15.2-1.pgdg110+1)) -Type "help" for help. - -postgres=# CREATE DATABASE nautilus; -CREATE DATABASE - -``` - -## Nautilus CLI developer guide - -## Introduction - -The Nautilus CLI is a command-line interface tool for interacting with the NautilusTrader ecosystem. -It offers commands for managing the PostgreSQL database and handling various trading operations. - -:::warning -On Linux systems with GNOME desktop, the `nautilus` command typically refers to the GNOME file manager (`/usr/bin/nautilus`). -After installing the NautilusTrader CLI, you may need to ensure the Cargo binary takes precedence by either: - -- Adding an alias to your shell config: `alias nautilus="$HOME/.cargo/bin/nautilus"` -- Using the full path: `~/.cargo/bin/nautilus` -- Ensuring `~/.cargo/bin` appears before `/usr/bin` in your `PATH` - -::: - -:::note -The Nautilus CLI command is only supported on UNIX-like systems. -::: - -## Install - -You can install the Nautilus CLI using the below Makefile target, which uses `cargo install` under the hood. -This will place the nautilus binary in your system's PATH, assuming Rust's `cargo` is properly configured. - -```bash -make install-cli -``` - -## Commands - -You can run `nautilus --help` to view the CLI structure and available command groups: - -### Database - -These commands handle bootstrapping the PostgreSQL database. -To use them, you need to provide the correct connection configuration, -either through command-line arguments or a `.env` file located in the root directory or the current working directory. - -- `--host` or `POSTGRES_HOST` for the database host -- `--port` or `POSTGRES_PORT` for the database port -- `--user` or `POSTGRES_USERNAME` for the root administrator (typically the postgres user) -- `--password` or `POSTGRES_PASSWORD` for the root administrator's password -- `--database` or `POSTGRES_DATABASE` for both the database **name and the new user** with privileges to that database - (e.g., if you provide `nautilus` as the value, a new user named nautilus will be created with the password from `POSTGRES_PASSWORD`, and the `nautilus` database will be bootstrapped with this user as the owner). - -Example of `.env` file - -``` -POSTGRES_HOST=localhost -POSTGRES_PORT=5432 -POSTGRES_USERNAME=postgres -POSTGRES_PASSWORD=pass -POSTGRES_DATABASE=nautilus -``` - -List of commands are: - -1. `nautilus database init`: Will bootstrap schema, roles and all sql files located in `schema` root directory (like `tables.sql`). -2. `nautilus database drop`: Will drop all tables, roles and data in target Postgres database. - -## Rust analyzer settings - -Rust analyzer is a popular language server for Rust and has integrations for many IDEs. It is recommended to configure rust analyzer to have same environment variables as `make build-debug` for faster compile times. Below tested configurations for VSCode and Astro Nvim are provided. For more information see [PR](https://github.com/nautechsystems/nautilus_trader/pull/2524) or rust analyzer [config docs](https://rust-analyzer.github.io/book/configuration.html). - -### VSCode - -You can add the following settings to your VSCode `settings.json` file: - -``` - "rust-analyzer.restartServerOnConfigChange": true, - "rust-analyzer.linkedProjects": [ - "Cargo.toml" - ], - "rust-analyzer.cargo.features": "all", - "rust-analyzer.check.workspace": false, - "rust-analyzer.check.extraEnv": { - "VIRTUAL_ENV": "/.venv", - "CC": "clang", - "CXX": "clang++" - }, - "rust-analyzer.cargo.extraEnv": { - "VIRTUAL_ENV": "/.venv", - "CC": "clang", - "CXX": "clang++" - }, - "rust-analyzer.runnables.extraEnv": { - "VIRTUAL_ENV": "/.venv", - "CC": "clang", - "CXX": "clang++" - }, - "rust-analyzer.check.features": "all", - "rust-analyzer.testExplorer": true -``` - -### Astro Nvim (Neovim + AstroLSP) - -You can add the following to your astro lsp config file: - -``` - config = { - rust_analyzer = { - settings = { - ["rust-analyzer"] = { - restartServerOnConfigChange = true, - linkedProjects = { "Cargo.toml" }, - cargo = { - features = "all", - extraEnv = { - VIRTUAL_ENV = "/.venv", - CC = "clang", - CXX = "clang++", - }, - }, - check = { - workspace = false, - command = "check", - features = "all", - extraEnv = { - VIRTUAL_ENV = "/.venv", - CC = "clang", - CXX = "clang++", - }, - }, - runnables = { - extraEnv = { - VIRTUAL_ENV = "/.venv", - CC = "clang", - CXX = "clang++", - }, - }, - testExplorer = true, - }, - }, - }, -``` diff --git a/nautilus-docs/docs/developer_guide/ffi.md b/nautilus-docs/docs/developer_guide/ffi.md deleted file mode 100644 index 6366708..0000000 --- a/nautilus-docs/docs/developer_guide/ffi.md +++ /dev/null @@ -1,142 +0,0 @@ -# FFI Memory Contract - -NautilusTrader exposes several **C-compatible** types so that compiled Rust code can be -consumed from C-extensions generated by Cython or by other native languages. The most -important of these is `CVec` – a *thin* wrapper around a Rust `Vec` that is passed across -the FFI boundary **by value**. - -The rules below are *strict*; violating them results in undefined behaviour (usually a double-free or a memory leak). - -## Fail-fast panics at the FFI boundary - -Rust panics must never unwind across `extern "C"` functions. Unwinding into C or Python is -undefined behaviour and can corrupt the foreign stack or leave partially-dropped resources -behind. To enforce the fail-fast architecture we wrap every exported symbol in -`crate::ffi::abort_on_panic`, which executes the body and calls `process::abort()` if a panic -occurs. The panic message is still logged before the abort, so debugging output is preserved -while avoiding undefined behaviour. - -When adding new FFI functions, call `abort_on_panic(|| { … })` around the implementation (or -use a helper that does so) to maintain this guarantee. - -## CVec lifecycle - -| Step | Owner | Action | -|-------|-------------------------------|--------| -| **1** | Rust | Build a `Vec` and convert it with `into()` – this *leaks* the vector and transfers ownership of the raw allocation to foreign code. | -| **2** | Foreign (Python / Cython / C) | Use the data while the `CVec` value is in scope. **Do not modify the fields `ptr`, `len`, `cap`.** | -| **3** | Foreign | Exactly once, call the *type-specific* drop helper exported by Rust (for example `vec_drop_book_levels`, `vec_drop_book_orders`, `vec_time_event_handlers_drop`). The helper reconstructs the original `Vec` with `Vec::from_raw_parts` and lets it drop, freeing the memory. | - -:::warning -If step **3** is forgotten the allocation is leaked for the remainder of the process; if it -is performed **twice** the program will double-free and likely crash. -::: - -## Capsules created on the Python side - -Several Cython helpers allocate temporary C buffers with `PyMem_Malloc`, wrap them into a -`CVec`, and return the address inside a `PyCapsule`. **Every such capsule is created with a -destructor** (`capsule_destructor` or `capsule_destructor_deltas`) that frees both the buffer -and the `CVec`. Callers must therefore *not* free the memory manually – doing so would double -free. - -## Capsules created on the Rust side *(PyO3 bindings)* - -When Rust code pushes a heap-allocated value into Python it **must** use -`PyCapsule::new_with_destructor` so that Python knows how to free the allocation -once the capsule becomes unreachable. The closure/destructor is responsible -for reconstructing the original `Box` or `Vec` and letting it drop. - -```rust -use pyo3::types::PyCapsule; - -Python::attach(|py| { - // Allocate the value on the heap - let my_data = Box::new(MyStruct::new()); - let ptr = Box::into_raw(my_data); - - // Move it into the capsule and register a destructor that frees the memory - let capsule = PyCapsule::new_with_destructor( - py, - ptr, - None, - |ptr, _| { - // Reconstruct the Box and let it drop, freeing the allocation - let _ = unsafe { Box::from_raw(ptr) }; - }, - ) - .expect("capsule creation failed"); - - // ... pass `capsule` back to Python ... -}); -``` - -Do **not** use `PyCapsule::new(…, None)`; that variant registers *no* destructor -and will leak memory unless the recipient manually extracts and frees the -pointer (something we never rely on). The codebase has been updated to follow -this rule everywhere – adding new FFI modules must follow the same pattern. - -## Why there is no generic `cvec_drop` anymore - -Earlier versions of the codebase shipped a generic `cvec_drop` function that always treated the -buffer as `Vec`. Using it with any other element type causes a size-mismatch during -deallocation and corrupts the allocator's bookkeeping. Because the helper was not referenced -anywhere inside the project it has been removed to avoid accidental misuse. - -Instead, use the **type-specific** drop helper for your element type (e.g., `vec_drop_book_levels`, -`vec_drop_book_orders`). If no helper exists for your type, add one following the pattern in -`crates/core/src/ffi/cvec.rs`. - -## Box-backed `*_API` wrappers (owned Rust objects) - -When the Rust core needs to hand a *complex* value (for example an -`OrderBook`, `SyntheticInstrument`, or `TimeEventAccumulator`) to foreign -code it allocates the value on the heap with `Box::new` and returns a -small `repr(C)` wrapper whose only field is that `Box`. - -```rust -#[repr(C)] -pub struct OrderBook_API(Box); - -#[unsafe(no_mangle)] -pub extern "C" fn orderbook_new(id: InstrumentId, book_type: BookType) -> OrderBook_API { - OrderBook_API(Box::new(OrderBook::new(id, book_type))) -} - -#[unsafe(no_mangle)] -pub extern "C" fn orderbook_drop(book: OrderBook_API) { - drop(book); // frees the heap allocation -} -``` - -Memory-safety requirements are therefore: - -1. Every constructor (`*_new`) **must** have a matching `*_drop` exported - next to it. -2. Validate parameters before heap allocation to fail fast and avoid allocating invalid objects. -3. The *Python/Cython* binding must guarantee that `*_drop` is invoked - exactly once. Two approaches exist: - - • **Preferred for new code**: Wrap the pointer in a `PyCapsule` created with - `PyCapsule::new_with_destructor`, passing a destructor that calls - the drop helper. - - • **Legacy pattern** (v1 Cython modules only): Call the helper explicitly in - `__del__`/`__dealloc__` on the Python side: - - ```python - cdef class OrderBook: - cdef OrderBook_API _mem - - def __cinit__(self, ...): - self._mem = orderbook_new(...) - - def __del__(self): - if self._mem._0 != NULL: - orderbook_drop(self._mem) - ``` - -Whichever style is used, remember: **forgetting the drop call leaks the -entire structure**, while calling it twice will double-free and crash. - -New FFI code must use `PyCapsule` with destructors and follow this template before it can be merged. diff --git a/nautilus-docs/docs/developer_guide/index.md b/nautilus-docs/docs/developer_guide/index.md deleted file mode 100644 index 6bf1c4f..0000000 --- a/nautilus-docs/docs/developer_guide/index.md +++ /dev/null @@ -1,27 +0,0 @@ -# Developer Guide - -Guidance on developing and extending NautilusTrader, or contributing back to the project. - -NautilusTrader uses a **Rust core with Python bindings** architecture: - -- **Rust** handles networking, data parsing, order matching, and other performance-critical operations. -- **Python** provides the user-facing API for strategy development, configuration, and system integration. -- **PyO3** bridges the two, exposing Rust functionality to Python with minimal overhead. - -This approach combines Python's simplicity and ecosystem with Rust's performance and memory safety. - -## Contents - -- [Environment Setup](environment_setup.md) -- [Coding Standards](coding_standards.md) -- [Rust](rust.md) -- [Python](python.md) -- [Testing](testing.md) -- [Test Datasets](test_datasets.md) -- [Docs Style](docs.md) -- [Release Notes](releases.md) -- [Adapters](adapters.md) -- [Data Testing Spec](spec_data_testing.md) -- [Execution Testing Spec](spec_exec_testing.md) -- [Benchmarking](benchmarking.md) -- [FFI Memory Contract](ffi.md) diff --git a/nautilus-docs/docs/developer_guide/python.md b/nautilus-docs/docs/developer_guide/python.md deleted file mode 100644 index 466a2b3..0000000 --- a/nautilus-docs/docs/developer_guide/python.md +++ /dev/null @@ -1,98 +0,0 @@ -# Python - -The [Python](https://www.python.org/) programming language is used for the majority of user-facing code in NautilusTrader. -Python provides a rich ecosystem of libraries and frameworks, making it ideal for strategy development, data analysis, and system integration. - -## Code style - -### PEP-8 - -The codebase generally follows the PEP-8 style guide. -One notable departure is that Python truthiness is not always taken advantage of to check if an argument is `None` for everything other than collections. - -As per the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), it's discouraged to use truthiness to check if an argument is/is not `None`, when there is a chance an unexpected object could be passed into the function or method which will yield an unexpected truthiness evaluation (which could result in a logical error type bug). - -*"Always use if foo is None: (or is not None) to check for a None value. E.g., when testing whether a variable or argument that defaults to None was set to some other value. The other value might be a value that's false in a boolean context!"* - -:::note -Use truthiness to check for empty collections (e.g., `if not my_list:`) rather than comparing explicitly to `None` or empty. -::: - -We welcome all feedback on where the codebase departs from PEP-8 for no apparent reason. - -### Type hints - -All function and method signatures *must* include type annotations: - -```python -def __init__(self, config: EMACrossConfig) -> None: -def on_bar(self, bar: Bar) -> None: -def on_save(self) -> dict[str, bytes]: -def on_load(self, state: dict[str, bytes]) -> None: -``` - -**Union syntax**: Use PEP 604 union syntax for optional types: - -```python -# Preferred -def get_instrument(self, id: InstrumentId) -> Instrument | None: - -# Avoid -def get_instrument(self, id: InstrumentId) -> Optional[Instrument]: -``` - -**Generic types**: Use `TypeVar` for reusable components: - -```python -T = TypeVar("T") -class ThrottledEnqueuer(Generic[T]): -``` - -### Docstrings - -The [NumPy docstring spec](https://numpydoc.readthedocs.io/en/latest/format.html) is used throughout the codebase. -This needs to be followed consistently so the docs build correctly. - -**Python** docstrings should be written in the **imperative mood** – e.g. *"Return a cached client."* - -This convention aligns with the prevailing style of the Python ecosystem and makes generated -documentation feel natural to end-users. - -#### Private methods - -Do not add docstrings to private methods (prefixed with `_`): - -- Docstrings generate public-facing API documentation. -- Docstrings on private methods incorrectly imply they are part of the public API. -- Private methods are implementation details not intended for end-users. - -Exceptions where docstrings are acceptable: - -- Very complex methods with non-trivial logic, multiple steps, or important edge cases. -- Methods requiring detailed parameter or return value documentation due to complexity. - -When a private method needs context (such as a tricky precondition or side effect), prefer a short inline comment (`#`) near the relevant logic rather than a docstring. - -### Test naming - -Descriptive names explaining the scenario: - -```python -def test_currency_with_negative_precision_raises_overflow_error(self): -def test_sma_with_no_inputs_returns_zero_count(self): -def test_sma_with_single_input_returns_expected_value(self): -``` - -### Ruff - -[ruff](https://astral.sh/ruff) is used to lint the codebase. Ruff rules can be found in the top-level `pyproject.toml`, with ignore justifications typically commented. - -## Cython (legacy) - -:::warning[Deprecation notice] -Cython is being phased out in favor of Rust implementations. New code should use Rust. This section documents legacy Cython code only. -::: - -For legacy `.pyx` and `.pxd` files, make sure all functions and methods returning `void` or a primitive C type (such as `bint`, `int`, `double`) include the `except *` keyword in the signature. Without it, Python exceptions are silently ignored. - -For more information, see the [Cython docs](https://cython.readthedocs.io/en/latest/index.html). diff --git a/nautilus-docs/docs/developer_guide/releases.md b/nautilus-docs/docs/developer_guide/releases.md deleted file mode 100644 index df89d9a..0000000 --- a/nautilus-docs/docs/developer_guide/releases.md +++ /dev/null @@ -1,271 +0,0 @@ -# Releases - -This guide covers the release process and the standards for writing release notes. - -## Overview - -NautilusTrader uses a three-branch model: - -- **`develop`**: active development; publishes dev wheels to Cloudflare R2 on every push. -- **`nightly`**: pre-release testing; publishes alpha wheels and CLI binaries. -- **`master`**: stable releases; triggers the full release pipeline. - -Pushing to `master` automatically tags the version from `pyproject.toml`, creates a GitHub -release, publishes wheels and sdist to PyPI, builds Docker images, and triggers a docs rebuild. - -## Versioning - -The project maintains two version numbers: - -| File | Scope | Example | -|--------------------------|----------------|-----------| -| `pyproject.toml` | Python package | `1.223.0` | -| `Cargo.toml` (workspace) | Rust crates | `0.53.0` | - -These are bumped independently. The Python version drives the release tag (`v1.223.0`). - -## Release checklist - -### Pre-release (on `develop`) - -- [ ] Finalize `RELEASES.md`: review all items, remove empty sections -- [ ] Ensure versions are set in `pyproject.toml` and `Cargo.toml` workspace -- [ ] Ensure all CI checks pass on `develop` - -### Release - -- [ ] Merge `develop` into `nightly`, verify nightly CI passes -- [ ] Merge `nightly` into `master` -- [ ] Verify the `build` workflow completes: - - Wheels built for Linux x86/ARM, macOS, Windows - - `cargo-deny` and `cargo-vet` pass - - Tag created and GitHub release published - - Wheels and sdist published to PyPI -- [ ] Verify the `docker` workflow completes (images built and pushed) -- [ ] Verify the `build-docs` workflow completes (docs rebuild triggered) - -### Post-release (on `develop`) - -- [ ] Update the release date in `RELEASES.md` for the published version -- [ ] Add horizontal separator `---` below the completed release -- [ ] Add the next version template at the top of `RELEASES.md` (see below) -- [ ] Bump `pyproject.toml` version to the next release number - -## Release notes - -This section documents the standards for writing release notes in `RELEASES.md`. - -### Sections - -Use the following sections in this order: - -1. Enhancements -2. Breaking Changes -3. Security -4. Fixes -5. Internal Improvements -6. Documentation Updates -7. Deprecations - -Omit sections that have no items for a given release. - -### Enhancements - -New features and user-visible improvements. - -**Format**: - -```markdown -- Added `subscribe_order_fills(...)` and `unsubscribe_order_fills(...)` for `Actor` -- Added BitMEX conditional orders support -- Added support for `OrderBookDepth10` requests (#2955), thanks @faysou -``` - -**Guidelines**: - -- Start with "Added". -- Use backticks for code elements. -- Be specific about what was added, not how. - -### Breaking Changes - -Changes that may break existing code. - -**Format**: - -```markdown -- Removed `nautilus_trader.analysis.statistics` subpackage - must import from `nautilus_trader.analysis` -- Renamed `BinanceAccountType.USDT_FUTURE` to `USDT_FUTURES` -- Changed `start` parameter to required for `Actor` data request methods -``` - -**Guidelines**: - -- Start with "Removed", "Renamed", or "Changed". -- Explain migration path briefly. - -### Security - -Security hardening and fixes that prevent crashes, undefined behavior, or data corruption. -Includes significant hardening improvements elevated from Internal Improvements. - -**Format**: - -```markdown -- Fixed non-executable stack for Cython extensions to support hardened Linux systems -- Fixed divide-by-zero and overflow bugs in model crate that could cause crashes -- Fixed core arithmetic operations to reject NaN/Infinity values and improve overflow handling -``` - -**Guidelines**: - -- Include overflow/underflow fixes, memory safety improvements, FFI guards, data integrity fixes. -- Focus on user impact: what could have happened. -- Exclude routine dependency updates, minor hardening, or test-only fixes. -- Omit this section entirely if there are no security items for the release. - -### Fixes - -Bug fixes that improve correctness but don't qualify as security issues. - -**Format**: - -```markdown -- Fixed reduce-only order panic when quantity exceeds position -- Fixed Binance order status parsing for external orders (#3006), thanks for reporting @bmlquant -``` - -**Guidelines**: - -- Start with "Fixed". - -### Internal Improvements - -Implementation details and infrastructure changes. - -**Format**: - -```markdown -- Added ARM64 support to Docker builds -- Ported `PortfolioAnalyzer` to Rust -- Improved clock and timer thread safety -- Upgraded Rust (MSRV) to 1.90.0 -- Upgraded `pyo3` crates to v0.26.0 -``` - -**Guidelines**: - -- Use "Added", "Implemented", "Improved", "Optimized", "Upgraded", "Refined", "Standardized". -- Include version numbers for dependency upgrades. - -### Documentation Updates - -Changes to guides and examples. - -**Format**: - -```markdown -- Added rate limit tables with links to official docs -- Improved dark and light themes for readability -- Fixed broken links -``` - -### Deprecations - -Features marked for removal. - -**Format**: - -```markdown -- Deprecated `convert_quote_qty_to_base`; disable (`False`) to maintain consistent behaviour. Will be removed in future version -``` - -**Guidelines**: - -- Explain migration path and provide alternatives. - -## Attribution - -- Credit external contributors: `thanks @username` or `thanks for reporting @username`. -- Include issue/PR numbers for community contributions and complex features: `(#1234)`. - -## Style - -- Use sentence case (capitalize first word only). -- Do not end with periods. -- Use backticks for code elements. -- Focus on **what** changed, not how. - -**Be specific**: - -```markdown -❌ Improved Binance adapter -✅ Improved Binance fill handling when instrument not cached -``` - -## Security classification - -Include in Security if the change addresses: - -- Memory safety (overflow, underflow, divide-by-zero that threatens stability). -- Undefined behavior or crashes that could corrupt state. -- Data integrity (NaN/Infinity propagation, race conditions leading to corruption). -- Input validation preventing injection or exploitation (SQL injection, command injection, path traversal). -- Build hardening (non-exec stack, FFI guards). -- Significant hardening that users should know about. - -Otherwise use Fixes (for logic bugs and panics) or Internal Improvements (for minor hardening). - -Note: Plain logic panics belong in Fixes unless they threaten system stability or data corruption. - -## Examples - -**Security** (could cause crashes/corruption): - -```markdown -- Fixed divide-by-zero in margin calculations that could crash the engine -- Fixed non-executable stack for Cython extensions to support hardened systems -``` - -**Fixes** (incorrect but safe): - -```markdown -- Fixed Binance order status parsing for external orders -- Fixed position purge logic to prevent purging re-opened position -``` - -**Enhancements** (user-facing): - -```markdown -- Added BitMEX conditional orders support -``` - -**Internal** (implementation): - -```markdown -- Implemented BitMEX ping/pong handling -``` - -## Release notes template - -```markdown -# NautilusTrader Beta - -Released on TBD (UTC). - -### Enhancements - -### Breaking Changes - -### Security - -### Fixes - -### Internal Improvements - -### Documentation Updates - -### Deprecations - ---- -``` diff --git a/nautilus-docs/docs/developer_guide/rust.md b/nautilus-docs/docs/developer_guide/rust.md deleted file mode 100644 index 1da78a3..0000000 --- a/nautilus-docs/docs/developer_guide/rust.md +++ /dev/null @@ -1,1265 +0,0 @@ -# Rust - -The [Rust](https://www.rust-lang.org/learn) programming language is an ideal fit for implementing the mission-critical core of the platform and systems. -Its strong type system, ownership model, and compile-time checks eliminate memory errors and data races by construction, -while zero-cost abstractions and the absence of a garbage collector deliver C-like performance, important for high-frequency trading workloads. - -## Cargo manifest conventions - -- In `[dependencies]`, list internal crates (`nautilus-*`) first in alphabetical order, insert a blank line, then external required dependencies alphabetically, followed by another blank line and the optional dependencies (those with `optional = true`) in alphabetical order. Preserve inline comments with their dependency. -- Add `"python"` to every `extension-module` feature list that builds a Python artefact, keeping it adjacent to `"pyo3/extension-module"` so the full Python stack is obvious. -- When a manifest groups adapters separately (for example `crates/pyo3`), keep the `# Adapters` block immediately below the internal crate list so downstream consumers can scan adapter coverage quickly. -- Always include a blank line before `[dev-dependencies]` and `[build-dependencies]` sections. -- Apply the same layout across related manifests when the feature or dependency sets change to avoid drift between crates. -- Use snake_case filenames for `bin/` sources (for example `bin/ws_data.rs`) and reflect those paths in each `[[bin]]` section. -- Keep `[[bin]] name` entries in kebab-case (for example `name = "hyperliquid-ws-data"`) so the compiled binaries retain their intended CLI names. - -## Versioning guidance - -- Use workspace inheritance for shared dependencies (for example `serde = { workspace = true }`). -- Only pin versions directly for crate-specific dependencies that are not part of the workspace. -- Group workspace-provided dependencies before crate-only dependencies so the inheritance is easy to audit. -- Keep related dependencies aligned: `capnp`/`capnpc` (exact), `arrow`/`parquet` (major.minor), - `datafusion`/`object_store`, and `dydx-proto`/`prost`/`tonic`. Pre-commit enforces this. -- Adapter-only dependencies belong in the "Adapter dependencies" section of the workspace - `Cargo.toml`. Pre-commit prevents core crates from using them. - -## Feature flag conventions - -- Prefer additive feature flags. Enabling a feature must not break existing functionality. -- Use descriptive flag names that explain what capability is enabled. -- Document every feature in the crate-level documentation so consumers know what they toggle. -- Common patterns: - - `high-precision`: switches the value-type backing (64-bit or 128-bit integers) to support domains that require extra precision. - - `default = []`: keep defaults minimal. - - `python`: enables Python bindings. - - `extension-module`: builds a Python extension module (always include `python`). - - `ffi`: enables C FFI bindings. - - `stubs`: exposes testing stubs. - -## Build configurations - -To avoid unnecessary rebuilds during development, align cargo features, profiles, and flags across different build targets. -Cargo's build cache is keyed by the exact combination of features, profiles, and flags. Any mismatch triggers a full rebuild. - -### Aligned targets (testing and linting) - -| Target | Features | Profile | `--all-targets` | `--no-deps` | Purpose | -|-----------------------------|----------------------------------|-----------|-----------------|-------------|----------------| -| `cargo-test` | `ffi,python,high-precision,defi` | `nextest` | ✓ (implicit) | n/a | Run tests. | -| `cargo-clippy` (pre-commit) | `ffi,python,high-precision,defi` | `nextest` | ✓ | n/a | Lint all code. | - -These targets share the same feature set and profile, allowing cargo to reuse compiled artifacts between linting and testing without rebuilds. -The `nextest` profile is used to align with the workflow of the majority of core maintainers who use cargo-nextest for running tests. - -### Documentation builds - -Documentation is built separately using `make docs-rust`, which runs: - -```bash -cargo +nightly doc --all-features --no-deps --workspace -``` - -This uses the nightly toolchain and `--all-features` rather than the aligned feature set above, so it does not share build artifacts with testing/linting. - -### Separate target (Python extension building) - -| Target | Features | Profile | Notes | -|---------------|--------------------------------------|-----------|-------| -| `build` | Includes `extension-module` + subset | `release` | Requires different features for PyO3 extension module. | -| `build-debug` | Includes `extension-module` + subset | `dev` | Requires different features for PyO3 extension module. | - -Python extension building intentionally uses different features (`extension-module` is required) and will trigger rebuilds. This is expected and unavoidable. - -### Rebuild triggers to avoid - -Mismatches in any of these cause full rebuilds: - -- Different feature combinations (e.g., `--features "a,b"` vs `--features "a,c"`). -- Different `--no-default-features` usage (enables/disables default features). -- Different profiles (e.g., `dev` vs `nextest` vs `release`). - -When adding new build targets or modifying existing ones, maintain alignment with the testing/linting group to preserve fast incremental builds. - -## Module organization - -- Keep modules focused on a single responsibility. -- Use `mod.rs` as the module root when defining submodules. -- Prefer relatively flat hierarchies over deep nesting to keep paths manageable. -- Re-export commonly used items from the crate root for convenience. - -## Code style and conventions - -### File header requirements - -All Rust files must include the standardized copyright header: - -```rust -// ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved. -// https://nautechsystems.io -// -// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ------------------------------------------------------------------------------------------------- -``` - -:::info[Automated enforcement] -The `check_copyright_year.sh` pre-commit hook verifies copyright headers include the current year. -::: - -### Code formatting - -Import formatting is automatically handled by rustfmt when running `make format`. -The tool organizes imports into groups (standard library, external crates, local imports) and sorts them alphabetically within each group. - -Within this section, follow these spacing rules: - -- Leave **one blank line between functions** (including tests) – this improves readability and -mirrors the default behavior of `rustfmt`. -- Leave **one blank line above every doc comment** (`///` or `//!`) so that the comment is clearly - detached from the previous code block. - -#### String formatting - -Prefer inline format strings over positional arguments: - -```rust -// Preferred - inline format with variable names -anyhow::bail!("Failed to subtract {n} months from {datetime}"); - -// Instead of - positional arguments -anyhow::bail!("Failed to subtract {} months from {}", n, datetime); -``` - -This makes messages more readable and self-documenting, especially when there are multiple variables. - -### Type qualification - -Follow these conventions for qualifying types in code: - -- **anyhow**: Always fully qualify `anyhow` macros (`anyhow::bail!`, `anyhow::anyhow!`) and the Result type (`anyhow::Result`). -- **Nautilus domain types**: Do not fully qualify Nautilus domain types. Use them directly after importing (e.g., `Symbol`, `InstrumentId`, `Price`). -- **tokio**: Generally fully qualify `tokio` types as they can have equivalents in std library and other crates (e.g., `tokio::spawn`, `tokio::time::timeout`). - -```rust -use nautilus_model::identifiers::Symbol; - -pub fn process_symbol(symbol: Symbol) -> anyhow::Result<()> { - if !symbol.is_valid() { - anyhow::bail!("Invalid symbol: {symbol}"); - } - - tokio::spawn(async move { - // Process symbol asynchronously - }); - - Ok(()) -} -``` - -:::info[Automated enforcement] -The `check_anyhow_usage.sh` pre-commit hook enforces these anyhow conventions automatically. -::: - -### Logging - -- Fully qualify logging macros so the backend is explicit: - - Use `log::…` (`log::debug!`, `log::info!`, `log::warn!`, etc.) for all Rust components. -- Start messages with a capitalised word, prefer complete sentences, and omit terminal periods (e.g. `"Processing batch"`, not `"Processing batch."`). - -:::info[Automated enforcement] -The `check_logging_macro_usage.sh` pre-commit hook enforces fully qualified logging macros. -::: - -### Error handling - -Use structured error handling patterns consistently: - -1. **Primary Pattern**: Use `anyhow::Result` for fallible functions: - - ```rust - pub fn calculate_balance(&mut self) -> anyhow::Result { - // Implementation - } - ``` - -2. **Custom Error Types**: Use `thiserror` for domain-specific errors: - - ```rust - #[derive(Error, Debug)] - pub enum NetworkError { - #[error("Connection failed: {0}")] - ConnectionFailed(String), - #[error("Timeout occurred")] - Timeout, - } - ``` - -3. **Error Propagation**: Use the `?` operator for clean error propagation. - -4. **Error Creation**: Prefer `anyhow::bail!` for early returns with errors: - - ```rust - // Preferred - using bail! for early returns - pub fn process_value(value: i32) -> anyhow::Result { - if value < 0 { - anyhow::bail!("Value cannot be negative: {value}"); - } - Ok(value * 2) - } - - // Instead of - verbose return statement - if value < 0 { - return Err(anyhow::anyhow!("Value cannot be negative: {value}")); - } - ``` - - **Note**: Use `anyhow::bail!` for early returns, but `anyhow::anyhow!` in closure contexts like `ok_or_else()` where early returns aren't possible. - -5. **Error Context**: Use lowercase for `.context()` messages to support error chaining (except proper nouns/acronyms): - - ```rust - // Good - lowercase chains naturally - parse_timestamp(value).context("failed to parse timestamp")?; - - // Exception - proper nouns stay capitalized - connect().context("BitMEX websocket did not become active")?; - ``` - -:::info[Automated enforcement] -The `check_error_conventions.sh` and `check_anyhow_usage.sh` pre-commit hooks enforce these error handling patterns. -::: - -### Async patterns - -Use consistent async/await patterns: - -1. **Async function naming**: No special suffix is required; prefer natural names. -2. **Tokio usage**: Fully qualify tokio types (e.g., `tokio::time::timeout`). See [Adapter runtime patterns](#adapter-runtime-patterns) for spawn rules. -3. **Error handling**: Return `anyhow::Result` from async functions to match the synchronous conventions. -4. **Cancellation safety**: Call out whether the function is cancellation-safe and what invariants still hold when it is cancelled. -5. **Stream handling**: Use `tokio_stream` (or `futures::Stream`) for async iterators to make back-pressure explicit. -6. **Timeout patterns**: Wrap network or long-running awaits with timeouts (`tokio::time::timeout`) and propagate or handle the timeout error. - -### Adapter runtime patterns - -Adapter crates (under `crates/adapters/`) require special handling for spawning async tasks due to Python FFI compatibility: - -1. **Use `get_runtime().spawn()` instead of `tokio::spawn()`**: When called from Python threads (which have no Tokio context), `tokio::spawn()` panics because it relies on thread-local storage. The global runtime pattern provides an explicit reference accessible from any thread. - - ```rust - use nautilus_common::live::get_runtime; - - // Correct - works from Python threads - get_runtime().spawn(async move { - // async work - }); - - // Incorrect - panics from Python threads - tokio::spawn(async move { - // async work - }); - ``` - -2. **Use the shorter import path**: Import `get_runtime` from the `live` module re-export, not the full path: - - ```rust - // Preferred - shorter path via re-export - use nautilus_common::live::get_runtime; - - // Avoid - unnecessarily verbose - use nautilus_common::live::runtime::get_runtime; - ``` - -3. **Use `get_runtime().block_on()` for sync-to-async bridges**: When synchronous code needs to call async functions in adapters: - - ```rust - fn sync_method(&self) -> anyhow::Result<()> { - get_runtime().block_on(self.async_implementation()) - } - ``` - -4. **Tests are exempt**: Test code using `#[tokio::test]` creates its own runtime context, so `tokio::spawn()` works correctly. The enforcement hook skips test files and test modules. - -:::info[Automated enforcement] -The `check_tokio_usage.sh` pre-commit hook enforces these adapter runtime patterns automatically. -::: - -### Attribute patterns - -Consistent attribute usage and ordering: - -```rust -#[repr(C)] -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr( - feature = "python", - pyo3::pyclass(module = "nautilus_trader.model") -)] -#[cfg_attr( - feature = "python", - pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.model") -)] -pub struct Symbol(Ustr); -``` - -For enums with extensive derive attributes: - -```rust -#[repr(C)] -#[derive( - Copy, - Clone, - Debug, - Display, - Hash, - PartialEq, - Eq, - PartialOrd, - Ord, - AsRefStr, - FromRepr, - EnumIter, - EnumString, -)] -#[strum(ascii_case_insensitive)] -#[strum(serialize_all = "SCREAMING_SNAKE_CASE")] -#[cfg_attr( - feature = "python", - pyo3::pyclass( - frozen, - eq, - eq_int, - module = "nautilus_trader.model", - from_py_object, - rename_all = "SCREAMING_SNAKE_CASE", - ) -)] -#[cfg_attr( - feature = "python", - pyo3_stub_gen::derive::gen_stub_pyclass_enum(module = "nautilus_trader.model") -)] -pub enum AccountType { - /// An account with unleveraged cash assets only. - Cash = 1, - /// An account which facilitates trading on margin, using account assets as collateral. - Margin = 2, -} -``` - -### Type stub annotations - -Python type stubs (`.pyi` files) are generated from Rust source using -[pyo3-stub-gen](https://github.com/Jij-Inc/pyo3-stub-gen). Every type and function -exposed to Python needs a matching stub annotation so the generated stubs stay in sync -with the bindings. - -**Annotation types:** - -| PyO3 construct | Stub annotation | -| ----------------- | ------------------------------------------------ | -| `#[pyclass]` | `pyo3_stub_gen::derive::gen_stub_pyclass` | -| enum `#[pyclass]` | `pyo3_stub_gen::derive::gen_stub_pyclass_enum` | -| `#[pymethods]` | `pyo3_stub_gen::derive::gen_stub_pymethods` | -| `#[pyfunction]` | `pyo3_stub_gen::derive::gen_stub_pyfunction` | - -**Placement rules:** - -- On structs and enums, use `#[cfg_attr(feature = "python", ...)]` and place the stub - annotation directly below the `pyo3::pyclass` attribute. -- On `#[pymethods]` impl blocks, place `#[pyo3_stub_gen::derive::gen_stub_pymethods]` - directly below `#[pymethods]`. -- On functions, place the stub annotation directly above `#[pyfunction]`, after any doc - comments. Fully qualify the path rather than importing it. - -```rust -/// Converts a list of `Bar` into Arrow IPC bytes. -#[pyo3_stub_gen::derive::gen_stub_pyfunction(module = "nautilus_trader.serialization")] -#[pyfunction(name = "bars_to_arrow")] -pub fn py_bars_to_arrow(data: Vec) -> PyResult> { - // ... -} -``` - -```rust -#[pymethods] -#[pyo3_stub_gen::derive::gen_stub_pymethods] -impl AccountState { - #[staticmethod] - #[pyo3(name = "from_dict")] - pub fn py_from_dict(values: &Bound<'_, PyDict>) -> PyResult { - // ... - } -} -``` - -**Module parameter:** set `module = "nautilus_trader."` to match the Python -package where the type is imported. For example, model types use -`nautilus_trader.model` and serialization functions use -`nautilus_trader.serialization`. - -**Cargo.toml:** add `pyo3-stub-gen` as an optional dependency and include it in the -`python` feature list: - -```toml -[features] -python = ["pyo3", "pyo3-stub-gen"] - -[dependencies] -pyo3-stub-gen = { workspace = true, optional = true } -``` - -**Regenerating stubs:** run `make py-stubs-v2` (or `python python/generate_stubs.py`) -after changing annotations. The post-processor handles `py_` prefix stripping, -`@property`/`@staticmethod`/`@classmethod` decoration, keyword escaping, deduplication, -and ruff formatting. - -### Constructor patterns - -Use the `new()` vs `new_checked()` convention consistently: - -```rust -/// Creates a new [`Symbol`] instance with correctness checking. -/// -/// # Errors -/// -/// Returns an error if `value` is not a valid string. -/// -/// # Notes -/// -/// PyO3 requires a `Result` type for proper error handling and stacktrace printing in Python. -pub fn new_checked>(value: T) -> anyhow::Result { - // Implementation -} - -/// Creates a new [`Symbol`] instance. -/// -/// # Panics -/// -/// Panics if `value` is not a valid string. -pub fn new>(value: T) -> Self { - Self::new_checked(value).expect(FAILED) -} -``` - -Always use the `FAILED` constant for `.expect()` messages related to correctness checks: - -```rust -use nautilus_core::correctness::FAILED; -``` - -### Type conversion patterns - -For types that parse from strings, provide both fallible and infallible conversions: - -1. **`FromStr`**: Fallible parsing via `.parse()` or `from_str()`. Returns `Result`. - -2. **`From>`**: Ergonomic infallible conversion that accepts `&str`, `String`, `Cow`, etc. directly without requiring `.as_str()`. - -```rust -impl FromStr for Symbol { - type Err = SymbolParseError; - - fn from_str(s: &str) -> Result { - // parsing logic - } -} - -impl> From for Symbol { - fn from(value: T) -> Self { - Self::from_str(value.as_ref()).expect(FAILED) - } -} -``` - -**Design note**: The `From` impl may panic on invalid input. This is intentional for API ergonomics. Use `FromStr` / `.parse()` when error handling is needed. The `From` impl provides convenience for cases where the input is known to be valid. - -**Constraint**: This pattern cannot be used for types that implement `AsRef` themselves (e.g., string wrapper types), as it would conflict with the blanket `impl From for T`. For such types, provide separate `From<&str>` and `From` impls instead. - -### Constants and naming conventions - -Use SCREAMING_SNAKE_CASE for constants with descriptive names: - -```rust -/// Number of nanoseconds in one second. -pub const NANOSECONDS_IN_SECOND: u64 = 1_000_000_000; - -/// Bar specification for 1-minute last price bars. -pub const BAR_SPEC_1_MINUTE_LAST: BarSpecification = BarSpecification { - step: NonZero::new(1).unwrap(), - aggregation: BarAggregation::Minute, - price_type: PriceType::Last, -}; -``` - -### Hash collections - -Use `AHashMap` and `AHashSet` from the `ahash` crate for performance-critical hot paths. -For non-performance-critical code, standard `HashMap`/`HashSet` are preferred for simplicity: - -```rust -// For hot paths - using AHashMap/AHashSet -use ahash::{AHashMap, AHashSet}; - -let mut symbols: AHashSet = AHashSet::new(); -let mut prices: AHashMap = AHashMap::new(); - -// For non-hot paths - standard library HashMap/HashSet -use std::collections::{HashMap, HashSet}; - -let mut symbols: HashSet = HashSet::new(); -let mut prices: HashMap = HashMap::new(); -``` - -**Why use `ahash`?** - -- **Superior performance**: AHash uses AES-NI hardware instructions when available, providing 2-3x faster hashing compared to the default SipHash. -- **Low collision rates**: Despite being non-cryptographic, AHash provides excellent distribution and low collision rates for typical data. -- **Drop-in replacement**: Fully compatible API with standard library collections. - -**When to use standard `HashMap`/`HashSet`:** - -- **Non-performance-critical code**: For simple cases where performance is not critical (e.g., factory registries, configuration maps, test fixtures), standard `HashMap`/`HashSet` are acceptable and even preferred for simplicity. -- **Cryptographic security required**: Use standard `HashMap` when hash flooding attacks are a concern (e.g., handling untrusted user input in network protocols). -- **Network clients**: Prefer standard `HashMap` for network-facing components where security considerations outweigh performance benefits. -- **External library boundaries**: Use standard `HashMap` when interfacing with external libraries that expect it (e.g., Arrow serialization metadata). - -### Thread-safe hash map patterns - -`AHashMap` is not thread-safe. Wrapping it in `Arc` only enables sharing the pointer across threads but does not coordinate mutation. Use `Arc` only when the map is immutable after construction, otherwise add proper synchronization. - -```rust -// Avoid: Data races when multiple threads mutate -let cache = Arc::new(AHashMap::new()); -let cache_clone = Arc::clone(&cache); -tokio::spawn(async move { - cache_clone.insert(key, value); // Data race -}); -cache.insert(other_key, other_value); // Data race -``` - -**Patterns:** - -1. **Immutable after construction** – Build the map once, then share it read-only: - - ```rust - let mut map = AHashMap::new(); - map.insert(key1, value1); - map.insert(key2, value2); - let shared_map = Arc::new(map); // Now immutable - - // Multiple threads can safely read - let map_clone = Arc::clone(&shared_map); - tokio::spawn(async move { - if let Some(value) = map_clone.get(&key1) { - // Safe read-only access - } - }); - ``` - -2. **Concurrent reads and writes** – Use `DashMap`: - - ```rust - use dashmap::DashMap; - - let cache: Arc> = Arc::new(DashMap::new()); - - // Multiple threads can safely read and write concurrently - cache.insert(key, value); - if let Some(entry) = cache.get(&key) { - // Safe concurrent access - } - ``` - - `DashMap` internally uses sharding and fine-grained locking for efficient concurrent access. - -3. **Single-threaded hot paths** – Use plain `AHashMap` in single-threaded contexts: - - ```rust - struct Handler { - instruments: AHashMap, - } - - impl Handler { - async fn next(&mut self) -> Option<()> { - // Handler runs on a single task, no concurrent access - self.instruments.insert(key, value); - Ok(()) - } - } - ``` - -**Decision tree:** - -- Immutable after construction → Use `Arc>` -- Concurrent access needed → Use `Arc>` -- Single-threaded access → Use plain `AHashMap` - -### Re-export patterns - -Organize re-exports alphabetically and place at the end of lib.rs files: - -```rust -// Re-exports -pub use crate::{ - nanos::UnixNanos, - time::AtomicTime, - uuid::UUID4, -}; - -// Module-level re-exports -pub use crate::identifiers::{ - account_id::AccountId, - actor_id::ActorId, - client_id::ClientId, -}; -``` - -### Documentation standards - -Use third-person declarative voice for all doc comments (e.g., "Returns the account ID" not "Return the account ID"). - -#### Section header casing - -Rustdoc section headers use Title Case, matching the Rust standard library convention: - -- `# Examples` -- `# Errors` -- `# Panics` -- `# Safety` -- `# Notes` -- `# Thread Safety` -- `# Feature Flags` - -#### Module-Level documentation - -All modules must have module-level documentation starting with a brief description: - -```rust -//! Functions for correctness checks similar to the *design by contract* philosophy. -//! -//! This module provides validation checking of function or method conditions. -//! -//! A condition is a predicate which must be true just prior to the execution of -//! some section of code - for correct behavior as per the design specification. -``` - -For modules with feature flags, document them clearly: - -```rust -//! # Feature flags -//! -//! This crate provides feature flags to control source code inclusion during compilation, -//! depending on the intended use case: -//! -//! - `ffi`: Enables the C foreign function interface (FFI) from [cbindgen](https://github.com/mozilla/cbindgen). -//! - `python`: Enables Python bindings from [PyO3](https://pyo3.rs). -//! - `extension-module`: Builds as a Python extension module (used with `python`). -//! - `stubs`: Enables type stubs for use in testing scenarios. -``` - -#### Field documentation - -All struct and enum fields must have documentation with terminating periods: - -```rust -pub struct Currency { - /// The currency code as an alpha-3 string (e.g., "USD", "EUR"). - pub code: Ustr, - /// The currency decimal precision. - pub precision: u8, - /// The ISO 4217 currency code. - pub iso4217: u16, - /// The full name of the currency. - pub name: Ustr, - /// The currency type, indicating its category (e.g. Fiat, Crypto). - pub currency_type: CurrencyType, -} -``` - -#### Function documentation - -Document all public functions with: - -- Purpose and behavior -- Explanation of input argument usage -- Error conditions (if applicable) -- Panic conditions (if applicable) - -```rust -/// Returns a reference to the `AccountBalance` for the specified currency, or `None` if absent. -/// -/// # Panics -/// -/// Panics if `currency` is `None` and `self.base_currency` is `None`. -pub fn base_balance(&self, currency: Option) -> Option<&AccountBalance> { - // Implementation -} -``` - -#### Errors and panics documentation format - -For single line errors and panics documentation, use sentence case with the following convention: - -```rust -/// Returns a reference to the `AccountBalance` for the specified currency, or `None` if absent. -/// -/// # Errors -/// -/// Returns an error if the currency conversion fails. -/// -/// # Panics -/// -/// Panics if `currency` is `None` and `self.base_currency` is `None`. -pub fn base_balance(&self, currency: Option) -> anyhow::Result> { - // Implementation -} -``` - -For multi-line errors and panics documentation, use sentence case with bullets and terminating periods: - -```rust -/// Calculates the unrealized profit and loss for the position. -/// -/// # Errors -/// -/// Returns an error if: -/// - The market price for the instrument cannot be found. -/// - The conversion rate calculation fails. -/// - Invalid position state is encountered. -/// -/// # Panics -/// -/// This function panics if: -/// - The instrument ID is invalid or uninitialized. -/// - Required market data is missing from the cache. -/// - Internal state consistency checks fail. -pub fn calculate_unrealized_pnl(&self, market_price: Price) -> anyhow::Result { - // Implementation -} -``` - -#### Safety documentation format - -For Safety documentation, use the `SAFETY:` prefix followed by a short description explaining why the unsafe operation is valid: - -```rust -/// Creates a new instance from raw components without validation. -/// -/// # Safety -/// -/// The caller must ensure that all input parameters are valid and properly initialized. -pub unsafe fn from_raw_parts(ptr: *const u8, len: usize) -> Self { - // SAFETY: Caller guarantees ptr is valid and len is correct - Self { - data: std::slice::from_raw_parts(ptr, len), - } -} -``` - -For inline unsafe blocks, use the `SAFETY:` comment directly above the unsafe code: - -```rust -impl Send for MessageBus { - fn send(&self) { - // SAFETY: Message bus is not meant to be passed between threads - unsafe { - // unsafe operation here - } - } -} -``` - -## Python bindings - -Python bindings are provided via [PyO3](https://pyo3.rs), allowing users to import NautilusTrader crates directly in Python without a Rust toolchain. - -### PyO3 naming conventions - -When exposing Rust functions to Python **via PyO3**: - -1. The Rust symbol **must** be prefixed with `py_*` to make its purpose explicit inside the Rust - codebase. -2. Use the `#[pyo3(name = "…")]` attribute to publish the *Python* name **without** the `py_` - prefix so the Python API remains clean. - -```rust -#[pyo3(name = "do_something")] -pub fn py_do_something() -> PyResult<()> { - // … -} -``` - -:::info[Automated enforcement] -The `check_pyo3_conventions.sh` pre-commit hook enforces the `py_` prefix for PyO3 functions. -::: - -### PyO3 enum conventions - -Enums exposed to Python should use the following `pyclass` attributes: - -- `frozen`: enums are immutable value types. -- `eq, eq_int`: enables equality with other enum instances and integer discriminants. -- `rename_all = "SCREAMING_SNAKE_CASE"`: standardizes Python variant names. -- `from_py_object`: enables conversion from Python objects. - -:::warning[Do not use the `hash` pyclass attribute with `eq_int` enums] -PyO3's auto-generated `__hash__` uses Rust's `DefaultHasher`, which produces different values -than Python's `hash()` on the equivalent integer. Since `eq_int` makes `MyEnum.VARIANT == 1` -true, the hash contract (`a == b` implies `hash(a) == hash(b)`) would be violated. Instead, -provide a manual `__hash__` returning the discriminant directly: -::: - -```rust -#[pymethods] -impl MyEnum { - const fn __hash__(&self) -> isize { - *self as isize - } -} -``` - -### Testing conventions - -- Use `mod tests` as the standard test module name unless you need to specifically compartmentalize. -- Use `#[rstest]` attributes consistently, this standardization reduces cognitive overhead. -- Do *not* use Arrange, Act, Assert separator comments in Rust tests. - -:::info[Automated enforcement] -The `check_testing_conventions.sh` pre-commit hook enforces the use of `#[rstest]` over `#[test]`. -::: - -#### Parameterized testing - -Use the `rstest` attribute consistently, and for parameterized tests: - -```rust -#[rstest] -#[case("AUDUSD", false)] -#[case("AUD/USD", false)] -#[case("CL.FUT", true)] -fn test_symbol_is_composite(#[case] input: &str, #[case] expected: bool) { - let symbol = Symbol::new(input); - assert_eq!(symbol.is_composite(), expected); -} -``` - -#### Property-based testing - -Use the `proptest` crate for property-based tests. Place these in a separate -`property_tests` module (not inside `mod tests`) to keep deterministic unit -tests separate from randomized property tests: - -```rust -#[cfg(test)] -mod property_tests { - use proptest::prelude::*; - use rstest::rstest; - - use super::*; - - // Define strategies for generating test inputs - fn my_strategy() -> impl Strategy { - prop_oneof![ - Just(MyType::VariantA), - Just(MyType::VariantB), - ] - } - - fn value_strategy() -> impl Strategy { - prop_oneof![ - -1000.0..1000.0, - Just(0.0), - ] - } - - // Group all property tests inside the proptest! macro - proptest! { - #[rstest] - fn prop_construction_roundtrip( - value in value_strategy(), - variant in my_strategy() - ) { - // Test invariants that should hold for all generated inputs - } - } -} -``` - -Conventions: - -- Name the module `property_tests`, separate from `mod tests`. -- Import `proptest::prelude::*` and `rstest::rstest`. -- Define strategy functions returning `impl Strategy`. -- Combine value ranges with edge cases using `prop_oneof!`. -- Filter invalid combinations with `prop_filter_map`. -- Prefix test names with `prop_`. -- Mark each test inside `proptest!` with `#[rstest]`. - -#### Test naming - -Use descriptive test names that explain the scenario: - -```rust -fn test_sma_with_no_inputs() -fn test_sma_with_single_input() -fn test_symbol_is_composite() -``` - -### Box-style banner comments - -Do not use box-style banner or separator comments. If code requires visual -separation, consider splitting it into separate modules or files. Instead use: - -- Clear function names that convey purpose. -- Module structure for logical groupings (`mod tests { mod fixtures { } }`). -- Impl blocks to group related methods. -- Doc comments (`///`) for semantic documentation. -- IDE navigation and code folding. - -Patterns to avoid: - -```rust -// ============================================================================ -// Some Section -// ============================================================================ - -// ========== Test Fixtures ========== -``` - -## Rust-Python memory management - -When working with PyO3 bindings, it's critical to understand and avoid reference cycles between Rust's `Arc` reference counting and Python's garbage collector. -This section documents best practices for handling Python objects in Rust callback-holding structures. - -### The reference cycle problem - -**Problem**: Using `Arc` in callback-holding structs creates circular references: - -1. **Rust `Arc` holds Python objects** → increases Python reference count. -2. **Python objects might reference Rust objects** → creates cycles. -3. **Neither side can be garbage collected** → memory leak. - -**Example of problematic pattern**: - -```rust -// AVOID: This creates reference cycles -struct CallbackHolder { - handler: Option>, // ❌ Arc wrapper causes cycles -} -``` - -### The solution: GIL-based cloning - -**Solution**: Use plain `PyObject` with proper GIL-based cloning via `clone_py_object()`: - -```rust -use nautilus_core::python::clone_py_object; - -// CORRECT: Use plain PyObject without Arc wrapper -struct CallbackHolder { - handler: Option, // ✅ No Arc wrapper -} - -// Manual Clone implementation using clone_py_object -impl Clone for CallbackHolder { - fn clone(&self) -> Self { - Self { - handler: self.handler.as_ref().map(clone_py_object), - } - } -} -``` - -### Best practices - -#### 1. Use `clone_py_object()` for Python object cloning - -```rust -// When cloning Python callbacks -let cloned_callback = clone_py_object(&original_callback); - -// In manual Clone implementations -self.py_handler.as_ref().map(clone_py_object) -``` - -#### 2. Remove `#[derive(Clone)]` from callback-holding structs - -```rust -// BEFORE: Automatic derive causes issues with PyObject -#[derive(Clone)] // ❌ Remove this -struct Config { - handler: Option, -} - -// AFTER: Manual implementation with proper cloning -struct Config { - handler: Option, -} - -impl Clone for Config { - fn clone(&self) -> Self { - Self { - // Clone regular fields normally - url: self.url.clone(), - // Use clone_py_object for Python objects - handler: self.handler.as_ref().map(clone_py_object), - } - } -} -``` - -#### 3. Update function signatures to accept `PyObject` - -```rust -// BEFORE: Arc wrapper in function signatures -fn spawn_task(handler: Arc) { ... } // ❌ - -// AFTER: Plain PyObject -fn spawn_task(handler: PyObject) { ... } // ✅ -``` - -#### 4. Avoid `Arc::new()` when creating Python callbacks - -```rust -// BEFORE: Wrapping in Arc -let callback = Arc::new(py_function); // ❌ - -// AFTER: Use directly -let callback = py_function; // ✅ -``` - -### Why this works - -The `clone_py_object()` function: - -- **Acquires the Python GIL** before performing clone operations. -- **Uses Python's native reference counting** via `clone_ref()`. -- **Avoids Rust Arc wrappers** that interfere with Python GC. -- **Maintains thread safety** through proper GIL management. - -This approach allows both Rust and Python garbage collectors to work correctly, eliminating memory leaks from reference cycles. - -## Common anti-patterns - -1. **Avoid `.clone()` in hot paths** – favour borrowing or shared ownership via `Arc`. -2. **Avoid `.unwrap()` in production code** – generally propagate errors with `?` or map them into domain errors, but unwrapping lock poisoning is acceptable because it signals a severe program state that should abort fast. -3. **Avoid `String` when `&str` suffices** – minimise allocations on tight loops. -4. **Avoid exposing interior mutability** – hide mutexes/`RefCell` behind safe APIs. -5. **Avoid large structs in `Result`** – box large error payloads (`Box`). - -## Unsafe Rust - -It will be necessary to write `unsafe` Rust code to be able to achieve the value -of interoperating between Cython and Rust. The ability to step outside the boundaries of safe Rust is what makes it possible to -implement many of the most fundamental features of the Rust language itself, just as C and C++ are used to implement -their own standard libraries. - -Great care will be taken with the use of Rusts `unsafe` facility - which enables a small set of additional language features, thereby changing -the contract between the interface and caller, shifting some responsibility for guaranteeing correctness -from the Rust compiler, and onto us. The goal is to realize the advantages of the `unsafe` facility, whilst avoiding *any* undefined behavior. -The definition for what the Rust language designers consider undefined behavior can be found in the [language reference](https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html). - -### Safety policy - -To maintain correctness, any use of `unsafe` Rust must follow our policy: - -- If a function is `unsafe` to call, there *must* be a `Safety` section in the documentation explaining why the function is `unsafe`, - covering the invariants which the function expects the callers to uphold, and how to meet their obligations in that contract. -- Document why each function is `unsafe` in its doc comment's Safety section, and cover all `unsafe` blocks with unit tests. -- Always include a `SAFETY:` comment explaining why the unsafe operation is valid. -- **Crate-level lint** – every crate that exposes FFI symbols enables - `#![deny(unsafe_op_in_unsafe_fn)]`. Even inside an `unsafe fn`, each pointer dereference or - other dangerous operation must be wrapped in its own `unsafe { … }` block. -- **CVec contract** – for raw vectors that cross the FFI boundary read the - [FFI Memory Contract](ffi.md). Foreign code becomes the owner of the allocation and **must** - call the matching `vec_drop_*` function exactly once. - -### Categories of unsafe code - -The codebase uses unsafe Rust in these categories: - -1. **FFI boundaries** – Raw pointer operations for C interop. See [FFI documentation](ffi.md). -2. **Interior mutability** – `UnsafeCell` for thread-local registries with controlled access patterns. -3. **Unsafe Send/Sync** – Types that are not inherently thread-safe but satisfy trait bounds - through runtime invariants (e.g., single-threaded access guaranteed by architecture). - -### Unsafe Send/Sync requirements - -When implementing `Send` or `Sync` unsafely: - -1. Document exactly which fields violate the trait requirements. -2. Explain the runtime mechanism that ensures safety (e.g., single-threaded event loop). -3. Include a `WARNING` stating that violating the invariant is undefined behavior. -4. Prefer runtime enforcement (assertions, `Result` returns) over documentation-only guarantees. - -```rust -// SAFETY: Contains Rc> which is not thread-safe. -// Single-threaded access guaranteed by the backtest engine architecture. -// WARNING: Actually sending across threads is undefined behavior. -#[allow(unsafe_code)] -unsafe impl Send for BacktestDataClient {} -``` - -### Defense in depth - -Where unsafe code relies on invariants, add defense mechanisms: - -- **Type verification**: Check types at runtime before casting (e.g., `TypeId` comparison). -- **Debug assertions**: Catch memory corruption early in debug builds. -- **RAII guards**: Ensure cleanup on both normal return and panic paths. -- **Runtime checks**: Fail fast when invariants are violated rather than proceeding unsafely. - -## Tooling configuration - -The project uses several tools for code quality: - -- **rustfmt**: Automatic code formatting (see `rustfmt.toml`). -- **clippy**: Linting and best practices (see `clippy.toml`). -- **cbindgen**: C header generation for FFI. - -## Rust version management - -The project pins to a specific Rust version via `rust-toolchain.toml`. - -**Keep your toolchain synchronized with CI:** - -```bash -rustup update # Update to latest stable Rust -rustup show # Verify correct toolchain is active -``` - -If pre-commit passes locally but fails in CI, clear the pre-commit cache and re-run: - -```bash -pre-commit clean # Clear cached environments -make pre-commit # Re-run all checks -``` - -This ensures you're using the same Rust and clippy versions as CI. - -## Resources - -- [The Rustonomicon](https://doc.rust-lang.org/nomicon/) – The Dark Arts of Unsafe Rust. -- [The Rust Reference – Unsafety](https://doc.rust-lang.org/stable/reference/unsafety.html). -- [Safe Bindings in Rust – Russell Johnston](https://www.abubalay.com/blog/2020/08/22/safe-bindings-in-rust). -- [Google – Rust and C interoperability](https://www.chromium.org/Home/chromium-security/memory-safety/rust-and-c-interoperability/). - -## Cap'n Proto serialization - -The `nautilus-serialization` crate provides optional Cap'n Proto serialization support for efficient data interchange. -This feature is opt-in to avoid requiring the Cap'n Proto compiler for standard builds. - -### Installing Cap'n Proto - -Install the Cap'n Proto compiler before working with schemas. The required version is -specified in the `capnp-version` file in the repository root. - -See the [Environment Setup](environment_setup.md#capn-proto) guide for detailed installation -instructions for each platform. - -:::warning -Ubuntu's default `capnproto` package is too old. Linux users must install from source. -::: - -Verify installation: - -```bash -capnp --version # Should match the version in capnp-version -``` - -### Schema development workflow - -Schema files live in `crates/serialization/schemas/capnp/`: - -- `common/` - Base types, identifiers, enums. -- `commands/` - Trading commands. -- `events/` - Order and position events. -- `data/` - Market data types. - -When modifying schemas: - -1. Edit the `.capnp` schema file in the appropriate subdirectory. -2. Regenerate Rust bindings: - - ```bash - make regen-capnp - # or - ./scripts/regen_capnp.sh - ``` - -3. Review changes: - - ```bash - git diff crates/serialization/generated/capnp - ``` - -4. Update conversions in `crates/serialization/src/capnp/conversions.rs` if needed. -5. Run tests: - - ```bash - make cargo-test EXTRA_FEATURES="capnp" - ``` - -### Generated code - -Generated Rust files are checked into `crates/serialization/generated/capnp/` for these reasons: - -- **docs.rs compatibility**: The documentation build environment lacks the Cap'n Proto - compiler. -- **Contributor convenience**: Most developers don't need to install capnp for standard - development. -- **Build reproducibility**: Ensures consistent code generation across environments. - -The generated files are automatically created during builds via `build.rs` when the `capnp` -feature is enabled, but we commit them to the repository to support builds without the -compiler installed. - -### Verifying schema consistency - -Before committing schema changes, ensure generated files are up-to-date: - -```bash -make check-capnp-schemas -``` - -This target: - -1. Skips with a warning if `capnp` is not installed (acceptable for local development). -2. Fails if regeneration errors occur (e.g., version mismatch). -3. Regenerates schemas and fails if generated files differ from committed versions. - -CI runs this check automatically to catch drift (capnp is always installed in CI). - -### Testing with capnp feature - -```bash -# Run workspace tests with capnp -make cargo-test EXTRA_FEATURES="capnp" - -# Run specific crate tests with capnp -make cargo-test-crate-nautilus-serialization FEATURES="capnp" - -# Run specific test -cargo test -p nautilus-serialization --features capnp test_price_roundtrip -``` - -### Schema evolution guidelines - -When evolving schemas: - -- **Additive changes only**: Add new fields at the end. -- **Never remove fields**: Mark deprecated fields in comments. -- **Never reuse field numbers**: Even after deprecation. -- **Test roundtrip compatibility**: Ensure old and new versions interoperate. - -Cap'n Proto's evolution rules allow schema changes without breaking binary compatibility, but -you must follow these constraints to maintain forward/backward compatibility. diff --git a/nautilus-docs/docs/developer_guide/spec_data_testing.md b/nautilus-docs/docs/developer_guide/spec_data_testing.md deleted file mode 100644 index d73f183..0000000 --- a/nautilus-docs/docs/developer_guide/spec_data_testing.md +++ /dev/null @@ -1,854 +0,0 @@ -# Data Testing Spec - -This section defines a rigorous test matrix for validating adapter data -functionality using the `DataTester` actor. Both Python -(`nautilus_trader.test_kit.strategies.tester_data`) and Rust -(`nautilus_testkit::testers`) provide the `DataTester`. Each test case is -identified by a prefixed ID (e.g. TC-D01) and grouped by functionality. - -**Each adapter must pass the subset of tests matching its supported data types.** - -Test groups are ordered from least derived to most derived data: instruments -and raw book data first, then quotes, trades, bars, and derivatives data. -An adapter that passes groups 1–4 is considered baseline data compliant. - -Document adapter-specific data behavior (custom channels, throttling, -snapshot semantics, etc.) in the adapter's own guide, not here. - -## Prerequisites - -Before running data tests: - -- Target instrument available and loadable via the instrument provider. -- API credentials set via environment variables (`{VENUE}_API_KEY`, `{VENUE}_API_SECRET`) when - the venue requires authentication for the data being tested. -- If the venue offers a demo/testnet mode (e.g. `is_demo=True`), use credentials created - for that environment. Demo and production API keys are typically separate and not - interchangeable; using the wrong credentials produces authentication errors (e.g. HTTP 401). - -**Python node setup** (reference: `examples/live/{adapter}/{adapter}_data_tester.py`): - -```python -from nautilus_trader.live.node import TradingNode -from nautilus_trader.test_kit.strategies.tester_data import DataTester, DataTesterConfig - -node = TradingNode(config=config_node) -tester = DataTester(config=config_tester) -node.trader.add_actor(tester) -# Register adapter factories, build, and run -``` - -**Rust node setup** (reference: `crates/adapters/{adapter}/examples/node_data_tester.rs`): - -```rust -use nautilus_testkit::testers::{DataTester, DataTesterConfig}; - -let tester_config = DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_quotes(true); -let tester = DataTester::new(tester_config); -node.add_actor(tester)?; -node.run().await?; -``` - -Each group below begins with a summary table, followed by detailed test cards. -Test IDs use spaced numbering to allow insertion without renumbering. - ---- - -## Group 1: Instruments - -Verify instrument loading and subscription before testing market data streams. - -| TC | Name | Description | Skip when | -|---------|-----------------------------|------------------------------------------------------|----------------------| -| TC-D01 | Request instruments | Load all instruments for a venue. | Never. | -| TC-D02 | Subscribe instrument | Subscribe to instrument updates. | No instrument sub. | -| TC-D03 | Load specific instrument | Load a single instrument by ID. | Never. | - -### TC-D01: Request instruments - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected. | -| **Action** | DataTester requests all instruments for the venue on start. | -| **Event sequence** | `on_instruments` callback receives instrument list. | -| **Pass criteria** | At least one instrument received; each has valid symbol, price precision, and size increment. | -| **Skip when** | Never. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - request_instruments=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_request_instruments(true) -``` - -### TC-D02: Subscribe instrument - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to instrument updates. | -| **Event sequence** | `on_instrument` callback receives instrument. | -| **Pass criteria** | Instrument received with correct `instrument_id`, valid fields. | -| **Skip when** | Adapter does not support instrument subscriptions. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_instrument=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_instrument(true) -``` - -### TC-D03: Load specific instrument - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected. | -| **Action** | Load a specific instrument by `InstrumentId` via the instrument provider. | -| **Event sequence** | Instrument available in cache after load. | -| **Pass criteria** | Instrument loaded with correct ID, price precision, size increment, and trading rules. | -| **Skip when** | Never. | - -**Considerations:** - -- This tests the instrument provider's `load` / `load_async` method directly. -- Verify the instrument is cached and available via `self.cache.instrument(instrument_id)`. - ---- - -## Group 2: Order book - -Test order book subscription modes and snapshot requests. - -| TC | Name | Description | Skip when | -|---------|--------------------------------|----------------------------------------------------|------------------------| -| TC-D10 | Subscribe book deltas | Stream `OrderBookDeltas` updates. | No book support. | -| TC-D11 | Subscribe book at interval | Periodic `OrderBook` snapshots. | No book support. | -| TC-D12 | Subscribe book depth | `OrderBookDepth10` snapshots. | No book depth. | -| TC-D13 | Request book snapshot | One-time book snapshot request. | No book snapshot. | -| TC-D14 | Managed book from deltas | Build local book from delta stream. | No book support. | -| TC-D15 | Request historical book deltas | Historical book deltas request. | No historical deltas. | - -### TC-D10: Subscribe book deltas - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to order book deltas. | -| **Event sequence** | `OrderBookDeltas` events received in `on_order_book_deltas`. | -| **Pass criteria** | Deltas received with valid instrument ID; at least one delta contains bid/ask updates. | -| **Skip when** | Adapter does not support order book data. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_book_deltas=True, - book_type=BookType.L2_MBP, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_book_deltas(true) - .with_book_type(BookType::L2_MBP) -``` - -### TC-D11: Subscribe book at interval - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to periodic order book snapshots. | -| **Event sequence** | `OrderBook` events received in `on_order_book` at configured interval. | -| **Pass criteria** | Book snapshots received with bid/ask levels; updates arrive at approximately the configured interval. | -| **Skip when** | Adapter does not support order book data. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_book_at_interval=True, - book_type=BookType.L2_MBP, - book_depth=10, - book_interval_ms=1000, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_book_at_interval(true) - .with_book_type(BookType::L2_MBP) - .with_book_depth(Some(NonZeroUsize::new(10).unwrap())) - .with_book_interval_ms(NonZeroUsize::new(1000).unwrap()) -``` - -### TC-D12: Subscribe book depth - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to `OrderBookDepth10` snapshots. | -| **Event sequence** | `OrderBookDepth10` events received in `on_order_book_depth`. | -| **Pass criteria** | Depth snapshots received with up to 10 bid/ask levels; prices are correctly ordered. | -| **Skip when** | Adapter does not support book depth subscriptions. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_book_depth=True, - book_type=BookType.L2_MBP, - book_depth=10, -) -``` - -**Rust config:** Not yet supported. Book depth subscription is TODO in the Rust `DataTester`. - -### TC-D13: Request book snapshot - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester requests a one-time order book snapshot. | -| **Event sequence** | Book snapshot received via historical data callback. | -| **Pass criteria** | Snapshot contains bid/ask levels with valid prices and sizes. | -| **Skip when** | Adapter does not support book snapshot requests. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - request_book_snapshot=True, - book_depth=10, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_request_book_snapshot(true) - .with_book_depth(Some(NonZeroUsize::new(10).unwrap())) -``` - -### TC-D14: Managed book from deltas - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, book deltas streaming. | -| **Action** | DataTester subscribes to deltas with `manage_book=True`; builds local order book from the delta stream. | -| **Event sequence** | `OrderBookDeltas` applied to local `OrderBook`; book logged with configured depth. | -| **Pass criteria** | Local book builds correctly from deltas; bid levels descend, ask levels ascend; book is not empty after initial snapshot. | -| **Skip when** | Adapter does not support order book data. | - -**Considerations:** - -- The managed book applies each delta to an `OrderBook` instance maintained by the actor. -- Use `book_levels_to_print` to control logging verbosity. - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_book_deltas=True, - manage_book=True, - book_type=BookType.L2_MBP, - book_levels_to_print=10, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_book_deltas(true) - .with_manage_book(true) - .with_book_type(BookType::L2_MBP) -``` - -### TC-D15: Request historical book deltas - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester requests historical order book deltas. | -| **Event sequence** | Historical deltas received via callback. | -| **Pass criteria** | Deltas received with valid timestamps and book actions. | -| **Skip when** | Adapter does not support historical book delta requests. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - request_book_deltas=True, -) -``` - -**Rust config:** Not yet supported. Historical book delta requests are TODO in the Rust `DataTester`. - ---- - -## Group 3: Quotes - -Test quote tick subscriptions and historical requests. - -| TC | Name | Description | Skip when | -|---------|---------------------------|-------------------------------------------------|------------------------| -| TC-D20 | Subscribe quotes | Verify `QuoteTick` events flow after start. | Never. | -| TC-D21 | Request historical quotes | Request historical quote ticks. | No historical quotes. | - -### TC-D20: Subscribe quotes - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to quotes on start. | -| **Event sequence** | `QuoteTick` events received in `on_quote_tick`. | -| **Pass criteria** | At least one `QuoteTick` received with valid bid/ask prices and sizes; bid < ask. | -| **Skip when** | Never. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_quotes=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_quotes(true) -``` - -### TC-D21: Request historical quotes - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester requests historical quote ticks. | -| **Event sequence** | Historical quotes received via `on_historical_data` callback. | -| **Pass criteria** | Quotes received with valid timestamps, bid/ask prices and sizes. | -| **Skip when** | Adapter does not support historical quote requests. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - request_quotes=True, - requests_start_delta=pd.Timedelta(hours=1), -) -``` - ---- - -## Group 4: Trades - -Test trade tick subscriptions and historical requests. - -| TC | Name | Description | Skip when | -|--------|---------------------------|-------------------------------------------------|------------------------| -| TC-D30 | Subscribe trades | Verify `TradeTick` events flow after start. | Never. | -| TC-D31 | Request historical trades | Request historical trade ticks. | No historical trades. | - -### TC-D30: Subscribe trades - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to trades on start. | -| **Event sequence** | `TradeTick` events received in `on_trade_tick`. | -| **Pass criteria** | At least one `TradeTick` received with valid price, size, and aggressor side. | -| **Skip when** | Never. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_trades=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_trades(true) -``` - -### TC-D31: Request historical trades - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester requests historical trade ticks. | -| **Event sequence** | Historical trades received via `on_historical_data` callback. | -| **Pass criteria** | Trades received with valid timestamps, prices, sizes, and trade IDs. | -| **Skip when** | Adapter does not support historical trade requests. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - request_trades=True, - requests_start_delta=pd.Timedelta(hours=1), -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_request_trades(true) -``` - ---- - -## Group 5: Bars - -Test bar subscriptions and historical requests. - -| TC | Name | Description | Skip when | -|---------|-------------------------|---------------------------------------------------|---------------------| -| TC-D40 | Subscribe bars | Verify `Bar` events flow after start. | No bar support. | -| TC-D41 | Request historical bars | Request historical OHLCV bars. | No historical bars. | - -### TC-D40: Subscribe bars - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, bar type configured. | -| **Action** | DataTester subscribes to bars for a configured `BarType`. | -| **Event sequence** | `Bar` events received in `on_bar`. | -| **Pass criteria** | At least one `Bar` received with valid OHLCV values; high >= low, high >= open, high >= close. | -| **Skip when** | Adapter does not support bar subscriptions. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - bar_types=[BarType.from_str("BTCUSDT-PERP.VENUE-1-MINUTE-LAST-EXTERNAL")], - subscribe_bars=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_bar_types(vec![bar_type]) - .with_subscribe_bars(true) -``` - -### TC-D41: Request historical bars - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, bar type configured. | -| **Action** | DataTester requests historical bars for a configured `BarType`. | -| **Event sequence** | Historical bars received via callback. | -| **Pass criteria** | Bars received with valid OHLCV values and ascending timestamps. | -| **Skip when** | Adapter does not support historical bar requests. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - bar_types=[BarType.from_str("BTCUSDT-PERP.VENUE-1-MINUTE-LAST-EXTERNAL")], - request_bars=True, - requests_start_delta=pd.Timedelta(hours=1), -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_bar_types(vec![bar_type]) - .with_request_bars(true) -``` - ---- - -## Group 6: Derivatives data - -Test derivatives-specific data streams: mark prices, index prices, and funding rates. - -| TC | Name | Description | Skip when | -|--------|----------------------------------|---------------------------------------------|-----------------------| -| TC-D50 | Subscribe mark prices | `MarkPriceUpdate` events. | Not a derivative. | -| TC-D51 | Subscribe index prices | `IndexPriceUpdate` events. | Not a derivative. | -| TC-D52 | Subscribe funding rates | `FundingRateUpdate` events. | Not a perpetual. | -| TC-D53 | Request historical funding rates | Historical funding rate data. | Not a perpetual. | - -### TC-D50: Subscribe mark prices - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, derivative instrument loaded. | -| **Action** | DataTester subscribes to mark price updates. | -| **Event sequence** | `MarkPriceUpdate` events received in `on_mark_price`. | -| **Pass criteria** | At least one `MarkPriceUpdate` received with valid instrument ID and mark price. | -| **Skip when** | Instrument is not a derivative, or adapter does not provide mark prices. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_mark_prices=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_mark_prices(true) -``` - -### TC-D51: Subscribe index prices - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, derivative instrument loaded. | -| **Action** | DataTester subscribes to index price updates. | -| **Event sequence** | `IndexPriceUpdate` events received in `on_index_price`. | -| **Pass criteria** | At least one `IndexPriceUpdate` received with valid instrument ID and index price. | -| **Skip when** | Instrument is not a derivative, or adapter does not provide index prices. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_index_prices=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_index_prices(true) -``` - -### TC-D52: Subscribe funding rates - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, perpetual instrument loaded. | -| **Action** | DataTester subscribes to funding rate updates. | -| **Event sequence** | `FundingRateUpdate` events received in `on_funding_rate`. | -| **Pass criteria** | At least one `FundingRateUpdate` received with valid instrument ID and rate. | -| **Skip when** | Instrument is not a perpetual, or adapter does not provide funding rates. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_funding_rates=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_funding_rates(true) -``` - -### TC-D53: Request historical funding rates - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, perpetual instrument loaded. | -| **Action** | DataTester requests historical funding rates (default 7-day lookback). | -| **Event sequence** | Historical funding rates received via callback. | -| **Pass criteria** | Funding rates received with valid timestamps and rate values. | -| **Skip when** | Instrument is not a perpetual, or adapter does not support historical funding rate requests. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - request_funding_rates=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_request_funding_rates(true) -``` - ---- - -## Group 7: Instrument status - -Test instrument status and close event subscriptions. - -| TC | Name | Description | Skip when | -|--------|-----------------------------|------------------------------------------------|-----------------------| -| TC-D60 | Subscribe instrument status | `InstrumentStatus` events. | No status support. | -| TC-D61 | Subscribe instrument close | `InstrumentClose` events. | No close support. | - -### TC-D60: Subscribe instrument status - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to instrument status updates. | -| **Event sequence** | `InstrumentStatus` events received in `on_instrument_status`. | -| **Pass criteria** | Status events received with valid `MarketStatusAction` (e.g. `Trading`). | -| **Skip when** | Adapter does not support instrument status subscriptions. | - -**Considerations:** - -- Status events may only fire on state changes (e.g. trading halt → resume). -- During normal trading hours, a `Trading` status may be received on subscribe. - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_instrument_status=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_instrument_status(true) -``` - -### TC-D61: Subscribe instrument close - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded. | -| **Action** | DataTester subscribes to instrument close events. | -| **Event sequence** | `InstrumentClose` events received in `on_instrument_close`. | -| **Pass criteria** | Close event received with valid close price and close type. | -| **Skip when** | Adapter does not support instrument close subscriptions. | - -**Considerations:** - -- Close events typically fire at end-of-session for traditional markets. -- May not fire for 24/7 crypto venues unless the adapter synthesizes a daily close. - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_instrument_close=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_instrument_close(true) -``` - ---- - -## Group 8: Option greeks - -Test option greeks and option chain subscriptions. - -| TC | Name | Description | Skip when | -|--------|-----------------------------|------------------------------------------------|------------------------| -| TC-D62 | Subscribe option greeks | `OptionGreeks` data for a single instrument. | No greeks support. | -| TC-D63 | Subscribe option chain | `OptionChainSlice` snapshots for a series. | No chain support. | - -### TC-D62: Subscribe option greeks - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, option instrument loaded. | -| **Action** | DataTester subscribes to option greeks updates. | -| **Event sequence** | `OptionGreeks` events received in `on_option_greeks`. | -| **Pass criteria** | Greeks received with valid delta, gamma, vega, theta values. | -| **Skip when** | Adapter does not support option greeks subscriptions. | - -**Considerations:** - -- Greeks are only available for option instruments. -- Values depend on the venue's pricing model and may update on every quote change. - -### TC-D63: Subscribe option chain - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, option instruments loaded for the series. | -| **Action** | DataTester subscribes to option chain snapshots for a series. | -| **Event sequence** | `OptionChainSlice` snapshots received in `on_option_chain`. | -| **Pass criteria** | Chain snapshot contains greeks for instruments matching the series. | -| **Skip when** | Adapter does not support option chain subscriptions. | - -**Considerations:** - -- Option chain subscriptions are managed by the DataEngine, which creates per-instrument - quote and greeks subscriptions internally. -- ATM-relative strike ranges require a forward price bootstrap before subscriptions begin. - ---- - -## Group 9: Lifecycle - -Test actor lifecycle behavior: unsubscribe handling and custom parameters. - -| TC | Name | Description | Skip when | -|--------|-------------------------|----------------------------------------------------|----------------------| -| TC-D70 | Unsubscribe on stop | Unsubscribe from data feeds on actor stop. | No unsub support. | -| TC-D71 | Custom subscribe params | Adapter-specific subscription parameters. | N/A. | -| TC-D72 | Custom request params | Adapter-specific request parameters. | N/A. | - -### TC-D70: Unsubscribe on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Active data subscriptions (quotes, trades, book). | -| **Action** | Stop the actor with `can_unsubscribe=True` (default). | -| **Event sequence** | Data subscriptions removed; no further data events received. | -| **Pass criteria** | Clean unsubscribe; no errors in logs; no data events after stop. | -| **Skip when** | Adapter does not support unsubscribe. | - -**Python config:** - -```python -DataTesterConfig( - instrument_ids=[instrument_id], - subscribe_quotes=True, - subscribe_trades=True, - can_unsubscribe=True, -) -``` - -**Rust config:** - -```rust -DataTesterConfig::new(client_id, vec![instrument_id]) - .with_subscribe_quotes(true) - .with_subscribe_trades(true) - .with_can_unsubscribe(true) -``` - -### TC-D71: Custom subscribe params - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, adapter accepts additional subscription parameters. | -| **Action** | Subscribe with `subscribe_params` dict containing adapter-specific parameters. | -| **Event sequence** | Subscription established with custom parameters applied. | -| **Pass criteria** | Data flows with adapter-specific parameters in effect. | -| **Skip when** | N/A (adapter-specific). | - -**Considerations:** - -- The `subscribe_params` dict is opaque to the DataTester and passed through to the adapter. -- Consult the adapter's guide for supported parameters. - -### TC-D72: Custom request params - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, adapter accepts additional request parameters. | -| **Action** | Request data with `request_params` dict containing adapter-specific parameters. | -| **Event sequence** | Request fulfilled with custom parameters applied. | -| **Pass criteria** | Historical data received with adapter-specific parameters in effect. | -| **Skip when** | N/A (adapter-specific). | - -**Considerations:** - -- The `request_params` dict is opaque to the DataTester and passed through to the adapter. -- Consult the adapter's guide for supported parameters. - ---- - -## DataTester configuration reference - -Quick reference for all `DataTesterConfig` parameters. Defaults shown are for the Python config. -Note: Rust `DataTesterConfig::new` sets `manage_book=true`, while Python defaults it to `False`. - -| Parameter | Type | Default | Affects groups | -|------------------------------|-------------------|-----------------|----------------| -| `instrument_ids` | list[InstrumentId]| *required* | All | -| `client_id` | ClientId? | None | All | -| `bar_types` | list[BarType]? | None | 5 | -| `subscribe_book_deltas` | bool | False | 2 | -| `subscribe_book_depth` | bool | False | 2 | -| `subscribe_book_at_interval` | bool | False | 2 | -| `subscribe_quotes` | bool | False | 3 | -| `subscribe_trades` | bool | False | 4 | -| `subscribe_mark_prices` | bool | False | 6 | -| `subscribe_index_prices` | bool | False | 6 | -| `subscribe_funding_rates` | bool | False | 6 | -| `subscribe_bars` | bool | False | 5 | -| `subscribe_instrument` | bool | False | 1 | -| `subscribe_instrument_status`| bool | False | 7 | -| `subscribe_instrument_close` | bool | False | 7 | -| `subscribe_params` | dict? | None | 8 | -| `can_unsubscribe` | bool | True | 8 | -| `request_instruments` | bool | False | 1 | -| `request_book_snapshot` | bool | False | 2 | -| `request_book_deltas` | bool | False | 2 | -| `request_quotes` | bool | False | 3 | -| `request_trades` | bool | False | 4 | -| `request_bars` | bool | False | 5 | -| `request_funding_rates` | bool | False | 6 | -| `request_params` | dict? | None | 8 | -| `requests_start_delta` | Timedelta? | 1 hour | 3, 4, 5 | -| `book_type` | BookType | L2_MBP | 2 | -| `book_depth` | PositiveInt? | None | 2 | -| `book_interval_ms` | PositiveInt | 1000 | 2 | -| `book_levels_to_print` | PositiveInt | 10 | 2 | -| `manage_book` | bool | False | 2 | -| `use_pyo3_book` | bool | False | 2 | -| `log_data` | bool | True | All | - ---- diff --git a/nautilus-docs/docs/developer_guide/spec_exec_testing.md b/nautilus-docs/docs/developer_guide/spec_exec_testing.md deleted file mode 100644 index d38e701..0000000 --- a/nautilus-docs/docs/developer_guide/spec_exec_testing.md +++ /dev/null @@ -1,1690 +0,0 @@ -# Execution Testing Spec - -This section defines a rigorous test matrix for validating adapter execution -functionality using the `ExecTester` strategy. Both Python -(`nautilus_trader.test_kit.strategies.tester_exec`) and Rust -(`nautilus_testkit::testers`) provide the `ExecTester`. Each test case is -identified by a prefixed ID (e.g. TC-E01) and grouped by functionality. - -**Each adapter must pass the subset of tests matching its supported capabilities.** - -Tests progress from simple (single market order) to complex (brackets, -modification chains, rejection handling). An adapter that passes groups 1–5 is -considered baseline compliant. Data connectivity should be verified first using -the [Data Testing Spec](spec_data_testing.md). - -Document adapter-specific behavior (how a venue simulates market orders, -handles TIF options, etc.) in the adapter's own guide, not here. Each adapter -guide should include a capability matrix showing which order types, time-in-force -options, actions, and flags it supports. - -## Prerequisites - -Before running execution tests: - -- Demo/testnet account with valid API credentials (preferred, not required). -- Account funded with sufficient margin for the test instrument and quantities. -- Target instrument available and loadable via the instrument provider. -- Environment variables set: `{VENUE}_API_KEY`, `{VENUE}_API_SECRET` (or sandbox variants). -- If the venue offers a demo/testnet mode (e.g. `is_demo=True`), use credentials created - for that environment. Demo and production API keys are typically separate and not - interchangeable; using the wrong credentials produces authentication errors (e.g. HTTP 401). -- Risk engine bypassed (`LiveRiskEngineConfig(bypass=True)`) to avoid interference. -- Reconciliation enabled to verify state consistency. - -**Python node setup** (reference: `examples/live/{adapter}/{adapter}_exec_tester.py`): - -```python -from nautilus_trader.live.node import TradingNode -from nautilus_trader.test_kit.strategies.tester_exec import ExecTester, ExecTesterConfig - -node = TradingNode(config=config_node) -strategy = ExecTester(config=config_tester) -node.trader.add_strategy(strategy) -# Register adapter factories, build, and run -``` - -**Rust node setup** (reference: `crates/adapters/{adapter}/examples/node_exec_tester.rs`): - -```rust -use nautilus_testkit::testers::{ExecTester, ExecTesterConfig}; - -let tester_config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, order_qty); -let tester = ExecTester::new(tester_config); -node.add_strategy(tester)?; -node.run().await?; -``` - -## Basic smoke test - -A quick sanity check that can run at any time, for example after adapter changes or between -development iterations. The tester opens a position with a market order on start, places a -buy and sell post-only limit order, waits 30 seconds, then stops (cancelling open orders and -closing the position). - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.001"), - open_position_on_start_qty=Decimal("0.001"), - enable_limit_buys=True, - enable_limit_sells=True, - use_post_only=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, dec!(0.001)) - .with_open_position_on_start_qty(Some(dec!(0.001))) - .with_enable_limit_buys(true) - .with_enable_limit_sells(true) - .with_use_post_only(true) -``` - -**Expected behavior:** - -1. On start: market order fills, opening a position. -2. Two limit orders placed at `tob_offset_ticks` away from best bid/ask (default 500 ticks). -3. Strategy idles for 30 seconds. Check logs for errors, rejected orders, or disconnections. -4. On stop: open limit orders cancelled, position closed with a market order. - -**Pass criteria:** No errors in logs, position opened and closed cleanly, limit orders -acknowledged by the venue. - ---- - -Each group below begins with a summary table, followed by detailed test cards. -Test IDs use spaced numbering to allow insertion without renumbering. - ---- - -## Group 1: Market orders - -Test market order submission and fills. Market orders should execute immediately. - -| TC | Name | Description | Skip when | -|--------|-------------------------------|-----------------------------------------------------|---------------------| -| TC-E01 | Market BUY - submit and fill | Open long position via market buy. | No market orders. | -| TC-E02 | Market SELL - submit and fill | Open short position via market sell. | No market orders. | -| TC-E03 | Market order with IOC TIF | Market order explicitly using IOC time in force. | No IOC. | -| TC-E04 | Market order with FOK TIF | Market order explicitly using FOK time in force. | No FOK. | -| TC-E05 | Market order with quote qty | Market order using quote currency quantity. | No quote quantity. | -| TC-E06 | Close position via market | Close an open position with a market order on stop. | No market orders. | - -### TC-E01: Market BUY - submit and fill - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, market data flowing, no open position. | -| **Action** | ExecTester opens a long position via `open_position_on_start_qty`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Position opened with side=LONG, quantity matches config, fill price within market range, `AccountState` updated. | -| **Skip when** | Adapter does not support market orders. | - -**Considerations:** - -- Some adapters simulate market orders as aggressive limit IOC orders (check adapter guide). -- The event sequence from the strategy's perspective should be identical regardless of the venue mechanism. -- Fill price should be within the recent bid/ask spread. -- Partial fills are valid; verify the cumulative filled quantity matches the order quantity. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) -``` - -### TC-E02: Market SELL - submit and fill - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, market data flowing, no open position. | -| **Action** | ExecTester opens a short position via negative `open_position_on_start_qty`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Position opened with side=SHORT, quantity matches config, fill price within market range. | -| **Skip when** | Adapter does not support market orders or short selling. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("-0.01"), - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(-1, 2))) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) -``` - -### TC-E03: Market order with IOC TIF - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, market data flowing. | -| **Action** | Open position with `open_position_time_in_force=IOC`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Same as TC-E01; the IOC TIF is explicitly set on the order. | -| **Skip when** | No IOC support. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), - open_position_time_in_force=TimeInForce.IOC, - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false); -config.open_position_time_in_force = TimeInForce::Ioc; -``` - -### TC-E04: Market order with FOK TIF - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, market data flowing. | -| **Action** | Open position with `open_position_time_in_force=FOK`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Same as TC-E01; the FOK TIF is explicitly set on the order. | -| **Skip when** | No FOK support. | - -**Considerations:** - -- FOK requires the entire quantity to be fillable immediately or the order is canceled. -- Use small test quantities so book depth is sufficient for a complete fill. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), - open_position_time_in_force=TimeInForce.FOK, - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false); -config.open_position_time_in_force = TimeInForce::Fok; -``` - -### TC-E05: Market order with quote quantity - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, adapter supports quote quantity. | -| **Action** | Open position with `use_quote_quantity=True`, quantity in quote currency. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Order submitted with quote currency quantity; fill quantity is in base currency. | -| **Skip when** | Adapter does not support quote quantity orders. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("100.0"), # Quote currency amount - open_position_on_start_qty=Decimal("100.0"), - use_quote_quantity=True, - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("100")) - .with_open_position_on_start(Some(Decimal::from(100))) - .with_use_quote_quantity(true) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) -``` - -### TC-E06: Close position via market order on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open position from TC-E01 or TC-E02. | -| **Action** | Stop the strategy; ExecTester closes position via market order. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled` (closing order). | -| **Pass criteria** | Position closed (net quantity = 0), no open orders remaining. | -| **Skip when** | Adapter does not support market orders. | - -**Considerations:** - -- This test naturally follows TC-E01 or TC-E02 as part of the same session. -- `close_positions_on_stop=True` is the default. -- The closing order should be on the opposite side of the position. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), - close_positions_on_stop=True, - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) - .with_close_positions_on_stop(true) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) -``` - ---- - -## Group 2: Limit orders - -Test limit order submission, acceptance, and behavior across time-in-force options. - -| TC | Name | Description | Skip when | -|--------|----------------------------|--------------------------------------------------|--------------------| -| TC-E10 | Limit BUY GTC | Place GTC limit buy below TOB, verify accepted. | Never. | -| TC-E11 | Limit SELL GTC | Place GTC limit sell above TOB, verify accepted. | Never. | -| TC-E12 | Limit BUY and SELL pair | Both sides simultaneously, verify both accepted. | Never. | -| TC-E13 | Limit IOC aggressive fill | Limit IOC at aggressive price, expect fill. | No IOC. | -| TC-E14 | Limit IOC passive no fill | Limit IOC away from market, expect cancel. | No IOC. | -| TC-E15 | Limit FOK fill | Limit FOK at aggressive price, expect fill. | No FOK. | -| TC-E16 | Limit FOK no fill | Limit FOK away from market, expect cancel. | No FOK. | -| TC-E17 | Limit GTD | Limit with expiry time, verify accepted. | No GTD. | -| TC-E18 | Limit GTD expiry | Wait for GTD expiry, verify `OrderExpired`. | No GTD. | -| TC-E19 | Limit DAY | Limit with DAY TIF, verify accepted. | No DAY. | - -### TC-E10: Limit BUY GTC - submit and accept - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places a limit buy at `best_bid - tob_offset_ticks`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order is open on the venue with correct price, quantity, side=BUY, TIF=GTC. | -| **Skip when** | Never. | - -**Considerations:** - -- The `tob_offset_ticks` (default 500) places the order well away from the market to avoid accidental fills. -- Verify the order appears in the cache with `OrderStatus.ACCEPTED`. -- The order should remain open until explicitly canceled. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false) -``` - -### TC-E11: Limit SELL GTC - submit and accept - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places a limit sell at `best_ask + tob_offset_ticks`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order is open on the venue with correct price, quantity, side=SELL, TIF=GTC. | -| **Skip when** | Never. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(true) -``` - -### TC-E12: Limit BUY and SELL pair - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places both a limit buy and limit sell. | -| **Event sequence** | Two independent sequences: each `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Both orders open on venue, buy below bid, sell above ask. | -| **Skip when** | Never. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(true) -``` - -### TC-E13: Limit IOC aggressive fill - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Submit a limit buy IOC at or above the best ask (aggressive price). | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Order fills immediately; position opened. | -| **Skip when** | Adapter does not support IOC TIF. | - -**Considerations:** - -- This test requires manual order creation or adapter-specific configuration, as the ExecTester's - default limit order placement uses GTC TIF. -- IOC orders that don't fill immediately are canceled by the venue. - -### TC-E14: Limit IOC passive - no fill - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Submit a limit buy IOC well below the market (passive price). | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderCanceled`. | -| **Pass criteria** | Order is immediately canceled by venue with no fill. | -| **Skip when** | Adapter does not support IOC TIF. | - -**Considerations:** - -- The venue should cancel the unfilled IOC order; verify `OrderCanceled` event (not `OrderExpired`). - -### TC-E15: Limit FOK fill - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing, sufficient book depth. | -| **Action** | Submit a limit buy FOK at aggressive price with quantity within top-of-book depth. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Order fills completely in a single fill event. | -| **Skip when** | Adapter does not support FOK TIF. | - -**Considerations:** - -- FOK requires the entire quantity to be fillable; use small quantities so book depth is sufficient. - -### TC-E16: Limit FOK no fill - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Submit a limit buy FOK at passive price (well below market). | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderCanceled`. | -| **Pass criteria** | Order is immediately canceled by venue with no fill. | -| **Skip when** | Adapter does not support FOK TIF. | - -### TC-E17: Limit GTD - submit and accept - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Place limit buy with `order_expire_time_delta_mins` set (e.g., 60 minutes). | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order accepted with GTD TIF and correct expiry timestamp. | -| **Skip when** | Adapter does not support GTD TIF. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - order_expire_time_delta_mins=60, - enable_limit_buys=True, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false); -config.order_expire_time_delta_mins = Some(60); -``` - -### TC-E18: Limit GTD expiry - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open GTD limit order from TC-E17 (or use a very short expiry). | -| **Action** | Wait for the GTD expiry time to elapse. | -| **Event sequence** | `OrderExpired`. | -| **Pass criteria** | Order transitions to expired status; `OrderExpired` event received. | -| **Skip when** | Adapter does not support GTD TIF. | - -**Considerations:** - -- Use a short `order_expire_time_delta_mins` (e.g., 1–2 minutes) to avoid long waits. -- Some venues may report expiry as a cancel; verify the adapter maps this to `OrderExpired`. - -### TC-E19: Limit DAY - submit and accept - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, market is in trading hours. | -| **Action** | Submit limit buy with DAY TIF. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order accepted with DAY TIF; will be automatically canceled at end of trading day. | -| **Skip when** | Adapter does not support DAY TIF. | - -**Considerations:** - -- DAY orders may behave differently on 24/7 crypto venues vs traditional markets. -- Verify behavior when submitted outside trading hours (if applicable). - ---- - -## Group 3: Stop and conditional orders - -Test stop and conditional order types. These orders rest on the venue until a trigger condition is met. - -| TC | Name | Description | Skip when | -|--------|------------------------|-------------------------------------------------------|---------------------| -| TC-E20 | StopMarket BUY | Stop buy above ask, verify accepted. | No `STOP_MARKET`. | -| TC-E21 | StopMarket SELL | Stop sell below bid, verify accepted. | No `STOP_MARKET`. | -| TC-E22 | StopLimit BUY | Stop-limit buy with trigger + limit price. | No `STOP_LIMIT`. | -| TC-E23 | StopLimit SELL | Stop-limit sell with trigger + limit price. | No `STOP_LIMIT`. | -| TC-E24 | MarketIfTouched BUY | MIT buy below bid. | No `MIT`. | -| TC-E25 | MarketIfTouched SELL | MIT sell above ask. | No `MIT`. | -| TC-E26 | LimitIfTouched BUY | LIT buy with trigger + limit price. | No `LIT`. | -| TC-E27 | LimitIfTouched SELL | LIT sell with trigger + limit price. | No `LIT`. | - -### TC-E20: StopMarket BUY - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places a stop-market buy above the current ask. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Stop order accepted on venue with correct trigger price and side=BUY. | -| **Skip when** | Adapter does not support `StopMarket` orders. | - -**Considerations:** - -- The trigger price should be above the current ask by `stop_offset_ticks`. -- The order should NOT trigger immediately (trigger price is above market). -- Verifying trigger and fill requires the market to move, which may not happen during the test. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=False, - enable_stop_buys=True, - enable_stop_sells=False, - stop_order_type=OrderType.STOP_MARKET, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) - .with_enable_stop_buys(true) - .with_enable_stop_sells(false) - .with_stop_order_type(OrderType::StopMarket) -``` - -### TC-E21: StopMarket SELL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places a stop-market sell below the current bid. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Stop order accepted on venue with correct trigger price and side=SELL. | -| **Skip when** | Adapter does not support `StopMarket` orders. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=False, - enable_stop_buys=False, - enable_stop_sells=True, - stop_order_type=OrderType.STOP_MARKET, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) - .with_enable_stop_buys(false) - .with_enable_stop_sells(true) - .with_stop_order_type(OrderType::StopMarket) -``` - -### TC-E22: StopLimit BUY - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places a stop-limit buy with trigger price above ask and limit offset. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Stop-limit order accepted with correct trigger price, limit price, and side=BUY. | -| **Skip when** | Adapter does not support `StopLimit` orders. | - -**Considerations:** - -- Requires `stop_limit_offset_ticks` to be set for the limit price offset from the trigger price. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=False, - enable_stop_buys=True, - enable_stop_sells=False, - stop_order_type=OrderType.STOP_LIMIT, - stop_limit_offset_ticks=50, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) - .with_enable_stop_buys(true) - .with_enable_stop_sells(false) - .with_stop_order_type(OrderType::StopLimit); -config.stop_limit_offset_ticks = Some(50); -``` - -### TC-E23: StopLimit SELL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places a stop-limit sell with trigger price below bid. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Stop-limit order accepted with correct trigger price, limit price, and side=SELL. | -| **Skip when** | Adapter does not support `StopLimit` orders. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=False, - enable_stop_buys=False, - enable_stop_sells=True, - stop_order_type=OrderType.STOP_LIMIT, - stop_limit_offset_ticks=50, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) - .with_enable_stop_buys(false) - .with_enable_stop_sells(true) - .with_stop_order_type(OrderType::StopLimit); -config.stop_limit_offset_ticks = Some(50); -``` - -### TC-E24: MarketIfTouched BUY - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Place MIT buy with trigger below current bid (buy on dip). | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | MIT order accepted on venue with correct trigger price. | -| **Skip when** | Adapter does not support `MarketIfTouched` orders. | - -### TC-E25: MarketIfTouched SELL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Place MIT sell with trigger above current ask (sell on rally). | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | MIT order accepted on venue with correct trigger price. | -| **Skip when** | Adapter does not support `MarketIfTouched` orders. | - -### TC-E26: LimitIfTouched BUY - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Place LIT buy with trigger below bid and limit price offset. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | LIT order accepted with correct trigger price and limit price. | -| **Skip when** | Adapter does not support `LimitIfTouched` orders. | - -### TC-E27: LimitIfTouched SELL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | Place LIT sell with trigger above ask and limit price offset. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | LIT order accepted with correct trigger price and limit price. | -| **Skip when** | Adapter does not support `LimitIfTouched` orders. | - ---- - -## Group 4: Order modification - -Test order modification (amend) and cancel-replace workflows. - -| TC | Name | Description | Skip when | -|-------|------------------------------|-----------------------------------------------------|-----------------------------| -| TC-E30 | Modify limit BUY price | Amend open limit buy to new price. | No modify support. | -| TC-E31 | Modify limit SELL price | Amend open limit sell to new price. | No modify support. | -| TC-E32 | Cancel-replace limit BUY | Cancel and resubmit limit buy at new price. | Never. | -| TC-E33 | Cancel-replace limit SELL | Cancel and resubmit limit sell at new price. | Never. | -| TC-E34 | Modify stop trigger price | Amend stop order trigger price. | No modify or no stop. | -| TC-E35 | Cancel-replace stop order | Cancel and resubmit stop at new trigger price. | No stop orders. | -| TC-E36 | Modify rejected | Modify on unsupported adapter. | Adapter supports modify. | - -### TC-E30: Modify limit BUY price - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open GTC limit buy from TC-E10. | -| **Action** | ExecTester modifies limit buy to a new price as market moves (`modify_orders_to_maintain_tob_offset=True`). | -| **Event sequence** | `OrderPendingUpdate` → `OrderUpdated`. | -| **Pass criteria** | Order price updated on venue; `OrderUpdated` event contains new price. | -| **Skip when** | Adapter does not support order modification. | - -**Considerations:** - -- Requires market movement to trigger the ExecTester's order maintenance logic. -- The modify is triggered when the order price drifts from the target TOB offset. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=False, - modify_orders_to_maintain_tob_offset=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false); -config.modify_orders_to_maintain_tob_offset = true; -``` - -### TC-E31: Modify limit SELL price - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open GTC limit sell from TC-E11. | -| **Action** | ExecTester modifies limit sell to new price as market moves. | -| **Event sequence** | `OrderPendingUpdate` → `OrderUpdated`. | -| **Pass criteria** | Order price updated on venue; `OrderUpdated` event contains new price. | -| **Skip when** | Adapter does not support order modification. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=True, - modify_orders_to_maintain_tob_offset=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(true); -config.modify_orders_to_maintain_tob_offset = true; -``` - -### TC-E32: Cancel-replace limit BUY - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open GTC limit buy. | -| **Action** | ExecTester cancels and resubmits limit buy at new price as market moves. | -| **Event sequence** | `OrderPendingCancel` → `OrderCanceled` → `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Original order canceled, new order accepted at updated price. | -| **Skip when** | Never (cancel-replace is always available). | - -**Considerations:** - -- This is the universal alternative when the adapter does not support native modify. -- Two distinct orders in the cache: the canceled original and the new replacement. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=False, - cancel_replace_orders_to_maintain_tob_offset=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false); -config.cancel_replace_orders_to_maintain_tob_offset = true; -``` - -### TC-E33: Cancel-replace limit SELL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open GTC limit sell. | -| **Action** | ExecTester cancels and resubmits limit sell at new price. | -| **Event sequence** | `OrderPendingCancel` → `OrderCanceled` → `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Original order canceled, new order accepted at updated price. | -| **Skip when** | Never. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=False, - enable_limit_sells=True, - cancel_replace_orders_to_maintain_tob_offset=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(false) - .with_enable_limit_sells(true); -config.cancel_replace_orders_to_maintain_tob_offset = true; -``` - -### TC-E34: Modify stop trigger price - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open stop order from TC-E20 or TC-E22. | -| **Action** | ExecTester modifies stop trigger price as market moves (`modify_stop_orders_to_maintain_offset=True`). | -| **Event sequence** | `OrderPendingUpdate` → `OrderUpdated`. | -| **Pass criteria** | Stop order trigger price updated on venue. | -| **Skip when** | Adapter does not support modify, or no stop order support. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_stop_buys=True, - modify_stop_orders_to_maintain_offset=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_stop_buys(true); -config.modify_stop_orders_to_maintain_offset = true; -``` - -### TC-E35: Cancel-replace stop order - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open stop order. | -| **Action** | ExecTester cancels and resubmits stop at new trigger price. | -| **Event sequence** | `OrderPendingCancel` → `OrderCanceled` → `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Original stop canceled, new stop accepted at updated trigger price. | -| **Skip when** | No stop order support. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_stop_buys=True, - cancel_replace_stop_orders_to_maintain_offset=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_stop_buys(true); -config.cancel_replace_stop_orders_to_maintain_offset = true; -``` - -### TC-E36: Modify rejected - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open limit order, adapter does NOT support modify. | -| **Action** | Attempt to modify the order (programmatically, not via ExecTester auto-maintain). | -| **Event sequence** | `OrderModifyRejected`. | -| **Pass criteria** | Modify attempt results in `OrderModifyRejected` event with reason; original order remains unchanged. | -| **Skip when** | Adapter supports order modification. | - -**Considerations:** - -- This tests the adapter's rejection path, not the ExecTester's cancel-replace logic. -- The rejection reason should indicate that modification is not supported. - ---- - -## Group 5: Order cancellation - -Test order cancellation workflows. - -| TC | Name | Description | Skip when | -|-------|----------------------------|------------------------------------------------------|----------------------| -| TC-E40 | Cancel single limit order | Cancel an open limit order. | Never. | -| TC-E41 | Cancel all on stop | Strategy stop cancels all open orders (default). | Never. | -| TC-E42 | Individual cancels on stop | Cancel orders one-by-one on stop. | Never. | -| TC-E43 | Batch cancel on stop | Cancel orders via batch API on stop. | No batch cancel. | -| TC-E44 | Cancel already-canceled | Cancel a non-open order. | Never. | - -### TC-E40: Cancel single limit order - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open GTC limit order from TC-E10 or TC-E11. | -| **Action** | Stop the strategy; ExecTester cancels the open limit order. | -| **Event sequence** | `OrderPendingCancel` → `OrderCanceled`. | -| **Pass criteria** | Order status transitions to CANCELED; no open orders remaining. | -| **Skip when** | Never. | - -**Considerations:** - -- `cancel_orders_on_stop=True` (default) triggers cancellation when the strategy stops. -- Verify the `OrderCanceled` event contains the correct `venue_order_id`. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=False, - cancel_orders_on_stop=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false) - .with_cancel_orders_on_stop(true) -``` - -### TC-E41: Cancel all on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Multiple open orders (limit buy + limit sell from TC-E12). | -| **Action** | Stop the strategy with `cancel_orders_on_stop=True` (default). | -| **Event sequence** | For each order: `OrderPendingCancel` → `OrderCanceled`. | -| **Pass criteria** | All open orders canceled; no open orders remaining. | -| **Skip when** | Never. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=True, - cancel_orders_on_stop=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(true) - .with_cancel_orders_on_stop(true) -``` - -### TC-E42: Individual cancels on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Multiple open orders. | -| **Action** | Stop with `use_individual_cancels_on_stop=True`. | -| **Event sequence** | Individual `OrderPendingCancel` → `OrderCanceled` for each order. | -| **Pass criteria** | Each order canceled individually; all orders reach CANCELED status. | -| **Skip when** | Never. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=True, - use_individual_cancels_on_stop=True, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(true); -config.use_individual_cancels_on_stop = true; -``` - -### TC-E43: Batch cancel on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Multiple open orders, adapter supports batch cancel. | -| **Action** | Stop with `use_batch_cancel_on_stop=True`. | -| **Event sequence** | Batch `OrderPendingCancel` → `OrderCanceled` for all orders. | -| **Pass criteria** | All orders canceled via single batch request; all reach CANCELED status. | -| **Skip when** | Adapter does not support batch cancel. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=True, - use_batch_cancel_on_stop=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(true) - .with_use_batch_cancel_on_stop(true) -``` - -### TC-E44: Cancel already-canceled order - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | A previously canceled order (from TC-E40). | -| **Action** | Attempt to cancel the same order again. | -| **Event sequence** | `OrderCancelRejected`. | -| **Pass criteria** | Cancel attempt is rejected; `OrderCancelRejected` event received with reason. | -| **Skip when** | Never. | - -**Considerations:** - -- This tests the adapter's error handling for invalid cancel requests. -- The rejection reason should indicate the order is not in a cancelable state. - ---- - -## Group 6: Bracket orders - -Test bracket order submission (entry + take-profit + stop-loss). - -| TC | Name | Description | Skip when | -|-------|-------------------------------|---------------------------------------------------|----------------------| -| TC-E50 | Bracket BUY | Entry limit buy + TP limit sell + SL stop sell. | No bracket support. | -| TC-E51 | Bracket SELL | Entry limit sell + TP limit buy + SL stop buy. | No bracket support. | -| TC-E52 | Bracket entry fill activates | Verify TP/SL become active after entry fill. | No bracket support. | -| TC-E53 | Bracket with post-only entry | Entry order uses post-only flag. | No bracket or PO. | - -### TC-E50: Bracket BUY - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester submits a bracket order: limit buy entry + take-profit sell + stop-loss sell. | -| **Event sequence** | Entry: `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`; TP and SL: `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Three orders created and accepted: entry below bid, TP above ask, SL below entry. | -| **Skip when** | Adapter does not support bracket orders. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_brackets=True, - bracket_entry_order_type=OrderType.LIMIT, - bracket_offset_ticks=500, - enable_limit_buys=True, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_brackets(true) - .with_bracket_entry_order_type(OrderType::Limit) - .with_bracket_offset_ticks(500) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false) -``` - -### TC-E51: Bracket SELL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester submits bracket: limit sell entry + TP buy + SL buy. | -| **Event sequence** | Same pattern as TC-E50 but for sell side. | -| **Pass criteria** | Three orders created and accepted on sell side. | -| **Skip when** | Adapter does not support bracket orders. | - -### TC-E52: Bracket entry fill activates TP/SL - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Bracket order from TC-E50 where entry order fills. | -| **Action** | Entry order fills; verify contingent TP and SL orders activate. | -| **Event sequence** | Entry: `OrderFilled`; TP and SL transition from contingent to active. | -| **Pass criteria** | After entry fill, TP and SL orders are live on the venue. | -| **Skip when** | Adapter does not support bracket orders. | - -**Considerations:** - -- This requires the entry order to actually fill, which may need aggressive pricing. -- The TP/SL activation mechanism varies by venue (some activate immediately, some are OCA groups). - -### TC-E53: Bracket with post-only entry - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter supports brackets and post-only. | -| **Action** | Submit bracket with `use_post_only=True` (applied to entry and TP). | -| **Event sequence** | Same as TC-E50 with post-only flag on entry. | -| **Pass criteria** | Entry and TP orders accepted as post-only (maker); SL is not post-only. | -| **Skip when** | No bracket support or no post-only support. | - ---- - -## Group 7: Order flags - -Test order-level flags and special parameters. - -| TC | Name | Description | Skip when | -|-------|----------------------|--------------------------------------------------------|----------------------| -| TC-E60 | PostOnly accepted | Limit with post-only, placed away from TOB. | No post-only. | -| TC-E61 | ReduceOnly on close | Close position with reduce-only flag. | No reduce-only. | -| TC-E62 | Display quantity | Iceberg order with visible quantity < total. | No display quantity. | -| TC-E63 | Custom order params | Adapter-specific params via `order_params`. | N/A. | - -### TC-E60: PostOnly accepted - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places limit buy with `use_post_only=True` at passive price. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order accepted as a maker order; post-only flag acknowledged by venue. | -| **Skip when** | Adapter does not support post-only flag. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=False, - use_post_only=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false) - .with_use_post_only(true) -``` - -### TC-E61: ReduceOnly on close - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open position (from TC-E01). | -| **Action** | Stop strategy with `reduce_only_on_stop=True`; closing order uses reduce-only flag. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled` (with reduce-only). | -| **Pass criteria** | Closing order has reduce-only flag; position fully closed. | -| **Skip when** | Adapter does not support reduce-only flag. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), - reduce_only_on_stop=True, - close_positions_on_stop=True, - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) - .with_reduce_only_on_stop(true) - .with_close_positions_on_stop(true) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) -``` - -### TC-E62: Display quantity (iceberg) - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, adapter supports display quantity. | -| **Action** | Place limit order with `order_display_qty` < `order_qty`. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order accepted with display quantity set; only display qty visible on the book. | -| **Skip when** | Adapter does not support display quantity / iceberg orders. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("1.0"), - order_display_qty=Decimal("0.1"), - enable_limit_buys=True, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -let mut config = ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("1.0")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false); -config.order_display_qty = Some(Quantity::from("0.1")); -``` - -### TC-E63: Custom order params - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, adapter accepts additional parameters. | -| **Action** | Place order with `order_params` dict containing adapter-specific parameters. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted`. | -| **Pass criteria** | Order accepted; adapter-specific parameters passed through to venue. | -| **Skip when** | N/A (adapter-specific). | - -**Considerations:** - -- The `order_params` dict is opaque to the ExecTester and passed through to the adapter. -- Consult the adapter's guide for supported parameters. - ---- - -## Group 8: Rejection handling - -Test that the adapter correctly handles and reports order rejections. - -| TC | Name | Description | Skip when | -|-------|-------------------------|------------------------------------------------------|-------------------------| -| TC-E70 | PostOnly rejection | Post-only order that would cross the spread. | No post-only. | -| TC-E71 | ReduceOnly rejection | Reduce-only order with no position to reduce. | No reduce-only. | -| TC-E72 | Unsupported order type | Submit order type not supported by adapter. | Never. | -| TC-E73 | Unsupported TIF | Submit order with unsupported time in force. | Never. | - -### TC-E70: PostOnly rejection - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, quotes flowing. | -| **Action** | ExecTester places post-only order on the wrong side of the book (`test_reject_post_only=True`), causing it to cross the spread. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderRejected`. | -| **Pass criteria** | Order rejected by venue; `OrderRejected` event received with reason indicating post-only violation. | -| **Skip when** | Adapter does not support post-only flag. | - -**Considerations:** - -- The ExecTester's `test_reject_post_only` mode intentionally prices the order to cross. -- Some venues may partially fill instead of rejecting; behavior is venue-specific. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - enable_limit_buys=True, - enable_limit_sells=False, - use_post_only=True, - test_reject_post_only=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_enable_limit_buys(true) - .with_enable_limit_sells(false) - .with_use_post_only(true) - .with_test_reject_post_only(true) -``` - -### TC-E71: ReduceOnly rejection - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, no open position for the instrument. | -| **Action** | ExecTester opens a market position with `reduce_only=True` via `test_reject_reduce_only=True` and `open_position_on_start_qty`, when no position exists to reduce. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderRejected`. | -| **Pass criteria** | Order rejected; `OrderRejected` event with reason indicating reduce-only violation. | -| **Skip when** | Adapter does not support reduce-only flag. | - -**Considerations:** - -- The `test_reject_reduce_only` flag only applies to the opening market order submitted via - `open_position_on_start_qty`. -- Verify no prior position exists for the instrument before running this test. - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), - test_reject_reduce_only=True, - enable_limit_buys=False, - enable_limit_sells=False, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) - .with_test_reject_reduce_only(true) - .with_enable_limit_buys(false) - .with_enable_limit_sells(false) -``` - -### TC-E72: Unsupported order type - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, order type not in adapter's supported set. | -| **Action** | Submit an order type the adapter does not support. | -| **Event sequence** | `OrderDenied` (pre-submission rejection by adapter). | -| **Pass criteria** | Order denied before reaching venue; `OrderDenied` event with reason. | -| **Skip when** | Never (every adapter has unsupported order types to test). | - -**Considerations:** - -- `OrderDenied` occurs at the adapter level before the order reaches the venue. -- This differs from `OrderRejected` which comes from the venue. -- Test by configuring a stop order type that the adapter does not support. - -### TC-E73: Unsupported TIF - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, TIF not in adapter's supported set. | -| **Action** | Submit an order with a TIF the adapter does not support. | -| **Event sequence** | `OrderDenied` (pre-submission rejection by adapter). | -| **Pass criteria** | Order denied before reaching venue; `OrderDenied` event with reason. | -| **Skip when** | Never (every adapter has unsupported TIF options to test). | - -**Considerations:** - -- Similar to TC-E72 but for time-in-force options. -- Test with TIF values from the Nautilus enum that the adapter does not map. - ---- - -## Group 9: Lifecycle (start/stop) - -Test strategy lifecycle behavior and state management on start and stop. - -| TC | Name | Description | Skip when | -|--------|-----------------------------|--------------------------------------------------------|----------------------| -| TC-E80 | Open position on start | Open a position immediately when strategy starts. | No market orders. | -| TC-E81 | Cancel orders on stop | Cancel all open orders when strategy stops. | Never. | -| TC-E82 | Close positions on stop | Close open positions when strategy stops. | No market orders. | -| TC-E83 | Unsubscribe on stop | Unsubscribe from data feeds on strategy stop. | No unsub support. | -| TC-E84 | Reconcile open orders | Reconcile existing open orders from a prior session. | Never. | -| TC-E85 | Reconcile filled orders | Reconcile previously filled orders from a prior session.| Never. | -| TC-E86 | Reconcile open long | Reconcile existing open long position. | Never. | -| TC-E87 | Reconcile open short | Reconcile existing open short position. | Never. | - -### TC-E80: Open position on start - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Adapter connected, instrument loaded, no existing position. | -| **Action** | Strategy starts with `open_position_on_start_qty` set. | -| **Event sequence** | `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | Position opened on start; market order submitted and filled before limit order maintenance begins. | -| **Skip when** | Adapter does not support market orders. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - open_position_on_start_qty=Decimal("0.01"), -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_open_position_on_start(Some(Decimal::new(1, 2))) -``` - -### TC-E81: Cancel orders on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open limit orders from the strategy session. | -| **Action** | Stop the strategy with `cancel_orders_on_stop=True` (default). | -| **Event sequence** | For each open order: `OrderPendingCancel` → `OrderCanceled`. | -| **Pass criteria** | All strategy-owned open orders canceled on stop. | -| **Skip when** | Never. | - -### TC-E82: Close positions on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Open position from the strategy session. | -| **Action** | Stop the strategy with `close_positions_on_stop=True` (default). | -| **Event sequence** | Closing order: `OrderInitialized` → `OrderSubmitted` → `OrderAccepted` → `OrderFilled`. | -| **Pass criteria** | All strategy-owned positions closed; net position = 0. | -| **Skip when** | Adapter does not support market orders. | - -### TC-E83: Unsubscribe on stop - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | Active data subscriptions (quotes, trades, book). | -| **Action** | Stop the strategy with `can_unsubscribe=True` (default). | -| **Event sequence** | Data subscriptions removed. | -| **Pass criteria** | No further data events received after stop; clean disconnection. | -| **Skip when** | Adapter does not support unsubscribe. | - -**Python config:** - -```python -ExecTesterConfig( - instrument_id=instrument_id, - order_qty=Decimal("0.01"), - can_unsubscribe=True, -) -``` - -**Rust config:** - -```rust -ExecTesterConfig::new(strategy_id, instrument_id, client_id, Quantity::from("0.01")) - .with_can_unsubscribe(true) -``` - -### TC-E84: Reconcile open orders - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | One or more open limit orders on the venue from a prior session. | -| **Action** | Start the node with `reconciliation=True`. | -| **Event sequence** | `OrderStatusReport` generated for each open order. | -| **Pass criteria** | Each open order is loaded into the cache with correct `venue_order_id`, status=ACCEPTED, price, quantity, side, and order type. | -| **Skip when** | Never. | - -**Considerations:** - -- Leave limit orders open from a prior test session (do not cancel on stop). -- Use `external_order_claims` to claim the instrument so the adapter reconciles orders for it. -- Verify that the reconciled order count matches the venue-reported count. - -### TC-E85: Reconcile filled orders - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | One or more filled orders on the venue from a prior session. | -| **Action** | Start the node with `reconciliation=True`. | -| **Event sequence** | `FillReport` generated for each historical fill. | -| **Pass criteria** | Each filled order is loaded into the cache with correct `venue_order_id`, status=FILLED, fill price, fill quantity, and commission. | -| **Skip when** | Never. | - -**Considerations:** - -- Requires orders that filled in a prior session. -- Verify fill price, quantity, and commission match the venue's reported values. -- Some adapters may only report fills within a lookback window. - -### TC-E86: Reconcile open long position - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | An open long position on the venue from a prior session. | -| **Action** | Start the node with `reconciliation=True`. | -| **Event sequence** | `PositionStatusReport` generated for the long position. | -| **Pass criteria** | Position loaded into cache with correct instrument, side=LONG, quantity, and entry price matching the venue. | -| **Skip when** | Never. | - -**Considerations:** - -- Open a long position in a prior session and stop the strategy without closing it - (`close_positions_on_stop=False`). -- Verify the reconciled position quantity and average entry price match the venue. -- After reconciliation, the strategy should be able to manage or close this position. - -### TC-E87: Reconcile open short position - -| Field | Value | -|--------------------|------------------------------------------------------------------------| -| **Prerequisite** | An open short position on the venue from a prior session. | -| **Action** | Start the node with `reconciliation=True`. | -| **Event sequence** | `PositionStatusReport` generated for the short position. | -| **Pass criteria** | Position loaded into cache with correct instrument, side=SHORT, quantity, and entry price matching the venue. | -| **Skip when** | Never. | - -**Considerations:** - -- Open a short position in a prior session and stop the strategy without closing it - (`close_positions_on_stop=False`). -- Verify the reconciled position quantity and average entry price match the venue. -- After reconciliation, the strategy should be able to manage or close this position. - ---- - -## ExecTester configuration reference - -Quick reference for all `ExecTesterConfig` parameters. Defaults shown are for the Python config; -the Rust builder uses equivalent defaults. - -| Parameter | Type | Default | Affects groups | -|-------------------------------------------------|-------------------|-----------------|----------------| -| `instrument_id` | InstrumentId | *required* | All | -| `order_qty` | Decimal | *required* | All | -| `order_display_qty` | Decimal? | None | 2, 7 | -| `order_expire_time_delta_mins` | PositiveInt? | None | 2 | -| `order_params` | dict? | None | 7 | -| `client_id` | ClientId? | None | All | -| `subscribe_quotes` | bool | True | — | -| `subscribe_trades` | bool | True | — | -| `subscribe_book` | bool | False | — | -| `book_type` | BookType | L2_MBP | — | -| `book_depth` | PositiveInt? | None | — | -| `book_interval_ms` | PositiveInt | 1000 | — | -| `book_levels_to_print` | PositiveInt | 10 | — | -| `open_position_on_start_qty` | Decimal? | None | 1, 9 | -| `open_position_time_in_force` | TimeInForce | GTC | 1 | -| `enable_limit_buys` | bool | True | 2, 4, 5, 6 | -| `enable_limit_sells` | bool | True | 2, 4, 5, 6 | -| `enable_stop_buys` | bool | False | 3, 4 | -| `enable_stop_sells` | bool | False | 3, 4 | -| `limit_time_in_force` | TimeInForce? | None | 2, 6 | -| `tob_offset_ticks` | PositiveInt | 500 | 2, 4 | -| `stop_order_type` | OrderType | STOP_MARKET | 3 | -| `stop_offset_ticks` | PositiveInt | 100 | 3 | -| `stop_limit_offset_ticks` | PositiveInt? | None | 3 | -| `stop_time_in_force` | TimeInForce? | None | 3 | -| `stop_trigger_type` | TriggerType? | None | 3 | -| `enable_brackets` | bool | False | 6 | -| `bracket_entry_order_type` | OrderType | LIMIT | 6 | -| `bracket_offset_ticks` | PositiveInt | 500 | 6 | -| `modify_orders_to_maintain_tob_offset` | bool | False | 4 | -| `modify_stop_orders_to_maintain_offset` | bool | False | 4 | -| `cancel_replace_orders_to_maintain_tob_offset` | bool | False | 4 | -| `cancel_replace_stop_orders_to_maintain_offset` | bool | False | 4 | -| `use_post_only` | bool | False | 2, 6, 7, 8 | -| `use_quote_quantity` | bool | False | 1, 7 | -| `emulation_trigger` | TriggerType? | None | 2, 3 | -| `cancel_orders_on_stop` | bool | True | 5, 9 | -| `close_positions_on_stop` | bool | True | 9 | -| `close_positions_time_in_force` | TimeInForce? | None | 9 | -| `reduce_only_on_stop` | bool | True | 7, 9 | -| `use_individual_cancels_on_stop` | bool | False | 5 | -| `use_batch_cancel_on_stop` | bool | False | 5 | -| `dry_run` | bool | False | — | -| `log_data` | bool | True | — | -| `test_reject_post_only` | bool | False | 8 | -| `test_reject_reduce_only` | bool | False | 8 | -| `can_unsubscribe` | bool | True | 9 | diff --git a/nautilus-docs/docs/developer_guide/test_datasets.md b/nautilus-docs/docs/developer_guide/test_datasets.md deleted file mode 100644 index debabc3..0000000 --- a/nautilus-docs/docs/developer_guide/test_datasets.md +++ /dev/null @@ -1,282 +0,0 @@ -# Test Datasets - -Target standards for curating, storing, and consuming external datasets used as test fixtures. -New datasets should follow these standards. Existing datasets are documented under -[legacy datasets](#legacy-datasets) and will be migrated incrementally. - -## Dataset categories - -**Small data** (< 1 MB) is checked directly into `tests/test_data//` -alongside a `metadata.json` file. These files are always available without network access. - -**Large data** (> 1 MB) is hosted as Parquet in the R2 test-data bucket. -A SHA-256 checksum is recorded in `tests/test_data/large/checksums.json`. -The `ensure_test_data_exists()` helper downloads the file on first use and verifies integrity. - -**User-fetched data** is used when a vendor license, entitlement model, or access control does not -allow NautilusTrader to redistribute the data through the public repo or the public R2 bucket. -In this model, the repo stores only a manifest, fetch instructions, and the transform code. Each -user downloads the source data with their own vendor account and converts it locally. - -Use the user-fetched model when any of the following apply: - -- The vendor requires each user to hold their own account, API key, or historical-data license. -- The license allows internal use but does not clearly allow redistribution of derived fixtures. -- The dataset is suitable for examples or opt-in integration tests, but not for default CI. - -## Required metadata - -Every curated dataset that stores or redistributes a concrete artifact must include a -`metadata.json` with at minimum: - -| Field | Description | -|----------------|-------------------------------------------------------------------| -| `file` | Filename of the dataset. | -| `sha256` | SHA-256 hash of the file. | -| `size_bytes` | File size in bytes. | -| `original_url` | Download URL of the original source data. | -| `licence` | License terms and any redistribution constraints. | -| `added_at` | ISO 8601 timestamp when the dataset was curated. | - -These fields match the output of `scripts/curate-dataset.sh`. Additional recommended -fields for richer provenance: - -| Field | Description | -|-----------------|------------------------------------------------------------------| -| `instrument` | Instrument symbol(s) covered. | -| `date` | Trading date(s) covered. | -| `format` | Storage format (e.g., "Nautilus OrderBookDelta Parquet"). | -| `original_file` | Original vendor filename before transformation. | -| `parser` | Parser used for transformation (e.g., "itchy 0.3.4"). | - -User-fetched datasets use the same metadata fields where they apply. They should also include: - -| Field | Description | -|-----------------------|-----------------------------------------------------------------------| -| `distribution` | Must be `"user-fetch"`. | -| `fetch_method` | How the user acquires the source data (API, web portal, CLI, etc.). | -| `fetch_reference` | URL or document reference for the user-facing download flow. | -| `auth` | Required credentials or entitlements, if any. | -| `transform_version` | Version of the local transform pipeline that builds the final files. | -| `redistribution` | Short note describing redistribution limits for the dataset. | -| `public_mirror` | Must be `false` for restricted vendor datasets. | - -For user-fetched datasets without a single committed or mirrored artifact, `file`, `sha256`, and -`size_bytes` may be omitted from `metadata.json`. In that case, `target_files` in `manifest.json` -is authoritative for the local output files. For user-fetched datasets, `original_url` may point -to the vendor download entry point rather than the exact file URL when the exact file is generated -per user account or per request. - -Other metadata fields remain recommended where they apply. In particular, `licence` and `added_at` -should still be recorded for user-fetched datasets. - -## Storage format - -New datasets should be stored as **Nautilus Parquet** (not raw vendor formats). -This ensures: - -- Consistent data types across all test datasets. -- No vendor format parsing at test time. -- Clear derivative-work status for licensing. - -Use ZSTD compression (level 3) with 1M row groups. - -User-fetched datasets should also end up as Nautilus Parquet after the local transform step. -Raw vendor files should stay outside the repo and outside the public R2 bucket. - -## Naming convention - -``` -___.parquet -``` - -Examples: - -- `itch_AAPL_2019-01-30_deltas.parquet` -- `tardis_BTCUSDT_2020-09-01_depth10.parquet` - -## Curation workflow - -### Simple files (single download) - -Use `scripts/curate-dataset.sh`: - -```bash -scripts/curate-dataset.sh -``` - -This creates a versioned directory (`v1//`) with the file, -`LICENSE.txt`, and `metadata.json` containing the required fields above. - -### Complex pipelines (parse + transform) - -For datasets requiring format conversion (e.g., binary ITCH to Parquet): - -1. Write a curation function in `crates/testkit/src//` gated behind - `#[cfg(test)]` or an `#[ignore]` test. -2. The function should: download, parse, filter, convert to NautilusTrader types, write Parquet. -3. Output the Parquet file and `metadata.json` to a local directory. -4. Upload to R2 manually, then add the checksum to `checksums.json`. - -### User-fetched pipelines (restricted redistribution) - -For datasets that NautilusTrader cannot redistribute: - -1. Commit a manifest and `metadata.json`, but do not commit the real vendor data or derived - Parquet output. -2. Provide a local fetch command or helper that uses the user's own vendor credentials, - entitlements, or purchased historical files. -3. Convert the vendor data locally into Nautilus Parquet. -4. Store the resulting files in a local cache path that is ignored by git. -5. Make tests and examples opt in. They should skip cleanly when the dataset is missing. - -The default distribution order for new datasets is: - -1. Checked in small data. -2. Public R2 large data. -3. User-fetched data. - -Choose user-fetched only when the first two options are not acceptable under the vendor's terms. - -Do not: - -- Upload restricted vendor datasets to the public R2 bucket. -- Commit real vendor-derived Parquet files to the repo when redistribution rights are unclear. -- Make default CI depend on vendor credentials or paid historical-data access. - -You may maintain a private mirror for internal CI or employees when the license permits internal -sharing. Treat this as a separate operational path, not as part of the public test-data standard. - -## Adding a new dataset - -1. Curate the data following the workflow above. -2. Write `metadata.json` with all required fields. -3. For small data: commit to `tests/test_data//`. -4. For large data: upload Parquet to R2, add checksum to `tests/test_data/large/checksums.json`. -5. For user-fetched data: commit the manifest and fetch instructions only. Keep the source and - derived data out of the repo and out of the public R2 bucket. -6. Add path helper functions to `crates/testkit/src/common.rs` when shared testkit access is needed. -7. Write tests that consume the dataset. - -For user-fetched data, prefer this layout: - -```text -tests/test_data/// - metadata.json - manifest.json - README.md -``` - -Use `tests/test_data/local///` as the standard local cache path for generated -artifacts. Keep raw vendor downloads in a sibling `vendor/` directory under the same cache path -when local retention is needed. - -The manifest should be machine-readable and stable. It should capture the minimum information needed -to reproduce the fetch and transform steps on another machine. - -`metadata.json` is authoritative for provenance, licensing, and redistribution rules. -`manifest.json` is authoritative for fetch inputs, commands, cache locations, and output files. - -Recommended manifest fields: - -| Field | Description | -|---------------------|--------------------------------------------------------------------| -| `slug` | Stable dataset identifier. | -| `vendor` | Vendor or venue name. | -| `source_type` | `api`, `portal-download`, `purchased-archive`, etc. | -| `source_filters` | Symbols, event IDs, market IDs, date ranges, or file names. | -| `target_files` | Output Nautilus Parquet files expected after conversion. | -| `cache_dir` | Local output location relative to `tests/test_data/local/`. | -| `fetch_command` | Suggested command or script entry point. | -| `transform_command` | Suggested local conversion command. | -| `env` | Required environment variables. | -| `notes` | Short operational notes for users. | - -Tests that rely on user-fetched data should: - -- Be marked or grouped separately from default CI tests. -- Skip with a clear message when the local dataset is absent. -- Avoid network access unless the user explicitly opts in. -- Reuse a stable local cache path so the fetch happens once per machine. - -For pytest-based tests, prefer a guard like: - -```python -if not filepath.exists(): - pytest.skip(f"User-fetched test data not found: {filepath}") -``` - -For Rust tests that require manual dataset preparation, prefer `#[ignore]` when the test is not -expected to run in default CI. - -## Test runner serialization - -Tests that download large data files share target paths across test binaries. -Because `nextest` runs each binary in a separate process, concurrent downloads -to the same path can race. The nextest config at `.config/nextest.toml` defines -a `large-data-tests` group with `max-threads = 1` to serialize these binaries. - -When adding a new test binary that downloads large shared files, add it to the -group filter: - -```toml -[[profile.default.overrides]] -filter = 'binary(grid_mm_itch) | binary(orderbook_integration) | binary(your_new_binary)' -test-group = 'large-data-tests' -``` - -## Regenerating datasets - -When a schema change invalidates a large Parquet file, regenerate it from the -original source data using the curation tests below. After regenerating: - -1. `sha256sum /tmp/.parquet` -2. Update `tests/test_data/large/checksums.json` with the new hash. -3. Update the corresponding `metadata.json` (sha256, size_bytes). -4. Upload the Parquet file to R2. -5. Commit `checksums.json` and `metadata.json` (this also busts the CI cache). - -### ITCH AAPL L3 deltas - -Source: `01302019.NASDAQ_ITCH50.gz` (~4.4 GB) from NASDAQ EMI. - -```bash -# Download source (keep a local copy, this is a large file) -wget -O ~/Downloads/01302019.NASDAQ_ITCH50.gz \ - "https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/01302019.NASDAQ_ITCH50.gz" - -# Curation test expects source at /tmp -ln -sf ~/Downloads/01302019.NASDAQ_ITCH50.gz /tmp/01302019.NASDAQ_ITCH50.gz - -# Regenerate parquet (output: /tmp/itch_AAPL.XNAS_2019-01-30_deltas.parquet) -cargo test -p nautilus-testkit --lib test_curate_aapl_itch -- --ignored --nocapture -``` - -### Tardis Deribit BTC-PERPETUAL L2 deltas - -Source: `tardis_deribit_incremental_book_L2_2020-04-01_BTC-PERPETUAL.csv.gz` from -[Tardis](https://tardis.dev/). First-of-month data is available as free samples -(no API key required). - -```bash -# Download source (free sample, no API key needed) -wget -O tests/test_data/large/tardis_deribit_incremental_book_L2_2020-04-01_BTC-PERPETUAL.csv.gz \ - "https://datasets.tardis.dev/v1/deribit/incremental_book_L2/2020/04/01/BTC-PERPETUAL.csv.gz" - -# Regenerate parquet (output: /tmp/tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet) -cargo test -p nautilus-tardis test_curate_deribit_deltas -- --ignored --nocapture -``` - -## Legacy datasets - -These datasets predate this policy and use raw vendor formats (CSV/CSV.gz) -without `metadata.json`. They remain valid for existing tests. New datasets -should follow the Parquet standard above. - -| Dataset | Source | Format | Location | Status | -|--------------------------|--------|------------------|---------------------------|---------| -| Tardis Deribit L2 deltas | Tardis | Parquet (large) | `tests/test_data/large/` | Curated | -| ITCH AAPL L3 deltas | NASDAQ | Parquet (large) | `tests/test_data/large/` | Curated | -| Tardis Deribit L2 | Tardis | CSV (checked in) | `tests/test_data/tardis/` | Legacy | -| Tardis Binance snapshots | Tardis | CSV.gz (large) | `tests/test_data/large/` | Legacy | -| Tardis Bitmex trades | Tardis | CSV.gz (large) | `tests/test_data/large/` | Legacy | diff --git a/nautilus-docs/docs/developer_guide/testing.md b/nautilus-docs/docs/developer_guide/testing.md deleted file mode 100644 index bbf767d..0000000 --- a/nautilus-docs/docs/developer_guide/testing.md +++ /dev/null @@ -1,289 +0,0 @@ -# Testing - -Our automated tests serve as executable specifications for the trading platform. -A healthy suite documents intended behaviour, gives contributors confidence to refactor, and catches regressions before they reach production. -Tests also double as living examples that clarify complex flows and provide rapid CI feedback so issues surface early. - -The suite covers these categories: - -- Unit tests -- Integration tests -- Acceptance tests -- Performance tests -- Property-based tests -- Fuzzing -- Memory leak tests - -## Property-based testing - -Property testing verifies that logic holds for *all* valid inputs, not just hand-picked examples. -We use [`proptest`](https://altsysrq.github.io/proptest-book/intro.html) in Rust to enforce invariants. - -- **Use cases:** Core domain types (`Price`, `Quantity`, `UnixNanos`), accounting engines, matching engines, and state machines. -- **Example invariants:** - - Round-trip serialization: `parse(to_string(value)) == value` - - Inverse operations: `(A + B) - B == A` - - Transitivity: `If A < B and B < C, then A < C` - -## Fuzzing - -Fuzzing introduces unstructured or malicious data to the system to verify it fails gracefully. - -- **Use cases:** Network boundaries, exchange data parsers (JSON, FIX, WebSocket feeds), and complex state machines. -- **Goal:** The system returns a `Result::Err` and never panics, hangs, or leaks memory when encountering malformed data. - -When building or modifying core types, write property tests to cover the mathematical boundaries. - -Performance tests help evolve performance-critical components. - -Run tests with [pytest](https://docs.pytest.org), our primary test runner. -Use parametrized tests and fixtures (e.g., `@pytest.mark.parametrize`) to avoid repetitive code and improve clarity. - -## Running tests - -### Python tests - -From the repository root: - -```bash -make pytest -# or -uv run --active --no-sync pytest --new-first --failed-first -# or -pytest -``` - -For performance tests: - -```bash -make test-performance -# or -uv run --active --no-sync pytest tests/performance_tests --benchmark-disable-gc --codspeed -``` - -The `--benchmark-disable-gc` flag prevents garbage collection from skewing results. Run performance tests in isolation (not with unit tests) to avoid interference. - -### Rust tests - -```bash -make cargo-test -# or -cargo nextest run --workspace --features "python,ffi,high-precision,defi" --cargo-profile nextest -``` - -#### Testing with optional features - -Use `EXTRA_FEATURES` to include optional features like `capnp` or `hypersync`: - -```bash -# Test with capnp feature -make cargo-test EXTRA_FEATURES="capnp" - -# Test with multiple features -make cargo-test EXTRA_FEATURES="capnp hypersync" - -# Legacy shorthand for hypersync -make cargo-test HYPERSYNC=true - -# Test specific crate with features -make cargo-test-crate-nautilus-serialization FEATURES="capnp" -``` - -### IDE integration - -- **PyCharm**: Right-click the tests folder or file → "Run pytest". -- **VS Code**: Use the Python Test Explorer extension. - -## Test style - -- Name test functions after what they exercise; you do not need to encode the expected assertions in the name. -- Add docstrings when they clarify setup, scenarios, or expectations. -- Prefer pytest-style free functions for Python tests instead of test classes with setup methods. -- **Group assertions** when possible: perform all setup/act steps first, then assert together to avoid the act-assert-act smell. -- Use `unwrap`, `expect`, or direct `panic!`/`assert` calls inside tests; clarity and conciseness matter more than defensive error handling here. - -For Rust-specific test conventions (module structure, `#[rstest]`, parameterization), see the [Rust guide](rust.md#testing-conventions). - -## Waiting for asynchronous effects - -When waiting for background work to complete, prefer the polling helpers `await eventually(...)` from `nautilus_trader.test_kit.functions` and `wait_until_async(...)` from `nautilus_common::testing` instead of arbitrary sleeps. They surface failures faster and reduce flakiness in CI because they stop as soon as the condition is satisfied or time out with a useful error. - -## Mocks - -Prefer hand-written stubs that return fixed values over mocking frameworks. Use `MagicMock` only when you need to assert call counts/arguments or simulate complex state changes. Avoid mocking the objects you're actually testing. - -## Code coverage - -We generate coverage reports with `coverage` and publish them to [codecov](https://about.codecov.io/). - -Aim for high coverage without sacrificing appropriate error handling or causing "test induced damage" to the architecture. - -Some branches remain untestable without modifying production behaviour. -For example, a final condition in a defensive if-else block may only trigger for unexpected values; leave these checks in place so future changes can exercise them if needed. - -Design-time exceptions can also be impractical to test, so 100% coverage is not the target. - -## Excluded code coverage - -We use `pragma: no cover` comments to [exclude code from coverage](https://coverage.readthedocs.io/en/coverage-4.3.3/excluding.html) when tests would be redundant. -Typical examples include: - -- Asserting an abstract method raises `NotImplementedError` when called. -- Asserting the final condition check of an if-else block when impossible to test (as above). - -Such tests are expensive to maintain because they must track refactors while providing little value. -Keep concrete implementations of abstract methods fully covered. -Remove `pragma: no cover` when it no longer applies and restrict its use to the cases above. - -## Debugging Rust tests - -Use the default test configuration to debug Rust tests. - -To run the full suite with debug symbols for later, run `make cargo-test-debug` instead of `make cargo-test`. - -In IntelliJ IDEA, adjust the run configuration for parametrised `#[rstest]` cases so it reads `test --package nautilus-model --lib data::bar::tests::test_get_time_bar_start::case_1` -(remove `-- --exact` and append `::case_n` where `n` starts at 1). This workaround matches the behaviour explained [here](https://github.com/rust-lang/rust-analyzer/issues/8964#issuecomment-871592851). - -In VS Code you can pick the specific test case to debug directly. - -## Python + Rust Mixed Debugging - -This workflow lets you debug Python and Rust code simultaneously from a Jupyter notebook inside VS Code. - -### Setup - -Install these VS Code extensions: Rust Analyzer, CodeLLDB, Python, Jupyter. - -### Step 0: Compile `nautilus_trader` with debug symbols - - ```bash - cd nautilus_trader && make build-debug-pyo3 - ``` - -### Step 1: Set up debugging configuration - -```python -from nautilus_trader.test_kit.debug_helpers import setup_debugging - -setup_debugging() -``` - -This command creates the required VS Code debugging configurations and starts a `debugpy` server for the Python debugger. - -By default `setup_debugging()` expects the `.vscode` folder one level above the `nautilus_trader` root directory. -Adjust the target location if your workspace layout differs. - -### Step 2: Set breakpoints - -- **Python breakpoints:** Set in VS Code in the Python source files. -- **Rust breakpoints:** Set in VS Code in the Rust source files. - -### Step 3: Start mixed debugging - -1. In VS Code select the **"Debug Jupyter + Rust (Mixed)"** configuration. -2. Start debugging (F5) or press the green run arrow. -3. Both Python and Rust debuggers attach to your Jupyter session. - -### Step 4: Execute code - -Run Jupyter notebook cells that call Rust functions. The debugger stops at breakpoints in both Python and Rust code. - -### Available configurations - -`setup_debugging()` creates these VS Code configurations: - -- **`Debug Jupyter + Rust (Mixed)`** - Mixed debugging for Jupyter notebooks. -- **`Jupyter Mixed Debugging (Python)`** - Python-only debugging for notebooks. -- **`Rust Debugger (for Jupyter debugging)`** - Rust-only debugging for notebooks. - -### Example - -Open and run the example notebook: `debug_mixed_jupyter.ipynb`. - -### Reference - -- [PyO3 debugging](https://pyo3.rs/v0.25.1/debugging.html?highlight=deb#debugging-from-jupyter-notebooks) - -## Data type testing - -Each data type flows through multiple layers of the platform. The table below shows where -existing types are tested, so new types can follow the same pattern. - -### Test layer matrix - -| Layer | Location | What it covers | -|------------------------|---------------------------------------------|------------------------------------------------------------| -| DataEngine subscribe | `crates/data/tests/engine.rs` | Engine processes subscribe/unsubscribe commands correctly. | -| DataEngine publish | `crates/data/tests/engine.rs` | Engine routes published data to the message bus. | -| DataActor subscribe | `crates/common/src/actor/tests.rs` | Actor subscribes and receives data via typed publish. | -| DataActor unsubscribe | `crates/common/src/actor/tests.rs` | Actor stops receiving data after unsubscribe. | -| PyO3 actor dispatch | `crates/common/src/python/actor.rs` | Rust handler dispatches to Python `on_*` method. | -| Python Actor subscribe | `tests/unit_tests/common/test_actor.py` | Python actor subscribes; command count increments. | -| Python Actor unsub | `tests/unit_tests/common/test_actor.py` | Python actor unsubscribes; subscription list clears. | -| Backtest client | `nautilus_trader/backtest/data_client.pyx` | Backtest client overrides base subscribe/unsubscribe. | -| Adapter live tests | `docs/developer_guide/spec_data_testing.md` | Live data acceptance tests (DataTester). | - -### Coverage per data type - -The following table shows which layers have test coverage for each data type. -Use this as a checklist when adding a new type. - -| Data type | Engine | Actor (Rust) | PyO3 dispatch | Actor (Python) | Backtest client | Adapter spec | -|---------------------|--------|--------------|---------------|----------------|-----------------|--------------| -| `InstrumentAny` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `OrderBookDeltas` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `OrderBook` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `QuoteTick` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `TradeTick` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `Bar` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `MarkPriceUpdate` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `IndexPriceUpdate` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `FundingRateUpdate` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `InstrumentStatus` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `InstrumentClose` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `OptionGreeks` | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| `OptionChainSlice` | - | ✓ | ✓ | ✓ | - | ✓ | -| `CustomData` | ✓ | ✓ | ✓ | ✓ | ✓ | - | - -`OptionChainSlice` is assembled by the DataEngine's `OptionChainManager` from per-instrument -greeks and quote subscriptions. It does not have its own engine subscribe command or -backtest client override. - -### Adding a new data type - -When introducing a new data type, add tests at each layer: - -1. **DataEngine** (`crates/data/tests/engine.rs`): Add `test_execute_subscribe_` and - `test_execute_unsubscribe_` tests. Follow the pattern in existing subscribe tests: - register client, build command, call `engine.execute`, assert subscription list. - -2. **DataActor Rust** (`crates/common/src/actor/tests.rs`): - - Add `received_: Vec` field to `TestDataActor`. - - Implement the `on_` handler in the `DataActor` trait impl. - - Add `test_subscribe_and_receive_` and `test_unsubscribe_` tests. - - Use the typed publish function (`msgbus::publish_`), not `publish_any`, - for types that use `TypedHandler` routing. - -3. **PyO3 actor dispatch** (`crates/common/src/python/actor.rs`): - - Add `dispatch_on_` method that calls `py_self.call_method1("on_", ...)`. - - Add `on_` in the `DataActor` trait impl that calls the dispatch method. - - Add `#[pyo3(name = "on_")]` method in the `#[pymethods]` block. - - Add `on_` to `RustTestDataActor` wrapper and the inline Python test class. - - Add handler test and dispatch test. - -4. **Python Actor** (`tests/unit_tests/common/test_actor.py`): - - Add `test_subscribe_` and `test_unsubscribe_` tests. - - Assert `actor.subscribed_()` returns expected entries after subscribe and - is empty after unsubscribe. - -5. **Backtest client** (`nautilus_trader/backtest/data_client.pyx`): Override - `subscribe_` and `unsubscribe_` if the base `MarketDataClient` raises - `NotImplementedError` for the method. - -6. **Documentation**: Add entries to `actors.md` callback table, `strategies.md` handler - signatures, `adapters.md` subscribe method stubs, and `spec_data_testing.md` test cards. - -:::tip -Search for an existing type like `instrument_close` or `funding_rate` across all six layers -to find concrete examples of the patterns described above. -::: diff --git a/nautilus-docs/docs/getting_started/backtest_high_level.py b/nautilus-docs/docs/getting_started/backtest_high_level.py deleted file mode 100644 index 9ba0e88..0000000 --- a/nautilus-docs/docs/getting_started/backtest_high_level.py +++ /dev/null @@ -1,251 +0,0 @@ -# %% [markdown] -# # Backtest (high-level API) -# -# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/getting_started/backtest_high_level.py). - -# %% [markdown] -# ## Overview -# -# This tutorial walks through how to use a `BacktestNode` to backtest a simple EMA cross strategy -# on a simulated FX ECN venue using historical quote tick data. -# -# The following points will be covered: -# - Load raw data (external to Nautilus) into the data catalog. -# - Set up configuration objects for a `BacktestNode`. -# - Run backtests with a `BacktestNode`. -# - -# %% [markdown] -# ## Prerequisites -# - Python 3.12+ installed. -# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`). - -# %% [markdown] -# ## Imports -# -# We'll start with all of our imports for the remainder of this tutorial. - -# %% -import shutil -from decimal import Decimal -from pathlib import Path - -import pandas as pd - -from nautilus_trader.backtest.node import BacktestDataConfig -from nautilus_trader.backtest.node import BacktestEngineConfig -from nautilus_trader.backtest.node import BacktestNode -from nautilus_trader.backtest.node import BacktestRunConfig -from nautilus_trader.backtest.node import BacktestVenueConfig -from nautilus_trader.config import ImportableStrategyConfig -from nautilus_trader.core.datetime import dt_to_unix_nanos -from nautilus_trader.model import QuoteTick -from nautilus_trader.persistence.catalog import ParquetDataCatalog -from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler -from nautilus_trader.test_kit.providers import CSVTickDataLoader -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# %% [markdown] -# Before starting, download some sample data for backtesting. -# -# For this example we will use FX data from `histdata.com`. Go to https://www.histdata.com/download-free-forex-historical-data/?/ascii/tick-data-quotes/ and select an FX pair, then select one or more months of data to download. -# -# Examples of downloaded files: -# -# - `DAT_ASCII_EURUSD_T_202410.csv` (EUR/USD data for month 2024-10) -# - `DAT_ASCII_EURUSD_T_202411.csv` (EUR/USD data for month 2024-11) -# -# Once you have downloaded the data: -# -# 1. Extract the CSV files and copy them into a folder, for example `~/Downloads/Data/HISTDATA/`. -# 2. Set the `DATA_DIR` variable below to the directory containing the CSV files. - -# %% -DATA_DIR = "~/Downloads/Data/HISTDATA/" - -# %% -path = Path(DATA_DIR).expanduser() -raw_files = [ - f for f in path.iterdir() if f.is_file() and (f.suffix == ".csv" or f.name.endswith(".csv.gz")) -] -assert raw_files, f"Unable to find any CSV files in directory {path}" -raw_files - -# %% [markdown] -# ## Loading data into the Parquet data catalog -# -# Histdata stores the FX data in CSV/text format with fields `timestamp, bid_price, ask_price`. -# First, load this raw data into a `pandas.DataFrame` with a schema compatible with Nautilus quotes. -# -# Then create Nautilus `QuoteTick` objects by processing the DataFrame with a `QuoteTickDataWrangler`. -# - -# %% -# Load the first CSV file into a pandas DataFrame -df = CSVTickDataLoader.load( - file_path=raw_files[0], - index_col=0, - header=None, - names=["timestamp", "bid_price", "ask_price", "volume"], - usecols=["timestamp", "bid_price", "ask_price"], - parse_dates=["timestamp"], - date_format="%Y%m%d %H%M%S%f", -) - -df = df.sort_index() -df.head(2) - -# %% -# Process quotes using a wrangler -EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD") -wrangler = QuoteTickDataWrangler(EURUSD) - -ticks = wrangler.process(df) - -# Preview: see first 2 ticks -ticks[0:2] - -# %% [markdown] -# See the [Loading data](../concepts/data) guide for more details. -# -# Instantiate a `ParquetDataCatalog` with a storage directory (here we use the current directory). -# Write the instrument and tick data to the catalog. -# - -# %% -CATALOG_PATH = Path.cwd() / "catalog" - -# Clear if it already exists, then create fresh -if CATALOG_PATH.exists(): - shutil.rmtree(CATALOG_PATH) -CATALOG_PATH.mkdir(parents=True) - -# Create a catalog instance -catalog = ParquetDataCatalog(CATALOG_PATH) - -# Write instrument to the catalog -catalog.write_data([EURUSD]) - -# Write ticks to catalog -catalog.write_data(ticks) - -# %% [markdown] -# ## Using the Data Catalog -# -# The catalog provides methods like `.instruments(...)` and `.quote_ticks(...)` to query stored data. -# - -# %% -# Get list of all instruments in catalog -catalog.instruments() - -# %% -# See 1st instrument from catalog -instrument = catalog.instruments()[0] -instrument - -# %% -# Query quote ticks from catalog to determine the data range -all_ticks = catalog.quote_ticks(instrument_ids=[EURUSD.id.value]) -print(f"Total ticks in catalog: {len(all_ticks)}") - -if all_ticks: - # Get timestamps from the data - first_tick_time = pd.Timestamp(all_ticks[0].ts_init, unit="ns", tz="UTC") - last_tick_time = pd.Timestamp(all_ticks[-1].ts_init, unit="ns", tz="UTC") - print(f"Data range: {first_tick_time} to {last_tick_time}") - - # Set backtest range to first 2 weeks of data (as ISO strings for BacktestDataConfig) - start_time = first_tick_time.isoformat() - end_time = (first_tick_time + pd.Timedelta(days=14)).isoformat() - print(f"Backtest range: {start_time} to {end_time}") - - # Preview selected data - start_ns = all_ticks[0].ts_init - end_ns = dt_to_unix_nanos(first_tick_time + pd.Timedelta(days=14)) - selected_quote_ticks = catalog.quote_ticks( - instrument_ids=[EURUSD.id.value], - start=start_ns, - end=end_ns, - ) - print(f"Selected ticks for backtest: {len(selected_quote_ticks)}") - selected_quote_ticks[:2] -else: - raise ValueError("No ticks found in catalog") - -# %% [markdown] -# ## Add venues - -# %% -venue_configs = [ - BacktestVenueConfig( - name="SIM", - oms_type="HEDGING", - account_type="MARGIN", - base_currency="USD", - starting_balances=["1_000_000 USD"], - ), -] - -# %% [markdown] -# ## Add data - -# %% -str(CATALOG_PATH) - -# %% -data_configs = [ - BacktestDataConfig( - catalog_path=str(CATALOG_PATH), - data_cls=QuoteTick, - instrument_id=instrument.id, - start_time=start_time, - end_time=end_time, - ), -] - -# %% [markdown] -# ## Add strategies - -# %% -strategies = [ - ImportableStrategyConfig( - strategy_path="nautilus_trader.examples.strategies.ema_cross:EMACross", - config_path="nautilus_trader.examples.strategies.ema_cross:EMACrossConfig", - config={ - "instrument_id": instrument.id, - "bar_type": "EUR/USD.SIM-15-MINUTE-BID-INTERNAL", - "fast_ema_period": 10, - "slow_ema_period": 20, - "trade_size": Decimal(1_000_000), - }, - ), -] - -# %% [markdown] -# ## Configure backtest -# -# Nautilus uses a `BacktestRunConfig` object to centralize backtest configuration. -# The `BacktestRunConfig` is Partialable, so you can configure it in stages. -# This design reduces boilerplate when you create multiple backtest runs (for example when performing a parameter grid search). -# - -# %% -config = BacktestRunConfig( - engine=BacktestEngineConfig(strategies=strategies), - data=data_configs, - venues=venue_configs, -) - -# %% [markdown] -# ## Run backtest -# -# Now we can run the backtest node, which will simulate trading across the entire data stream. - -# %% -node = BacktestNode(configs=[config]) - -results = node.run() -results diff --git a/nautilus-docs/docs/getting_started/backtest_low_level.py b/nautilus-docs/docs/getting_started/backtest_low_level.py deleted file mode 100644 index 32069e1..0000000 --- a/nautilus-docs/docs/getting_started/backtest_low_level.py +++ /dev/null @@ -1,220 +0,0 @@ -# %% [markdown] -# # Backtest (low-level API) -# -# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/getting_started/backtest_low_level.py). - -# %% [markdown] -# ## Overview -# -# This tutorial walks through how to use a `BacktestEngine` to backtest a simple EMA cross strategy -# with a TWAP execution algorithm on a simulated Binance Spot exchange using historical trade tick data. -# -# The following points will be covered: -# - Load raw data (external to Nautilus) using data loaders and wranglers. -# - Add this data to a `BacktestEngine`. -# - Add venues, strategies, and execution algorithms to a `BacktestEngine`. -# - Run backtests with a `BacktestEngine`. -# - Perform post-run analysis and repeated runs. -# - -# %% [markdown] -# ## Prerequisites -# - Python 3.12+ installed. -# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`). - -# %% [markdown] -# ## Imports -# -# We'll start with all of our imports for the remainder of this tutorial. - -# %% -from decimal import Decimal - -from nautilus_trader.backtest.config import BacktestEngineConfig -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.examples.algorithms.twap import TWAPExecAlgorithm -from nautilus_trader.examples.strategies.ema_cross_twap import EMACrossTWAP -from nautilus_trader.examples.strategies.ema_cross_twap import EMACrossTWAPConfig -from nautilus_trader.model import BarType -from nautilus_trader.model import Money -from nautilus_trader.model import TraderId -from nautilus_trader.model import Venue -from nautilus_trader.model.currencies import ETH -from nautilus_trader.model.currencies import USDT -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.persistence.wranglers import TradeTickDataWrangler -from nautilus_trader.test_kit.providers import TestDataProvider -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# %% [markdown] -# ## Loading data -# -# For this tutorial we use stub test data from the NautilusTrader repository (the automated test suite also uses this data to verify platform correctness). -# -# First, instantiate a data provider to read raw CSV trade tick data into a `pd.DataFrame`. -# Next, initialize the matching instrument (`ETHUSDT` spot on Binance). -# Then wrangle the data into Nautilus `TradeTick` objects to add to the `BacktestEngine`. -# - -# %% -# Load stub test data -provider = TestDataProvider() -trades_df = provider.read_csv_ticks("binance/ethusdt-trades.csv") - -# Initialize the instrument which matches the data -ETHUSDT_BINANCE = TestInstrumentProvider.ethusdt_binance() - -# Process into Nautilus objects -wrangler = TradeTickDataWrangler(instrument=ETHUSDT_BINANCE) -ticks = wrangler.process(trades_df) - -# %% [markdown] -# See the [Loading External Data](https://nautilustrader.io/docs/latest/concepts/data#loading-data) guide for details on the data processing pipeline. - -# %% [markdown] -# ## Initialize a backtest engine -# -# Create a `BacktestEngine`. Here we pass a `BacktestEngineConfig` with a custom `trader_id` to show the configuration pattern. -# -# See the [Configuration](https://nautilustrader.io/docs/api_reference/config) API reference for all available options. -# - -# %% -# Configure backtest engine -config = BacktestEngineConfig(trader_id=TraderId("BACKTESTER-001")) - -# Build the backtest engine -engine = BacktestEngine(config=config) - -# %% [markdown] -# ## Add venues -# -# Create a venue to trade on that matches the market data you add to the engine. -# -# In this case we set up a simulated Binance Spot exchange. -# - -# %% -# Add a trading venue (multiple venues possible) -BINANCE = Venue("BINANCE") -engine.add_venue( - venue=BINANCE, - oms_type=OmsType.NETTING, - account_type=AccountType.CASH, # Spot CASH account (not for perpetuals or futures) - base_currency=None, # Multi-currency account - starting_balances=[Money(1_000_000.0, USDT), Money(10.0, ETH)], -) - -# %% [markdown] -# ## Add data -# -# Add data to the backtest engine. Start by adding the `Instrument` object we initialized earlier to match the data. -# -# Then add the trades we wrangled earlier. -# - -# %% -# Add instrument(s) -engine.add_instrument(ETHUSDT_BINANCE) - -# Add data -engine.add_data(ticks) - -# %% [markdown] -# :::note -# You can add multiple data types (including custom types) and backtest across multiple venues. -# ::: -# - -# %% [markdown] -# ## Add strategies -# -# Add the trading strategies you plan to run as part of the system. -# -# Initialize a strategy configuration, then create and add the strategy: -# - -# %% -# Configure your strategy -strategy_config = EMACrossTWAPConfig( - instrument_id=ETHUSDT_BINANCE.id, - bar_type=BarType.from_str("ETHUSDT.BINANCE-250-TICK-LAST-INTERNAL"), - trade_size=Decimal("0.10"), - fast_ema_period=10, - slow_ema_period=20, - twap_horizon_secs=10.0, - twap_interval_secs=2.5, -) - -# Instantiate and add your strategy -strategy = EMACrossTWAP(config=strategy_config) -engine.add_strategy(strategy=strategy) - -# %% [markdown] -# The strategy config above includes TWAP parameters, but we still need to add the `ExecAlgorithm` component. -# -# ## Add execution algorithms -# -# Add a TWAP execution algorithm to the engine, following the same pattern as strategies. -# - -# %% -# Instantiate and add your execution algorithm -exec_algorithm = TWAPExecAlgorithm() # Using defaults -engine.add_exec_algorithm(exec_algorithm) - -# %% [markdown] -# ## Run backtest -# -# After configuring the data, venues, and trading system, run a backtest. -# Call the `.run(...)` method to process all available data by default. -# -# See the [BacktestEngineConfig](https://nautilustrader.io/docs/latest/api_reference/config) API reference for all available options. -# - -# %% -# Run the engine (from start to end of data) -engine.run() - -# %% [markdown] -# ## Post-run and analysis -# -# The engine logs a post-run tearsheet with default statistics. You can load custom statistics too; see the [Portfolio statistics](../concepts/portfolio.md#portfolio-statistics) guide. -# -# The engine retains data and execution objects in memory for generating reports. -# - -# %% -engine.trader.generate_account_report(BINANCE) - -# %% -engine.trader.generate_order_fills_report() - -# %% -engine.trader.generate_positions_report() - -# %% [markdown] -# ## Repeated runs -# -# You can reset the engine for repeated runs with different strategy and component configurations. -# -# Instruments and data persist across resets by default, so you don't need to reload them. - -# %% -# For repeated backtest runs, reset the engine -engine.reset() - -# Instruments and data persist, just add new components and run again - -# %% [markdown] -# Remove and add individual components (actors, strategies, execution algorithms) as required. -# -# See the [Trader](../api_reference/trading.md) API reference for a description of all methods available to achieve this. -# - -# %% -# Once done, good practice to dispose of the object if the script continues -engine.dispose() diff --git a/nautilus-docs/docs/getting_started/index.md b/nautilus-docs/docs/getting_started/index.md deleted file mode 100644 index 249dba6..0000000 --- a/nautilus-docs/docs/getting_started/index.md +++ /dev/null @@ -1,75 +0,0 @@ -# Getting Started - -To get started with NautilusTrader, you will need: - -- A Python 3.12–3.14 environment with the `nautilus_trader` package installed. -- A way to run Python scripts or Jupyter notebooks for backtesting and/or live trading. - -## Installation - -How to install NautilusTrader on your machine. - -## Quickstart - -Run your first backtest in five minutes. No data downloads, no catalog setup. - -## Examples in repository - -The [online documentation](https://nautilustrader.io/docs/latest/) shows just a subset of examples. For the full set, see this repository on GitHub. - -The following table lists example locations ordered by recommended learning progression: - -| Directory | Contains | -|:----------------------------|:----------------------------------------------------------------------------------------------------------------------------| -| [examples/](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples) | Fully runnable, self-contained Python examples. | -| [docs/tutorials/](../tutorials/) | Tutorials demonstrating common workflows. | -| [docs/concepts/](../concepts/) | Concept guides with concise code snippets illustrating key features. | -| [nautilus_trader/examples/](../../nautilus_trader/examples/) | Pure-Python examples of basic strategies, indicators, and execution algorithms. | -| [tests/unit_tests/](../../tests/unit_tests/) | Unit tests covering core functionality and edge cases. | - -## Backtesting API levels - -NautilusTrader provides two different API levels for backtesting: - -| API Level | Description | Characteristics | -|:---------------|:--------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| High-Level API | Uses `BacktestNode` and `TradingNode` | Recommended for production: easier transition to live trading; requires a Parquet-based data catalog. | -| Low-Level API | Uses `BacktestEngine` | Intended for library development: no live-trading path; direct component access; may encourage non-live-compatible patterns. | - -:::warning[One node per process] -Running multiple `BacktestNode` or `TradingNode` instances concurrently in the same process is not supported due to global singleton state. -Sequential execution with proper disposal between runs is supported. - -See [Processes and threads](../concepts/architecture.md#processes-and-threads) for details. -::: - -See the [Backtesting](../concepts/backtesting.md) guide for help choosing an API level. - -### Backtest (low-level API) - -This tutorial runs through how to load raw data (external to Nautilus) using data loaders and wranglers, -and then use this data with a `BacktestEngine` to run a single backtest. - -### Backtest (high-level API) - -This tutorial runs through how to load raw data (external to Nautilus) into the data catalog, -and then use this data with a `BacktestNode` to run a single backtest. - -## Running in docker - -Alternatively, you can download a self-contained dockerized Jupyter notebook server, which requires no setup or -installation. This is the fastest way to get up and running to try out NautilusTrader. Note that deleting the container will also delete any data. - -- To get started, install docker: - - Go to [Docker installation guide](https://docs.docker.com/get-docker/) and follow the instructions. -- From a terminal, download the latest image: - - `docker pull ghcr.io/nautechsystems/jupyterlab:nightly --platform linux/amd64` -- Run the docker container, exposing the Jupyter port: - - `docker run -p 8888:8888 ghcr.io/nautechsystems/jupyterlab:nightly` -- Open your web browser to `localhost:{port}`: - - - -:::warning -Examples use `log_level="ERROR"` because Nautilus logging exceeds Jupyter's stdout rate limit, -causing notebooks to hang at lower log levels. -::: diff --git a/nautilus-docs/docs/getting_started/installation.md b/nautilus-docs/docs/getting_started/installation.md deleted file mode 100644 index c956313..0000000 --- a/nautilus-docs/docs/getting_started/installation.md +++ /dev/null @@ -1,361 +0,0 @@ -# Installation - -NautilusTrader is officially supported for Python 3.12-3.14 on the following 64-bit platforms: - -| Operating System | Supported Versions | CPU Architecture | -|------------------------|--------------------|-------------------| -| Linux (Ubuntu) | 22.04 and later | x86_64 | -| Linux (Ubuntu) | 22.04 and later | ARM64 | -| macOS | 15.0 and later | ARM64 | -| Windows Server | 2022 and later | x86_64 | - -:::note -NautilusTrader may work on other platforms, but only those listed above are regularly used by developers and tested in CI. -::: - -Continuous CI coverage comes from the GitHub Actions runners we build on: - -- `Linux (Ubuntu)` builds currently pin to `ubuntu-22.04` to keep glibc 2.35 compatibility even as `ubuntu-latest` moves ahead. -- `macOS (ARM64)` builds run on `macos-latest`, so support tracks that runner image as it moves ahead. -- `Windows (x86_64)` builds run on `windows-latest`, so support tracks that runner image as it moves ahead. - -On Linux, confirm your glibc version with `ldd --version` and ensure it reports 2.35 or newer before proceeding. - -We recommend using the latest supported version of Python and installing [nautilus_trader](https://pypi.org/project/nautilus_trader/) inside a virtual environment to isolate dependencies. - -**There are two supported ways to install**: - -1. Pre-built binary wheel from PyPI *or* the Nautech Systems package index. -2. Build from source. - -:::tip -We highly recommend installing using the [uv](https://docs.astral.sh/uv) package manager with a "vanilla" CPython. - -Conda and other Python distributions *may* work but aren’t officially supported. -::: - -## From PyPI - -To install the latest [nautilus_trader](https://pypi.org/project/nautilus_trader/) binary wheel (or sdist package) from PyPI: - -```bash -uv pip install nautilus_trader -``` - -## Extras - -Install optional dependencies as 'extras' for specific integrations: - -- `betfair`: Betfair adapter (integration) dependencies. -- `docker`: Needed for Docker when using the IB gateway (with the Interactive Brokers adapter). -- `ib`: Interactive Brokers adapter (integration) dependencies. -- `polymarket`: Polymarket adapter (integration) dependencies. -- `visualization`: Plotly-based interactive tearsheets and charts. - -To install with specific extras: - -```bash -uv pip install "nautilus_trader[docker,ib]" -``` - -## From the Nautech Systems package index - -The Nautech Systems package index (`packages.nautechsystems.io`) complies with [PEP-503](https://peps.python.org/pep-0503/) and hosts both stable and development binary wheels for `nautilus_trader`. -This enables users to install either the latest stable release or pre-release versions for testing. - -### Stable wheels - -Stable wheels correspond to official releases of `nautilus_trader` on PyPI, and use standard versioning. - -To install the latest stable release: - -```bash -uv pip install nautilus_trader --index-url=https://packages.nautechsystems.io/simple -``` - -:::tip -Use `--extra-index-url` instead of `--index-url` if you want uv to fall back to PyPI automatically: - -::: - -### Development wheels - -Development wheels are published from both the `nightly` and `develop` branches, -allowing users to test features and fixes ahead of stable releases. - -This process also helps preserve compute resources and provides easy access to the exact binaries tested in CI pipelines, -while adhering to [PEP-440](https://peps.python.org/pep-0440/) versioning standards: - -- `develop` wheels use the version format `dev{date}+{build_number}` (e.g., `1.208.0.dev20241212+7001`). -- `nightly` wheels use the version format `a{date}` (alpha) (e.g., `1.208.0a20241212`). - -| Platform | Nightly | Develop | -| :----------------- | :------ | :------ | -| `Linux (x86_64)` | ✓ | ✓ | -| `Linux (ARM64)` | ✓ | - | -| `macOS (ARM64)` | ✓ | ✓ | -| `Windows (x86_64)` | ✓ | ✓ | - -**Note**: Development wheels from the `develop` branch publish for every supported platform except Linux ARM64. -Skipping that target keeps CI feedback fast while avoiding unnecessary build resource usage. - -:::warning -We do not recommend using development wheels in production environments, such as live trading controlling real capital. -::: - -### Installation commands - -By default, uv will install the latest stable release. Adding the `--pre` flag ensures that pre-release versions, including development wheels, are considered. - -To install the latest available pre-release (including development wheels): - -```bash -uv pip install nautilus_trader --pre --index-url=https://packages.nautechsystems.io/simple -``` - -To install a specific development wheel (e.g., `1.221.0a20250912` for September 12, 2025): - -```bash -uv pip install nautilus_trader==1.221.0a20250912 --index-url=https://packages.nautechsystems.io/simple -``` - -### Available versions - -You can view all available versions of `nautilus_trader` on the [package index](https://packages.nautechsystems.io/simple/nautilus-trader/index.html). - -To programmatically request and list available versions: - -```bash -curl -s https://packages.nautechsystems.io/simple/nautilus-trader/index.html | grep -oP '(?<=.whl -``` - -## Versioning and releases - -NautilusTrader is still under active development. Some features may be incomplete, and while -the API is becoming more stable, breaking changes can occur between releases. -We strive to document these changes in the release notes on a **best-effort basis**. - -We aim to follow a **bi-weekly release schedule**, though experimental or larger features may cause delays. - -Use NautilusTrader only if you are prepared to adapt to these changes. - -## Redis - -Using [Redis](https://redis.io) with NautilusTrader is **optional** and only required if configured as the backend for a cache database or [message bus](../concepts/message_bus.md). - -:::info -The minimum supported Redis version is 6.2 (required for [streams](https://redis.io/docs/latest/develop/data-types/streams/) functionality). -::: - -For a quick setup, we recommend using a [Redis Docker container](https://hub.docker.com/_/redis/). You can find an example setup in the `.docker` directory, -or run the following command to start a container: - -```bash -docker run -d --name redis -p 6379:6379 redis:latest -``` - -This command will: - -- Pull the latest version of Redis from Docker Hub if it's not already downloaded. -- Run the container in detached mode (`-d`). -- Name the container `redis` for easy reference. -- Expose Redis on the default port 6379, making it accessible to NautilusTrader on your machine. - -To manage the Redis container: - -- Start it with `docker start redis` -- Stop it with `docker stop redis` - -:::tip -We recommend using [Redis Insight](https://redis.io/insight/) as a GUI to visualize and debug Redis data efficiently. -::: - -## Precision mode - -NautilusTrader supports two precision modes for its core value types (`Price`, `Quantity`, `Money`), -which differ in their internal bit-width and maximum decimal precision. - -- **High-precision**: 128-bit integers with up to 16 decimals of precision, and a larger value range. -- **Standard-precision**: 64-bit integers with up to 9 decimals of precision, and a smaller value range. - -:::note -By default, the official Python wheels ship in high-precision (128-bit) mode on Linux and macOS. -On Windows, only standard-precision (64-bit) Python wheels are available because MSVC's C/C++ frontend -does not support `__int128`, preventing the Cython/FFI layer from handling 128-bit integers. - -For pure Rust crates, high-precision works on all platforms (including Windows) since Rust handles -`i128`/`u128` via software emulation. The default is standard-precision unless you explicitly enable -the `high-precision` feature flag. -::: - -The performance tradeoff is that standard-precision is ~3–5% faster in typical backtests, -but has lower decimal precision and a smaller representable value range. - -:::note -Performance benchmarks comparing the modes are pending. -::: - -### Build configuration - -The precision mode is determined by: - -- Setting the `HIGH_PRECISION` environment variable during compilation, **and/or** -- Enabling the `high-precision` Rust feature flag explicitly. - -#### High-precision mode (128-bit) - -```bash -export HIGH_PRECISION=true -make install-debug -``` - -#### Standard-precision mode (64-bit) - -```bash -export HIGH_PRECISION=false -make install-debug -``` - -### Rust feature flag - -To enable high-precision (128-bit) mode in Rust, add the `high-precision` feature to your `Cargo.toml`: - -```toml -[dependencies] -nautilus_core = { version = "*", features = ["high-precision"] } -``` - -:::info -See the [Value Types](../concepts/overview.md#value-types) specifications for more details. -::: diff --git a/nautilus-docs/docs/getting_started/quickstart.py b/nautilus-docs/docs/getting_started/quickstart.py deleted file mode 100644 index 474b45e..0000000 --- a/nautilus-docs/docs/getting_started/quickstart.py +++ /dev/null @@ -1,211 +0,0 @@ -# %% [markdown] -# # Quickstart -# -# Run your first backtest in under five minutes. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/getting_started/quickstart.py). - -# %% [markdown] -# ## Prerequisites -# -# - Python 3.12+ -# - `pip install nautilus_trader` - -# %% [markdown] -# ## Write a strategy -# -# A strategy extends the `Strategy` base class and overrides event handlers to -# react to market data. This one trades an EMA crossover: buy when a fast -# exponential moving average crosses above a slow one, sell when it crosses below. - -# %% -from decimal import Decimal - -from nautilus_trader.config import StrategyConfig -from nautilus_trader.indicators import ExponentialMovingAverage -from nautilus_trader.model.data import Bar -from nautilus_trader.model.data import BarType -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.identifiers import InstrumentId -from nautilus_trader.trading.strategy import Strategy - - -class EMACrossConfig(StrategyConfig, frozen=True): - instrument_id: InstrumentId - bar_type: BarType - trade_size: Decimal - fast_ema_period: int = 10 - slow_ema_period: int = 20 - - -class EMACross(Strategy): - def __init__(self, config: EMACrossConfig): - super().__init__(config) - self.fast_ema = ExponentialMovingAverage(config.fast_ema_period) - self.slow_ema = ExponentialMovingAverage(config.slow_ema_period) - - def on_start(self): - self.register_indicator_for_bars(self.config.bar_type, self.fast_ema) - self.register_indicator_for_bars(self.config.bar_type, self.slow_ema) - self.subscribe_bars(self.config.bar_type) - - def on_bar(self, bar: Bar): - if not self.indicators_initialized(): - return - - if self.fast_ema.value >= self.slow_ema.value: - if self.portfolio.is_flat(self.config.instrument_id): - self.buy() - elif self.portfolio.is_net_short(self.config.instrument_id): - self.close_all_positions(self.config.instrument_id) - self.buy() - elif self.fast_ema.value < self.slow_ema.value: - if self.portfolio.is_flat(self.config.instrument_id): - self.sell() - elif self.portfolio.is_net_long(self.config.instrument_id): - self.close_all_positions(self.config.instrument_id) - self.sell() - - def buy(self): - instrument = self.cache.instrument(self.config.instrument_id) - order = self.order_factory.market( - self.config.instrument_id, - OrderSide.BUY, - instrument.make_qty(self.config.trade_size), - ) - self.submit_order(order) - - def sell(self): - instrument = self.cache.instrument(self.config.instrument_id) - order = self.order_factory.market( - self.config.instrument_id, - OrderSide.SELL, - instrument.make_qty(self.config.trade_size), - ) - self.submit_order(order) - - def on_stop(self): - self.close_all_positions(self.config.instrument_id) - - -# %% [markdown] -# `on_start` registers the two EMA indicators so the engine updates them -# automatically with each new bar. `on_bar` waits for the indicators to warm up, -# then enters or reverses a position based on the crossover signal. - -# %% [markdown] -# ## Set up the backtest -# -# Generate synthetic EUR/USD 1-minute bars, configure a `BacktestEngine`, and run. - -# %% -import numpy as np -import pandas as pd - -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.config import BacktestEngineConfig -from nautilus_trader.config import LoggingConfig -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.model.identifiers import Venue -from nautilus_trader.model.objects import Money -from nautilus_trader.persistence.wranglers import BarDataWrangler -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# Create a EUR/USD instrument on the SIM venue -EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD") - -# Generate synthetic 1-minute bars (random walk around 1.10) -rng = np.random.default_rng(42) -n = 10_000 -price = 1.10 + np.cumsum(rng.normal(0, 0.0002, n)) -spread = np.abs(rng.normal(0, 0.0003, n)) -bars_df = pd.DataFrame( - { - "open": price, - "high": price + spread, - "low": price - spread, - "close": price + rng.normal(0, 0.00005, n), - }, - index=pd.date_range("2024-01-01", periods=n, freq="1min", tz="UTC"), -) -bars_df["high"] = bars_df[["open", "high", "close"]].max(axis=1) -bars_df["low"] = bars_df[["open", "low", "close"]].min(axis=1) - -bar_type = BarType.from_str("EUR/USD.SIM-1-MINUTE-LAST-EXTERNAL") -bars = BarDataWrangler(bar_type, EURUSD).process(bars_df) - -# %% [markdown] -# `BarDataWrangler` converts a pandas DataFrame with OHLCV columns into Nautilus -# `Bar` objects. The bar type string encodes the instrument, aggregation period, -# price source, and data origin. - -# %% -# Configure the engine -engine = BacktestEngine( - config=BacktestEngineConfig( - logging=LoggingConfig(log_level="ERROR"), - ), -) - -# Add a simulated FX venue -SIM = Venue("SIM") -engine.add_venue( - venue=SIM, - oms_type=OmsType.NETTING, - account_type=AccountType.MARGIN, - starting_balances=[Money(1_000_000, USD)], - base_currency=USD, - default_leverage=Decimal(1), -) - -# Add instrument, data, and strategy -engine.add_instrument(EURUSD) -engine.add_data(bars) - -strategy = EMACross( - EMACrossConfig( - instrument_id=EURUSD.id, - bar_type=bar_type, - trade_size=Decimal(100000), - ), -) -engine.add_strategy(strategy) - -# Run the backtest -engine.run() - -# %% [markdown] -# The engine processes all 10,000 bars in timestamp order. Each bar updates the -# registered indicators, then triggers `on_bar`. The simulated exchange fills -# market orders at the current price. - -# %% [markdown] -# ## See results -# -# The engine generates reports from the completed backtest. The account report -# shows balance changes over time. The positions report lists each round-trip -# trade with its realized PnL. - -# %% -engine.trader.generate_account_report(SIM) - -# %% -engine.trader.generate_positions_report() - -# %% -engine.trader.generate_order_fills_report() - -# %% [markdown] -# ## Next steps -# -# - [Backtest (low-level API)](backtest_low_level) for direct `BacktestEngine` usage -# with real market data and execution algorithms. -# - [Backtest (high-level API)](backtest_high_level) for config-driven backtesting -# with `BacktestNode` and the Parquet data catalog. -# - [Tutorials](../tutorials/) for venue-specific walkthroughs covering Binance, -# Bybit, Databento, and more. - -# %% -engine.dispose() diff --git a/nautilus-docs/docs/integrations/architect_ax.md b/nautilus-docs/docs/integrations/architect_ax.md deleted file mode 100644 index 40a754a..0000000 --- a/nautilus-docs/docs/integrations/architect_ax.md +++ /dev/null @@ -1,400 +0,0 @@ -# AX Exchange - -[AX Exchange](https://architect.exchange) is the world's first centralized and regulated exchange -for perpetual futures on traditional underlying asset classes. Operated by Architect Bermuda Ltd. -and licensed by the [Bermuda Monetary Authority (BMA)](https://www.bma.bm/), AX brings crypto-style -perpetual contracts to traditional financial markets including foreign exchange, metals, energy, -equity indices, and interest rates. - -This integration supports live market data ingest and order execution with AX Exchange. - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/architect_ax/). - -## Overview - -This guide assumes a trader is setting up for both live market data feeds, and trade execution. -The AX Exchange adapter includes multiple components, which can be used together or separately -depending on the use case. - -- `AxHttpClient`: Low-level HTTP API connectivity. -- `AxMdWebSocketClient`: Market data WebSocket connectivity. -- `AxOrdersWebSocketClient`: Orders WebSocket connectivity. -- `AxInstrumentProvider`: Instrument parsing and loading functionality. -- `AxDataClient`: A market data feed manager. -- `AxExecutionClient`: An account management and trade execution gateway. -- `AxLiveDataClientFactory`: Factory for AX data clients (used by the trading node builder). -- `AxLiveExecClientFactory`: Factory for AX execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), -and won't need to necessarily work with these lower level components directly. -::: - -## AX Exchange documentation - -AX Exchange provides documentation for users which can be found at the -[Architect documentation site](https://docs.architect.exchange/). -It's recommended you also refer to the AX Exchange documentation in conjunction with this -NautilusTrader integration guide. - -## Products - -AX Exchange specializes in perpetual futures contracts on traditional asset classes. Perpetual -contracts never expire, eliminating rollover costs associated with standard futures. - -| Asset Class | Examples | Notes | -|------------------|-------------------------------------|------------------------------| -| Foreign exchange | GBPUSD-PERP, EURUSD-PERP. | Major and minor FX pairs. | -| Stock indices | Equity index perpetuals. | | -| Metals | XAU-PERP (gold), XAG-PERP (silver). | Precious metals perpetuals. | -| Energy | Crude oil, natural gas. | Energy commodity perpetuals. | -| Interest rates | SOFR, treasury yields. | Rate perpetuals. | - -### Perpetual contracts - -A perpetual contract (perpetual swap) is a derivative that tracks the price of an underlying -asset without expiring. Unlike standard futures, there is no settlement date, which eliminates -rollover costs and simplifies position management. A funding rate mechanism keeps the contract -price aligned with the underlying index price through periodic payments between long and short -holders. See the [Architect documentation](https://docs.architect.exchange/) for details on -funding rate mechanics and contract specifications. - -Characteristics of AX perpetual contracts: - -- **Cash-settled in USD**: No physical delivery. All profit and loss is settled in USD. -- **Funding rates**: Periodic payments keep the contract price aligned with the underlying. -- **Multiplier of 1**: Each contract represents one unit of exposure to the underlying. -- **Whole contracts only**: Fractional quantities are not supported. -- **Margin**: Initial margin is required to open a position; maintenance margin to keep it open. - -In NautilusTrader, all AX instruments are represented as `PerpetualContract`, an asset-class -agnostic perpetual swap type. The asset class (FX, commodity, equity, etc.) is inferred -automatically from the underlying. The adapter uses `MARGIN` account type and `NETTING` order -management. - -## Symbology - -AX Exchange uses a straightforward naming convention. All instruments are perpetual futures -identified by the `-PERP` suffix appended to the underlying asset symbol. - -**Format**: `{SYMBOL}-PERP` - -| Underlying | AX Symbol | Nautilus InstrumentId | -|----------------|----------------|-----------------------| -| GBP/USD | `GBPUSD-PERP` | `GBPUSD-PERP.AX` | -| EUR/USD | `EURUSD-PERP` | `EURUSD-PERP.AX` | -| Gold | `XAU-PERP` | `XAU-PERP.AX` | -| Silver | `XAG-PERP` | `XAG-PERP.AX` | - -The venue identifier is `AX`. To construct a Nautilus `InstrumentId`: - -```python -from nautilus_trader.model.identifiers import InstrumentId - -instrument_id = InstrumentId.from_str("GBPUSD-PERP.AX") -``` - -## Environments - -AX Exchange provides two trading environments. Configure the appropriate environment using the -`environment` parameter in your client configuration. - -| Environment | Config | Description | -|----------------|----------------------------------------|----------------------------------------| -| **Sandbox** | `environment=AxEnvironment.SANDBOX` | Test environment with simulated funds. | -| **Production** | `environment=AxEnvironment.PRODUCTION` | Live trading with real funds. | - -### Sandbox - -The sandbox is the default environment for development and testing with simulated funds. -All sandbox endpoints are resolved automatically when `environment=AxEnvironment.SANDBOX`. - -#### 1. Create a sandbox account - -Follow the [Architect documentation](https://docs.architect.exchange/) to create a sandbox -account. An invite code is required during registration. - -#### 2. Create API keys and fund the account - -Use the AX sandbox UI to generate API keys and deposit simulated funds into your account. -Store the `api_key` and `api_secret` securely. - -#### 3. Set environment variables - -```bash -export AX_API_KEY="your-sandbox-api-key" -export AX_API_SECRET="your-sandbox-api-secret" -``` - -#### 4. Configure the trading node - -```python -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - AX: AxDataClientConfig( - environment=AxEnvironment.SANDBOX, - instrument_provider=InstrumentProviderConfig(load_all=True), - ), - }, - exec_clients={ - AX: AxExecClientConfig( - environment=AxEnvironment.SANDBOX, - instrument_provider=InstrumentProviderConfig(load_all=True), - ), - }, -) -``` - -### Production - -For live trading with real funds. Requires a verified AX Exchange account. - -```python -config = AxExecClientConfig( - environment=AxEnvironment.PRODUCTION, -) -``` - -:::warning -Ensure you are using the correct environment before placing orders. -The sandbox environment is the default to prevent accidental live trading. -::: - -## Market data - -The adapter provides real-time market data via WebSocket subscriptions, with HTTP endpoints -for historical data backfill. - -### Data types - -| AX Data | Nautilus Data Type | Notes | -|-----------------|---------------------|-------------------------------------------------------------| -| Order book (L1) | `QuoteTick` | Best bid/ask top-of-book from L1 book subscription. | -| Order book (L2) | `OrderBookDelta` | Aggregated price levels. | -| Order book (L3) | `OrderBookDelta` | Individual order quantities. | -| Trades | `TradeTick` | Real-time trade events from L1 subscription. | -| Bars/candles | `Bar` | OHLCV data (total volume only, no buy/sell breakdown). | -| Funding rates | `FundingRateUpdate` | Polled via HTTP (not real-time WebSocket); interval configurable. | - -:::note -Historical quote tick requests are not supported by AX Exchange. Only real-time quote -data is available via WebSocket L1 book subscriptions. -::: - -### Bar intervals - -| Interval | Description | -|----------|-------------| -| `1s` | 1-second | -| `5s` | 5-second | -| `1m` | 1-minute | -| `5m` | 5-minute | -| `15m` | 15-minute | -| `1h` | 1-hour | -| `1d` | 1-day | - -## Orders capability - -AX Exchange supports market and limit order types with stop triggers. - -### Order types - -| Order Type | Supported | Notes | -|------------------------|-----------|----------------------------------------------------| -| `MARKET` | ✓ | Execute immediately at best available price. | -| `LIMIT` | ✓ | Execute at specified price or better. | -| `STOP_LIMIT` | ✓ | Trigger a limit order when stop price is breached. | -| `LIMIT_IF_TOUCHED` | - | *Not currently implemented by AX Exchange*. | -| `STOP_MARKET` | - | *Not supported*. | -| `MARKET_IF_TOUCHED` | - | *Not supported*. | -| `TRAILING_STOP_MARKET` | - | *Not supported*. | - -### Execution instructions - -| Instruction | Supported | Notes | -|---------------|-----------|-----------------------------------------------------| -| `post_only` | ✓ | Maker-only; rejected if order would take liquidity. | -| `reduce_only` | - | *Not supported*. | - -### Time in force - -| Time in Force | Supported | Notes | -|---------------|-----------|----------------------------------------------| -| `GTC` | ✓ | Good Till Canceled. | -| `GTD` | ✓ | Good Till Date. | -| `DAY` | ✓ | Valid until end of trading day. | -| `IOC` | ✓ | Immediate or Cancel. | -| `FOK` | ✓ | Fill or Kill. | -| `AT_THE_OPEN` | ✓ | Execute at market open or expire. | -| `AT_THE_CLOSE`| ✓ | Execute at market close or expire. | - -### Advanced order features - -| Feature | Supported | Notes | -|--------------------|-----------|--------------------------------------------------------------------| -| Order modification | - | *Not supported by AX*. Cancel and resubmit instead. | -| Cancel order | ✓ | Single order cancellation. | -| Cancel all orders | ✓ | Cancel all open orders for an instrument. | -| Batch cancel | ✓ | Cancel multiple specified orders. | -| Order lists | ✓ | Sequential submission (orders submitted individually, non-atomic). | - -### Position management - -| Feature | Supported | Notes | -|------------------|-----------|--------------------------------------| -| Query positions | ✓ | Real-time position updates. | -| Position mode | - | Netting mode only. | -| Cross margin | ✓ | Cross-margin across all instruments. | - -### Order querying - -| Feature | Supported | Notes | -|----------------------|-----------|---------------------------------------------------------| -| Query open orders | ✓ | List all active orders. | -| Query single order | ✓ | By venue order ID or client order ID (any order state). | -| Order status reports | ✓ | Reconciliation from open orders; see note below. | -| Fill reports | ✓ | Execution and fill history. | - -:::note -Order status reports for reconciliation are generated from the open orders endpoint. -Filled or canceled orders are not included in the reconciliation snapshot. Single-order -queries via `query_order` use the dedicated `/order-status` endpoint which works for -any order state. -::: - -## Authentication - -AX Exchange uses bearer token authentication: - -1. API key and secret obtain a session token via `/authenticate`. -2. The session token is used as a bearer token for subsequent REST and WebSocket requests. -3. Session tokens expire after a configurable period (default: 86400 seconds). - -## Configuration - -### Environments and endpoints - -| Environment | HTTP API (market data) | HTTP API (orders) | Market Data WS | Orders WS | -|-------------|--------------------------------------------------|-----------------------------------------------------|--------------------------------------------------|------------------------------------------------------| -| Sandbox | `https://gateway.sandbox.architect.exchange/api` | `https://gateway.sandbox.architect.exchange/orders` | `wss://gateway.sandbox.architect.exchange/md/ws` | `wss://gateway.sandbox.architect.exchange/orders/ws` | -| Production | `https://gateway.architect.exchange/api` | `https://gateway.architect.exchange/orders` | `wss://gateway.architect.exchange/md/ws` | `wss://gateway.architect.exchange/orders/ws` | - -:::info -Order management HTTP endpoints (place, cancel, order status) use a separate base URL -from market data endpoints. This is handled automatically by the adapter configuration. -::: - -### Data client configuration options - -| Option | Default | Description | -|------------------------------------|-----------|---------------------------------------------------------------------| -| `api_key` | `None` | API key; loaded from `AX_API_KEY` env var when omitted. | -| `api_secret` | `None` | API secret; loaded from `AX_API_SECRET` env var when omitted. | -| `environment` | `SANDBOX` | Trading environment (`SANDBOX` or `PRODUCTION`). | -| `base_url_http` | `None` | Override for the REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket URL. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `http_timeout_secs` | `60` | Timeout (seconds) for REST requests. | -| `max_retries` | `3` | Maximum retry attempts for REST requests. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries (exponential backoff). | -| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalog refreshes. | -| `funding_rate_poll_interval_mins` | `15` | Interval (minutes) between funding rate poll requests. | - -### Execution client configuration options - -| Option | Default | Description | -|--------------------------|-----------|---------------------------------------------------------------------| -| `api_key` | `None` | API key; loaded from `AX_API_KEY` env var when omitted. | -| `api_secret` | `None` | API secret; loaded from `AX_API_SECRET` env var when omitted. | -| `environment` | `SANDBOX` | Trading environment (`SANDBOX` or `PRODUCTION`). | -| `base_url_http` | `None` | Override for the REST base URL. | -| `base_url_ws` | `None` | Override for the orders WebSocket URL. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `http_timeout_secs` | `60` | Timeout (seconds) for REST requests. | -| `max_retries` | `3` | Maximum retry attempts for REST requests. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries (exponential backoff). | - -The most common use case is to configure a live `TradingNode` to include AX Exchange -data and execution clients. To achieve this, add an `AX` section to your client -configuration(s): - -```python -from nautilus_trader.adapters.architect_ax import AX -from nautilus_trader.adapters.architect_ax import AxDataClientConfig -from nautilus_trader.adapters.architect_ax import AxEnvironment -from nautilus_trader.adapters.architect_ax import AxExecClientConfig -from nautilus_trader.config import InstrumentProviderConfig -from nautilus_trader.config import TradingNodeConfig - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - AX: AxDataClientConfig( - environment=AxEnvironment.SANDBOX, - instrument_provider=InstrumentProviderConfig(load_all=True), - ), - }, - exec_clients={ - AX: AxExecClientConfig( - environment=AxEnvironment.SANDBOX, - instrument_provider=InstrumentProviderConfig(load_all=True), - ), - }, -) -``` - -Then, create a `TradingNode` and add the client factories: - -```python -from nautilus_trader.adapters.architect_ax import AX -from nautilus_trader.adapters.architect_ax import AxLiveDataClientFactory -from nautilus_trader.adapters.architect_ax import AxLiveExecClientFactory -from nautilus_trader.live.node import TradingNode - -# Instantiate the live trading node with a configuration -node = TradingNode(config=config) - -# Register the client factories with the node -node.add_data_client_factory(AX, AxLiveDataClientFactory) -node.add_exec_client_factory(AX, AxLiveExecClientFactory) - -# Finally build the node -node.build() -``` - -### API credentials - -There are two options for supplying your credentials to the AX Exchange clients. -Either pass the corresponding `api_key` and `api_secret` values to the configuration objects, or -set the following environment variables: - -- `AX_API_KEY` -- `AX_API_SECRET` - -:::tip -We recommend using environment variables to manage your credentials. -::: - -When starting the trading node, you'll receive immediate confirmation of whether your -credentials are valid and have trading permissions. - -## Implementation notes - -- **Whole contracts only**: AX Exchange uses integer contract quantities. Fractional quantities - are not supported and will be rejected. -- **Rate limiting**: The adapter applies a conservative rate limit of 10 requests/second with - automatic exponential backoff on rate limit responses. -- **Market orders**: AX does not support native market orders. The adapter uses a preview endpoint - to determine the take-through price and submits an aggressive IOC limit order. - -## Contributing - -:::info -For additional features or to contribute to the AX Exchange adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/betfair.md b/nautilus-docs/docs/integrations/betfair.md deleted file mode 100644 index 3859907..0000000 --- a/nautilus-docs/docs/integrations/betfair.md +++ /dev/null @@ -1,574 +0,0 @@ -# Betfair - -Founded in 2000, Betfair operates the world’s largest online betting exchange, -with its headquarters in London and satellite offices across the globe. - -NautilusTrader provides an adapter for integrating with the Betfair REST API and -Exchange Streaming API. - -## Installation - -Install NautilusTrader with Betfair support: - -```bash -uv pip install "nautilus_trader[betfair]" -``` - -To build from source with Betfair extras: - -```bash -uv sync --all-extras -``` - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/betfair/). - -## Betfair documentation - -Betfair provides documentation for developers: - -- [Betfair Developer Portal](https://developer.betfair.com/): Main entry point for API access and documentation. -- [Exchange API Guide](https://developer.betfair.com/exchange-api/): Overview of the Betting, Accounts, and Streaming APIs. - -## Application keys - -Betfair requires an Application Key to authenticate API requests. After registering and funding your account, -obtain your key using the [API-NG Developer AppKeys Tool](https://apps.betfair.com/visualisers/api-ng-account-operations/). - -Two App Keys are assigned per account: a **Live** key (requires a one-time activation fee) and a -**Delayed** key for development and testing. - -:::info -See the [Application Keys](https://betfair-developer-docs.atlassian.net/wiki/spaces/1smk3cen4v3lu3yomq5qye0ni/pages/2687105/Application+Keys) documentation for detailed setup instructions. -::: - -## API credentials - -Supply your Betfair credentials via environment variables or client configuration: - -```bash -export BETFAIR_USERNAME= -export BETFAIR_PASSWORD= -export BETFAIR_APP_KEY= -export BETFAIR_CERTS_DIR= -``` - -:::tip -We recommend using environment variables to manage your credentials. -::: - -## SSL Certificates - -Betfair recommends [non-interactive (bot) login](https://betfair-developer-docs.atlassian.net/wiki/spaces/1smk3cen4v3lu3yomq5qye0ni/pages/2687915/Non-Interactive+bot+login) -with SSL certificates for automated trading systems. The `certs_dir` configuration is optional, -but certificates are recommended for production deployments. - -### Generating certificates - -Create a 2048-bit RSA certificate using OpenSSL: - -```bash -# Generate private key and certificate signing request -openssl genrsa -out client-2048.key 2048 -openssl req -new -key client-2048.key -out client-2048.csr - -# Self-sign the certificate (valid for 365 days) -openssl x509 -req -days 365 -in client-2048.csr -signkey client-2048.key -out client-2048.crt -``` - -### Uploading to Betfair - -Before using the certificate, attach it to your Betfair account: - -1. Navigate to [My Betfair Account Security](https://myaccount.betfair.com/accountdetails/mysecurity?showAPI=1). -2. Scroll to **Automated Betting Program Access** and click **Edit**. -3. Upload your `client-2048.crt` file. - -### Directory structure - -Place your certificate files in a directory and set `BETFAIR_CERTS_DIR` to that path: - -``` -/path/to/certs/ -├── client-2048.crt -└── client-2048.key -``` - -:::info -SSL certificates are used for the Exchange Streaming API connection. The REST API uses -username/password authentication with your Application Key. -::: - -:::warning -Enabling 2-Step Authentication on the Betfair website does not affect API access. -Certificate-based login remains functional regardless of 2FA settings. -::: - -## Overview - -The Betfair adapter provides three primary components: - -- `BetfairInstrumentProvider`: loads Betfair markets and converts them into Nautilus instruments. -- `BetfairDataClient`: streams real-time market data from the Exchange Streaming API. -- `BetfairExecutionClient`: submits orders (bets) and tracks execution status via the REST API. - -## Orders capability - -Betfair operates as a betting exchange with unique characteristics compared to traditional financial exchanges: - -### Order types - -| Order Type | Supported | Notes | -|------------------------|-----------|-------------------------------------| -| `MARKET` | - | Not applicable to betting exchange. | -| `LIMIT` | ✓ | Orders placed at specific odds. | -| `STOP_MARKET` | - | *Not supported*. | -| `STOP_LIMIT` | - | *Not supported*. | -| `MARKET_IF_TOUCHED` | - | *Not supported*. | -| `LIMIT_IF_TOUCHED` | - | *Not supported*. | -| `TRAILING_STOP_MARKET` | - | *Not supported*. | - -### Execution instructions - -| Instruction | Supported | Notes | -|---------------|-----------|-------------------------------------| -| `post_only` | - | Not applicable to betting exchange. | -| `reduce_only` | - | Not applicable to betting exchange. | - -### Time in force options - -| Time in force | Supported | Notes | -|---------------|-----------|--------------------------------------------| -| `GTC` | ✓ | Maps to Betfair `PERSIST` persistence. | -| `GTD` | - | *Not supported*. | -| `DAY` | ✓ | Maps to Betfair `LAPSE` persistence. | -| `FOK` | ✓ | Maps to Betfair `FILL_OR_KILL`. | -| `IOC` | ✓ | Maps to `FILL_OR_KILL` with partial fills. | - -:::note -Betfair uses a persistence model rather than traditional time-in-force. The adapter maps `FOK` to -Betfair's `FILL_OR_KILL`, while `IOC` uses `FILL_OR_KILL` with `min_fill_size=0` to allow partial fills. -::: - -### Advanced order features - -| Feature | Supported | Notes | -|--------------------|-----------|------------------------------------------| -| Order Modification | ✓ | Limited to non-exposure changing fields. | -| Bracket/OCO Orders | - | *Not supported*. | -| Iceberg Orders | - | *Not supported*. | - -### Batch operations - -| Operation | Supported | Notes | -|--------------------|-----------|----------------------| -| Batch Submit | - | *Not supported*. | -| Batch Modify | - | *Not supported*. | -| Batch Cancel | - | *Not supported*. | - -### Position management - -| Feature | Supported | Notes | -|---------------------|-----------|-----------------------------------------| -| Query positions | - | Betting exchange model differs. | -| Position mode | - | Not applicable to betting exchange. | -| Leverage control | - | No leverage in betting exchange. | -| Margin mode | - | No margin in betting exchange. | - -### Order querying - -| Feature | Supported | Notes | -|----------------------|-----------|----------------------------------------| -| Query open orders | ✓ | List all active bets. | -| Query order history | ✓ | Historical betting data. | -| Order status updates | ✓ | Real-time bet state changes. | -| Trade history | ✓ | Bet matching and settlement reports. | - -### Contingent orders - -| Feature | Supported | Notes | -|---------------------|-----------|-----------------------------------------| -| Order lists | - | *Not supported*. | -| OCO orders | - | *Not supported*. | -| Bracket orders | - | *Not supported*. | -| Conditional orders | - | Basic bet conditions only. | - -## Tick scheme and pricing - -Betfair uses a tiered tick scheme with varying increments across price ranges: - -| Price Range | Tick Size | -|---------------|-----------| -| 1.01 - 2.00 | 0.01 | -| 2.00 - 3.00 | 0.02 | -| 3.00 - 4.00 | 0.05 | -| 4.00 - 6.00 | 0.10 | -| 6.00 - 10.00 | 0.20 | -| 10.00 - 20.00 | 0.50 | -| 20.00 - 30.00 | 1.00 | -| 30.00 - 50.00 | 2.00 | -| 50.00 - 100.00 | 5.00 | -| 100.00 - 1000.00 | 10.00 | - -The minimum price is 1.01 and the maximum is 1000.00. - -## Order modification - -Order modification on Betfair has specific constraints: - -- **Price and size cannot be changed atomically** - these require separate operations. -- **Price modification** uses `ReplaceOrders` (cancel + new order at new price). -- **Size reduction** uses `CancelOrders` with a `size_reduction` parameter. -- **Size increase** is not supported - submit a new order instead. - -:::warning -A replace operation generates both a cancel event for the original order and an accepted event -for the replacement order. The adapter tracks pending replacements to suppress synthetic cancel events. -::: - -## Order stream fill handling - -The execution client processes order updates from the Betfair Exchange Streaming API. -Two configuration options control how updates are filtered: - -- **`stream_market_ids_filter`**: Filters at the market level (early exit, silent skip). -- **`ignore_external_orders`**: Filters at the order level (controls warning vs debug logging). - -Note that `stream_market_ids_filter` is independent of reconciliation scope (`reconcile_market_ids_only`). -Stream filtering affects live updates only; reconciliation uses its own market filter. - -```mermaid -flowchart TD - A[Stream update arrives] --> B{Market in
stream_market_ids_filter?} - B -->|No filter set| C{Instrument loaded?} - B -->|Yes| C - B -->|No| D[Skip silently] - C -->|No| E[Warning: Instrument not loaded] - C -->|Yes| F{Known order?
rfo or cache} - F -->|Yes| G[Process order update] - F -->|No| H{ignore_external_orders?} - H -->|True| I[Debug log, skip] - H -->|False| J[Warning log, skip] -``` - -The same `stream_market_ids_filter` is also applied during full-image reconciliation in `check_cache_against_order_image`. - -When `ignore_external_orders=True`, the client silently skips orders and fills not found in cache: - -| Scenario | Description | -|--------------------------------|-----------------------------------------------------| -| Unknown order in stream update | No venue-to-client order ID mapping exists. | -| Unknown order in full image | Order not found in cache during image sync. | -| Unknown fill in full image | Fill does not match any known order during sync. | - -:::info -For multi-node setups sharing a Betfair account, set both `stream_market_ids_filter` (your markets only) -and `ignore_external_orders=True` to avoid warnings about orders managed by other nodes. -::: - -### Fill handling - -The adapter handles several edge cases when processing fills from the stream: - -- **Incremental fills**: Betfair reports cumulative matched sizes. The adapter calculates incremental - fills by tracking the last known filled quantity per order. -- **Overfill protection**: Fills that would exceed the order quantity are rejected. -- **Deduplication**: A cache of published trade IDs prevents duplicate fill events from late messages - or stream reconnection replays. -- **Race conditions**: When stream fills arrive before the HTTP order response, the adapter caches - the venue order ID immediately to ensure correct order matching. -- **Network error recovery**: When an HTTP order submission fails with a network error (timeout, - connection reset), the order may still have been placed on the venue. The adapter leaves the - order in SUBMITTED status and retains the customer order reference so the stream can confirm - the order when it reconnects. API errors (where Betfair explicitly rejected) still reject - immediately. - -## Rate limiting - -The adapter uses separate rate limit buckets so that account state polling and -reconciliation do not throttle order placement: - -| Bucket | Default | Endpoints | Configurable | -|---------|---------|------------------------------------------------------|----------------------------------| -| General | 5/s | Account state, reconciliation, keep-alive. | | -| Orders | 20/s | `placeOrders`, `replaceOrders`, `cancelOrders`. | `order_request_rate_per_second`. | - -Order status and fill report queries retry once on `TOO_MANY_REQUESTS` errors -after a 1-second delay; order operations reject with the error message. - -Betfair's actual API limits are more nuanced: - -| Category | Limit | Notes | -|--------------------------|----------------------|------------------------------------------------------| -| Order operations | 1,000 transactions/s | Total instructions across `placeOrders`, `cancelOrders`, `replaceOrders`. | -| Order projection queries | 3 concurrent | `listMarketBook` (with `OrderProjection`), `listCurrentOrders`, `listMarketProfitAndLoss`. | -| Best practice | 5 requests/s | Recommended for `listMarketBook` per market. | - -:::info -For details on rate limits, see [Why am I receiving the TOO_MANY_REQUESTS error?](https://support.developer.betfair.com/hc/en-us/articles/360000406111) -and [Market Data Request Limits](https://docs.developer.betfair.com/display/1smk3cen4v3lu3yomq5qye0ni/Market+Data+Request+Limits). -::: - -## Custom data types - -The Betfair adapter provides several custom data types that flow through the market stream. -All custom data is delivered automatically when subscribed to markets - no explicit subscription is required, -though strategies can register handlers for specific data types. - -### BetfairTicker - -Real-time ticker data for a betting selection. - -| Field | Type | Description | -|-----------------------|---------|---------------------------------| -| `instrument_id` | str | Nautilus instrument identifier. | -| `last_traded_price` | float | Last matched price (odds). | -| `traded_volume` | float | Total matched volume. | -| `starting_price_near` | float | Near-side BSP indicator. | -| `starting_price_far` | float | Far-side BSP indicator. | - -### BetfairStartingPrice - -The realized Betfair Starting Price (BSP) after market close. - -| Field | Type | Description | -|-----------------|-------|---------------------------------| -| `instrument_id` | str | Nautilus instrument identifier. | -| `bsp` | float | Final starting price (odds). | - -### BetfairRaceRunnerData - -Live GPS tracking data for individual horses (Total Performance Data). -Available for supported UK and Irish races. - -| Field | Type | Description | -|--------------------|-------|-----------------------------------------| -| `race_id` | str | Betfair race identifier. | -| `market_id` | str | Betfair market identifier. | -| `selection_id` | int | Betfair selection (runner) identifier. | -| `latitude` | float | GPS latitude. | -| `longitude` | float | GPS longitude. | -| `speed` | float | Current speed in m/s (Doppler-derived). | -| `progress` | float | Distance to finish line in meters. | -| `stride_frequency` | float | Stride frequency in Hz. | - -### BetfairRaceProgress - -Race summary data with sectional times and running order. - -| Field | Type | Description | -|------------------|------------|-----------------------------------------------| -| `race_id` | str | Betfair race identifier. | -| `market_id` | str | Betfair market identifier. | -| `gate_name` | str | Timing gate (e.g., "1f", "2f", "Finish"). | -| `sectional_time` | float | Time for this section in seconds. | -| `running_time` | float | Total time since race start in seconds. | -| `speed` | float | Lead horse speed in m/s. | -| `progress` | float | Lead horse distance to finish in meters. | -| `order` | list[int] | Selection IDs in current race position order. | -| `jumps` | list[dict] | Jump obstacle data for National Hunt races. | - -### Subscribing to custom data - -Custom data flows automatically through the Betfair market stream when you subscribe to markets. -To receive custom data in your strategy or actor, register a handler with the Betfair client ID: - -```python -from nautilus_trader.adapters.betfair.constants import BETFAIR_CLIENT_ID -from nautilus_trader.adapters.betfair.data_types import BetfairRaceRunnerData -from nautilus_trader.adapters.betfair.data_types import BetfairRaceProgress -from nautilus_trader.adapters.betfair.data_types import BetfairTicker -from nautilus_trader.model.data import DataType - -class MyStrategy(Strategy): - def on_start(self): - # Subscribe to ticker data - self.subscribe_data(DataType(BetfairTicker), client_id=BETFAIR_CLIENT_ID) - - # Subscribe to ALL race runner data (wildcard) - self.subscribe_data(DataType(BetfairRaceRunnerData), client_id=BETFAIR_CLIENT_ID) - - # Or subscribe to a specific runner by selection_id - self.subscribe_data( - DataType(BetfairRaceRunnerData, metadata={"selection_id": 49411491}), - client_id=BETFAIR_CLIENT_ID, - ) - - # Subscribe to ALL race progress updates (wildcard) - self.subscribe_data(DataType(BetfairRaceProgress), client_id=BETFAIR_CLIENT_ID) - - # Or subscribe to a specific race by race_id - self.subscribe_data( - DataType(BetfairRaceProgress, metadata={"race_id": "35278018.1617"}), - client_id=BETFAIR_CLIENT_ID, - ) - - def on_data(self, data): - if isinstance(data, BetfairRaceRunnerData): - self.log.info( - f"Runner {data.selection_id}: speed={data.speed} m/s, " - f"progress={data.progress}m to finish" - ) - elif isinstance(data, BetfairRaceProgress): - self.log.info(f"Race order: {data.order}") - elif isinstance(data, BetfairTicker): - self.log.info(f"LTP: {data.last_traded_price}") -``` - -:::info -Subscribing with `DataType(BetfairRaceRunnerData)` (no metadata) receives data for -**all** runners. Adding `metadata={"selection_id": }` filters to a specific runner. -Similarly, `DataType(BetfairRaceProgress)` receives progress for all races, while -`metadata={"race_id": }` filters to a specific race. - -Race data (RCM messages) requires Total Performance Data (TPD) coverage and a Betfair -API key with TPD access. Not all races have GPS tracking enabled. -::: - -### Loading race data from files - -For backtesting with recorded race data, use the file parser: - -```python -from nautilus_trader.adapters.betfair.parsing.core import parse_betfair_rcm_file - -for data in parse_betfair_rcm_file("path/to/rcm_data.json"): - if isinstance(data, BetfairRaceRunnerData): - print(f"Runner {data.selection_id} at {data.latitude}, {data.longitude}") -``` - -## Configuration - -### Data client configuration options - -| Option | Default | Description | -|---------------------------|-----------|-------------| -| `account_currency` | Required | Betfair account currency for data and price feeds. | -| `username` | `None` | Betfair account username; taken from environment when omitted. | -| `password` | `None` | Betfair account password; taken from environment when omitted. | -| `app_key` | `None` | Betfair application key used for API authentication. | -| `certs_dir` | `None` | Directory containing Betfair SSL certificates for login. | -| `instrument_config` | `None` | Optional `BetfairInstrumentProviderConfig` to scope available markets. | -| `subscription_delay_secs` | `3` | Delay (seconds) before initial market subscription request is sent. | -| `keep_alive_secs` | `36,000` | Keep-alive interval (seconds) for the Betfair session. | -| `subscribe_race_data` | `False` | When `True`, subscribe to Race Change Messages (RCM) for live GPS tracking data. | -| `stream_conflate_ms` | `None` | Explicit stream conflation interval in milliseconds (`0` disables conflation). | -| `stream_heartbeat_ms` | `5,000` | Stream heartbeat interval in milliseconds (500-5000). `None` to omit. | -| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | - -:::warning -When `stream_conflate_ms` is `None`, Betfair applies its default conflation behavior (typically enabled). -Set `stream_conflate_ms=0` explicitly to guarantee no conflation and receive every price update. -::: - -### Execution client configuration options - -| Option | Default | Description | -|------------------------------|----------|-------------| -| `account_currency` | Required | Betfair account currency for order placement and balances. | -| `username` | `None` | Betfair account username; taken from environment when omitted. | -| `password` | `None` | Betfair account password; taken from environment when omitted. | -| `app_key` | `None` | Betfair application key used for API authentication. | -| `certs_dir` | `None` | Directory containing Betfair SSL certificates for login. | -| `instrument_config` | `None` | Optional `BetfairInstrumentProviderConfig` to scope reconciliation. | -| `calculate_account_state` | `True` | Calculate account state locally from events when `True`. | -| `request_account_state_secs` | `300` | Interval (seconds) to poll Betfair for account state (`0` disables). | -| `reconcile_market_ids_only` | `False` | When `True`, reconciliation only covers `instrument_config.market_ids` (no effect if unset). | -| `stream_market_ids_filter` | `None` | List of market IDs to process from stream; others are silently skipped. | -| `ignore_external_orders` | `False` | When `True`, ignore stream orders missing from the local cache. | -| `use_market_version` | `False` | When `True`, attach the latest market version to order requests for price protection. | -| `order_request_rate_per_second` | `20` | Rate limit (requests/second) for order endpoints, separate from general API endpoints. | -| `stream_heartbeat_ms` | `5,000` | Order stream heartbeat interval in milliseconds (500-5000). `None` to omit. | -| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | - -:::warning -If you set `stream_market_ids_filter`, ensure it includes all markets you trade. Orders placed on -markets excluded from this filter will miss live fill and cancel updates from the stream. -::: - -## Session management - -Betfair sessions typically expire every 12-24 hours. The adapter automatically handles session -reconnection when `NO_SESSION` or `INVALID_SESSION_INFORMATION` errors occur: - -- The HTTP client reconnects and obtains a new session token. -- The streaming client re-authenticates and resubscribes to markets. -- The keep-alive mechanism (`keep_alive_secs`, default 10 hours) proactively extends sessions. - -:::info -Session errors during account state polling or keep-alive trigger automatic reconnection. -No manual intervention is required for normal session expiry. -::: - -## Market version price protection - -Betfair markets have a `version` number that increments whenever the market book changes -(e.g., a new price level appears, a bet is matched). The adapter can attach this version -to `placeOrders` and `replaceOrders` requests, providing price protection against stale orders. - -When `use_market_version=True`, each order request includes the market version last seen -by the adapter. If the market has advanced beyond that version by the time Betfair processes -the order, Betfair **lapses** the bet rather than matching it against a changed book. - -```python -from nautilus_trader.adapters.betfair.config import BetfairExecClientConfig - -exec_config = BetfairExecClientConfig( - account_currency="GBP", - use_market_version=True, -) -``` - -The adapter reads the market version from the instrument's `info` dictionary, which -the Exchange Streaming API's `MarketDefinition` updates populate. This means: - -- The version reflects the most recent stream update, not the HTTP API snapshot. -- There is inherent latency between a market change and the adapter receiving the updated version. -- Orders submitted before the first stream `MarketDefinition` is received will not include a version. - -:::warning -Market version protection is conservative. In fast-moving markets, the version may advance -between your order signal and submission, causing the bet to lapse even though the price -is still acceptable. Consider this trade-off between protection and fill rate. -::: - -## Multi-node deployment - -When multiple trading nodes share a single Betfair account across different markets, configure -each node to avoid interference: - -1. Set `stream_market_ids_filter` to include only that node's markets. -2. Set `ignore_external_orders=True` to suppress warnings about orders from other nodes. -3. Set `reconcile_market_ids_only=True` to limit reconciliation scope. - -This prevents warning spam and ensures each node processes only its own orders and fills. - -Here is a minimal example showing how to configure a live `TradingNode` with Betfair clients: - -```python -from nautilus_trader.adapters.betfair import BETFAIR -from nautilus_trader.adapters.betfair import BetfairLiveDataClientFactory -from nautilus_trader.adapters.betfair import BetfairLiveExecClientFactory -from nautilus_trader.config import TradingNodeConfig -from nautilus_trader.live.node import TradingNode - -# Configure Betfair data and execution clients (using AUD account currency) -config = TradingNodeConfig( - data_clients={BETFAIR: {"account_currency": "AUD"}}, - exec_clients={BETFAIR: {"account_currency": "AUD"}}, -) - -# Build the TradingNode with Betfair adapter factories -node = TradingNode(config) -node.add_data_client_factory(BETFAIR, BetfairLiveDataClientFactory) -node.add_exec_client_factory(BETFAIR, BetfairLiveExecClientFactory) -node.build() -``` - -## Contributing - -:::info -For additional features or to contribute to the Betfair adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/binance.md b/nautilus-docs/docs/integrations/binance.md deleted file mode 100644 index bc91f88..0000000 --- a/nautilus-docs/docs/integrations/binance.md +++ /dev/null @@ -1,857 +0,0 @@ -# Binance - -Founded in 2017, Binance is one of the largest cryptocurrency exchanges in terms -of daily trading volume, and open interest of crypto assets and crypto -derivative products. - -This integration supports live market data ingest and order execution for: - -- **Binance Spot** (including Binance US) -- **Binance USDT-Margined Futures** (perpetuals and delivery contracts) -- **Binance Coin-Margined Futures** - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/binance/). - -## Overview - -This guide assumes a trader is setting up for both live market data feeds, and trade execution. -The Binance adapter includes multiple components, which can be used together or separately depending -on the use case. - -- `BinanceHttpClient`: Low-level HTTP API connectivity. -- `BinanceWebSocketClient`: Low-level WebSocket API connectivity. -- `BinanceInstrumentProvider`: Instrument parsing and loading functionality. -- `BinanceSpotDataClient`/`BinanceFuturesDataClient`: A market data feed manager. -- `BinanceSpotExecutionClient`/`BinanceFuturesExecutionClient`: An account management and trade execution gateway. -- `BinanceLiveDataClientFactory`: Factory for Binance data clients (used by the trading node builder). -- `BinanceLiveExecClientFactory`: Factory for Binance execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), -and won't need to necessarily work with these lower level components directly. -::: - -### Product support - -| Product Type | Supported | Notes | -|------------------------------------------|-----------|-------------------------------------| -| Spot Markets (incl. Binance US) | ✓ | | -| Margin Accounts (Cross & Isolated) | - | Margin trading not implemented. | -| USDT-Margined Futures (PERP & Delivery) | ✓ | | -| Coin-Margined Futures | ✓ | | - -:::note -Margin trading (cross & isolated) is not implemented at this time. -Contributions via [GitHub issue #2631](https://github.com/nautechsystems/nautilus_trader/issues/2631) -or pull requests to add margin trading functionality are welcome. -::: - -## Data types - -To provide complete API functionality to traders, the integration includes several -custom data types: - -- `BinanceTicker`: Represents data returned for Binance 24-hour ticker subscriptions, including price and statistical information. -- `BinanceBar`: Represents data for historical requests or real-time subscriptions to Binance bars, with additional volume metrics. -- `BinanceFuturesMarkPriceUpdate`: Represents mark price updates for Binance Futures subscriptions. - -See the Binance [API Reference](/docs/python-api-latest/adapters/binance.html) for full definitions. - -## Symbology - -As per the Nautilus unification policy for symbols, the native Binance symbols are used where possible including for -spot assets and futures contracts. Because NautilusTrader is capable of multi-venue + multi-account -trading, it's necessary to explicitly clarify the difference between `BTCUSDT` as the spot and margin traded -pair, and the `BTCUSDT` perpetual futures contract (this symbol is used for *both* natively by Binance). - -Therefore, Nautilus appends the suffix `-PERP` to all perpetual symbols. -E.g. for Binance Futures, the `BTCUSDT` perpetual futures contract symbol would be `BTCUSDT-PERP` within the Nautilus system boundary. - -## Order capability - -The following tables detail the order types, execution instructions, and time-in-force options supported across different Binance account types: - -### Order types - -| Order Type | Spot | Margin | USDT Futures | Coin Futures | Notes | -|------------------------|------|--------|--------------|--------------|-------------------------| -| `MARKET` | ✓ | - | ✓ | ✓ | Quote quantity support: Spot only. | -| `LIMIT` | ✓ | - | ✓ | ✓ | | -| `STOP_MARKET` | - | - | ✓ | ✓ | Futures only. | -| `STOP_LIMIT` | ✓ | - | ✓ | ✓ | | -| `MARKET_IF_TOUCHED` | - | - | ✓ | ✓ | Futures only. | -| `LIMIT_IF_TOUCHED` | ✓ | - | ✓ | ✓ | | -| `TRAILING_STOP_MARKET` | - | - | ✓ | ✓ | Futures only. | - -### Execution instructions - -| Instruction | Spot | Margin | USDT Futures | Coin Futures | Notes | -|---------------|------|--------|--------------|--------------|---------------------------------------| -| `post_only` | ✓ | - | ✓ | ✓ | See restrictions below. | -| `reduce_only` | - | - | ✓ | ✓ | Futures only; disabled in Hedge Mode. | - -#### Post-only restrictions - -Only *limit* order types support `post_only`. - -| Order Type | Spot | Margin | USDT Futures | Coin Futures | Notes | -|--------------------------|------|--------|--------------|--------------|-----------------------------------------------------| -| `LIMIT` | ✓ | - | ✓ | ✓ | Uses `LIMIT_MAKER` for Spot, `GTX` TIF for Futures. | -| `STOP_LIMIT` | - | - | ✓ | ✓ | Futures only. | - -### Time in force - -| Time in force | Spot | Margin | USDT Futures | Coin Futures | Notes | -|---------------|------|--------|--------------|--------------|--------------------------------------------| -| `GTC` | ✓ | - | ✓ | ✓ | Good Till Canceled. | -| `GTD` | ✓* | - | ✓ | ✓ | *Converted to GTC for Spot with warning. | -| `FOK` | ✓ | - | ✓ | ✓ | Fill or Kill. | -| `IOC` | ✓ | - | ✓ | ✓ | Immediate or Cancel. | - -### Advanced order features - -| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | -|--------------------|------|--------|--------------|--------------|----------------------------------------------| -| Order Modification | ✓ | - | ✓ | ✓ | Price and quantity for `LIMIT` orders only. | -| Bracket/OCO Orders | - | - | - | - | *Planned*. Currently denied at submission. | -| Iceberg Orders | ✓ | - | ✓ | ✓ | Large orders split into visible portions. | - -### Batch operations - -| Operation | Spot | Margin | USDT Futures | Coin Futures | Notes | -|--------------------|------|--------|--------------|--------------|----------------------------------------------| -| Batch Submit | ✓ | - | ✓ | ✓ | Orders submitted individually (no batch API call). | -| Batch Modify | - | - | - | - | Not implemented. | -| Batch Cancel | -* | - | ✓ | ✓ | *Spot falls back to individual cancels. | - -#### Cancel all orders behavior - -When calling `cancel_all_orders()` from a strategy, the adapter includes orders in both open and inflight (SUBMITTED) states. -This ensures that orders submitted but not yet acknowledged by Binance are also canceled. - -**Multi-strategy safety**: When multiple strategies trade the same instrument, the adapter compares orders owned by the requesting strategy against all orders for that instrument. If the strategy owns all orders, a single cancel-all API call is used. Otherwise, per-strategy cancels are sent (batch for regular orders, individual for algo orders) to avoid affecting other strategies' orders. - -**Futures algo orders**: For Binance Futures, conditional order types (STOP_MARKET, STOP_LIMIT, TAKE_PROFIT, TAKE_PROFIT_MARKET, TRAILING_STOP_MARKET) require a different cancel endpoint than regular orders. -The adapter automatically routes these "algo" orders through the correct endpoint. Once an algo order triggers and becomes a regular order, it uses the standard cancel endpoint. - -**Endpoints used**: - -| Account Type | Regular Orders | Algo Orders (batch) | Algo Orders (individual) | -|--------------|---------------------------------|----------------------------------|-----------------------------| -| Spot/Margin | `DELETE /api/v3/openOrders` | N/A | N/A | -| USDT Futures | `DELETE /fapi/v1/allOpenOrders` | `DELETE /fapi/v1/algoOpenOrders` | `DELETE /fapi/v1/algoOrder` | -| Coin Futures | `DELETE /dapi/v1/allOpenOrders` | `DELETE /dapi/v1/algoOpenOrders` | `DELETE /dapi/v1/algoOrder` | - -### Position management - -| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | -|---------------------|------|--------|--------------|--------------|---------------------------------------------| -| Query positions | - | - | ✓ | ✓ | Real-time position updates. | -| Position mode | - | - | ✓ | ✓ | One-Way vs Hedge mode (position IDs). | -| Leverage control | - | - | ✓ | ✓ | Dynamic leverage adjustment per symbol. | -| Margin mode | - | - | ✓ | ✓ | Cross vs Isolated margin per symbol. | - -### Risk events - -| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | -|----------------------|------|--------|--------------|--------------|---------------------------------------------| -| Liquidation handling | - | - | ✓ | ✓ | Exchange-forced position closures. | -| ADL handling | - | - | ✓ | ✓ | Auto-Deleveraging events. | - -Binance Futures can trigger exchange-generated orders in response to risk events: - -- **Liquidations**: When insufficient margin exists to maintain a position, Binance forcibly closes it at the bankruptcy price. These orders have client IDs starting with `autoclose-`. -- **ADL (Auto-Deleveraging)**: When the insurance fund is depleted, Binance closes profitable positions to cover losses. These orders use client ID `adl_autoclose`. -- **Settlements**: Quarterly contract deliveries use client IDs starting with `settlement_autoclose-`. - -The adapter detects these special order types via their client ID patterns and execution type (`CALCULATED`), then: - -1. Logs a warning with order details for monitoring. -2. Generates an `OrderStatusReport` to seed the cache. -3. Generates a `FillReport` with correct fill details and TAKER liquidity side. - -This ensures liquidation and ADL events are properly reflected in portfolio state and PnL calculations. - -### Order querying - -| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | -|---------------------|------|--------|--------------|--------------|---------------------------------------------| -| Query open orders | ✓ | ✓ | ✓ | ✓ | List all active orders. | -| Query order history | ✓ | ✓ | ✓ | ✓ | Historical order data. | -| Order status updates| ✓ | ✓ | ✓ | ✓ | Real-time order state changes. | -| Trade history | ✓ | ✓ | ✓ | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Spot | Margin | USDT Futures | Coin Futures | Notes | -|---------------------|------|--------|--------------|--------------|----------------------------------------------| -| Order lists | - | - | - | - | *Not supported*. | -| OCO orders | - | - | - | - | *Planned*. Currently denied at submission. | -| Bracket orders | - | - | - | - | *Planned*. Currently denied at submission. | -| Conditional orders | ✓ | ✓ | ✓ | ✓ | Stop and market-if-touched orders. | - -### Order parameters - -Customize individual orders by supplying a `params` dictionary when calling `Strategy.submit_order`. The Binance execution clients currently recognise: - -| Parameter | Type | Account types | Description | -|-----------------|--------|-------------------|-------------| -| `price_match` | `str` | USDT/COIN Futures | Set one of Binance's `priceMatch` modes (see Price match section below) to delegate price selection to the exchange. Cannot be combined with `post_only` or iceberg (`display_qty`) instructions. | - -### Price match - -Binance Futures supports BBO (Best Bid/Offer) price matching via the `priceMatch` parameter, which delegates price selection to the exchange. This feature allows limit orders to dynamically join the order book at optimal prices without manually specifying the exact price level. - -When using `price_match`, you submit a limit order with a reference price (for local risk checks), but Binance determines the actual working price based on the current market state and the selected price match mode. - -#### Valid price match values - -Valid `priceMatch` values for Binance Futures: - -| Value | Behaviour | -|---------------|----------------------------------------------------------------| -| `OPPONENT` | Join the best price on the opposing side of the book. | -| `OPPONENT_5` | Join the opposing side price but allow up to a 5-tick offset. | -| `OPPONENT_10` | Join the opposing side price but allow up to a 10-tick offset. | -| `OPPONENT_20` | Join the opposing side price but allow up to a 20-tick offset. | -| `QUEUE` | Join the best price on the same side (stay maker). | -| `QUEUE_5` | Join the same-side queue but offset up to 5 ticks. | -| `QUEUE_10` | Join the same-side queue but offset up to 10 ticks. | -| `QUEUE_20` | Join the same-side queue but offset up to 20 ticks. | - -:::info -For more details, see the [official documentation](https://developers.binance.com/docs/derivatives/usds-margined-futures/trade/rest-api). -::: - -#### Event sequence - -When an order is submitted with `price_match`, the following sequence of events occurs: - -1. **Order submission**: Nautilus sends the order to Binance with the `priceMatch` parameter but omits the limit price in the API request. -2. **Order acceptance**: Binance accepts the order and determines the actual working price based on the current market and the specified price match mode. -3. **OrderAccepted event**: Nautilus generates an `OrderAccepted` event when the order is confirmed. -4. **OrderUpdated event**: If the Binance-accepted price differs from the original reference price, Nautilus immediately generates an `OrderUpdated` event with the actual working price. -5. **Price synchronization**: The order's limit price in the Nautilus cache is now synchronized with the actual price accepted by Binance. - -This ensures that the order price in your system accurately reflects what Binance has accepted, which is important for position management, risk calculations, and strategy logic. - -#### Example - -```python -order = strategy.order_factory.limit( - instrument_id=InstrumentId.from_str("BTCUSDT-PERP.BINANCE"), - order_side=OrderSide.BUY, - quantity=Quantity.from_int(1), - price=Price.from_str("65000"), # Reference price for local risk checks -) - -strategy.submit_order( - order, - params={"price_match": "QUEUE"}, -) -``` - -:::note -After submission, if Binance accepts the order at a different price (e.g., 64,995.50), you will receive both an `OrderAccepted` event followed by an `OrderUpdated` event with the new price. -::: - -### Trailing stops - -For trailing stop market orders on Binance: - -- Use `activation_price` (optional) to specify when the trailing mechanism activates -- When omitted, Binance uses the current market price at submission time -- Use `trailing_offset` for the callback rate (in basis points) - -:::warning -Do not use `trigger_price` for trailing stop orders - it will fail with an error. Use `activation_price` instead. -::: - -## Link & Trade - -The NautilusTrader integration ID is automatically prefixed to all -system-generated client order IDs for every order placed through the Binance -Rust adapter. This provides transparent order attribution through Binance's -[Link and Trade](https://developers.binance.com/docs/binance_link/link-and-trade) -program without requiring any user configuration. - -The adapter encodes outgoing `ClientOrderId` values into a compact format that -fits within Binance's 36-character `newClientOrderId` limit, and decodes -incoming order events back to the original ID before they reach strategies. -This transformation is fully transparent: strategies see only their original -`ClientOrderId` values at all times. - -:::note -The integration ID prefix applies to all order operations including -submissions, modifications, cancellations, and status queries. Orders placed -before this support was added are handled gracefully through passthrough -decoding. -::: - -:::info -This feature is currently available in the Rust adapter only. Users can opt out -by passing a custom `client_order_id` on their orders, or by removing the -encoding calls and recompiling. There is no technical limitation preventing -either approach. -::: - -## Order books - -Order books can be maintained at full or partial depths based on the subscription settings. -WebSocket stream update rates differ between Spot and Futures exchanges, with Nautilus using the -highest available streaming rate: - -- **Spot**: 100ms -- **Futures**: 0ms (unthrottled) - -There is a limitation of one order book per instrument per trader instance. -As stream subscriptions may vary, the latest order book data (deltas or snapshots) -subscription will be used by the Binance data client. - -Order book snapshot rebuilds will be triggered on: - -- Initial subscription of the order book data. -- Data websocket reconnects. - -The sequence of events is as follows: - -- Deltas will start buffered. -- Snapshot is requested and awaited. -- Snapshot response is parsed to `OrderBookDeltas`. -- Snapshot deltas are sent to the `DataEngine`. -- Buffered deltas are iterated, dropping those where the sequence number is not greater than the last delta in the snapshot. -- Deltas will stop buffering. -- Remaining deltas are sent to the `DataEngine`. - -## Binance data differences - -The `ts_event` field value for `QuoteTick` objects will differ between Spot and Futures exchanges, -where the former does not provide an event timestamp, so the `ts_init` is used (which means `ts_event` and `ts_init` are identical). - -## Binance specific data - -It's possible to subscribe to Binance specific data streams as they become available to the -adapter over time. - -:::note -Bars are not considered 'Binance specific' and can be subscribed to in the normal way. -As more adapters are built out which need for example mark price and funding rate updates, then these -methods may eventually become first-class (not requiring custom/generic subscriptions as below). -::: - -### `BinanceFuturesMarkPriceUpdate` - -You can subscribe to `BinanceFuturesMarkPriceUpdate` (including funding rate info) -data streams by subscribing in the following way from your actor or strategy: - -```python -from nautilus_trader.adapters.binance import BinanceFuturesMarkPriceUpdate -from nautilus_trader.model import DataType -from nautilus_trader.model import ClientId - -# In your `on_start` method -self.subscribe_data( - data_type=DataType(BinanceFuturesMarkPriceUpdate, metadata={"instrument_id": self.instrument.id}), - client_id=ClientId("BINANCE"), -) -``` - -This will result in your actor/strategy passing these received `BinanceFuturesMarkPriceUpdate` -objects to your `on_data` method. You will need to check the type, as this -method acts as a flexible handler for all custom/generic data. - -```python -from nautilus_trader.core import Data - -def on_data(self, data: Data): - # First check the type of data - if isinstance(data, BinanceFuturesMarkPriceUpdate): - # Do something with the data -``` - -## Funding rates - -The adapter subscribes to funding rate data through the -[Mark Price Stream](https://developers.binance.com/docs/derivatives/usds-margined-futures/websocket-market-streams/Mark-Price-Stream) -WebSocket endpoint, which provides the current funding rate and next funding time. - -The `interval` field on `FundingRateUpdate` is `None` for Binance because the Mark Price Stream -does not include a funding interval field. Binance exposes `fundingIntervalHours` through the -[Get Funding Rate Info](https://developers.binance.com/docs/derivatives/usds-margined-futures/market-data/rest-api/Get-Funding-Rate-Info) -REST endpoint, but this is not currently consumed by the adapter. - -## Rate limiting - -Binance uses an interval-based rate limiting system where request weight is tracked per fixed time window (every minute, resetting at :00 seconds). Each API endpoint has an assigned weight cost, and your total weight usage is tracked per IP address. - -### Global weight limits - -These are the primary limits shared across all endpoints: - -| Account Type | Weight Limit | Interval | -|--------------|--------------|----------| -| Spot/Margin | 6,000 | 1 minute | -| Futures | 2,400 | 1 minute | - -### Endpoint weight costs - -Some endpoints have higher weight costs per request: - -| Endpoint | Weight | Notes | -|---------------------------|--------|--------------------------------------------------------| -| `/api/v3/order` | 1 | Spot order placement. | -| `/api/v3/allOrders` | 20 | Spot historical orders (expensive). | -| `/api/v3/klines` | 2+ | Scales with `limit` parameter. | -| `/fapi/v1/order` | 1 | Futures order placement. | -| `/fapi/v1/allOrders` | 20 | Futures historical orders (expensive). | -| `/fapi/v1/commissionRate` | 20 | Futures commission rate query. | -| `/fapi/v1/klines` | 5+ | Scales with `limit` parameter. | - -### WebSocket API limits - -The WebSocket API (used for user data streams) shares the same weight quota as the REST API: - -| Limit Type | Value | Notes | -|------------------|--------|--------------------------------------------| -| Request weight | Shared | Counts against REST API weight quota. | -| Handshake | 5 | Weight cost per connection attempt. | -| Ping/pong frames | 5/sec | Maximum ping/pong rate. | - -### Adapter behavior - -The adapter uses token bucket rate limiters to approximate Binance's interval-based limits. This reduces the risk of quota violations while maintaining throughput for normal operations. - -For endpoints with dynamic weight (e.g., `/klines` scales with the `limit` parameter), the adapter draws a single token per call. Large history requests may need manual pacing. Monitor the `X-MBX-USED-WEIGHT-*` response headers to track actual usage. - -:::warning -Binance returns HTTP 429 when you exceed the allowed weight. Repeated violations trigger temporary IP bans (escalating from 2 minutes to 3 days for repeat offenders). -::: - -:::info -For the latest rate limits, query `/api/v3/exchangeInfo` (Spot) or `/fapi/v1/exchangeInfo` (Futures), or see: - -- [Spot API Limits](https://developers.binance.com/docs/binance-spot-api-docs/rest-api/limits) -- [Futures API Limits](https://developers.binance.com/docs/derivatives/usds-margined-futures/general-info) - -::: - -## Configuration - -### Data client configuration options - -| Option | Default | Description | -|------------------------------------|-----------|-------------| -| `venue` | `BINANCE` | Venue identifier used when registering the client. | -| `api_key` | `None` | Binance API key; loaded from environment variables when omitted. | -| `api_secret` | `None` | Binance API secret; loaded from environment variables when omitted. | -| `key_type` | `HMAC` | **Deprecated**: key type is now auto-detected from the API secret format. Only needed to force `RSA`. | -| `account_type` | `SPOT` | Account type for data endpoints (spot, margin, USDT futures, coin futures). | -| `base_url_http` | `None` | Override for the HTTP REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | -| `us` | `False` | Route requests to Binance US endpoints when `True`. | -| `environment` | `None` | Binance environment: `LIVE`, `TESTNET`, or `DEMO`. Defaults to `LIVE` when `None`. | -| `testnet` | `False` | **Deprecated**: use `environment=BinanceEnvironment.TESTNET` instead. | -| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalogue refreshes. | -| `use_agg_trade_ticks` | `False` | When `True`, subscribe to aggregated trade ticks instead of raw trades. | - -### Execution client configuration options - -| Option | Default | Description | -|--------------------------------------|-----------|-------------| -| `venue` | `BINANCE` | Venue identifier used when registering the client. | -| `api_key` | `None` | Binance API key; loaded from environment variables when omitted. | -| `api_secret` | `None` | Binance API secret; loaded from environment variables when omitted. | -| `key_type` | `HMAC` | **Deprecated**: key type is now auto-detected from the API secret format. Only needed to force `RSA` (data clients only, RSA is not supported for execution). | -| `account_type` | `SPOT` | Account type for order placement (spot, margin, USDT futures, coin futures). | -| `base_url_http` | `None` | Override for the HTTP REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket API base URL. | -| `base_url_ws_stream` | `None` | Override for the WebSocket stream URL (futures user data event delivery). | -| `proxy_url` | `None` | Optional proxy URL for HTTP requests. | -| `us` | `False` | Route requests to Binance US endpoints when `True`. | -| `environment` | `None` | Binance environment: `LIVE`, `TESTNET`, or `DEMO`. Defaults to `LIVE` when `None`. | -| `testnet` | `False` | **Deprecated**: use `environment=BinanceEnvironment.TESTNET` instead. | -| `use_gtd` | `True` | When `False`, remaps GTD orders to GTC for local expiry management. | -| `use_reduce_only` | `True` | When `True`, passes through `reduce_only` instructions to Binance. | -| `use_position_ids` | `True` | Enable Binance hedging position IDs; set `False` for virtual hedging. | -| `use_trade_lite` | `False` | Use TRADE_LITE execution events that include derived fees. | -| `treat_expired_as_canceled` | `False` | Treat `EXPIRED` execution types as `CANCELED` when `True`. | -| `recv_window_ms` | `5,000` | Receive window (milliseconds) for signed REST requests. | -| `max_retries` | `None` | Maximum retry attempts for order submission/cancel/modify calls. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retry attempts. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retry attempts. | -| `futures_leverages` | `None` | Mapping of `BinanceSymbol` to initial leverage for futures accounts. | -| `futures_margin_types` | `None` | Mapping of `BinanceSymbol` to futures margin type (isolated/cross). | -| `log_rejected_due_post_only_as_warning` | `True` | Log post-only rejections as warnings when `True`; otherwise as errors. | - -The most common use case is to configure a live `TradingNode` to include Binance -data and execution clients. To achieve this, add a `BINANCE` section to your client -configuration(s): - -```python -from nautilus_trader.adapters.binance import BINANCE -from nautilus_trader.live.node import TradingNode - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - BINANCE: { - "api_key": "YOUR_BINANCE_API_KEY", - "api_secret": "YOUR_BINANCE_API_SECRET", - "account_type": "spot", # {spot, usdt_futures, coin_futures} - "base_url_http": None, # Override with custom endpoint - "base_url_ws": None, # Override with custom endpoint - "us": False, # If client is for Binance US - }, - }, - exec_clients={ - BINANCE: { - "api_key": "YOUR_BINANCE_API_KEY", - "api_secret": "YOUR_BINANCE_API_SECRET", - "account_type": "spot", # {spot, usdt_futures, coin_futures} - "base_url_http": None, # Override with custom endpoint - "base_url_ws": None, # Override with custom endpoint - "us": False, # If client is for Binance US - }, - }, -) -``` - -Then, create a `TradingNode` and add the client factories: - -```python -from nautilus_trader.adapters.binance import BINANCE -from nautilus_trader.adapters.binance import BinanceLiveDataClientFactory -from nautilus_trader.adapters.binance import BinanceLiveExecClientFactory -from nautilus_trader.live.node import TradingNode - -# Instantiate the live trading node with a configuration -node = TradingNode(config=config) - -# Register the client factories with the node -node.add_data_client_factory(BINANCE, BinanceLiveDataClientFactory) -node.add_exec_client_factory(BINANCE, BinanceLiveExecClientFactory) - -# Finally build the node -node.build() -``` - -### Key types - -Binance supports three API key types: **Ed25519**, **HMAC-SHA256**, and **RSA**. -The adapter auto-detects the key type from your API secret format, so no configuration is needed. - -**Ed25519 is strongly recommended** for all API access. Binance recommends Ed25519 for -its superior performance and security, and a future version of NautilusTrader will -require Ed25519 exclusively. - -| Key Type | Data Clients | Execution Clients | Status | -|----------|--------------|-------------------|--------| -| Ed25519 | ✓ | ✓ | **Recommended** | -| HMAC | ✓ | ✓ | Deprecated, will be removed in a future version. | -| RSA | ✓ | - | Deprecated, not supported for execution. | - -:::tip -**We strongly recommend switching to Ed25519 keys now.** Generate an Ed25519 keypair and register -it with Binance. See [Generating Ed25519 keys](#generating-ed25519-keys) below for instructions. -::: - -:::note -Ed25519 keys must be provided in unencrypted PEM format (base64-encoded ASN.1/DER). -The implementation automatically extracts the 32-byte seed from the DER structure. -Encrypted (password-protected) PEM keys are not supported. If your key is encrypted, -decrypt it first: `openssl pkey -in encrypted.pem -out decrypted.pem` -::: - -#### Generating Ed25519 keys - -**Option 1: OpenSSL (recommended)** - -```bash -# Generate private key (PKCS#8 PEM format) -openssl genpkey -algorithm ed25519 -out binance_ed25519_private.pem - -# Extract public key -openssl pkey -in binance_ed25519_private.pem -pubout -out binance_ed25519_public.pem -``` - -**Option 2: Binance Key Generator** - -Download the [Binance Asymmetric Key Generator](https://github.com/binance/asymmetric-key-generator) from the releases page and run it to generate a keypair. - -**Registering with Binance** - -1. Log in to Binance and go to **Profile** → **API Management** -2. Click **Create API** and select **Self-generated** -3. Paste the contents of your public key file (including the `-----BEGIN PUBLIC KEY-----` header/footer) -4. Configure permissions (Enable Spot & Margin Trading, etc.) - -**Using with NautilusTrader** - -Set the private key as your API secret: - -```bash -export BINANCE_API_KEY="your-api-key-from-binance" -export BINANCE_API_SECRET="$(cat binance_ed25519_private.pem)" -``` - -Or pass the PEM content directly in your configuration. - -:::warning -Keep your private key secure. Never share it or commit it to version control. -::: - -### API credentials - -There are multiple options for supplying your credentials to the Binance clients. -Either pass the corresponding values to the configuration objects, or -set the appropriate environment variables (see [Environments](#environments) for per-environment variables). - -:::tip -We recommend using Ed25519 keys for all clients. HMAC keys still work for both data and -execution clients, but Ed25519 offers better performance and will become the only supported -key type in a future version. See [Key types](#key-types) for details. -::: - -:::warning -The `BINANCE_ED25519_*` and `BINANCE_*_ED25519_*` environment variables have been removed -for Spot/Margin. For Futures, they are deprecated with a warning and will be removed in a -future version. Rename them to the standard `BINANCE_API_KEY`/`BINANCE_API_SECRET` variables -(Ed25519 keys are now auto-detected). -::: - -When starting the trading node, you'll receive immediate confirmation of whether your -credentials are valid and have trading permissions. - -### Account type - -Set the `account_type` using the `BinanceAccountType` enum. The supported account types are: - -- `SPOT` -- `USDT_FUTURES` (USDT or BUSD stablecoins as collateral) -- `COIN_FUTURES` (other cryptocurrency as collateral) - -:::note -`MARGIN` and `ISOLATED_MARGIN` account types exist in the enum but margin trading -is not yet implemented. See [Product support](#product-support). -::: - -### Base URL overrides - -It's possible to override the default base URLs for both HTTP Rest and -WebSocket APIs. This is useful for configuring API clusters for performance reasons, -or when Binance has provided you with specialized endpoints. - -### Binance US - -There is support for Binance US accounts by setting the `us` option in the configs -to `True` (this is `False` by default). All functionality available to US accounts -should behave identically to standard Binance. - -### Environments - -Binance provides three trading environments, configured via the `environment` option: - -| Environment | Config | Description | -|--------------|--------------------------|---------------------------------------------------------------------| -| **Live** | `environment="LIVE"` | Production trading with real funds (default). | -| **Demo** | `environment="DEMO"` | Practice trading with simulated funds on production infrastructure. | -| **Testnet** | `environment="TESTNET"` | Separate test network for development and integration testing. | - -#### Live (production) - -The default environment for live trading with real funds. - -```python -config = BinanceExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - account_type=BinanceAccountType.SPOT, - # environment=BinanceEnvironment.LIVE (default) -) -``` - -Environment variables: `BINANCE_API_KEY`, `BINANCE_API_SECRET` - -#### Demo trading - -Practice trading with simulated funds. Spot demo uses `demo-api.binance.com`, USD-M Futures demo uses `demo-fapi.binance.com`, and COIN-M Futures demo shares testnet endpoints. Create demo API keys from the [Binance Demo Trading page](https://www.binance.com/en/demo-trading). - -```python -config = BinanceExecClientConfig( - api_key="YOUR_DEMO_API_KEY", - api_secret="YOUR_DEMO_API_SECRET", - account_type=BinanceAccountType.SPOT, - environment=BinanceEnvironment.DEMO, -) -``` - -Environment variables: `BINANCE_DEMO_API_KEY`, `BINANCE_DEMO_API_SECRET` (shared across Spot and Futures) - -:::warning -**Demo environment limitations:** - -- COIN-M Futures are **not supported** in demo mode. -- Futures demo shares testnet infrastructure, so market data and liquidity may differ from production. - -::: - -#### Testnet - -A separate test network for development and integration testing. Spot testnet is at `testnet.binance.vision`, Futures testnet at `testnet.binancefuture.com`. - -```python -config = BinanceExecClientConfig( - api_key="YOUR_TESTNET_API_KEY", - api_secret="YOUR_TESTNET_API_SECRET", - account_type=BinanceAccountType.SPOT, - environment=BinanceEnvironment.TESTNET, -) -``` - -Environment variables (Spot/Margin): `BINANCE_TESTNET_API_KEY`, `BINANCE_TESTNET_API_SECRET` -Environment variables (Futures): `BINANCE_FUTURES_TESTNET_API_KEY`, `BINANCE_FUTURES_TESTNET_API_SECRET` - -:::note -Testnet uses completely separate infrastructure from production. Market data and liquidity differ significantly from live. -::: - -:::warning -The `testnet` config option is deprecated and will be removed in a future version. Use `environment="TESTNET"` instead. -::: - -### Aggregated trades - -Binance provides aggregated trade data endpoints as an alternative source of trades. -In comparison to the default trade endpoints, aggregated trade data endpoints can return all -ticks between a `start_time` and `end_time`. - -To use aggregated trades and the endpoint features, set the `use_agg_trade_ticks` option -to `True` (this is `False` by default.) - -### Commission rate queries - -By default, Binance Futures instruments use fee tier tables based on your VIP level. -For market maker accounts with negative maker fees or when precise rates are required, -enable per-symbol commission rate queries: - -```python -from nautilus_trader.adapters.binance import BinanceInstrumentProviderConfig - -instrument_provider=BinanceInstrumentProviderConfig( - load_all=True, - query_commission_rates=True, # Query accurate rates per symbol -) -``` - -When enabled, the adapter queries Binance's `/fapi/v1/commissionRate` endpoint for -each symbol in parallel during instrument loading. This is particularly useful for: - -- Market maker accounts with negative maker fees. -- Accounts with custom fee arrangements. -- Ensuring exact commission rates for PnL calculations. - -The adapter uses parallel requests with proper rate limiting (120 requests/minute -accounting for the endpoint's weight of 20). If a query fails, it automatically -falls back to the fee tier table. - -### Parser warnings - -Some Binance instruments are unable to be parsed into Nautilus objects if they -contain enormous field values beyond what can be handled by the platform. -In these cases, a *warn and continue* approach is taken (the instrument will not -be available). - -These warnings may cause unnecessary log noise, and so it's possible to -configure the provider to not log the warnings, as per the client configuration -example below: - -```python -from nautilus_trader.config import InstrumentProviderConfig - -instrument_provider=InstrumentProviderConfig( - load_all=True, - log_warnings=False, -) -``` - -### Futures hedge mode - -Binance Futures Hedge mode is a position mode where a trader opens positions in both long and short -directions to mitigate risk and potentially profit from market volatility. - -To use Binance Future Hedge mode, you need to follow the three items below: - -- 1. Before starting the strategy, ensure that hedge mode is configured on Binance. -- 2. Set the `use_reduce_only` option to `False` in BinanceExecClientConfig (this is `True` by default). - - ```python - from nautilus_trader.adapters.binance import BINANCE - - config = TradingNodeConfig( - ..., # Omitted - data_clients={ - BINANCE: BinanceDataClientConfig( - api_key=None, # 'BINANCE_API_KEY' env var - api_secret=None, # 'BINANCE_API_SECRET' env var - account_type=BinanceAccountType.USDT_FUTURES, - base_url_http=None, # Override with custom endpoint - base_url_ws=None, # Override with custom endpoint - ), - }, - exec_clients={ - BINANCE: BinanceExecClientConfig( - api_key=None, # 'BINANCE_API_KEY' env var - api_secret=None, # 'BINANCE_API_SECRET' env var - account_type=BinanceAccountType.USDT_FUTURES, - base_url_http=None, # Override with custom endpoint - base_url_ws=None, # Override with custom endpoint - use_reduce_only=False, # Must be disabled for Hedge mode - ), - } - ) - ``` - -- 3. When submitting an order, use a suffix (`LONG` or `SHORT` ) in the `position_id` to indicate the position direction. - - ```python - class EMACrossHedgeMode(Strategy): - ..., # Omitted - def buy(self) -> None: - """ - Users simple buy method (example). - """ - order: MarketOrder = self.order_factory.market( - instrument_id=self.instrument_id, - order_side=OrderSide.BUY, - quantity=self.instrument.make_qty(self.trade_size), - # time_in_force=TimeInForce.FOK, - ) - - # LONG suffix is recognized as a long position by Binance adapter. - position_id = PositionId(f"{self.instrument_id}-LONG") - self.submit_order(order, position_id) - - def sell(self) -> None: - """ - Users simple sell method (example). - """ - order: MarketOrder = self.order_factory.market( - instrument_id=self.instrument_id, - order_side=OrderSide.SELL, - quantity=self.instrument.make_qty(self.trade_size), - # time_in_force=TimeInForce.FOK, - ) - # SHORT suffix is recognized as a short position by Binance adapter. - position_id = PositionId(f"{self.instrument_id}-SHORT") - self.submit_order(order, position_id) - ``` - -## Contributing - -:::info -For additional features or to contribute to the Binance adapter, see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/bitmex.md b/nautilus-docs/docs/integrations/bitmex.md deleted file mode 100644 index 4f3d2e7..0000000 --- a/nautilus-docs/docs/integrations/bitmex.md +++ /dev/null @@ -1,871 +0,0 @@ -# BitMEX - -Founded in 2014, BitMEX (Bitcoin Mercantile Exchange) is a cryptocurrency derivatives -trading platform offering spot, perpetual contracts, traditional futures, prediction -markets, and other advanced trading products. This integration supports live market data -ingest and order execution with BitMEX. - -## Overview - -This adapter is implemented in Rust, with optional Python bindings for ease of use in Python-based workflows. -It does not require external BitMEX client libraries; the core components are compiled as a static library and linked automatically during the build. - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/bitmex/). - -## Components - -This guide assumes a trader is setting up for both live market data feeds, and trade execution. -The BitMEX adapter includes multiple components, which can be used together or separately depending -on the use case. - -- `BitmexHttpClient`: Low-level HTTP API connectivity. -- `BitmexWebSocketClient`: Low-level WebSocket API connectivity. -- `BitmexInstrumentProvider`: Instrument parsing and loading functionality. -- `BitmexDataClient`: A market data feed manager. -- `BitmexExecutionClient`: An account management and trade execution gateway. -- `BitmexLiveDataClientFactory`: Factory for BitMEX data clients (used by the trading node builder). -- `BitmexLiveExecClientFactory`: Factory for BitMEX execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), -and won't need to necessarily work with these lower level components directly. -::: - -## BitMEX documentation - -BitMEX provides extensive documentation for users: - -- [BitMEX API Explorer](https://www.bitmex.com/app/restAPI) - Interactive API documentation. -- [BitMEX API Documentation](https://www.bitmex.com/app/apiOverview) - Complete API reference. -- [BitMEX Exchange Rules](https://www.bitmex.com/exchange-rules) - Official exchange rules and regulations. -- [Contract Guides](https://www.bitmex.com/app/contract) - Detailed contract specifications. -- [Spot Trading Guide](https://www.bitmex.com/app/spotGuide) - Spot trading overview. -- [Perpetual Contracts Guide](https://www.bitmex.com/app/perpetualContractsGuide) - Perpetual swaps explained. -- [Futures Contracts Guide](https://www.bitmex.com/app/futuresGuide) - Traditional futures information. - -It's recommended you refer to the BitMEX documentation in conjunction with this -NautilusTrader integration guide. - -## Product support - -| Product Type | Data Feed | Trading | Notes | -|-------------------|-----------|---------|-----------------------------------------------------| -| Spot | ✓ | ✓ | Limited pairs, unified wallet with derivatives. | -| Perpetual Swaps | ✓ | ✓ | Inverse and linear contracts available. | -| Stock Perpetuals | - | - | *Not yet supported*. Currently on testnet only. | -| Futures | ✓ | ✓ | Traditional fixed expiration contracts. | -| Quanto Futures | ✓ | ✓ | Settled in different currency than underlying. | -| Prediction Markets| ✓ | ✓ | Event-based contracts, 0-100 pricing, USDT settled. | -| Options | - | - | *Not provided by BitMEX*. | - -:::note -BitMEX has discontinued their options products to focus on their core derivatives and spot offerings. -::: - -### Spot trading - -- Direct token/coin trading with immediate settlement. -- Major pairs including XBT/USDT, ETH/USDT, ETH/XBT. -- Additional altcoin pairs (LINK, SOL, UNI, APE, AXS, BMEX against USDT). - -### Derivatives - -- **Perpetual contracts**: Inverse (e.g., XBTUSD) and linear (e.g., ETHUSDT). -- **Traditional futures**: Fixed expiration date contracts. -- **Quanto futures**: Contracts settled in a different currency than the underlying. -- **Prediction markets**: Event-based derivatives (e.g., P_FTXZ26, P_SBFJAILZ26) allowing traders to speculate on outcomes across crypto, finance, and other events. No leverage, priced 0-100, settled in USDT. -- **Stock perpetuals**: Equity-based perpetuals (e.g., SPYUSDT, CRCLUSDT). *Currently on testnet only; not yet supported by this adapter.* - -### Instrument type codes (CFI) - -BitMEX uses CFI (Classification of Financial Instruments) codes following the ISO 10962 standard. -The adapter recognizes the following instrument type codes: - -| Code | Type | Status | Description | -|----------|----------------------|-------------|-------------------------------------------------| -| `FFWCSX` | Perpetual Contract | Supported | Crypto-based perpetual swaps (e.g., XBTUSD). | -| `FFWCSF` | Perpetual FX | Supported | FX-based perpetual contracts. | -| `FFCCSX` | Futures | Supported | Calendar futures with fixed expiration. | -| `FFICSX` | Prediction Market | Supported | Event-based prediction contracts. | -| `IFXXXP` | Spot | Supported | Spot trading pairs. | -| `FFSCSX` | Stock Perpetual | Unsupported | Stock/equity-based perpetuals. Testnet only. | -| `SRMCSX` | Swap Rate | Unsupported | Yield-based swap products (historical). | -| `MR****` | Index | Reference | BitMEX indices (non-tradeable, for price ref). | - -See [BitMEX Typ Values](https://support.bitmex.com/hc/en-gb/articles/6299296145565-What-are-the-Typ-Values-for-Instrument-endpoint) for more details. - -## Symbology - -BitMEX uses a specific naming convention for its trading symbols. Understanding this -convention is crucial for correctly identifying and trading instruments. - -### Symbol format - -BitMEX symbols typically follow these patterns: - -- **Spot pairs**: Base currency + Quote currency (e.g., `XBT/USDT`, `ETH/USDT`). -- **Perpetual contracts**: Base currency + Quote currency (e.g., `XBTUSD`, `ETHUSD`). -- **Futures contracts**: Base currency + Expiry code (e.g., `XBTM24`, `ETHH25`). -- **Quanto contracts**: Special naming for non-USD settled contracts. -- **Prediction markets**: `P_` prefix + Event identifier + Expiry code (e.g., `P_POWELLK26`, `P_FTXZ26`). - -:::info -BitMEX uses `XBT` as the symbol for Bitcoin instead of `BTC`. This follows the ISO 4217 -currency code standard where "X" denotes non-national currencies. XBT and BTC refer to -the same asset - Bitcoin. -::: - -### Expiry codes - -Futures contracts use standard futures month codes: - -- `F` = January -- `G` = February -- `H` = March -- `J` = April -- `K` = May -- `M` = June -- `N` = July -- `Q` = August -- `U` = September -- `V` = October -- `X` = November -- `Z` = December - -Followed by the year (e.g., `24` for 2024, `25` for 2025). - -### NautilusTrader instrument IDs - -Within NautilusTrader, BitMEX instruments are identified using the native BitMEX symbol -directly, combined with the venue identifier: - -```python -from nautilus_trader.model.identifiers import InstrumentId - -# Spot pairs (note: no slash in the symbol) -spot_id = InstrumentId.from_str("XBTUSDT.BITMEX") # XBT/USDT spot -eth_spot_id = InstrumentId.from_str("ETHUSDT.BITMEX") # ETH/USDT spot - -# Perpetual contracts -perp_id = InstrumentId.from_str("XBTUSD.BITMEX") # Bitcoin perpetual (inverse) -linear_perp_id = InstrumentId.from_str("ETHUSDT.BITMEX") # Ethereum perpetual (linear) - -# Futures contract (June 2024) -futures_id = InstrumentId.from_str("XBTM24.BITMEX") # Bitcoin futures expiring June 2024 - -# Prediction market contracts -prediction_id = InstrumentId.from_str("P_XBTETFV23.BITMEX") # Bitcoin ETF SEC approval prediction expiring October 2023 -``` - -:::note -BitMEX spot symbols in NautilusTrader don't include the slash (/) that appears in the -BitMEX UI. Use `XBTUSDT` instead of `XBT/USDT`. -::: - -### Quantity scaling - -BitMEX reports spot and derivative quantities in *contract* units. The actual asset size per -contract is exchange-specific and published on the instrument definition: - -- `lotSize` – minimum number of contracts you can trade. -- `underlyingToPositionMultiplier` – number of contracts per unit of the underlying asset. - -For example, the SOL/USDT spot instrument (`SOLUSDT`) exposes `lotSize = 1000` and -`underlyingToPositionMultiplier = 10000`, meaning one contract represents `1 / 10000 = 0.0001` -SOL, and the minimum order (`lotSize * contract_size`) is `0.1` SOL. The adapter now derives the -contract size directly from these fields and scales both inbound market data and outbound orders -accordingly, so quantities in Nautilus are always expressed in base units (SOL, ETH, etc.). - -See the BitMEX API documentation for details on these fields: . - -## Orders capability - -The BitMEX integration supports the following order types and execution features. - -### Order types - -| Order Type | Supported | Notes | -|------------------------|-----------|-----------------------------------------------| -| `MARKET` | ✓ | Executed immediately at current market price. Quote quantity not supported. | -| `LIMIT` | ✓ | Executed only at specified price or better. | -| `STOP_MARKET` | ✓ | Supported (set `trigger_price`). | -| `STOP_LIMIT` | ✓ | Supported (set `price` and `trigger_price`). | -| `MARKET_IF_TOUCHED` | ✓ | Supported (set `trigger_price`). | -| `LIMIT_IF_TOUCHED` | ✓ | Supported (set `price` and `trigger_price`). | -| `TRAILING_STOP_MARKET` | ✓ | Supported (set `trailing_offset`). Price offset type only. | - -### Execution instructions - -| Instruction | Supported | Notes | -|---------------|-----------|-----------------------------------------------------------------------------------| -| `post_only` | ✓ | Supported via `ParticipateDoNotInitiate` execution instruction on `LIMIT` orders. | -| `reduce_only` | ✓ | Supported via `ReduceOnly` execution instruction. | - -:::note -Post-only orders that would cross the spread are canceled by BitMEX rather than rejected. The -integration surfaces these as rejections with `due_post_only=True` so strategies can handle them -consistently. -::: - -### Trigger types - -BitMEX supports multiple reference prices to evaluate stop/conditional order triggers for: - -- `STOP_MARKET` -- `STOP_LIMIT` -- `MARKET_IF_TOUCHED` -- `LIMIT_IF_TOUCHED` - -Choose the trigger type that matches your strategy and/or risk preferences. - -| Reference price | Nautilus `TriggerType` | BitMEX value | Notes | -|-----------------|------------------------|---------------|---------------------------------------------------------------------------------| -| Last trade | `LAST_PRICE` | `LastPrice` | BitMEX default; triggers on the last traded price. | -| Mark price | `MARK_PRICE` | `MarkPrice` | Recommended for many stop-loss use cases to reduce stop-outs from price spikes. | -| Index price | `INDEX_PRICE` | `IndexPrice` | Tracks the external index; useful for some contracts. | - -- If no `trigger_type` is provided, BitMEX uses its venue default (`LastPrice`). -- These trigger references are exchange-evaluated; the order remains resting at the venue until triggered. - -**Example**: - -```python -from nautilus_trader.model.enums import TriggerType - -order = self.order_factory.stop_market( - instrument_id=instrument_id, - order_side=order_side, - quantity=qty, - trigger_price=trigger, - trigger_type=TriggerType.MARK_PRICE, # Use BitMEX Mark Price as reference -) -``` - -`ExecTester` example configuration also demonstrates setting `stop_trigger_type=TriggerType.MARK_PRICE` -in `examples/live/bitmex/bitmex_exec_tester.py`. - -### Trailing stops - -BitMEX supports trailing stop orders that automatically adjust the stop price as the market moves -favorably. The adapter maps `TRAILING_STOP_MARKET` orders to BitMEX's pegged orders with -`TrailingStopPeg` price type. - -**Limitations:** - -- Only `PRICE` trailing offset type is supported (absolute price offset, not basis points or ticks). -- The offset sign is handled automatically: sell stops use negative offset, buy stops use positive. -- Trigger type can be combined with trailing stops for additional control. - -**Example**: - -```python -from nautilus_trader.model.enums import TrailingOffsetType - -order = self.order_factory.trailing_stop_market( - instrument_id=instrument_id, - order_side=OrderSide.SELL, - quantity=qty, - trailing_offset=Decimal("100"), # $100 trailing offset - trailing_offset_type=TrailingOffsetType.PRICE, - trigger_type=TriggerType.LAST_PRICE, # Optional -) -``` - -:::note -BitMEX updates trailing stop prices periodically as the market moves. -The stop price freezes when the market moves toward the trigger level. -See the [BitMEX API documentation](https://www.bitmex.com/app/perpetualContractsGuide) for current update cadence details. -::: - -### Pegged orders - -BitMEX supports pegged orders (BBO) that automatically track a reference price. The adapter -supports pegged orders via the `params` dict on `submit_order`, which overrides the order type -to `Pegged` on the exchange side. - -| Peg price type | Description | -|----------------|------------------------------------------------------------------| -| `PrimaryPeg` | Pegs to the best bid (buy) or best ask (sell). | -| `MarketPeg` | Pegs to the opposite side (best ask for buy, best bid for sell). | -| `MidPricePeg` | Pegs to the mid-price between bid and ask. | -| `LastPeg` | Pegs to the last traded price. | - -**Requirements**: - -- The underlying order must be a `LIMIT` order. Other order types are rejected. -- `peg_price_type` is required; `peg_offset_value` is optional (defaults to 0). -- `peg_offset_value` can be negative (e.g., sell-side offsets) or fractional. - -**Example**: - -```python -# Pegged to best bid with zero offset (BBO) -order = self.order_factory.limit( - instrument_id=instrument_id, - order_side=OrderSide.BUY, - quantity=qty, - price=price, # Required for LIMIT order, but overridden by peg -) -self.submit_order(order, params={"peg_price_type": "PrimaryPeg", "peg_offset_value": "0"}) - -# Pegged to mid-price with a -0.5 offset -self.submit_order(order, params={"peg_price_type": "MidPricePeg", "peg_offset_value": "-0.5"}) -``` - -:::note -The `price` field is still required when constructing the `LimitOrder`, but BitMEX ignores it -for pegged orders and instead continuously tracks the reference price plus offset. -::: - -### Time in force - -| Time in force | Supported | Notes | -|----------------|-----------|-----------------------------------------------------| -| `GTC` | ✓ | Good Till Canceled (default). | -| `GTD` | - | *Not supported by BitMEX*. | -| `FOK` | ✓ | Fill or Kill - fills entire order or cancels. | -| `IOC` | ✓ | Immediate or Cancel - partial fill allowed. | -| `DAY` | ✓ | Expires at 00:00 UTC (BitMEX trading day boundary). | - -:::note -`DAY` orders expire at 12:00am UTC, which marks the BitMEX trading day boundary (end of trading hours for that day). -See the [BitMEX Exchange Rules](https://www.bitmex.com/exchange-rules) and [API documentation](https://www.bitmex.com/api/explorer/) for complete details. -::: - -### Advanced order features - -| Feature | Supported | Notes | -|--------------------|-----------|--------------------------------------------------------------------------| -| Order Modification | ✓ | Modify price, quantity, and trigger price. | -| Bracket Orders | ✓ | Use `contingency_type` and `linked_order_ids`. | -| Iceberg Orders | ✓ | Use `display_qty`. | -| Trailing Stops | ✓ | Use `trailing_offset`. Price offset type only. | -| Pegged Orders | ✓ | Use `params` with `peg_price_type`. See [Pegged orders](#pegged-orders). | - -### Batch operations - -| Operation | Supported | Notes | -|--------------------|-----------|---------------------------------------------| -| Batch Submit | - | *Not supported by BitMEX*. | -| Batch Modify | - | *Not supported by BitMEX*. | -| Batch Cancel | ✓ | Cancel multiple orders in a single request. | - -### Position management - -| Feature | Supported | Notes | -|---------------------|-----------|----------------------------------------------------| -| Query positions | ✓ | REST and real-time position updates via WebSocket. | -| Cross margin | ✓ | Default margin mode. | -| Isolated margin | ✓ | | - -### Order querying - -| Feature | Supported | Notes | -|----------------------|-----------|----------------------------------------------| -| Query open orders | ✓ | List all active orders. | -| Query order history | ✓ | Historical order data. | -| Order status updates | ✓ | Real-time order state changes via WebSocket. | -| Trade history | ✓ | Execution and fill reports. | - -## Market data - -- Order book deltas: `L2_MBP` only; `depth` 0 (full book) or 25. -- Order book depth10 snapshots: fixed 10 levels via `orderBook10` channel. -- Quotes, trades, and instrument updates are supported via WebSocket. -- Funding rates, mark prices, and index prices are supported where applicable. -- Historical requests via REST: - - Trade ticks with optional `start`, `end`, and `limit` filters (up to 1,000 results per call). - - Time bars (`1m`, `5m`, `1h`, `1d`) for externally aggregated LAST prices, including optional partial bins. - -:::note -BitMEX caps each REST response at 1,000 rows and requires manual pagination via `start`/`startTime`. The current adapter returns only the -first page; wider pagination support is scheduled for a future update. -::: - -## Connection management - -### HTTP Keep-Alive - -The BitMEX adapter uses HTTP keep-alive for optimal performance: - -- **Connection pooling**: Connections are automatically pooled and reused. -- **Keep-alive timeout**: 90 seconds (matches BitMEX server-side timeout). -- **Automatic reconnection**: Failed connections are automatically re-established. -- **SSL session caching**: Reduces handshake overhead for subsequent requests. - -This configuration ensures low-latency communication with BitMEX servers by maintaining -persistent connections and avoiding the overhead of establishing new connections for each request. - -### Request authentication and expiration - -BitMEX uses an `api-expires` header for request authentication to prevent replay attacks: - -- Signed requests include an `api-expires` Unix timestamp set `recv_window_ms / 1000` seconds ahead (10 seconds by default). -- BitMEX rejects any request once that timestamp has passed, so keep latency within your configured window. - -## Funding rates - -The adapter receives funding rate data from the -[Funding](https://www.bitmex.com/app/wsAPI#Funding) -WebSocket stream. BitMEX returns a `fundingInterval` datetime field in each message, -and the adapter reads the hours and minutes to compute the `interval` field on -`FundingRateUpdate`. - -## Rate limiting - -BitMEX implements a dual-layer rate limiting system: - -### REST limits - -- **Burst limit**: 10 requests per second for authenticated users (applies to order placement, modification, and cancel endpoints). -- **Rolling minute limit**: 120 requests per minute for authenticated users (30 requests per minute for unauthenticated users). -- **Order caps**: 200 open orders and 10 stop orders per symbol; exceeding these caps triggers exchange-side rejections. - -The adapter enforces these quotas locally using the configured `max_requests_per_second` and `max_requests_per_minute` values. - -### WebSocket limits - -- Connection requests: follow the exchange guidance (currently 3 connections per second per IP). -- Private streams require authentication; the adapter reconnects automatically if a limit is exceeded. - -:::warning -Exceeding BitMEX rate limits returns HTTP 429 and may trigger temporary IP bans; persistent 4xx/5xx errors can extend the lockout period. -::: - -### Configurable rate limits - -The rate limits can be configured if your account has different limits than the defaults: - -| Parameter | Default (authenticated) | Default (unauthenticated) | Description | -|----------------------------|-------------------------|---------------------------|-----------------------------------------------------| -| `max_requests_per_second` | 10 | 10 | Maximum requests per second (burst limit). | -| `max_requests_per_minute` | 120 | 30 | Maximum requests per minute (rolling window). | - -:::info -For more details on rate limiting, see the [BitMEX API documentation on rate limits](https://www.bitmex.com/app/restAPI#Limits). -::: - -:::warning -**Cancel Broadcaster Rate Limit Considerations** - -The cancel broadcaster (when `canceller_pool_size > 1`) fans out each cancel request to multiple independent HTTP clients in parallel. Each client maintains its own rate limiter, which means the effective request rate is multiplied by the pool size. - -**Example**: With `canceller_pool_size=3` and `max_requests_per_second=10`, a single cancel operation consumes **3 requests** (one per client), potentially reaching **30 requests/second** if canceling rapidly. - -Since BitMEX enforces rate limits **at the account level** (not per connection), the broadcaster can push you over the exchange's default limits of 10 req/s burst and 120 req/min rolling window. - -**Mitigations**: Reduce `max_requests_per_second` and `max_requests_per_minute` proportionally (divide by `canceller_pool_size`), or adjust the pool size itself (see [Cancel broadcaster configuration](#cancel-broadcaster)). -Future versions may support shared rate limiters across the pool. -::: - -### Rate-limit headers - -BitMEX exposes the current allowance via response headers: - -- `x-ratelimit-limit`: total requests permitted in the current window. -- `x-ratelimit-remaining`: remaining requests before throttling occurs. -- `x-ratelimit-reset`: UNIX timestamp when the allowance resets. -- `retry-after`: seconds to wait after a 429 response. - -## Submit broadcaster - -The BitMEX execution client includes a submit broadcaster that provides higher assurance of market and limit orders being accepted at target prices through parallel request fanout, trading lower minimum latency for the risk of duplicate submissions. - -### Concepts - -Order submissions are time-critical operations - when a strategy decides to enter a position, any delay can result in missed opportunities or adverse pricing. The submit broadcaster addresses this by: - -- **Parallel fanout**: Submit requests are simultaneously broadcast to multiple independent HTTP client instances. -- **First-success short-circuiting**: The first successful response wins, minimizing latency to acceptance. -- **Shared client_order_id**: All transports use the same `client_order_id`. BitMEX rejects duplicate submissions with "duplicate clOrdID" (tracked as expected rejections). -- **Latency vs. duplicates tradeoff**: Accepts the risk of potential duplicate fills (if multiple transports succeed before rejection) in exchange for lower minimum latency and higher assurance of acceptance. - -This architecture reduces the minimum latency to order acceptance by parallelizing across multiple network paths. - -### Usage - -The submit broadcaster is opt-in and controlled via the `submit_tries` parameter when submitting orders. By default, orders are submitted through a single HTTP client. To enable broadcasting: - -```python -# Single submission (default behavior) -self.submit_order(order) - -# Broadcast to 2 parallel HTTP clients for redundancy -self.submit_order(order, params={"submit_tries": 2}) - -# Broadcast to 3 parallel HTTP clients (maximum recommended) -self.submit_order(order, params={"submit_tries": 3}) -``` - -**Key points**: - -- `submit_tries` must be a positive integer. -- Broadcasting only occurs when `submit_tries > 1`. Default submits go through a single HTTP client. -- If `submit_tries` exceeds `submitter_pool_size`, it will be capped at the pool size with a warning. -- All transports use the same `client_order_id`; BitMEX rejects duplicates as expected rejections. - -### Health monitoring - -Each HTTP client in the broadcaster pool maintains health metrics: - -- Successful submissions mark a client as healthy. -- Failed requests increment error counters. -- Background health checks periodically verify client connectivity. -- Degraded clients are tracked but remain in the pool to maintain fault tolerance. - -The broadcaster exposes metrics including total submits, successful submits, failed submits, and expected rejects for operational monitoring and debugging. - -#### Tracked metrics - -| Metric | Type | Description | -|--------------------------|--------|-----------------------------------------------------------------------------------------------------------------------| -| `total_submits` | `u64` | Total number of submit operations initiated. | -| `successful_submits` | `u64` | Number of submit operations that successfully received acknowledgement from BitMEX. | -| `failed_submits` | `u64` | Number of submit operations where all HTTP clients in the pool failed (no healthy clients or all requests failed). | -| `expected_rejects` | `u64` | Number of expected rejection patterns detected (e.g., duplicate clOrdID from parallel submissions). | -| `healthy_clients` | `usize`| Current number of healthy HTTP clients in the pool (clients that passed recent health checks). | -| `total_clients` | `usize`| Total number of HTTP clients configured in the pool (`submitter_pool_size`). | - -These metrics can be accessed programmatically via the `get_metrics()` method on the `SubmitBroadcaster` instance. - -### Configuration - -The submit broadcaster is configured via the execution client configuration: - -| Option | Default | Description | -|------------------------|---------|-------------------------------------------------------------------------------------------| -| `submitter_pool_size` | `None` | Size of the HTTP client pool. `None` resolves to 1 (single client, no redundancy). | -| `submitter_proxy_urls` | `None` | Optional list of proxy URLs for submit broadcaster path diversity. *Not yet wired through Python integration.* | - -**Example configuration**: - -```python -from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig - -exec_config = BitmexExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - submitter_pool_size=3, # Recommended pool size for redundancy -) -``` - -:::tip -For HFT strategies without higher rate limits, consider the advantages of using the submit broadcaster against potentially hitting rate limits, as each client has an independent rate limit budget. -The default `submitter_pool_size=None` disables the broadcaster. The recommended setting of `submitter_pool_size=3` broadcasts each submit request to 3 parallel HTTP clients for fault tolerance, which consumes 3× the rate limit quota per submit operation but provides higher assurance against network or exchange issues. -::: - -The broadcaster is automatically started when the execution client connects and stopped when it disconnects. Submit operations are routed through the broadcaster only when `submit_tries > 1`; default submits use a single HTTP client directly. - -## Cancel broadcaster - -The BitMEX execution client includes a cancel broadcaster that provides fault-tolerant order cancellation through parallel request fanout. - -### Concepts - -Order cancellations are time-critical operations - when a strategy decides to cancel an order, any delay or failure can result in unintended fills, slippage, or unwanted position exposure. The cancel broadcaster addresses this by: - -- **Parallel fanout**: Cancel requests are simultaneously broadcast to multiple independent HTTP client instances. -- **First-success short-circuiting**: The first successful response wins, and remaining in-flight requests are immediately aborted. -- **Fault tolerance**: If one HTTP client experiences network issues, DNS failures, or connection timeouts, other clients in the pool continue processing. -- **Idempotent success handling**: Responses indicating the order was already canceled (such as "orderID not found" or similar idempotent states) are treated as success rather than failure, preventing unnecessary error propagation. - -This architecture ensures that a single network path failure or slow connection doesn't block cancel operations, improving the reliability of risk management and position control in live trading. - -### Health monitoring - -Each HTTP client in the broadcaster pool maintains health metrics: - -- Successful cancellations mark a client as healthy. -- Failed requests increment error counters. -- Background health checks periodically verify client connectivity. -- Degraded clients are tracked but remain in the pool to maintain fault tolerance. - -The broadcaster exposes metrics including total cancels, successful cancels, failed cancels, expected rejects (already canceled orders), and idempotent successes for operational monitoring and debugging. - -#### Tracked metrics - -| Metric | Type | Description | -|--------------------------|--------|-----------------------------------------------------------------------------------------------------------------------| -| `total_cancels` | `u64` | Total number of cancel operations initiated (includes single, batch, and cancel-all requests). | -| `successful_cancels` | `u64` | Number of cancel operations that successfully received acknowledgement from BitMEX. | -| `failed_cancels` | `u64` | Number of cancel operations where all HTTP clients in the pool failed (no healthy clients or all requests failed). | -| `expected_rejects` | `u64` | Number of expected rejection patterns detected (e.g., post-only order rejections). | -| `idempotent_successes` | `u64` | Number of idempotent success responses (order already cancelled, order not found, unable to cancel due to state). | -| `healthy_clients` | `usize`| Current number of healthy HTTP clients in the pool (clients that passed recent health checks). | -| `total_clients` | `usize`| Total number of HTTP clients configured in the pool (`canceller_pool_size`). | - -These metrics can be accessed programmatically via the `get_metrics()` method on the `CancelBroadcaster` instance. - -### Configuration - -The cancel broadcaster is configured via the execution client configuration: - -| Option | Default | Description | -|------------------------|---------|-------------------------------------------------------------------------------------------| -| `canceller_pool_size` | `None` | Size of the HTTP client pool. `None` resolves to 1 (single client, no redundancy). | -| `canceller_proxy_urls` | `None` | Optional list of proxy URLs for cancel broadcaster path diversity. *Not yet wired through Python integration.* | - -**Example configuration**: - -```python -from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig - -exec_config = BitmexExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - canceller_pool_size=3, # Recommended pool size for redundancy -) -``` - -:::tip -For HFT strategies without higher rate limits, consider the advantages of using the cancel broadcaster against potentially hitting rate limits, as each client has an independent rate limit budget. -The default `canceller_pool_size=None` disables the broadcaster. The recommended setting of `canceller_pool_size=3` broadcasts each cancel request to 3 parallel HTTP clients for fault tolerance, which consumes 3× the rate limit quota per cancel operation but provides higher assurance against network or exchange issues. -::: - -The broadcaster is automatically started when the execution client connects and stopped when it disconnects. All cancel operations (`cancel_order`, `cancel_all_orders`, `batch_cancel_orders`) are automatically routed through the broadcaster without requiring any changes to strategy code. - -## Dead man's switch - -The adapter supports BitMEX's [dead man's switch](https://www.bitmex.com/app/restAPI#OrdercancelAllAfter) -(`cancelAllAfter`), which provides automatic order cancellation as a safety net against -connectivity failures. - -### How it works - -When enabled, a server-side timer is set on BitMEX. If the timer expires without being -refreshed, BitMEX cancels **all** open orders on the account. The adapter keeps the timer -alive by sending periodic heartbeat requests. If the adapter loses connectivity (network -failure, process crash, etc.), the heartbeats stop and BitMEX cancels the orders after the -configured timeout. - -The flow: - -1. On **connect**, the adapter calls `POST /api/v1/order/cancelAllAfter` with the configured - timeout (in milliseconds) to arm the server-side timer. -2. A background task sends the same request at a **refresh interval** of `timeout / 4` - (minimum 1 second) to keep resetting the timer before it expires. -3. On **disconnect**, the adapter waits for the background heartbeat task to fully shut - down, then calls `cancelAllAfter` with `timeout=0` to **disarm** the server-side timer. - -For example, with a 60-second timeout the adapter sends a heartbeat every 15 seconds. -If four consecutive heartbeats fail (60 seconds of lost connectivity), BitMEX cancels -all open orders. - -### Disconnect ordering - -Disarming the dead man's switch during disconnect requires careful ordering. The disarm -request (`timeout=0`) should be the last `cancelAllAfter` call to reach BitMEX. If an -in-flight heartbeat were processed after the disarm, it would re-arm the server-side timer -and orders could be unexpectedly cancelled after the timeout expires, even though the -adapter disconnected gracefully. - -The adapter mitigates this in both implementations: - -- **Rust**: The heartbeat task is immediately stopped (abort + await) so disconnect does - not stall waiting for a sleep or HTTP timeout to elapse. The disarm request is then sent - after the task has exited. -- **Python**: The heartbeat task is cancelled and awaited, ensuring the coroutine fully - unwinds before the disarm request is sent. - -In a force-stop scenario (e.g., process shutdown via `stop()`), the heartbeat task is -aborted without disarming. This is intentional, as the server-side timer provides the -desired safety behavior when the process exits unexpectedly. - -:::note -Each heartbeat consumes one REST rate limit token. A 60-second timeout uses approximately -4 requests per minute from the 120/min budget. -::: - -### Configuration - -Enable the dead man's switch by setting `deadmans_switch_timeout_secs` on the execution -client config: - -```python -from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig - -exec_config = BitmexExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - deadmans_switch_timeout_secs=60, # Cancel all orders after 60s of lost connectivity -) -``` - -When enabled, the adapter logs: - -``` -Starting dead man's switch: timeout=60s, refresh_interval=15s -``` - -on connect, and: - -``` -Disarming dead man's switch -``` - -on disconnect. - -:::tip -A timeout of **60 seconds** is the recommended starting point. Shorter timeouts provide -faster protection but are more sensitive to transient network blips. Longer timeouts are -more tolerant of brief outages but leave orders exposed longer during a real failure. -::: - -:::warning -The dead man's switch applies to **all** open orders on the account, not just orders placed -by the adapter. If other systems place orders on the same account, enabling the dead man's -switch will affect those orders too. -::: - -## Configuration - -### API credentials - -BitMEX API credentials can be provided either directly in the configuration or via environment variables: - -- `BITMEX_API_KEY`: Your BitMEX API key for production. -- `BITMEX_API_SECRET`: Your BitMEX API secret for production. -- `BITMEX_TESTNET_API_KEY`: Your BitMEX API key for testnet (when `testnet=True`). -- `BITMEX_TESTNET_API_SECRET`: Your BitMEX API secret for testnet (when `testnet=True`). - -To generate API keys: - -1. Log in to your BitMEX account. -2. Navigate to Account & Security → API Keys. -3. Create a new API key with appropriate permissions. -4. For testnet, use [testnet.bitmex.com](https://testnet.bitmex.com). - -:::note -**Testnet API endpoints**: - -- REST API: `https://testnet.bitmex.com/api/v1` -- WebSocket: `wss://ws.testnet.bitmex.com/realtime` - -The adapter automatically routes requests to the correct endpoints when `testnet=True` is configured. -::: - -### Data client configuration options - -The BitMEX data client provides the following configuration options: - -| Option | Default | Description | -|-----------------------------------|----------|-------------| -| `api_key` | `None` | Optional API key; if `None`, loaded from `BITMEX_API_KEY` or `BITMEX_TESTNET_API_KEY` (when `testnet=True`). | -| `api_secret` | `None` | Optional API secret; if `None`, loaded from `BITMEX_API_SECRET` or `BITMEX_TESTNET_API_SECRET` (when `testnet=True`). | -| `base_url_http` | `None` | Override for the REST base URL (defaults to production). | -| `base_url_ws` | `None` | Override for the WebSocket base URL (defaults to production). | -| `testnet` | `False` | Route requests to the BitMEX testnet when `True`. | -| `http_timeout_secs` | `60` | Request timeout applied to HTTP calls. | -| `max_retries` | `None` | Maximum retry attempts for HTTP calls (disabled when `None`). | -| `retry_delay_initial_ms` | `1,000` | Initial backoff delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `5,000` | Maximum backoff delay (milliseconds) between retries. | -| `recv_window_ms` | `10,000` | Expiration window (milliseconds) for signed requests. See [Request authentication](#request-authentication-and-expiration). | -| `update_instruments_interval_mins`| `60` | Interval (minutes) between instrument catalogue refreshes. | -| `max_requests_per_second` | `10` | Burst rate limit enforced by the adapter for REST calls. | -| `max_requests_per_minute` | `120` | Rolling minute rate limit enforced by the adapter for REST calls. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. *Not yet implemented; reserved for future use.* | - -### Execution client configuration options - -The BitMEX execution client provides the following configuration options: - -| Option | Default | Description | -|--------------------------|----------|-------------| -| `api_key` | `None` | Optional API key; if `None`, loaded from `BITMEX_API_KEY` or `BITMEX_TESTNET_API_KEY` (when `testnet=True`). | -| `api_secret` | `None` | Optional API secret; if `None`, loaded from `BITMEX_API_SECRET` or `BITMEX_TESTNET_API_SECRET` (when `testnet=True`). | -| `base_url_http` | `None` | Override for the REST base URL (defaults to production). | -| `base_url_ws` | `None` | Override for the WebSocket base URL (defaults to production). | -| `testnet` | `False` | Route orders to the BitMEX testnet when `True`. | -| `http_timeout_secs` | `60` | Request timeout applied to HTTP calls. | -| `max_retries` | `None` | Maximum retry attempts for HTTP calls (disabled when `None`). | -| `retry_delay_initial_ms` | `1,000` | Initial backoff delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `5,000` | Maximum backoff delay (milliseconds) between retries. | -| `recv_window_ms` | `10,000` | Expiration window (milliseconds) for signed requests. See [Request authentication](#request-authentication-and-expiration). | -| `max_requests_per_second`| `10` | Burst rate limit enforced by the adapter for REST calls. | -| `max_requests_per_minute`| `120` | Rolling minute rate limit enforced by the adapter for REST calls. | -| `deadmans_switch_timeout_secs` | `None` | Timeout in seconds for the dead man's switch. `None` disables. See [Dead man's switch](#dead-mans-switch). | -| `canceller_pool_size` | `None` | Number of HTTP clients in the cancel broadcaster pool. `None` resolves to 1. See [Cancel broadcaster](#cancel-broadcaster). | -| `submitter_pool_size` | `None` | Number of HTTP clients in the submit broadcaster pool. `None` resolves to 1. See [Submit broadcaster](#submit-broadcaster). | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. *Not yet implemented; reserved for future use.* | -| `submitter_proxy_urls` | `None` | Optional list of proxy URLs for submit broadcaster path diversity. *Not yet wired through Python integration.* | -| `canceller_proxy_urls` | `None` | Optional list of proxy URLs for cancel broadcaster path diversity. *Not yet wired through Python integration.* | - -### Configuration examples - -A typical BitMEX configuration for live trading includes both testnet and mainnet options: - -```python -from nautilus_trader.adapters.bitmex.config import BitmexDataClientConfig -from nautilus_trader.adapters.bitmex.config import BitmexExecClientConfig - -# Using environment variables (recommended) -testnet_data_config = BitmexDataClientConfig( - testnet=True, # API credentials loaded from BITMEX_TESTNET_API_KEY and BITMEX_TESTNET_API_SECRET -) - -# Using explicit credentials -mainnet_data_config = BitmexDataClientConfig( - api_key="YOUR_API_KEY", # Or use os.getenv("BITMEX_API_KEY") - api_secret="YOUR_API_SECRET", # Or use os.getenv("BITMEX_API_SECRET") - testnet=False, -) - -mainnet_exec_config = BitmexExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - testnet=False, -) -``` - -## Trading considerations - -### Contingent orders - -The BitMEX execution adapter now maps Nautilus contingent order lists to the exchange's -native `clOrdLinkID`/`contingencyType` mechanics. When the engine submits -`ContingencyType::Oco` or `ContingencyType::Oto` orders the adapter will: - -- Create/maintain the linked order group on BitMEX so child stops and targets inherit the - parent order status. -- Propagate order list updates and cancellations so that contingent peers stay aligned with - the current position state. -- Surface execution reports with the appropriate contingency metadata, enabling strategy-level - tracking without additional manual wiring. - -This means common bracket flows (entry + stop + take-profit) and multi-leg stop structures can -now be managed directly by BitMEX instead of being emulated client-side. When defining -strategies, continue to use Nautilus `OrderList`/`ContingencyType` abstractions. The adapter -handles the required BitMEX wiring automatically. - -### Contract specifications - -- **Inverse contracts**: Settled in cryptocurrency (e.g., XBTUSD settled in XBT). -- **Linear contracts**: Settled in stablecoin (e.g., ETHUSDT settled in USDT). -- **Contract size**: Varies by instrument, check specifications carefully. -- **Tick size**: Minimum price increment varies by contract. - -### Margin requirements - -- Initial margin requirements vary by contract and market conditions. -- Maintenance margin is typically lower than initial margin. -- Liquidation occurs when maintenance margin requirement is not satisfied. -- BitMEX supports both isolated margin and cross margin modes. -- Risk limits can be adjusted based on position size per the [Exchange Rules](https://www.bitmex.com/exchange-rules). - -### Fees - -- **Maker fees**: Typically negative (rebate) for providing liquidity. -- **Taker fees**: Positive fee for taking liquidity. -- **Funding rates**: Apply to perpetual contracts every 8 hours. -- **Prediction market fees**: Maker 0.00%, Taker 0.25% (no leverage allowed). - -## Contributing - -:::info -For additional features or to contribute to the BitMEX adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/blockchain.md b/nautilus-docs/docs/integrations/blockchain.md deleted file mode 100644 index c116184..0000000 --- a/nautilus-docs/docs/integrations/blockchain.md +++ /dev/null @@ -1,92 +0,0 @@ -# Blockchain - -## Core Primitives - -Nautilus Trader's blockchain integration is built on foundational primitives defined in the DeFi domain model (`nautilus_model::defi`). These building blocks provide type-safe abstractions for working with EVM-based blockchains. - -### Chain - -The `Chain` struct represents a blockchain network with its connection endpoints and metadata. Each chain instance contains: - -**Fields:** - -- `name` (`Blockchain`): The blockchain network type (enum of 80+ supported chains) -- `chain_id` (`u32`): Unique EVM chain identifier (e.g., 1 for Ethereum, 42161 for Arbitrum) -- `hypersync_url` (`String`): Endpoint for high-performance Hypersync data streaming -- `rpc_url` (`Option`): Optional HTTP/WSS RPC endpoint for direct node communication -- `native_currency_decimals` (`u8`): Decimal precision for the chain's native gas token (typically 18) - -**Chain Retrieval:** - -Chains can be retrieved by numeric ID or string name (case-insensitive): - -- **By Chain ID:** Lookup using EVM chain identifier with `from_chain_id` -- **By Name:** Lookup using blockchain name (case-insensitive: "ethereum", "Ethereum", "ETHEREUM" all work) with `from_chain_name` -- **Static Instances:** Pre-configured chains available as constants - -Each chain has a native currency used for gas fees. The `native_currency()` method returns a properly configured Currency instance: - -| Chain Family | Code | Name | Decimals | -|-------------------------------------------------|------|--------------|----------| -| Ethereum & L2s (Arbitrum, Base, Optimism, etc.) | ETH | Ethereum | 18 | -| Polygon | POL | Polygon | 18 | -| Avalanche | AVAX | Avalanche | 18 | -| BSC | BNB | Binance Coin | 18 | - -## Contracts - -High-performance interface for querying EVM smart contracts with type-safe Rust abstractions. Supports token metadata, DEX pools, and DeFi protocols through efficient batch operations. - -### Base (Multicall3) - -Batches multiple contract calls into a single RPC request using Multicall3 (`0xcA11bde05977b3631167028862bE2a173976CA11`). - -- Always uses `allow_failure: true` for partial success and detailed errors -- Executes atomically in the same block -- Errors: `RpcError` (network issues), `AbiDecodingError` (decode failures) - -### ERC20 - -Inherits from `BaseContract` to use Multicall3 for efficient batch operations. Fetches token metadata, handling non-standard implementations. - -**Methods:** - -- `fetch_token_info`: Single token metadata (uses multicall internally for name, symbol, decimals) -- `batch_fetch_token_info`: Multiple tokens in one multicall (3 calls per token) -- `enforce_token_fields`: Validate non-empty name/symbol - -**Error Types:** - -1. **`CallFailed`** - Contract missing or function not implemented → Skip token -2. **`DecodingError`** - Raw bytes instead of ABI encoding (e.g., `0x5269636f...`) → Skip token -3. **`EmptyTokenField`** - Function returns empty string → Skip if enforced - -**Best Practices:** - -- Skip pools with any token errors -- `raw_data` field preserves original response for debugging -- Non-standard tokens often have other issues (transfer fees, rebasing) - -## Configuration - -| Option | Default | Description | -|---------------------------------|---------|-------------| -| `chain` | Required | `nautilus_trader.model.Chain` to synchronize (e.g., `Chain.ETHEREUM`). | -| `dex_ids` | Required | Sequence of `DexType` identifiers describing which DEX integrations to enable. | -| `http_rpc_url` | Required | HTTPS RPC endpoint used for EVM calls and Multicall requests. | -| `wss_rpc_url` | `None` | Optional WSS endpoint for streaming live updates. | -| `rpc_requests_per_second` | `None` | Optional throttle for outbound RPC calls (requests per second). | -| `multicall_calls_per_rpc_request` | `200` | Maximum number of Multicall targets batched per RPC request. | -| `use_hypersync_for_live_data` | `True` | When `True`, bootstrap and stream using Hypersync for lower-latency diffs. | -| `from_block` | `None` | Optional starting block height for historical backfill. | -| `pool_filters` | `DexPoolFilters()` | Filtering rules applied when selecting DEX pools to monitor. | -| `postgres_cache_database_config`| `None` | Optional `PostgresConnectOptions` enabling on-disk caching of decoded pool state. | -| `http_proxy_url` | `None` | Reserved for future use; not yet configurable via the constructor. | -| `ws_proxy_url` | `None` | Reserved for future use; not yet configurable via the constructor. | - -## Contributing - -:::info -For additional features or to contribute to the Blockchain adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/bybit.md b/nautilus-docs/docs/integrations/bybit.md deleted file mode 100644 index 7fecb58..0000000 --- a/nautilus-docs/docs/integrations/bybit.md +++ /dev/null @@ -1,673 +0,0 @@ -# Bybit - -Founded in 2018, Bybit is one of the largest cryptocurrency exchanges in terms -of daily trading volume, and open interest of crypto assets and crypto -derivative products. This integration supports live market data ingest and order -execution with Bybit. - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/bybit/). - -## Overview - -This guide assumes a trader is setting up for both live market data feeds, and trade execution. -The Bybit adapter includes multiple components, which can be used together or separately depending -on the use case. - -- `BybitHttpClient`: Low-level HTTP API connectivity. -- `BybitWebSocketClient`: Low-level WebSocket API connectivity. -- `BybitInstrumentProvider`: Instrument parsing and loading functionality. -- `BybitDataClient`: A market data feed manager. -- `BybitExecutionClient`: An account management and trade execution gateway. -- `BybitLiveDataClientFactory`: Factory for Bybit data clients (used by the trading node builder). -- `BybitLiveExecClientFactory`: Factory for Bybit execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), -and won't need to necessarily work with these lower level components directly. -::: - -## Bybit documentation - -Bybit provides extensive documentation for users which can be found in the [Bybit help center](https://www.bybit.com/en/help-center). -It’s recommended you also refer to the Bybit documentation in conjunction with this NautilusTrader integration guide. - -## Products - -A product is an umbrella term for a group of related instrument types. - -:::note -Product is also referred to as `category` in the Bybit v5 API. -::: - -The following product types are supported on Bybit: - -| Product Type | Supported | Notes | -|-----------------------------|-----------|------------------------------------------| -| Spot cryptocurrencies | ✓ | Native spot markets with margin support. | -| Linear perpetual contracts | ✓ | USDT/USDC margined perpetual swaps. | -| Linear futures contracts | ✓ | Delivery-settled linear futures. | -| Inverse perpetual contracts | ✓ | Coin-margined perpetual swaps. | -| Inverse futures contracts | ✓ | Coin-margined delivery futures. | -| Option contracts | ✓ | USDC-settled options. | - -## Symbology - -To distinguish between different product types on Bybit, Nautilus uses specific product category suffixes for symbols: - -- `-SPOT`: Spot cryptocurrencies -- `-LINEAR`: Perpetual and futures contracts -- `-INVERSE`: Inverse perpetual and inverse futures contracts -- `-OPTION`: Option contracts - -These suffixes must be appended to the Bybit raw symbol string to identify the specific product type -for the instrument ID. For example: - -- The Ether/Tether spot currency pair is identified with `-SPOT`, such as `ETHUSDT-SPOT`. -- The BTCUSDT perpetual futures contract is identified with `-LINEAR`, such as `BTCUSDT-LINEAR`. -- The BTCUSD inverse perpetual futures contract is identified with `-INVERSE`, such as `BTCUSD-INVERSE`. - -## Environments - -Bybit provides three trading environments. Configure the appropriate environment using the `demo` and `testnet` flags in your client configuration. - -| Environment | Config | Description | -|--------------|-------------------------------|------------------------------------------------------------------| -| **Mainnet** | `demo=False, testnet=False` | Production trading with real funds. | -| **Demo** | `demo=True` | Practice trading with simulated funds on mainnet infrastructure. | -| **Testnet** | `testnet=True` | Separate test network for development and integration testing. | - -### Mainnet (Production) - -The default environment for live trading with real funds. - -```python -config = BybitExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - # demo=False (default) - # testnet=False (default) -) -``` - -Environment variables: `BYBIT_API_KEY`, `BYBIT_API_SECRET` - -### Demo trading - -Demo trading uses Bybit's mainnet infrastructure with simulated funds. Create demo API keys from the [Bybit demo trading page](https://www.bybit.com/en/demo-trading). - -```python -config = BybitExecClientConfig( - api_key="YOUR_DEMO_API_KEY", - api_secret="YOUR_DEMO_API_SECRET", - demo=True, -) -``` - -Environment variables: `BYBIT_DEMO_API_KEY`, `BYBIT_DEMO_API_SECRET` - -:::warning -**Demo environment limitations:** - -- The WebSocket Trade API is **not supported** for demo trading. NautilusTrader automatically uses the HTTP REST API for order operations in demo mode. -- Some advanced order features available via WebSocket (trigger orders, post-only with is_quote_quantity) are not available in demo mode. - -::: - -### Testnet - -A separate test network for development. Create testnet API keys from [testnet.bybit.com](https://testnet.bybit.com). - -```python -config = BybitExecClientConfig( - api_key="YOUR_TESTNET_API_KEY", - api_secret="YOUR_TESTNET_API_SECRET", - testnet=True, -) -``` - -Environment variables: `BYBIT_TESTNET_API_KEY`, `BYBIT_TESTNET_API_SECRET` - -:::note -Testnet supports all trading features including the WebSocket Trade API. It uses completely separate infrastructure from mainnet, so market data and liquidity differ significantly from production. -::: - -### Environment priority - -If both `demo` and `testnet` are set to `True`, demo takes priority. - -## Orders capability - -Bybit offers a flexible combination of trigger types, enabling a broader range of Nautilus orders. -All the order types listed below can be used as *either* entries or exits, except for trailing stops -(which use a position-related API). - -### Order types - -| Order Type | Spot | Linear | Inverse | Notes | -|------------------------|------|--------|---------|---------------------------| -| `MARKET` | ✓ | ✓ | ✓ | Supports quote quantity. | -| `LIMIT` | ✓ | ✓ | ✓ | | -| `STOP_MARKET` | ✓ | ✓ | ✓ | | -| `STOP_LIMIT` | ✓ | ✓ | ✓ | | -| `MARKET_IF_TOUCHED` | ✓ | ✓ | ✓ | | -| `LIMIT_IF_TOUCHED` | ✓ | ✓ | ✓ | | -| `TRAILING_STOP_MARKET` | - | ✓ | ✓ | *Not supported for Spot*. | - -### Execution instructions - -| Instruction | Spot | Linear | Inverse | Notes | -|---------------|------|--------|---------|------------------------------------| -| `post_only` | ✓ | ✓ | ✓ | Only supported on `LIMIT` orders. | -| `reduce_only` | - | ✓ | ✓ | *Not supported for Spot*. | - -### Time in force - -| Time in force | Spot | Linear | Inverse | Notes | -|---------------|------|--------|---------|------------------------------| -| `GTC` | ✓ | ✓ | ✓ | Good Till Canceled. | -| `GTD` | - | - | - | *Not supported*. | -| `FOK` | ✓ | ✓ | ✓ | Fill or Kill. | -| `IOC` | ✓ | ✓ | ✓ | Immediate or Cancel. | - -### Advanced order features - -| Feature | Spot | Linear | Inverse | Notes | -|--------------------|------|--------|---------|----------------------------------------| -| Order Modification | ✓ | ✓ | ✓ | Price and quantity modification. | -| Bracket/OCO Orders | ✓ | ✓ | ✓ | UI only; API users implement manually. | -| Iceberg Orders | ✓ | ✓ | ✓ | Max 10 per account, 1 per symbol. | - -### Batch operations - -| Operation | Spot | Linear | Inverse | Notes | -|--------------------|------|--------|---------|-------------------------------------------| -| Batch Submit | ✓ | ✓ | ✓ | Submit multiple orders in single request. | -| Batch Modify | ✓ | ✓ | ✓ | Modify multiple orders in single request. | -| Batch Cancel | ✓ | ✓ | ✓ | Cancel multiple orders in single request. | - -### Position management - -| Feature | Spot | Linear | Inverse | Notes | -|---------------------|------|--------|---------|------------------------------------------| -| Query positions | - | ✓ | ✓ | Real-time position updates. | -| Position mode | - | ✓ | ✓ | One-Way vs Hedge mode. | -| Leverage control | - | ✓ | ✓ | Dynamic leverage adjustment per symbol. | -| Margin mode | - | ✓ | ✓ | Cross vs Isolated margin. | - -### Order querying - -| Feature | Spot | Linear | Inverse | Notes | -|---------------------|------|--------|---------|-----------------------------------------| -| Query open orders | ✓ | ✓ | ✓ | List all active orders. | -| Query order history | ✓ | ✓ | ✓ | Historical order data. | -| Order status updates| ✓ | ✓ | ✓ | Real-time order state changes. | -| Trade history | ✓ | ✓ | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Spot | Linear | Inverse | Notes | -|---------------------|------|--------|---------|-----------------------------------------| -| Order lists | - | - | - | *Not supported*. | -| OCO orders | ✓ | ✓ | ✓ | UI only; API users implement manually. | -| Bracket orders | ✓ | ✓ | ✓ | UI only; API users implement manually. | -| Conditional orders | ✓ | ✓ | ✓ | Stop and limit-if-touched orders. | - -### Order parameters - -Individual orders can be customized using the `params` dictionary when submitting orders: - -| Parameter | Type | Description | -|---------------|--------|--------------------------------------------------------------------------------| -| `is_leverage` | `bool` | For Spot products only. If `True`, enables margin trading (borrowing) for the order. Default: `False`. See [Bybit's isLeverage documentation](https://bybit-exchange.github.io/docs/v5/order/create-order#request-parameters). | - -#### Example: Spot margin trading - -```python -# Submit a Spot order with margin enabled -order = strategy.order_factory.market( - instrument_id=InstrumentId.from_str("BTCUSDT-SPOT.BYBIT"), - order_side=OrderSide.BUY, - quantity=Quantity.from_str("0.1"), - params={"is_leverage": True} # Enable margin for this order -) -strategy.submit_order(order) -``` - -:::note -Without `is_leverage=True` in the params, Spot orders will only use your available balance -and won't borrow funds, even if you have auto-borrow enabled on your Bybit account. -::: - -For a complete example of using order parameters including `is_leverage`, see the -[bybit_exec_tester.py](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/bybit/bybit_exec_tester.py) example. - -## Spot margin borrowing and repayment - -NautilusTrader provides automated spot margin borrow repayment functionality to prevent interest accrual after closing short positions on Bybit. - -### Background - -When trading Spot with margin enabled (`is_leverage=True`), Bybit automatically borrows coins when you execute short positions. -However, after you close the short position (BUY order fills), the borrowed coins are **NOT automatically repaid** - they continue accruing hourly interest charges until manually repaid. -This can result in significant interest costs if left unattended. - -### Automatic repayment (recommended) - -NautilusTrader automatically repays spot margin borrows immediately after BUY orders fill on Spot instruments. -This feature is **enabled by default** via the `auto_repay_spot_borrows` configuration flag. - -**How it works:** - -1. When a Spot BUY order fills, the execution client automatically attempts to repay any outstanding borrows for that coin. -2. The repayment uses Bybit's `no-convert-repay` endpoint, which repays the full outstanding borrow amount. -3. If the repayment fails (e.g., API error), it logs the error but does not crash the execution client. -4. Repayments are automatically skipped during Bybit's UTC blackout window (see below). - -**Example:** - -```python -from nautilus_trader.adapters.bybit import BybitExecClientConfig - -config = BybitExecClientConfig( - api_key="YOUR_API_KEY", - api_secret="YOUR_API_SECRET", - product_types=[BybitProductType.SPOT], - auto_repay_spot_borrows=True, # Default is True -) -``` - -### Manual margin operations - -Strategies can control margin borrowing and repayment directly via `query_account` with the -`BybitMarginAction` enum: - -| Action | Description | -|---------------------------------------|--------------------------------------| -| `BybitMarginAction.BORROW` | Borrow funds for margin trading. | -| `BybitMarginAction.REPAY` | Repay borrowed funds. | -| `BybitMarginAction.GET_BORROW_AMOUNT` | Query current borrowed amount. | - -#### Borrow - -```python -self.query_account( - account_id=self.account_id, - params={"action": BybitMarginAction.BORROW, "coin": "USDT", "amount": 1000}, -) -``` - -#### Repay - -```python -# Repay specific amount -self.query_account( - account_id=self.account_id, - params={"action": BybitMarginAction.REPAY, "coin": "USDT", "amount": 500}, -) - -# Repay all (omit amount) -self.query_account( - account_id=self.account_id, - params={"action": BybitMarginAction.REPAY, "coin": "USDT"}, -) -``` - -#### Query borrow amount - -```python -self.query_account( - account_id=self.account_id, - params={"action": BybitMarginAction.GET_BORROW_AMOUNT, "coin": "USDT"}, -) -``` - -:::note -The `account_id` can be obtained from `self.portfolio.account(BYBIT_VENUE).id` or stored -during strategy initialization via the config. -::: - -#### Receiving results - -Results are published as custom data on the message bus. Subscribe in your strategy to receive them: - -```python -from nautilus_trader.adapters.bybit import BybitMarginAction -from nautilus_trader.adapters.bybit import BybitMarginBorrowResult -from nautilus_trader.adapters.bybit import BybitMarginRepayResult -from nautilus_trader.adapters.bybit import BybitMarginStatusResult -from nautilus_trader.model.data import DataType - - -class MyStrategy(Strategy): - def on_start(self): - self.subscribe_data(DataType(BybitMarginBorrowResult)) - self.subscribe_data(DataType(BybitMarginRepayResult)) - self.subscribe_data(DataType(BybitMarginStatusResult)) - - def on_data(self, data): - if isinstance(data, BybitMarginBorrowResult): - if data.success: - self.log.info(f"Borrowed {data.amount} {data.coin}") - else: - self.log.error(f"Borrow failed: {data.message}") - elif isinstance(data, BybitMarginRepayResult): - if data.success: - self.log.info(f"Repaid {data.amount or 'all'} {data.coin}") - else: - self.log.error(f"Repay failed: {data.message}") - elif isinstance(data, BybitMarginStatusResult): - self.log.info(f"Borrow amount for {data.coin}: {data.borrow_amount}") -``` - -### UTC blackout window - -Bybit blocks `no-convert-repay` operations daily during **04:00-05:30 UTC** for interest calculation processing. NautilusTrader automatically detects this window and skips repayment attempts, logging a warning instead. - -During the blackout window, any BUY order fills will trigger a warning like: - -``` -Skipping borrow repayment for BTC due to Bybit blackout window (04:00-05:30 UTC daily). Will need manual repayment. -``` - -**Important:** If your BUY orders fill during the blackout window, you'll need to manually repay the borrows after 05:30 UTC to stop interest accrual, or wait for the next BUY order fill outside the blackout window. - -### Configuration options - -| Option | Type | Default | Description | -|---------------------------|--------|---------|-----------------------------------------------------------------------------| -| `auto_repay_spot_borrows` | `bool` | `True` | If `True`, automatically repay spot margin borrows after BUY orders fill. Prevents interest accrual on borrowed coins. Repayment is skipped during blackout window. | - -### Important notes - -- Auto-repayment only triggers on **Spot BUY orders**, not derivatives. -- Repayment uses the `no-convert-repay` endpoint which repays the full outstanding borrow by default. -- The feature gracefully handles API errors and logs failures without crashing. -- Bybit is planning to release an auto-repay mode at the venue level (end of month), which may make this feature redundant in the future. -- Manual borrowing is still required before opening short positions unless auto-borrow is enabled on your Bybit account. - -### Spot trading limitations - -The following limitations apply to Spot products, as positions are not tracked on the venue side: - -- `reduce_only` orders are *not supported*. -- Trailing stop orders are *not supported*. - -### Trailing stops - -Trailing stops on Bybit do not have a client order ID on the venue side (though there is a `venue_order_id`). -This is because trailing stops are associated with a netted position for an instrument. -Consider the following points when using trailing stops on Bybit: - -- `reduce_only` instruction is available -- When the position associated with a trailing stop is closed, the trailing stop is automatically "deactivated" (closed) on the venue side. -- You cannot query trailing stop orders that are not already open (the `venue_order_id` is unknown until then). -- You can manually adjust the trigger price in the GUI, which will update the Nautilus order. - -## Funding rates - -The adapter receives funding rate data from the -[Linear Ticker](https://bybit-exchange.github.io/docs/v5/websocket/public/ticker#linear-inverse-perpetual-response) -WebSocket stream. Bybit provides the `fundingIntervalHour` field in ticker updates, -which the adapter uses to populate the `interval` field on `FundingRateUpdate`. - -The adapter caches the last known `fundingIntervalHour` per symbol so that partial -ticker updates (which may omit the field) still carry the correct interval. - -For historical funding rate requests, the adapter computes the interval from consecutive -funding timestamps. - -## Rate limiting - -Every HTTP call consumes the global token bucket as well as any keyed quota(s). When usage exceeds a bucket, requests are queued automatically, so manual throttling is rarely required. - -| Key / Endpoint | Limit (requests/sec) | Notes | -|---------------------------|----------------------|------------------------------------------------------| -| `bybit:global` | 120 | Exchange-wide 600 req / 5 s ceiling. | -| `/v5/market/kline` | 20 | Historical sweeps throttled slightly below global. | -| `/v5/market/trades` | 24 | Matches the global quota. | -| `/v5/order/create` | 10 | Standard order placement. | -| `/v5/order/cancel` | 10 | Single-order cancellation. | -| `/v5/order/create-batch` | 5 | Batch placement endpoints. | -| `/v5/order/cancel-batch` | 5 | Batch cancellation endpoints. | -| `/v5/order/cancel-all` | 2 | Full book cancel to mirror Bybit guidance. | - -:::warning -Bybit responds with error code `10016` when the rate limit is exceeded and may temporarily block the IP if requests continue without back-off. -::: - -:::info -For more details on rate limiting, see the official documentation: . -::: - -### Data clients - -If no product types are specified then all product types will be loaded and available. - -### Execution clients - -The adapter automatically determines the account type based on configured product types: - -- **Spot only**: Uses `CASH` account type with borrowing support enabled -- **Derivatives or mixed products**: Uses `MARGIN` account type (UTA - Unified Trading Account) - -This allows you to trade Spot alongside derivatives in a single Unified Trading Account, which is the standard account type for most Bybit users. - -:::info -**Unified Trading Accounts (UTA) and Spot margin trading** - -Most Bybit users now have Unified Trading Accounts (UTA) as Bybit steers new users to this account type. -Classic accounts are considered legacy. - -For Spot margin trading on UTA accounts: - -- Borrowing is **NOT automatically enabled** - it requires explicit API configuration -- To use Spot margin via API, you must submit orders with `is_leverage=True` in the parameters (see [Bybit docs](https://bybit-exchange.github.io/docs/v5/order/create-order#request-parameters)) -- If auto-borrow/auto-repay is enabled on your Bybit account, the venue will automatically borrow/repay funds for those margin orders -- Without auto-borrow enabled, you'll need to manually manage borrowing through Bybit's interface - -**Important**: The Nautilus Bybit adapter defaults to `is_leverage=False` for Spot orders, -meaning they won't use margin unless you explicitly enable it. -::: - -## Fee currency logic - -Understanding how Bybit determines the currency for trading fees is important for accurate accounting and position tracking. The fee currency rules vary between Spot and derivatives products. - -### Spot trading fees - -For Spot trading, the fee currency depends on the order side and whether the fee is a rebate (negative fee for maker orders): - -#### Normal fees (positive) - -- **BUY orders**: Fee is charged in the **base currency** (e.g., BTC for BTCUSDT) -- **SELL orders**: Fee is charged in the **quote currency** (e.g., USDT for BTCUSDT) - -#### Maker rebates (negative fees) - -When maker fees are negative (rebates), the currency logic is **inverted**: - -- **BUY orders with maker rebate**: Rebate is paid in the **quote currency** (e.g., USDT for BTCUSDT) -- **SELL orders with maker rebate**: Rebate is paid in the **base currency** (e.g., BTC for BTCUSDT) - -:::note -**Taker orders never have inverted logic**, even if the maker fee rate is negative. Taker fees always follow the normal fee currency rules. -::: - -#### Example: BTCUSDT Spot - -- **Buy 1 BTC as taker (0.1% fee)**: Pay 0.001 BTC in fees -- **Sell 1 BTC as taker (0.1% fee)**: Pay equivalent USDT in fees -- **Buy 1 BTC as maker (-0.01% rebate)**: Receive USDT rebate (inverted) -- **Sell 1 BTC as maker (-0.01% rebate)**: Receive BTC rebate (inverted) - -### Derivatives trading fees - -For all derivatives products (LINEAR, INVERSE, OPTION), fees are always charged in the **settlement currency**: - -| Product Type | Settlement Currency | Fee Currency | -|--------------|---------------------------------------|--------------| -| LINEAR | USDT (typically) | USDT | -| INVERSE | Base coin (e.g., BTC for BTCUSD) | Base coin | -| OPTION | USDC (legacy) or USDT (post Feb 2025) | USDC/USDT | - -### Fee calculation - -When the WebSocket execution message doesn't provide the exact fee amount (`execFee`), the adapter calculates fees as follows: - -#### Spot products - -- **BUY orders**: `fee = base_quantity × fee_rate` -- **SELL orders**: `fee = notional_value × fee_rate` (where `notional_value = quantity × price`) - -#### Derivatives - -- All derivatives: `fee = notional_value × fee_rate` - -### Official documentation - -For complete details on Bybit's fee structure and currency rules, refer to: - -- [Bybit WebSocket Private Execution](https://bybit-exchange.github.io/docs/v5/websocket/private/execution) -- [Bybit Spot Fee Currency Instruction](https://bybit-exchange.github.io/docs/v5/enum#spot-fee-currency-instruction) - -## Configuration - -The product types for each client must be specified in the configurations. - -### Data client configuration options - -| Option | Default | Description | -|----------------------------------|---------|-------------| -| `api_key` | `None` | API key; loaded from `BYBIT_API_KEY`/`BYBIT_TESTNET_API_KEY` when omitted. | -| `api_secret` | `None` | API secret; loaded from `BYBIT_API_SECRET`/`BYBIT_TESTNET_API_SECRET` when omitted. | -| `product_types` | `None` | Sequence of `BybitProductType` values to enable; loads all products when `None`. | -| `base_url_http` | `None` | Override for the REST base URL. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Optional WebSocket proxy URL (not yet implemented). | -| `demo` | `False` | Connect to the Bybit demo environment when `True`. | -| `testnet` | `False` | Connect to the Bybit testnet when `True`. | -| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalogue refreshes. | -| `recv_window_ms` | `5,000`| Receive window (milliseconds) for signed REST requests. | -| `bars_timestamp_on_close` | `True` | Timestamp bars on the close (`True`) or open (`False`) of the interval. | -| `max_retries` | `None` | Maximum retry attempts for REST/WebSocket recovery. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | - -### Execution client configuration options - -| Option | Default | Description | -|----------------------------------|---------|-------------| -| `api_key` | `None` | API key; loaded from `BYBIT_API_KEY`/`BYBIT_TESTNET_API_KEY` when omitted. | -| `api_secret` | `None` | API secret; loaded from `BYBIT_API_SECRET`/`BYBIT_TESTNET_API_SECRET` when omitted. | -| `product_types` | `None` | Sequence of `BybitProductType` values to enable (Spot cannot be mixed with derivatives for execution). | -| `base_url_http` | `None` | Override for the REST base URL. | -| `base_url_ws_private` | `None` | Override for the private WebSocket base URL. | -| `base_url_ws_trade` | `None` | Override for the trade WebSocket base URL. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Optional WebSocket proxy URL (not yet implemented). | -| `demo` | `False` | Connect to the Bybit demo environment when `True`. | -| `testnet` | `False` | Connect to the Bybit testnet when `True`. | -| `use_gtd` | `False` | Remap GTD orders to GTC when `True` (Bybit lacks native GTD support). | -| `use_ws_execution_fast` | `False` | Subscribe to the low-latency execution stream. | -| `use_http_batch_api` | `False` | Use Bybit's HTTP batch trading API (deprecated). | -| `use_spot_position_reports` | `False` | Report Spot wallet balances as positions when `True`. | -| `auto_repay_spot_borrows` | `True` | Automatically repay Spot margin borrows after BUY orders fully fill (Spot only). | -| `repay_queue_interval_secs` | `1.0` | Interval (seconds) between processing repayment queues for spot borrows. | -| `ignore_uncached_instrument_executions` | `False` | Ignore execution messages for instruments not yet cached. | -| `max_retries` | `None` | Maximum retry attempts for order submission/cancel/modify calls. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | -| `recv_window_ms` | `5,000`| Receive window (milliseconds) for signed REST requests. | -| `ws_trade_timeout_secs` | `5.0` | Timeout (seconds) waiting for trade WebSocket acknowledgements. | -| `ws_auth_timeout_secs` | `5.0` | Timeout (seconds) waiting for auth WebSocket acknowledgements. | -| `futures_leverages` | `None` | Mapping of `BybitSymbol` to leverage settings. | -| `position_mode` | `None` | Mapping of `BybitSymbol` to position mode (one-way vs hedge). | -| `margin_mode` | `None` | Margin mode setting for the account. | - -The most common use case is to configure a live `TradingNode` to include Bybit -data and execution clients. To achieve this, add a `BYBIT` section to your client -configuration(s): - -```python -from nautilus_trader.adapters.bybit import BYBIT -from nautilus_trader.adapters.bybit import BybitProductType -from nautilus_trader.live.node import TradingNode - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - BYBIT: { - "api_key": "YOUR_BYBIT_API_KEY", - "api_secret": "YOUR_BYBIT_API_SECRET", - "base_url_http": None, # Override with custom endpoint - "product_types": [BybitProductType.LINEAR], - "testnet": False, - }, - }, - exec_clients={ - BYBIT: { - "api_key": "YOUR_BYBIT_API_KEY", - "api_secret": "YOUR_BYBIT_API_SECRET", - "base_url_http": None, # Override with custom endpoint - "product_types": [BybitProductType.LINEAR], - "testnet": False, - }, - }, -) -``` - -Then, create a `TradingNode` and add the client factories: - -```python -from nautilus_trader.adapters.bybit import BYBIT -from nautilus_trader.adapters.bybit import BybitLiveDataClientFactory -from nautilus_trader.adapters.bybit import BybitLiveExecClientFactory -from nautilus_trader.live.node import TradingNode - -# Instantiate the live trading node with a configuration -node = TradingNode(config=config) - -# Register the client factories with the node -node.add_data_client_factory(BYBIT, BybitLiveDataClientFactory) -node.add_exec_client_factory(BYBIT, BybitLiveExecClientFactory) - -# Finally build the node -node.build() -``` - -### API credentials - -There are two options for supplying your credentials to the Bybit clients. -Either pass the corresponding `api_key` and `api_secret` values to the configuration objects, or -set the following environment variables: - -For Bybit live clients, you can set: - -- `BYBIT_API_KEY` -- `BYBIT_API_SECRET` - -For Bybit demo clients, you can set: - -- `BYBIT_DEMO_API_KEY` -- `BYBIT_DEMO_API_SECRET` - -For Bybit testnet clients, you can set: - -- `BYBIT_TESTNET_API_KEY` -- `BYBIT_TESTNET_API_SECRET` - -:::tip -We recommend using environment variables to manage your credentials. -::: - -When starting the trading node, you'll receive immediate confirmation of whether your -credentials are valid and have trading permissions. - -## Contributing - -:::info -For additional features or to contribute to the Bybit adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/databento.md b/nautilus-docs/docs/integrations/databento.md deleted file mode 100644 index 09172d2..0000000 --- a/nautilus-docs/docs/integrations/databento.md +++ /dev/null @@ -1,881 +0,0 @@ -# Databento - -NautilusTrader includes an adapter for the [Databento](https://databento.com/) API and -[Databento Binary Encoding (DBN)](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) format data. -Databento is a market data provider only. The adapter does not include an execution client, -but you can pair it with a sandbox for simulated execution. -You can also match Databento data with Interactive Brokers execution, -or calculate traditional asset class signals for crypto trading. - -The adapter supports: - -- Loading historical data from DBN files and decoding to Nautilus objects for backtesting or catalog storage. -- Requesting historical data decoded to Nautilus objects for live trading and backtesting. -- Subscribing to real-time data feeds decoded to Nautilus objects for live trading and sandbox environments. - -:::tip -[Databento](https://databento.com/signup) offers 125 USD in free data credits (historical only) for new sign-ups. - -With careful requests, this covers testing and evaluation. -Check the [/metadata.get_cost](https://databento.com/docs/api-reference-historical/metadata/metadata-get-cost) -endpoint before requesting data. -::: - -## Overview - -The adapter uses the [databento-rs](https://crates.io/crates/databento) crate, -Databento's official Rust client library. - -:::info -No separate `databento` installation is needed. The adapter compiles as a static -library and links automatically during the build. -::: - -The following adapter classes are available: - -- `DatabentoDataLoader`: Loads DBN data from files. -- `DatabentoInstrumentProvider`: Fetches latest or historical instrument definitions via the Databento HTTP API. -- `DatabentoHistoricalClient`: Fetches historical market data via the Databento HTTP API. -- `DatabentoLiveClient`: Subscribes to real-time data feeds via Databento's raw TCP API. -- `DatabentoDataClient`: `LiveMarketDataClient` implementation for live trading nodes. - -:::info -Most users configure a live trading node (covered below) and do not work with -these components directly. -::: - -## Examples - -Live example scripts are available [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/databento/). - -## Databento documentation - -See the [Databento new users guide](https://databento.com/docs/quickstart/new-user-guides). -Refer to it alongside this integration guide. - -## Databento Binary Encoding (DBN) - -Databento Binary Encoding (DBN) is a fast message encoding and storage format for -normalized market data. The [DBN specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) -includes a self-describing metadata header and a fixed set of struct definitions -that standardize how market data is normalized. - -The adapter decodes DBN data to Nautilus objects. The same Rust decoder handles: - -- Loading and decoding DBN files from disk. -- Decoding historical and live data in real time. - -## Supported schemas - -The following Databento schemas are supported by NautilusTrader: - -| Databento schema | Nautilus data type | Description | -|:------------------------------------------------------------------------------|:----------------------------------|:--------------------------------| -| [MBO](https://databento.com/docs/schemas-and-data-formats/mbo) | `OrderBookDelta` | Market by order (L3). | -| [MBP_1](https://databento.com/docs/schemas-and-data-formats/mbp-1) | `(QuoteTick, TradeTick \| None)` | Market by price (L1). | -| [MBP_10](https://databento.com/docs/schemas-and-data-formats/mbp-10) | `OrderBookDepth10` | Market depth (L2). | -| [BBO_1S](https://databento.com/docs/schemas-and-data-formats/bbo-1s) | `QuoteTick` | 1-second best bid/offer. | -| [BBO_1M](https://databento.com/docs/schemas-and-data-formats/bbo-1m) | `QuoteTick` | 1-minute best bid/offer. | -| [CMBP_1](https://databento.com/docs/schemas-and-data-formats/cmbp-1) | `(QuoteTick, TradeTick \| None)` | Consolidated MBP across venues. | -| [CBBO_1S](https://databento.com/docs/schemas-and-data-formats/cbbo-1s) | `QuoteTick` | Consolidated 1-second BBO. | -| [CBBO_1M](https://databento.com/docs/schemas-and-data-formats/cbbo-1m) | `QuoteTick` | Consolidated 1-minute BBO. | -| [TCBBO](https://databento.com/docs/schemas-and-data-formats/tcbbo) | `(QuoteTick, TradeTick)` | Trade-sampled consolidated BBO. | -| [TBBO](https://databento.com/docs/schemas-and-data-formats/tbbo) | `(QuoteTick, TradeTick)` | Trade-sampled best bid/offer. | -| [TRADES](https://databento.com/docs/schemas-and-data-formats/trades) | `TradeTick` | Trade ticks. | -| [OHLCV_1S](https://databento.com/docs/schemas-and-data-formats/ohlcv-1s) | `Bar` | 1-second bars. | -| [OHLCV_1M](https://databento.com/docs/schemas-and-data-formats/ohlcv-1m) | `Bar` | 1-minute bars. | -| [OHLCV_1H](https://databento.com/docs/schemas-and-data-formats/ohlcv-1h) | `Bar` | 1-hour bars. | -| [OHLCV_1D](https://databento.com/docs/schemas-and-data-formats/ohlcv-1d) | `Bar` | Daily bars. | -| [OHLCV_EOD](https://databento.com/docs/schemas-and-data-formats/ohlcv-eod) | `Bar` | End-of-day bars. | -| [DEFINITION](https://databento.com/docs/schemas-and-data-formats/definition) | `Instrument` (various types) | Instrument definitions. | -| [IMBALANCE](https://databento.com/docs/schemas-and-data-formats/imbalance) | `DatabentoImbalance` | Auction imbalance data. | -| [STATISTICS](https://databento.com/docs/schemas-and-data-formats/statistics) | `DatabentoStatistics` | Market statistics. | -| [STATUS](https://databento.com/docs/schemas-and-data-formats/status) | `InstrumentStatus` | Market status updates. | - -### Schema considerations - -- **TBBO and TCBBO**: Trade-sampled feeds that pair every trade with the BBO immediately *before* the trade's effect (TBBO per-venue, TCBBO consolidated). Use when you need trades aligned with contemporaneous quotes without managing two streams. -- **MBP-1 and CMBP-1 (L1)**: Event-level updates; emit trades only on trade events. Choose for a complete top-of-book event tape. For quote+trade alignment, prefer TBBO/TCBBO; otherwise use TRADES. -- **MBP-10 (L2)**: Top 10 levels with trades. Lighter than MBO for depth-aware strategies. Includes orders per level. -- **MBO (L3)**: Per-order events for queue position modeling and exact book reconstruction. Highest volume/cost; start at node initialization for proper replay context. -- **BBO_1S/BBO_1M and CBBO_1S/CBBO_1M**: Sampled top-of-book quotes at fixed intervals (1s/1m), no trades. Good for monitoring, spreads, and low-cost signals. Not suited for microstructure work. -- **TRADES**: Trades only. Pair with MBP-1 (`include_trades=True`) or use TBBO/TCBBO for quote context with trades. -- **OHLCV_ (incl. OHLCV_EOD)**: Aggregated bars from trades. Use for higher-timeframe analytics. Set `bars_timestamp_on_close=True` for close timestamps. -- **Imbalance / Statistics / Status**: Venue operational data; subscribe via `subscribe_data` with a `DataType` carrying `instrument_id` metadata. - -:::tip -Consolidated schemas (CMBP_1, CBBO_1S, CBBO_1M, TCBBO) aggregate data across -multiple venues. Useful for cross-venue analysis. -::: - -:::info -See also the Databento [Schemas and data formats](https://databento.com/docs/schemas-and-data-formats) guide. -::: - -## Schema selection for live subscriptions - -Nautilus subscription methods map to Databento schemas as follows: - -| Nautilus Subscription Method | Default Schema | Available Databento Schemas | Nautilus Data Type | -|:--------------------------------|:---------------|:-----------------------------------------------------------------------------|:-------------------| -| `subscribe_quote_ticks()` | `mbp-1` | `mbp-1`, `bbo-1s`, `bbo-1m`, `cmbp-1`, `cbbo-1s`, `cbbo-1m`, `tbbo`, `tcbbo` | `QuoteTick` | -| `subscribe_trade_ticks()` | `trades` | `trades`, `tbbo`, `tcbbo`, `mbp-1`, `cmbp-1` | `TradeTick` | -| `subscribe_order_book_depth()` | `mbp-10` | `mbp-10` | `OrderBookDepth10` | -| `subscribe_order_book_deltas()` | `mbo` | `mbo` | `OrderBookDeltas` | -| `subscribe_bars()` | varies | `ohlcv-1s`, `ohlcv-1m`, `ohlcv-1h`, `ohlcv-1d` | `Bar` | - -:::note -The examples below assume a `Strategy` or `Actor` context where `self` has -subscription methods. Import the required types: - -```python -from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID -from nautilus_trader.model import BarType -from nautilus_trader.model.enums import BookType -from nautilus_trader.model.identifiers import InstrumentId -``` - -::: - -### Quote subscriptions (MBP / L1) - -```python -# Default MBP-1 quotes (may include trades) -self.subscribe_quote_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) - -# Explicit MBP-1 schema -self.subscribe_quote_ticks( - instrument_id=instrument_id, - params={"schema": "mbp-1"}, - client_id=DATABENTO_CLIENT_ID, -) - -# 1-second BBO snapshots (quotes only, no trades) -self.subscribe_quote_ticks( - instrument_id=instrument_id, - params={"schema": "bbo-1s"}, - client_id=DATABENTO_CLIENT_ID, -) - -# Consolidated quotes across venues -self.subscribe_quote_ticks( - instrument_id=instrument_id, - params={"schema": "cbbo-1s"}, # or "cmbp-1" for consolidated MBP - client_id=DATABENTO_CLIENT_ID, -) - -# Trade-sampled BBO (includes both quotes AND trades) -self.subscribe_quote_ticks( - instrument_id=instrument_id, - params={"schema": "tbbo"}, # Will receive both QuoteTick and TradeTick onto the message bus - client_id=DATABENTO_CLIENT_ID, -) -``` - -### Trade subscriptions - -```python -# Trade ticks only -self.subscribe_trade_ticks(instrument_id, client_id=DATABENTO_CLIENT_ID) - -# Trades from MBP-1 feed (only when trade events occur) -self.subscribe_trade_ticks( - instrument_id=instrument_id, - params={"schema": "mbp-1"}, - client_id=DATABENTO_CLIENT_ID, -) - -# Trade-sampled data (includes quotes at trade time) -self.subscribe_trade_ticks( - instrument_id=instrument_id, - params={"schema": "tbbo"}, # Also provides quotes at trade events - client_id=DATABENTO_CLIENT_ID, -) -``` - -### Order book depth subscriptions (MBP / L2) - -```python -# Subscribe to top 10 levels of market depth -self.subscribe_order_book_depth( - instrument_id=instrument_id, - depth=10 # MBP-10 schema is automatically selected -) - -# The depth parameter must be 10 for Databento -# This will receive OrderBookDepth10 updates -``` - -### Order book deltas subscriptions (MBO / L3) - -```python -# Subscribe to full order book updates (market by order) -self.subscribe_order_book_deltas( - instrument_id=instrument_id, - book_type=BookType.L3_MBO # Uses MBO schema -) - -# Note: MBO subscriptions must be made at node startup for Databento -# to ensure proper replay from session start -``` - -### Bar subscriptions - -```python -# Subscribe to 1-minute bars (automatically uses ohlcv-1m schema) -self.subscribe_bars( - bar_type=BarType.from_str(f"{instrument_id}-1-MINUTE-LAST-EXTERNAL") -) - -# Subscribe to 1-second bars (automatically uses ohlcv-1s schema) -self.subscribe_bars( - bar_type=BarType.from_str(f"{instrument_id}-1-SECOND-LAST-EXTERNAL") -) - -# Subscribe to hourly bars (automatically uses ohlcv-1h schema) -self.subscribe_bars( - bar_type=BarType.from_str(f"{instrument_id}-1-HOUR-LAST-EXTERNAL") -) - -# Subscribe to daily bars (automatically uses ohlcv-1d schema) -self.subscribe_bars( - bar_type=BarType.from_str(f"{instrument_id}-1-DAY-LAST-EXTERNAL") -) - -# Subscribe to daily bars with end-of-day schema (only valid for DAY aggregation) -self.subscribe_bars( - bar_type=BarType.from_str(f"{instrument_id}-1-DAY-LAST-EXTERNAL"), - params={"schema": "ohlcv-eod"}, # Override to use end-of-day bars -) -``` - -### Custom data type subscriptions - -Imbalance, statistics, and status data require the generic `subscribe_data` method: - -```python -from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID -from nautilus_trader.adapters.databento import DatabentoImbalance -from nautilus_trader.adapters.databento import DatabentoStatistics -from nautilus_trader.model import DataType - -# Subscribe to imbalance data -self.subscribe_data( - data_type=DataType(DatabentoImbalance, metadata={"instrument_id": instrument_id}), - client_id=DATABENTO_CLIENT_ID, -) - -# Subscribe to statistics data -self.subscribe_data( - data_type=DataType(DatabentoStatistics, metadata={"instrument_id": instrument_id}), - client_id=DATABENTO_CLIENT_ID, -) - -# Subscribe to instrument status updates -from nautilus_trader.model.data import InstrumentStatus -self.subscribe_data( - data_type=DataType(InstrumentStatus, metadata={"instrument_id": instrument_id}), - client_id=DATABENTO_CLIENT_ID, -) -``` - -## Instrument IDs and symbology - -Databento market data includes an `instrument_id` field: an integer assigned by -the source venue or by Databento during normalization. This differs from the -Nautilus `InstrumentId`, a string of symbol + venue separated by a period: -`"{symbol}.{venue}"`. - -The decoder maps the Databento `raw_symbol` to the Nautilus `symbol` and uses an -[ISO 10383 MIC](https://www.iso20022.org/market-identifier-codes) (Market Identifier Code) from the -definition message for the Nautilus `venue`. - -Databento identifies datasets with a *dataset ID*, separate from venue identifiers. -See [Databento dataset naming conventions](https://databento.com/docs/api-reference-historical/basics/datasets) -for details. - -For CME Globex MDP 3.0 (`GLBX.MDP3`), these exchanges group under the `GLBX` venue. -The instrument's `exchange` field determines the mapping: - -- `CBCM`: XCME-XCBT inter-exchange spread -- `NYUM`: XNYM-DUMX inter-exchange spread -- `XCBT`: Chicago Board of Trade (CBOT) -- `XCEC`: Commodities Exchange Center (COMEX) -- `XCME`: Chicago Mercantile Exchange (CME) -- `XFXS`: CME FX Link spread -- `XNYM`: New York Mercantile Exchange (NYMEX) - -:::info -Other venue MICs are in the `venue` field of responses from -the [metadata.list_publishers](https://databento.com/docs/api-reference-historical/metadata/metadata-list-publishers) endpoint. -::: - -## Timestamps - -Databento data includes these timestamp fields: - -- `ts_event`: Matching-engine-received timestamp in nanoseconds since the UNIX epoch. -- `ts_in_delta`: Matching-engine-sending timestamp in nanoseconds before `ts_recv`. -- `ts_recv`: Capture-server-received timestamp in nanoseconds since the UNIX epoch. -- `ts_out`: Databento sending timestamp. - -Nautilus data requires at least two timestamps (per the `Data` contract): - -- `ts_event`: UNIX timestamp (nanoseconds) when the data event occurred. -- `ts_init`: UNIX timestamp (nanoseconds) when the data instance was created. - -The decoder maps Databento `ts_recv` to Nautilus `ts_event`. This timestamp is -more reliable and monotonically increases per instrument. The exceptions are -`DatabentoImbalance` and `DatabentoStatistics`, which carry all timestamp fields -since they are adapter-specific types. - -:::info -See the following Databento docs for further information: - -- [Databento standards and conventions - timestamps](https://databento.com/docs/standards-and-conventions/common-fields-enums-types#timestamps) -- [Databento timestamping guide](https://databento.com/docs/architecture/timestamping-guide) - -::: - -## Data types - -This section covers Databento schema to Nautilus data type mapping. - -:::info -See Databento [schemas and data formats](https://databento.com/docs/schemas-and-data-formats). -::: - -### Instrument definitions - -Databento uses a single schema for all instrument classes. The decoder maps each -to the appropriate Nautilus `Instrument` type. - -| Databento instrument class | Code | Nautilus instrument type | -|----------------------------|------|--------------------------| -| Stock | `K` | `Equity` | -| Future | `F` | `FuturesContract` | -| Call | `C` | `OptionContract` | -| Put | `P` | `OptionContract` | -| Future spread | `S` | `FuturesSpread` | -| Option spread | `T` | `OptionSpread` | -| Mixed spread | `M` | `OptionSpread` | -| FX spot | `X` | `CurrencyPair` | -| Bond | `B` | Not yet available | - -### Price precision - -Databento raw prices are fixed-point integers scaled by 1e-9. The adapter derives -price precision from the instrument's tick size in the definition message. - -For live feeds, the feed handler maintains a per-instrument precision map populated -from `InstrumentDefMsg` records as they arrive. Market data handlers look up -precision from this map. Without a prior definition, precision falls back to 2 -(USD default). - -**Instrument definitions must arrive before market data** for correct precision on -instruments with non-standard tick sizes (e.g., treasury futures with fractional -ticks like 1/256). Subscribe to `DEFINITION` schema for your instruments before -or alongside market data subscriptions. - -For historical and file-based loading, pass an explicit `price_precision` parameter -to override the default. - -:::tip -The Python adapter automatically subscribes to instrument definitions before -market data, so the precision map populates without extra configuration. For -direct Rust client usage, subscribe to `DEFINITION` schema before market data. -::: - -### MBO (market by order) - -MBO is the highest granularity data from Databento, representing full order book -depth. Some messages include trade data. The decoder produces an `OrderBookDelta` -and optionally a `TradeTick`. - -The live client buffers MBO messages until it sees an `F_LAST` flag, then passes -an `OrderBookDeltas` container to the handler. - -The client also buffers order book snapshots into `OrderBookDeltas` during the -replay startup sequence. - -### MBP-1 (market by price, top-of-book) - -MBP-1 represents top-of-book quotes and trades. Some messages carry trade data. -The decoder produces a `QuoteTick` and also a `TradeTick` when the message is -a trade. - -### TBBO and TCBBO (top-of-book with trades) - -TBBO and TCBBO provide both quote and trade data in each message. Both schemas -emit `QuoteTick` and `TradeTick` per message, more efficient than separate quote -and trade subscriptions. TCBBO provides consolidated data across venues. - -### OHLCV (bar aggregates) - -Databento timestamps bar messages at the **open** of the interval. The decoder -normalizes `ts_event` to the bar **close** (original `ts_event` + interval). - -### Imbalance & Statistics - -The `imbalance` and `statistics` schemas have no built-in Nautilus equivalents. -The adapter defines `DatabentoImbalance` and `DatabentoStatistics` in Rust. - -PyO3 bindings expose these types in Python. Their attributes are PyO3 objects -and may not be compatible with methods expecting Cython types. See the API -reference for PyO3 to Cython conversion methods. - -Convert a PyO3 `Price` to a Cython `Price`: - -```python -price = Price.from_raw(pyo3_price.raw, pyo3_price.precision) -``` - -Requesting and subscribing to these types requires the generic `subscribe_data` -method. Subscribe to `imbalance` for `AAPL.XNAS`: - -```python -from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID -from nautilus_trader.adapters.databento import DatabentoImbalance -from nautilus_trader.model import DataType - -instrument_id = InstrumentId.from_str("AAPL.XNAS") -self.subscribe_data( - data_type=DataType(DatabentoImbalance, metadata={"instrument_id": instrument_id}), - client_id=DATABENTO_CLIENT_ID, -) -``` - -Request the previous day's `statistics` for the `ES.FUT` parent symbol -(all active E-mini S&P 500 futures): - -```python -from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID -from nautilus_trader.adapters.databento import DatabentoStatistics -from nautilus_trader.model import DataType - -instrument_id = InstrumentId.from_str("ES.FUT.GLBX") -metadata = { - "instrument_id": instrument_id, - "start": "2024-03-06", -} -self.request_data( - data_type=DataType(DatabentoStatistics, metadata=metadata), - client_id=DATABENTO_CLIENT_ID, -) -``` - -### Catalog persistence - -Both types support Arrow serialization for catalog storage. The Arrow serializers -register automatically when you import the adapter package. - -#### Writing to the catalog - -```python -from nautilus_trader.adapters.databento import DatabentoDataLoader -from nautilus_trader.model.identifiers import InstrumentId -from nautilus_trader.persistence.catalog import ParquetDataCatalog - -catalog = ParquetDataCatalog.from_env() -loader = DatabentoDataLoader() - -imbalances = loader.from_dbn_file( - path="aapl-imbalance.dbn.zst", - instrument_id=InstrumentId.from_str("AAPL.XNAS"), - as_legacy_cython=False, # Required for Databento-specific types -) - -catalog.write_data(imbalances) -``` - -#### Reading from the catalog - -```python -from nautilus_trader.adapters.databento import DatabentoImbalance - -results = catalog.query(DatabentoImbalance, identifiers=["AAPL.XNAS"]) - -for imbalance in results: - print(imbalance.ref_price) # DatabentoImbalance fields -``` - -:::warning -Catalog persistence supports writing and querying these types, but streaming -them through `BacktestNode` or `BacktestEngine` is not yet supported. For -backtesting with imbalance or statistics data, query the catalog directly and -process the results in your strategy or analysis code. -::: - -#### Encoding and decoding in Rust - -The `nautilus_databento::arrow` module provides Arrow record batch encoding and -decoding. Requires the `arrow` feature flag. - -```rust -use nautilus_databento::arrow::imbalance::{ - decode_imbalance_batch, - imbalance_to_arrow_record_batch, -}; - -let batch = imbalance_to_arrow_record_batch(imbalances)?; - -let metadata = batch.schema().metadata().clone(); -let decoded = decode_imbalance_batch(&metadata, batch)?; -``` - -The `statistics` module follows the same pattern with -`decode_statistics_batch` and `statistics_to_arrow_record_batch`. - -## Performance considerations - -Two options for backtesting with DBN data: - -- Store data as DBN (`.dbn.zst`) files and decode to Nautilus objects every run. -- Convert DBN files to Nautilus objects once and write to the data catalog (Nautilus Parquet format). - -The DBN decoder is optimized Rust, but writing to the catalog once gives the -best backtest performance. - -[DataFusion](https://arrow.apache.org/datafusion/) streams Nautilus Parquet data -from disk at high throughput, at least an order of magnitude faster than -decoding DBN per run. - -:::note -Performance benchmarks are under development. -::: - -## Loading DBN data - -The `DatabentoDataLoader` class loads DBN files and converts records to Nautilus -objects. Two primary uses: - -- Pass data to `BacktestEngine.add_data` for backtesting. -- Write data to `ParquetDataCatalog` for streaming with a `BacktestNode`. - -### DBN data to a BacktestEngine - -Load DBN data and pass to a `BacktestEngine`. The engine requires an instrument. -This example uses `TestInstrumentProvider` (an instrument parsed from a DBN -file also works). The data covers one month of TSLA trades on Nasdaq: - -```python -# Add instrument -TSLA_NASDAQ = TestInstrumentProvider.equity(symbol="TSLA") -engine.add_instrument(TSLA_NASDAQ) - -# Decode data to legacy Cython objects -loader = DatabentoDataLoader() -trades = loader.from_dbn_file( - path=TEST_DATA_DIR / "databento" / "temp" / "tsla-xnas-20240107-20240206.trades.dbn.zst", - instrument_id=TSLA_NASDAQ.id, -) - -# Add data -engine.add_data(trades) -``` - -### DBN data to a ParquetDataCatalog - -Load DBN data and write to a `ParquetDataCatalog`. Set `as_legacy_cython=False` -to decode as PyO3 objects. Cython objects also work with `write_data` but require -conversion under the hood, so PyO3 objects are faster. - -### Loading instruments - -**Important**: Load instrument definitions from DEFINITION schema files before -loading market data into a catalog. The catalog requires instruments before it -can store market data. Market data files do not contain instrument definitions. - -```python -# Initialize the catalog interface -# (will use the `NAUTILUS_PATH` env var as the path) -catalog = ParquetDataCatalog.from_env() - -loader = DatabentoDataLoader() - -# Step 1: Load instrument definitions FIRST -# You must obtain DEFINITION schema files from Databento for your instruments -instruments = loader.from_dbn_file( - path=TEST_DATA_DIR / "databento" / "temp" / "tsla-xnas-definition.dbn.zst", - as_legacy_cython=False, # Use PyO3 for optimal performance -) - -# Write instruments to catalog -catalog.write_data(instruments) - -# Step 2: Now load and write market data -instrument_id = InstrumentId.from_str("TSLA.XNAS") - -# Decode trades to pyo3 objects -trades = loader.from_dbn_file( - path=TEST_DATA_DIR / "databento" / "temp" / "tsla-xnas-20240107-20240206.trades.dbn.zst", - instrument_id=instrument_id, - as_legacy_cython=False, # This is an optimization for writing to the catalog -) - -# Write market data -catalog.write_data(trades) -``` - -#### Loading multiple data types for backtesting - -Always load instruments before market data: - -```python -from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader -from nautilus_trader.model.identifiers import InstrumentId -from nautilus_trader.persistence.catalog import ParquetDataCatalog - -catalog = ParquetDataCatalog.from_env() -loader = DatabentoDataLoader() - -# Step 1: Load instrument definitions from DEFINITION files -instruments = loader.from_dbn_file( - path="equity-definitions.dbn.zst", - as_legacy_cython=False, -) -catalog.write_data(instruments) - -# Step 2: Load market data (MBO, trades, quotes, etc.) -instrument_id = InstrumentId.from_str("AAPL.XNAS") - -# Load MBO order book deltas -deltas = loader.from_dbn_file( - path="aapl-mbo.dbn.zst", - instrument_id=instrument_id, # Optional but improves performance - as_legacy_cython=False, -) -catalog.write_data(deltas) - -# Load trades -trades = loader.from_dbn_file( - path="aapl-trades.dbn.zst", - instrument_id=instrument_id, - as_legacy_cython=False, -) -catalog.write_data(trades) - -# Verify instruments are in the catalog -print(catalog.instruments()) # Should show your loaded instruments -``` - -:::tip -Call `catalog.instruments()` to verify. An empty list means you need to load -DEFINITION files first. -::: - -:::info -Download DEFINITION schema files through the Databento API or CLI for your -symbols and date ranges. See the -[Databento documentation](https://databento.com/docs/api-reference-historical/timeseries/timeseries-get-range) -for details. -::: - -:::info -See also the [Data concepts guide](../concepts/data.md). -::: - -### Historical loader options - -Parameters for `from_dbn_file`: - -- `instrument_id`: Speeds up decoding by skipping symbology lookup. -- `price_precision`: Overrides the default price precision. -- `include_trades`: For MBP-1/CMBP-1 schemas, `True` emits both `QuoteTick` and `TradeTick` when trade data is present. -- `as_legacy_cython`: Set to `False` for IMBALANCE/STATISTICS schemas (required) or for better catalog write performance. - -:::warning -IMBALANCE and STATISTICS schemas require `as_legacy_cython=False` (PyO3-only -types). `True` raises a `ValueError`. -::: - -### Loading consolidated data - -Consolidated schemas aggregate data across multiple venues: - -```python -# Load consolidated MBP-1 quotes -loader = DatabentoDataLoader() -cmbp_quotes = loader.from_dbn_file( - path="consolidated.cmbp-1.dbn.zst", - instrument_id=InstrumentId.from_str("AAPL.XNAS"), - include_trades=True, # Get both quotes and trades if available - as_legacy_cython=True, -) - -# Load consolidated BBO quotes -cbbo_quotes = loader.from_dbn_file( - path="consolidated.cbbo-1s.dbn.zst", - instrument_id=InstrumentId.from_str("AAPL.XNAS"), - as_legacy_cython=False, # Use PyO3 for better performance -) - -# Load TCBBO (trade-sampled consolidated BBO) - provides both quotes and trades -# Note: include_trades=True loads quotes, include_trades=False loads trades -tcbbo_quotes = loader.from_dbn_file( - path="consolidated.tcbbo.dbn.zst", - instrument_id=InstrumentId.from_str("AAPL.XNAS"), - include_trades=True, # Loads quotes - as_legacy_cython=True, -) - -tcbbo_trades = loader.from_dbn_file( - path="consolidated.tcbbo.dbn.zst", - instrument_id=InstrumentId.from_str("AAPL.XNAS"), - include_trades=False, # Loads trades - as_legacy_cython=True, -) -``` - -:::tip -Avoid subscribing to both TBBO/TCBBO and separate trade feeds for the same -instrument. These schemas already include trades. Duplicating wastes cost and -creates duplicate data. -::: - -## Real-time client architecture - -The `DatabentoDataClient` wraps the other Databento adapter classes. Each -dataset uses two `DatabentoLiveClient` instances: - -- One for MBO (order book deltas) real-time feeds -- One for all other real-time feeds - -:::warning -All MBO subscriptions for a dataset must be made at node startup to replay from -session start. Subscriptions after start are logged as errors and ignored. - -This limitation does not apply to other schemas. -::: - -A single `DatabentoHistoricalClient` serves both `DatabentoInstrumentProvider` -and `DatabentoDataClient` for historical requests. - -## Configuration - -Add a `DATABENTO` section to your `TradingNode` client configuration: - -```python -from nautilus_trader.adapters.databento import DATABENTO -from nautilus_trader.live.node import TradingNode - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - DATABENTO: { - "api_key": None, # 'DATABENTO_API_KEY' env var - "http_gateway": None, # Override for the default HTTP historical gateway - "live_gateway": None, # Override for the default raw TCP real-time gateway - "instrument_provider": InstrumentProviderConfig(load_all=True), - "instrument_ids": None, # Nautilus instrument IDs to load on start - "parent_symbols": None, # Databento parent symbols to load on start - }, - }, - ..., # Omitted -) -``` - -Create the `TradingNode` and register the factory: - -```python -from nautilus_trader.adapters.databento.factories import DatabentoLiveDataClientFactory -from nautilus_trader.live.node import TradingNode - -# Instantiate the live trading node with a configuration -node = TradingNode(config=config) - -# Register the client factory with the node -node.add_data_client_factory(DATABENTO, DatabentoLiveDataClientFactory) - -# Finally build the node -node.build() -``` - -### Configuration parameters - -| Option | Default | Description | -|---------------------------|---------|----------------------------------------------------------------------------------------------------------------------| -| `api_key` | `None` | Databento API secret. Falls back to the `DATABENTO_API_KEY` environment variable when `None`. | -| `http_gateway` | `None` | Historical HTTP gateway override for testing custom endpoints. | -| `live_gateway` | `None` | Raw TCP real-time gateway override, typically for testing only. | -| `use_exchange_as_venue` | `True` | Use the exchange MIC for Nautilus venues (e.g., `XCME`). `False` retains the default GLBX mapping. | -| `timeout_initial_load` | `15.0` | Seconds to wait for instrument definitions per dataset before proceeding. | -| `mbo_subscriptions_delay` | `3.0` | Seconds to buffer before enabling MBO/L3 streams so initial snapshots replay in order. | -| `bars_timestamp_on_close` | `True` | Timestamp bars on the close (`ts_event`/`ts_init`). `False` timestamps on the open. | -| `reconnect_timeout_mins` | `10` | Minutes to attempt reconnection before giving up. `None` retries indefinitely. See [Connection stability](#connection-stability). | -| `venue_dataset_map` | `None` | Optional Nautilus venue to Databento dataset code mapping. | -| `parent_symbols` | `None` | Optional `{dataset: {parent symbols}}` to preload definition trees (e.g., `{"GLBX.MDP3": {"ES.FUT", "ES.OPT"}}`). | -| `instrument_ids` | `None` | Nautilus `InstrumentId` values to preload definitions for at startup. | - -:::tip -Use environment variables for credentials. -::: - -### Connection stability - -The live client reconnects automatically on: - -- **Network interruptions**: Temporary connectivity issues. -- **Gateway restarts**: Databento Sunday maintenance (see [Maintenance Schedule](https://databento.com/docs/api-reference-live/basics#maintenance-schedule)). -- **Market closures**: Sessions ending during off-hours. - -#### Reconnection strategy - -Backoff strategy depends on the timeout configuration: - -**With timeout** (default 10 minutes): - -- Exponential backoff capped at **60 seconds**. -- Pattern: 1s, 2s, 4s, 8s, 16s, 32s, 60s, 60s... (with jitter). -- Reconnects quickly within the timeout window. - -**Without timeout** (`reconnect_timeout_mins=None`): - -- Exponential backoff capped at **10 minutes**. -- Pattern: 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 512s, 600s, 600s... (with jitter). -- Suited for unattended systems through overnight closures and scheduled maintenance. - -All reconnections include: - -- **Jitter**: Random delay (up to 1 second) to prevent simultaneous reconnection storms. -- **Automatic resubscription**: Restores all active subscriptions after reconnecting. -- **Cycle reset**: Each successful session (>60s) resets the timeout clock. - -#### Timeout configuration - -The `reconnect_timeout_mins` parameter controls how long the client attempts reconnection: - -**Default (10 minutes)**: Suitable for most use cases. - -- Handles transient network issues. -- Survives scheduled gateway restarts. -- Stops retrying overnight when markets close. -- Requires manual intervention for longer outages. - -:::warning -Setting `reconnect_timeout_mins=None` retries indefinitely. Use only for -unattended systems that must survive overnight market closures. This can mask -persistent configuration or authentication issues. -::: - -#### Scheduled maintenance - -Databento restarts live gateways every Sunday (all clients disconnect): - -| Dataset | Maintenance Time (UTC) | -|--------------------|------------------------| -| CME Globex | 09:30 | -| All ICE venues | 09:45 | -| All other datasets | 10:30 | - -The default 10-minute timeout covers typical restarts. For unattended systems, -use `reconnect_timeout_mins=None` or a longer value. See the -[Databento Maintenance Schedule](https://databento.com/docs/api-reference-live/basics/maintenance-schedule) -for details. - -## Contributing - -:::info -To contribute, see the -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/deribit.md b/nautilus-docs/docs/integrations/deribit.md deleted file mode 100644 index b1e4109..0000000 --- a/nautilus-docs/docs/integrations/deribit.md +++ /dev/null @@ -1,648 +0,0 @@ -# Deribit - -Founded in 2016, Deribit is a cryptocurrency derivatives exchange specializing in Bitcoin and -Ethereum options and futures. It is one of the largest crypto options exchanges by volume, -and a leading platform for crypto derivatives trading. - -This integration supports live market data ingest and order execution with Deribit. - -## Overview - -This adapter is implemented in Rust, with optional Python bindings for use in Python-based workflows. -Deribit uses JSON-RPC 2.0 over both HTTP and WebSocket transports. -WebSocket is preferred for subscriptions and real-time data. - -The official Deribit API reference can be found at [docs.deribit.com](https://docs.deribit.com/). - -The Deribit adapter includes multiple components, which can be used together or separately depending -on your use case: - -- `DeribitHttpClient`: Low-level HTTP API connectivity (JSON-RPC over HTTP). -- `DeribitWebSocketClient`: Low-level WebSocket API connectivity (JSON-RPC over WebSocket). -- `DeribitInstrumentProvider`: Instrument parsing and loading functionality. -- `DeribitDataClient`: Market data feed manager. -- `DeribitExecutionClient`: Account management and trade execution gateway. -- `DeribitLiveDataClientFactory`: Factory for Deribit data clients (used by the trading node builder). -- `DeribitLiveExecClientFactory`: Factory for Deribit execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as shown below), -and won't need to work directly with these lower-level components. -::: - -### Product support - -| Product Type | Data Feed | Trading | Notes | -|-------------------|-----------|---------|----------------------------------| -| Perpetual Futures | ✓ | ✓ | BTC-PERPETUAL, ETH-PERPETUAL. | -| Dated Futures | ✓ | ✓ | Futures with fixed expiry dates. | -| Options | ✓ | ✓ | BTC and ETH options. | -| Spot | ✓ | ✓ | BTC_USDC, ETH_USDC pairs. | -| Future Combos | ✓ | ✓ | Calendar spreads for futures. | -| Option Combos | ✓ | ✓ | Option spread strategies. | - -## Symbology - -Deribit uses specific symbol conventions for different instrument types. -All instrument IDs should include the `.DERIBIT` suffix when referencing them -(e.g., `BTC-PERPETUAL.DERIBIT` for BTC perpetual). - -### Perpetual Futures - -Format: `{Currency}-PERPETUAL` - -Examples: - -- `BTC-PERPETUAL` - Bitcoin perpetual swap -- `ETH-PERPETUAL` - Ethereum perpetual swap - -To subscribe to BTC perpetual in your strategy: - -```python -InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") -``` - -### Dated Futures - -Format: `{Currency}-{DDMMMYY}` - -Examples: - -- `BTC-28MAR25` - Bitcoin futures expiring March 28, 2025 -- `ETH-27JUN25` - Ethereum futures expiring June 27, 2025 - -```python -InstrumentId.from_str("BTC-28MAR25.DERIBIT") -``` - -### Options - -Format: `{Currency}-{DDMMMYY}-{Strike}-{Type}` - -Examples: - -- `BTC-28MAR25-100000-C` - Bitcoin call option, $100,000 strike, expiring March 28, 2025 -- `BTC-28MAR25-80000-P` - Bitcoin put option, $80,000 strike, expiring March 28, 2025 -- `ETH-28MAR25-4000-C` - Ethereum call option, $4,000 strike - -Where: - -- `C` = Call option -- `P` = Put option - -```python -InstrumentId.from_str("BTC-28MAR25-100000-C.DERIBIT") -``` - -### Spot - -Format: `{BaseCurrency}_{QuoteCurrency}` - -Examples: - -- `BTC_USDC` - Bitcoin against USDC -- `ETH_USDC` - Ethereum against USDC - -```python -InstrumentId.from_str("BTC_USDC.DERIBIT") -``` - -## Order book subscriptions - -Deribit provides two types of order book feeds, each suited for different use cases. - -### Raw feeds (tick-by-tick) - -Raw channels deliver every single update as an individual message. Subscribing to a raw order book -gives you a notification for every order insertion, update, or deletion in the book. - -- Requires authenticated connection (safeguard against abuse). -- Use when you need every price level change for HFT or market making. -- Higher message volume. - -### Aggregated feeds (batched) - -Aggregated channels deliver updates in batches at a fixed interval (e.g., every 100ms). -This groups multiple order book changes into single messages. - -- Available without authentication. -- Recommended for most use cases. -- Lower message volume, easier to process. -- Default interval: 100ms. - -### Subscription parameters - -The Nautilus adapter supports both feed types via subscription parameters: - -| Parameter | Values | Notes | -|-----------|--------|-------| -| `interval` | `raw`, `100ms`, `agg2` | Default: `100ms`. `agg2` batches at ~1 second intervals. `raw` requires auth. | -| `depth` | `1`, `10`, `20` | Default: `10`. Number of price levels per side. | - -```python -from nautilus_trader.model.identifiers import InstrumentId - -instrument_id = InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") - -# Default: 100ms aggregated feed (no authentication required) -strategy.subscribe_order_book_deltas(instrument_id) - -# Raw feed (requires API credentials) -strategy.subscribe_order_book_deltas( - instrument_id, - params={"interval": "raw"}, -) -``` - -:::note -Raw order book feeds require an authenticated WebSocket connection. Ensure API credentials are -configured before subscribing to raw feeds. -::: - -:::tip -For most strategies, the default 100ms aggregated feed provides sufficient granularity with -significantly lower message overhead. Only use raw feeds when tick-by-tick precision is essential. -::: - -### Sequence gap recovery - -The adapter tracks `change_id` / `prev_change_id` sequence numbers on every book update. -When a gap is detected (missed message), the adapter automatically: - -1. Drops all incoming deltas for the affected instrument. -2. Unsubscribes from the book channel. -3. Resubscribes to obtain a fresh snapshot. -4. Resumes normal processing once the snapshot arrives. - -During resync, the strategy will not receive stale or incomplete book updates. - -## Orders capability - -Below are the order types, execution instructions, and time-in-force options supported on Deribit. - -### Order types - -| Order Type | Supported | Notes | -|---------------|-----------|------------------------------------------| -| `MARKET` | ✓ | Immediate execution at market price. | -| `LIMIT` | ✓ | Execution at specified price or better. | -| `STOP_MARKET` | ✓ | Conditional market order on trigger. | -| `STOP_LIMIT` | ✓ | Conditional limit order on trigger. | - -### Execution instructions - -| Instruction | Supported | Notes | -|----------------|-----------|------------------------------------------------| -| `post_only` | ✓ | Order will be rejected if it would take liquidity. Uses `reject_post_only=true`. | -| `reduce_only` | ✓ | Order can only reduce an existing position. | - -### Time in force - -| Time in force | Supported | Notes | -|---------------|-----------|-----------------------------------------------------| -| `GTC` | ✓ | Good Till Canceled (`good_til_cancelled`). | -| `GTD` | ✓ | Good Till Day - expires at 8:00 UTC (`good_til_day`). | -| `IOC` | ✓ | Immediate or Cancel (`immediate_or_cancel`). | -| `FOK` | ✓ | Fill or Kill (`fill_or_kill`). | - -:::note -**GTD on Deribit**: Unlike other exchanges where GTD accepts an arbitrary expiry time, -Deribit's `good_til_day` always expires at 8:00 UTC the same or next day. Custom expiry times -will be logged as warnings and the order will use the exchange's fixed expiry behavior. -::: - -### Trigger types - -Conditional orders (stop orders) support different trigger price sources: - -| Trigger Type | Supported | Notes | -|---------------|-----------|------------------------------------------| -| `last_price` | ✓ | Uses the last traded price (default). | -| `mark_price` | ✓ | Uses the mark price. | -| `index_price` | ✓ | Uses the underlying index price. | - -```python -# Example: Stop loss using mark price trigger -stop_order = order_factory.stop_market( - instrument_id=instrument_id, - order_side=OrderSide.SELL, - quantity=Quantity.from_str("0.1"), - trigger_price=Price.from_str("45000.0"), - trigger_type=TriggerType.MARK_PRICE, # Use mark price for trigger -) -strategy.submit_order(stop_order) -``` - -### Batch operations - -| Operation | Supported | Notes | -|---------------|-----------|--------------------------------------------| -| Batch Submit | - | *Not yet implemented*. | -| Batch Modify | - | *Not yet implemented*. | -| Batch Cancel | - | *Not yet implemented*. | - -### Post-only behavior - -Deribit offers two post-only modes: - -1. **Price adjustment (Deribit default)**: If a post-only order would cross the spread and execute, - Deribit automatically adjusts the price to one tick inside the spread. -2. **Reject mode**: Order is immediately rejected if it would cross the spread. - -The Nautilus adapter uses **reject mode** (`reject_post_only=true`) for deterministic behavior. -If a post-only order would take liquidity, it is rejected with error code `11054`, and an `OrderRejected` -event is emitted with the `due_post_only` flag set to `true`. - -This allows strategies to differentiate between: - -- Orders rejected due to post-only violation (attempted to take liquidity). -- Orders rejected for other reasons (insufficient margin, invalid price, etc.). - -### Order modification - -The adapter uses Deribit's native `private/edit` endpoint rather than cancel-and-replace. -This provides several advantages: - -| Benefit | Description | -|----------------------------|--------------------------------------------------------------------| -| Single request | Faster execution, lower latency than cancel + new order. | -| Queue priority preservation | Keeps position when only reducing quantity or keeping same price. | -| Fill history maintained | Partial fills remain linked to the same order ID. | - -**Queue priority rules:** - -- **Decreasing quantity only**: Keeps queue position. -- **Same price**: Keeps queue position. -- **Increasing quantity or changing price**: Loses queue position (treated as new order). - -### Position management - -| Feature | Supported | Notes | -|-------------------|-----------|-------------------------------------------| -| Query positions | ✓ | Real-time position updates. | -| Position mode | - | Deribit uses net position mode only. | -| Leverage control | - | Leverage set at account level via UI. | -| Margin mode | - | Portfolio margin via Deribit UI settings. | - -### Order querying - -| Feature | Supported | Notes | -|----------------------|-----------|------------------------------------| -| Query open orders | ✓ | List all active orders. | -| Query order history | ✓ | Historical order data. | -| Order status updates | ✓ | Real-time order state changes. | -| Trade history | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Supported | Notes | -|---------------------|-----------|------------------------------------| -| Order lists | - | *Not supported*. | -| OCO orders | - | *Not supported*. | -| Bracket orders | - | *Not supported*. | -| Conditional orders | ✓ | Stop market and stop limit orders. | - -## Funding rates - -Deribit exchanges funding continuously (every few seconds) rather than at fixed intervals -like most other exchanges. The `interval` field on `FundingRateUpdate` is `None` for -Deribit because this continuous model does not map to a discrete period. - -## Rate limiting - -Deribit uses a credit-based rate limiting system. Each API request consumes credits, which are replenished -over time. The adapter enforces these quotas to prevent rate limit violations. - -### REST limits - -| Bucket / Key | Limit | Notes | -|---------------------|------------------|---------------------------------------------| -| `deribit:global` | 20 req/sec (100 burst) | Default bucket for all REST requests. | -| `deribit:orders` | 5 req/sec (20 burst) | Matching engine operations (buy, sell, edit, cancel). | -| `deribit:account` | 5 req/sec | Account information endpoints. | - -### WebSocket limits - -| Operation | Limit | Notes | -|---------------------|------------------|---------------------------------------------| -| Subscribe/unsubscribe | 3 req/sec (10 burst) | Subscription operations. | -| Order operations | 5 req/sec (20 burst) | Buy, sell, edit, cancel via WebSocket. | - -:::note -The Nautilus adapter uses WebSocket for order submission (not REST) for lower latency. -Order operations are rate-limited by `DERIBIT_WS_ORDER_QUOTA` (5 req/sec, 20 burst). -::: - -### Credit-based system details - -Deribit uses a sophisticated credit-based rate limiting system where credits are replenished -continuously at a fixed rate. Each second, credits "drip" back into your sub-account's credit pool. - -**Non-matching engine requests:** - -| Parameter | Value | Notes | -|------------------|--------------------|---------------------------------| -| Cost per request | 500 credits | Each API call consumes credits. | -| Maximum pool | 50,000 credits | Allows 100 request burst. | -| Refill rate | 10,000 credits/sec | ~20 sustained requests/second. | - -**Matching engine requests (default tier):** - -| Parameter | Value | Notes | -|----------------|----------------|------------------------------------| -| Sustained rate | 5 requests/sec | Continuous rate limit. | -| Burst capacity | 20 requests | Maximum burst before throttling. | - -Higher matching engine limits are available for market makers and high-volume traders based on -7-day trading volume tiers. - -The Nautilus adapter implements this using token bucket rate limiters configured as: - -- `DERIBIT_HTTP_REST_QUOTA`: 20 req/sec with 100 burst (non-matching REST) -- `DERIBIT_HTTP_ORDER_QUOTA`: 5 req/sec with 20 burst (matching engine REST) -- `DERIBIT_WS_ORDER_QUOTA`: 5 req/sec with 20 burst (matching engine WebSocket) -- `DERIBIT_WS_SUBSCRIPTION_QUOTA`: 3 req/sec with 10 burst (subscribe/unsubscribe) - -For more details, see the [Rate Limits article](https://support.deribit.com/hc/en-us/articles/25944617523357-Rate-Limits). - -:::warning -Deribit returns error code `10028` (too_many_requests) when you exceed the allowed quota. -Repeated violations may result in temporary throttling. -::: - -## Connection management - -### Platform limits - -| Limit | Value | -|-----------------------------------|-------| -| Maximum connections per IP | 32 | -| Maximum sessions per API key | 16 | -| Maximum API keys per (sub)account | 8 | - -### Session-based authentication - -The adapter uses **separate WebSocket sessions** for data and execution clients, each with its own -authentication scope: - -| Client | Session Name | Purpose | -|------------------|----------------------|------------------------------------------------------| -| Data client | `nautilus-data` | Market data subscriptions (raw feeds require auth). | -| Execution client | `nautilus-execution` | Order operations (buy, sell, edit, cancel). | - -**Authentication flow:** - -1. WebSocket connects to Deribit. -2. Client authenticates using `client_signature` grant type with session scope. -3. Tokens are automatically refreshed at 80% of expiry time (continuous refresh cycle). -4. On reconnection, re-authentication is retried with exponential backoff (up to 3 attempts). - If all attempts fail, only public channel subscriptions are restored. - -This session-based approach allows: - -- Independent token management per client type. -- Isolated failure domains (data auth failure does not affect execution). -- Clear audit trail in Deribit's session logs. - -### Best practices - -The adapter follows Deribit's -[recommended connection practices](https://support.deribit.com/hc/en-us/articles/25944603459613): - -1. **Uses WebSocket subscriptions** for real-time data instead of REST polling, resulting in fewer requests, - lower latency, and reduced rate limit consumption. -2. **Authenticates all connections** when credentials are provided. Authenticated users benefit - from higher rate limits and are less likely to be IP rate-limited. -3. **Implements heartbeats** (30 second interval) to maintain connection health and detect - disconnections early. -4. **Handles reconnection** automatically with re-authentication and subscription recovery. - -:::tip -Always provide API credentials even for public data access. Authenticated connections have higher -rate limits, and Deribit contacts authenticated clients before applying restrictions during -high-load periods. -::: - -:::note -The adapter uses a 30 second heartbeat interval, which is the lower bound of Deribit's recommended -30-60 second range. More frequent heartbeats may trigger stricter rate limits. -::: - -## Authentication - -Deribit uses API key authentication with HMAC-SHA256 signatures for private endpoints. - -To create API credentials: - -1. Log into your Deribit account at [deribit.com](https://www.deribit.com) (or [test.deribit.com](https://test.deribit.com) for testnet). -2. Navigate to **Account** → **API**. -3. Click **Add new key** and configure permissions: - - Enable **read** for market data access - - Enable **trade** for order execution - - Enable **wallet** if you need account balance access -4. Note down your **Client ID** (API key) and **Client Secret** (API secret). - -:::warning -Keep your API secret secure. Never share it or commit it to version control. -::: - -### API key scopes - -Each API key on Deribit is assigned a default access scope, which defines the maximum permissions. -Configure appropriate permissions when -[creating your API key](https://support.deribit.com/hc/en-us/articles/26268257333661): - -| Scope | Required For | -|--------------------|----------------------------------------| -| `account:read` | Account information, portfolio data. | -| `trade:read` | View orders and positions. | -| `trade:read_write` | Place, modify, and cancel orders. | -| `wallet:read` | View balances and transaction history. | - -**Recommended minimum for trading:** `account:read`, `trade:read_write`, `wallet:read` - -:::tip -Follow the principle of least privilege. For data-only access (market data, no trading), -create a read-only key without `trade:read_write`. -::: - -## Testnet - -Deribit provides a testnet environment for testing strategies without real funds. -To use the testnet, set `is_testnet=True` in your client configuration: - -```python -config = TradingNodeConfig( - data_clients={ - DERIBIT: DeribitDataClientConfig( - is_testnet=True, # Enable testnet mode - # ... other config - ), - }, - exec_clients={ - DERIBIT: DeribitExecClientConfig( - is_testnet=True, # Enable testnet mode - # ... other config - ), - }, -) -``` - -When testnet mode is enabled: - -- HTTP requests use `https://test.deribit.com`. -- WebSocket connections use `wss://test.deribit.com/ws/api/v2`. -- Loads credentials from `DERIBIT_TESTNET_API_KEY` and `DERIBIT_TESTNET_API_SECRET` environment variables. - -:::note -Testnet API keys are separate from production keys. Create API keys specifically -for the testnet through the testnet interface at [test.deribit.com](https://test.deribit.com). -::: - -## Configuration - -### Data client configuration options - -| Option | Default | Description | -|------------------------------------|------------|-------------| -| `api_key` | `None` | Deribit API key; loads from environment variables when omitted. | -| `api_secret` | `None` | Deribit API secret; loads from environment variables when omitted. | -| `product_types` | `None` | Product types to load (Future, Option, Spot, etc.). If `None`, defaults to Future. | -| `base_url_http` | `None` | Override for the HTTP REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `is_testnet` | `False` | Use Deribit testnet endpoints when `True`. | -| `http_timeout_secs` | `60` | Request timeout (seconds) for REST calls. | -| `max_retries` | `3` | Maximum retry attempts for recoverable errors. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) before retrying. | -| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | -| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument refreshes. | - -### Execution client configuration options - -| Option | Default | Description | -|--------------------------|------------|-------------| -| `api_key` | `None` | Deribit API key; loads from environment variables when omitted. | -| `api_secret` | `None` | Deribit API secret; loads from environment variables when omitted. | -| `product_types` | `None` | Product types to load (Future, Option, Spot, etc.). If `None`, defaults to Future. | -| `base_url_http` | `None` | Override for the HTTP REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `is_testnet` | `False` | Use Deribit testnet endpoints when `True`. | -| `http_timeout_secs` | `60` | Request timeout (seconds) for REST calls. | -| `max_retries` | `3` | Maximum retry attempts for recoverable errors. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) before retrying. | -| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | - -### Production configuration - -Below is an example configuration for a live trading node using Deribit data and execution clients: - -```python -from nautilus_trader.adapters.deribit import DERIBIT -from nautilus_trader.adapters.deribit import DeribitDataClientConfig -from nautilus_trader.adapters.deribit import DeribitExecClientConfig -from nautilus_trader.adapters.deribit import DeribitLiveDataClientFactory -from nautilus_trader.adapters.deribit import DeribitLiveExecClientFactory -from nautilus_trader.config import InstrumentProviderConfig -from nautilus_trader.config import TradingNodeConfig -from nautilus_trader.core.nautilus_pyo3 import DeribitProductType -from nautilus_trader.live.node import TradingNode - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - DERIBIT: DeribitDataClientConfig( - api_key=None, # Uses DERIBIT_API_KEY env var - api_secret=None, # Uses DERIBIT_API_SECRET env var - product_types=(DeribitProductType.Future,), - instrument_provider=InstrumentProviderConfig(load_all=True), - is_testnet=False, - ), - }, - exec_clients={ - DERIBIT: DeribitExecClientConfig( - api_key=None, - api_secret=None, - product_types=(DeribitProductType.Future,), - instrument_provider=InstrumentProviderConfig(load_all=True), - is_testnet=False, - ), - }, -) - -node = TradingNode(config=config) -node.add_data_client_factory(DERIBIT, DeribitLiveDataClientFactory) -node.add_exec_client_factory(DERIBIT, DeribitLiveExecClientFactory) -node.build() -``` - -### API credentials - -There are multiple options for supplying your credentials to the Deribit clients. -Either pass the corresponding values to the configuration objects, or -set the following environment variables: - -For Deribit live (production) clients: - -- `DERIBIT_API_KEY` -- `DERIBIT_API_SECRET` - -For Deribit testnet clients: - -- `DERIBIT_TESTNET_API_KEY` -- `DERIBIT_TESTNET_API_SECRET` - -:::tip -We recommend using environment variables to manage your credentials. -::: - -### Product types - -The `product_types` configuration option controls which Deribit product families are loaded. -Available options via the `DeribitProductType` enum: - -- `DeribitProductType.Future` - Perpetual and dated futures. -- `DeribitProductType.Option` - Call and put options. -- `DeribitProductType.Spot` - Spot trading pairs. -- `DeribitProductType.FutureCombo` - Future spread instruments. -- `DeribitProductType.OptionCombo` - Option spread instruments. - -Example loading multiple product types: - -```python -from nautilus_trader.core.nautilus_pyo3 import DeribitProductType - -config = DeribitDataClientConfig( - product_types=( - DeribitProductType.Future, - DeribitProductType.Option, - ), - # ... other config -) -``` - -### Base URL overrides - -It's possible to override the default base URLs for both HTTP and WebSocket APIs: - -| Environment | HTTP URL | WebSocket URL | -|-------------|----------------------------|------------------------------------| -| Production | `https://www.deribit.com` | `wss://www.deribit.com/ws/api/v2` | -| Testnet | `https://test.deribit.com` | `wss://test.deribit.com/ws/api/v2` | - -## Server infrastructure - -Deribit's matching engine is located in **Equinix LD4, Slough, UK**. For latency-sensitive strategies, -consider hosting in or near London. Colocation and cross-connect options are available directly -from Deribit for institutional clients. - -For most users connecting via internet, the adapter's built-in retry logic, heartbeat monitoring, -and automatic reconnection handling provide reliable connectivity. - -For more details, see the [Server Infrastructure article](https://support.deribit.com/hc/en-us/articles/25944617582877). - -## Contributing - -:::info -For additional features or to contribute to the Deribit adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/dydx.md b/nautilus-docs/docs/integrations/dydx.md deleted file mode 100644 index ea0d4e4..0000000 --- a/nautilus-docs/docs/integrations/dydx.md +++ /dev/null @@ -1,803 +0,0 @@ -# dYdX v4 - -dYdX is one of the largest decentralized cryptocurrency exchanges for crypto derivative products. -This integration supports live market data ingestion and order execution with dYdX v4, running on -its own Cosmos SDK application-specific blockchain (dYdX Chain) with CometBFT consensus. The order -book and matching engine run on-chain as part of the validator process. Orders are submitted as -Cosmos transactions via gRPC and settled each block. An Indexer service exposes REST and WebSocket -APIs for market data and account state. - -This is the Rust-backed adapter with Python bindings. - -## Installation - -:::note -No additional installation extras are required. The adapter is implemented in Rust and -compiled into the core `nautilus_trader` package automatically during the build. -::: - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/dydx/). - -## Overview - -This adapter is implemented in Rust with Python bindings via PyO3. It provides direct integration -with dYdX's Indexer API (REST/WebSocket) for market data and gRPC for Cosmos SDK transaction -submission, without requiring external client libraries. - -### Product support - -| Product Type | Data Feed | Trading | Notes | -|-------------------|-----------|---------|----------------------------------------| -| Perpetual Futures | ✓ | ✓ | All perpetuals are USDC-settled. | -| Spot | - | - | dYdX offers spot on Solana; not supported by this adapter. | -| Options | - | - | *Not available on dYdX*. | - -:::note -This adapter supports perpetual futures only. All markets are quoted in USD and settled in USDC. -::: - -## Chain architecture - -Unlike centralized exchanges (CEXs) that expose a single REST/WebSocket API, dYdX v4 runs on its -own **Cosmos SDK application-specific blockchain**. This means every trade is a Cosmos transaction -that goes through consensus, and the adapter must manage sequences, gas, and block-height-based -expiration. - -### Transport layers - -The adapter communicates through three independent transport layers: - -``` - ┌─────────────────────────────────────────────┐ - │ dYdX v4 Chain │ - │ │ - ┌──────────┐ HTTP │ ┌──────────────────────┐ │ - │ │───────────►│ │ Indexer (read-only) │ │ - │ │ WebSocket │ │ - REST API │ │ - │ Nautilus │───────────►│ │ - Streaming API │ │ - │ Adapter │ │ └──────────────────────┘ │ - │ │ gRPC │ ┌──────────────────────┐ │ - │ │───────────►│ │ Validator (write) │ │ - └──────────┘ │ │ - Cosmos Tx submit │ │ - │ │ - Sequence mgmt │ │ - │ └──────────────────────┘ │ - └─────────────────────────────────────────────┘ -``` - -| Layer | Target | Direction | Purpose | -|-----------|-----------|------------|------------------------------------------------------| -| HTTP | Indexer | Read-only | Instrument metadata, historical data, account state. | -| WebSocket | Indexer | Read-only | Real-time market data, order/fill/position updates. | -| gRPC | Validator | Write | Order placement, cancellation, and batch operations. | - -### Block-based settlement - -dYdX blocks are produced approximately every **~0.5 seconds** (actual times vary). The adapter -includes a `BlockTimeMonitor` that tracks observed block times from the WebSocket feed to -dynamically estimate `seconds_per_block`. This estimate is used to convert time-based order expiry -into block-height offsets for short-term orders. - -## Architecture - -The dYdX v4 adapter includes multiple components which can be used together or separately: - -- `DydxHttpClient`: Rust-backed HTTP client for Indexer REST API queries. -- `DydxWebSocketClient`: Rust-backed WebSocket client for real-time market data and account updates. -- `DydxGrpcClient`: Rust-backed gRPC client for Cosmos SDK transaction submission. -- `DydxInstrumentProvider`: Instrument parsing and loading functionality. -- `DydxDataClient`: Market data feed manager. -- `DydxExecutionClient`: Account management and trade execution gateway. -- `DydxLiveDataClientFactory`: Factory for dYdX v4 data clients (used by the trading node builder). -- `DydxLiveExecClientFactory`: Factory for dYdX v4 execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), -and won't need to work with these lower level components directly. -::: - -:::warning[First-time account activation] -A dYdX v4 trading account (sub-account 0) is created only after the wallet's first deposit or trade. -Until then, every gRPC/Indexer query returns `NOT_FOUND`, so `DydxExecutionClient.connect()` fails. - -Before starting a live `TradingNode`, send any positive amount of USDC or other supported collateral -from the same wallet on the same network (mainnet/testnet). Once the transaction has finalised -(a few blocks), restart the node and the client will connect cleanly. -::: - -## Troubleshooting - -### `StatusCode.NOT_FOUND` account not found - -**Cause:** The wallet/sub-account has never been funded and therefore does not yet exist on-chain. - -**Fix:** - -1. Deposit any positive amount of USDC to sub-account 0 on the correct network. -2. Wait for finality (roughly 30 seconds on mainnet, longer on testnet). -3. Restart the `TradingNode`; the connection should now succeed. - -:::tip -In unattended deployments, wrap the `connect()` call in an exponential-backoff loop so the -client retries until the deposit appears. -::: - -## Symbology - -dYdX uses specific symbol conventions for perpetual futures contracts. - -### Symbol format - -Format: `{Base}-USD-PERP` - -All perpetuals on dYdX are: - -- Quoted in USD -- Settled in USDC -- Use the `.DYDX` venue suffix in Nautilus - -Examples: - -- `BTC-USD-PERP.DYDX` - Bitcoin perpetual futures -- `ETH-USD-PERP.DYDX` - Ethereum perpetual futures -- `SOL-USD-PERP.DYDX` - Solana perpetual futures - -To subscribe in your strategy: - -```python -InstrumentId.from_str("BTC-USD-PERP.DYDX") -InstrumentId.from_str("ETH-USD-PERP.DYDX") -``` - -:::info -The `-PERP` suffix is appended for consistency with other adapters and future-proofing. While dYdX -currently only supports perpetuals, this naming convention allows for potential expansion to other -product types. -::: - -## Orders capability - -dYdX supports perpetual futures trading with a full set of order types and execution -features. The Rust adapter automatically classifies orders as short-term or long-term based on -time-in-force and expiry, so no manual tagging is needed (unlike the legacy Python adapter). - -### Order types - -| Order Type | Perpetuals | Notes | -|------------------------|------------|----------------------------------------------------| -| `MARKET` | ✓ | Immediate execution at best available price. | -| `LIMIT` | ✓ | | -| `STOP_MARKET` | ✓ | Stop-loss conditional order, always long-term. | -| `STOP_LIMIT` | ✓ | Conditional order, always long-term. | -| `MARKET_IF_TOUCHED` | ✓ | Take-profit market order, triggers on price touch. | -| `LIMIT_IF_TOUCHED` | ✓ | Take-profit limit order, triggers on price touch. | -| `TRAILING_STOP_MARKET` | - | *Not supported*. | - -### Execution instructions - -| Instruction | Perpetuals | Notes | -|---------------|------------|--------------------------------------------------------------| -| `post_only` | ✓ | Supported on LIMIT, STOP_LIMIT, and LIMIT_IF_TOUCHED orders. | -| `reduce_only` | ✓ | Passed for all order types. Venue support may vary by order type. | - -### Time in force options - -| Time in force | Perpetuals | Notes | -|---------------|------------|----------------------| -| `GTC` | ✓ | Good Till Canceled. | -| `GTD` | ✓ | Good Till Date. | -| `FOK` | ✓ | Fill or Kill. | -| `IOC` | ✓ | Immediate or Cancel. | - -### Advanced order features - -| Feature | Perpetuals | Notes | -|--------------------|------------|------------------| -| Order modification | - | Not supported. dYdX supports short-term order [replacement](https://docs.dydx.xyz/concepts/trading/limit-orderbook#replacements) (same ID, higher GTB); not yet exposed as `ModifyOrder`. | -| Bracket/OCO orders | - | *Not supported*. | -| Iceberg orders | - | *Not supported*. | - -### Batch operations - -| Operation | Perpetuals | Notes | -|--------------|------------|------------------------------------------------------------------------------------------------------------------------| -| Batch submit | - | *Not supported*. | -| Batch modify | - | *Not supported*. | -| Batch cancel | ✓ | Partitioned: short-term orders use `MsgBatchCancel` (single gRPC call), long-term orders use batched `MsgCancelOrder`. | - -### Position management - -| Feature | Perpetuals | Notes | -|------------------|------------|-------------------------------| -| Query positions | ✓ | Real-time position updates. | -| Position mode | - | Netting only (see below). | -| Leverage control | ✓ | Per-market leverage settings. | -| Margin mode | - | Cross margin only. | - -:::note -dYdX supports netting (one position per instrument) at the venue level. The adapter currently -operates in `NETTING` mode only. Hedging support is planned for a future version. -::: - -### Order querying - -| Feature | Perpetuals | Notes | -|----------------------|------------|--------------------------------| -| Query open orders | ✓ | List all active orders. | -| Query order history | ✓ | Historical order data. | -| Order status updates | ✓ | Real-time order state changes. | -| Trade history | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Perpetuals | Notes | -|--------------------|------------|--------------------------------------------------| -| Order lists | - | *Not supported*. | -| OCO orders | - | *Not supported*. | -| Bracket orders | - | *Not supported*. | -| Conditional orders | ✓ | Stop, take-profit market, and take-profit limit. | - -### Order classification - -dYdX classifies every order into one of three on-chain categories. The Rust adapter -automatically determines the category based on time-in-force and expiry, so no manual -configuration is required. - -| Category | Placement | Expiry | Typical use | -|-----------------|-------------|-------------------|-----------------------------------------------| -| Short-term | In-memory | Block height | IOC/FOK, or orders expiring within 40 blocks. | -| Long-term | On-chain | Timestamp (UTC) | GTC/GTD with expiry beyond the short-term window (~20s at ~0.5s/block). | -| Conditional | On-chain | Timestamp (UTC) | Stop-loss and take-profit triggers. | - -At the protocol level, **all dYdX orders are limit orders**. The `MARKET` order type -is a Nautilus convenience that the adapter implements as an aggressive IOC limit order -priced well through the book. This means market orders follow the same -`Submitted > Accepted > Filled` lifecycle as limit orders (an `OrderAccepted` event is -expected before the fill). - -See the [dYdX order documentation](https://docs.dydx.xyz/concepts/trading/orders) -for full protocol-level details on short-term vs stateful order mechanics. - -#### Short-term orders - -Short-term orders live **in validator memory only** and expire by block height (max 40 blocks, -roughly ~20 seconds at ~0.5s/block). They are the fastest order type on dYdX because they skip -on-chain storage. - -Key properties: - -- **IOC and FOK are always short-term**, regardless of other parameters -- **GTD orders** are automatically classified as short-term when the expiry falls within the - dynamic short-term window (`40 blocks × seconds_per_block`) -- Use Good-Til-Block (GTB) for replay protection instead of Cosmos SDK sequences -- Can be broadcast **concurrently** (no semaphore, cached sequence) -- Expire silently without generating cancel events -- Cannot be batched in a single transaction (one `MsgPlaceOrder` per tx) - -#### Long-term orders - -Long-term (stateful) orders are **stored on-chain** and expire by UTC timestamp. They generate -explicit cancel events when they expire or are cancelled. - -Key properties: - -- **GTC** orders default to 90-day expiration (protocol limit is 95 days) -- **GTD** orders use the user-provided expiry timestamp -- Require proper Cosmos SDK sequence management (serialized via semaphore) -- Must be broadcast **serially** with incrementing sequence numbers -- Can be batched in a single transaction - -#### Conditional orders - -Conditional orders (stop-loss, take-profit) are **always stored on-chain** and triggered by -price conditions on the validator. - -Key properties: - -- Always use timestamp-based expiry (default 90 days for GTC, protocol limit 95 days) -- Always use the long-term broadcast path (serialized with semaphore) -- Include `StopMarket`, `StopLimit`, `TakeProfitMarket`, and `TakeProfitLimit` - -#### Automatic routing - -The adapter determines order lifetime automatically using the `BlockTimeMonitor`: - -``` -max_short_term_secs = SHORT_TERM_ORDER_MAXIMUM_LIFETIME (40) × seconds_per_block -``` - -If the order's time until expiry is within `max_short_term_secs`, it is routed as short-term. -Otherwise, it is routed as long-term. No manual configuration is needed. - -#### MARKET order implementation - -dYdX has no native market order type. The adapter implements `MARKET` orders as aggressive -**IOC limit orders** priced at: - -- **Buy**: `oracle_price × (1 + 0.05)` (5% above oracle) -- **Sell**: `oracle_price × (1 - 0.05)` (5% below oracle) - -This 5% slippage buffer (`DEFAULT_MARKET_ORDER_SLIPPAGE = 0.05`) sets the worst-case price -(the "pay-through price"). Because the order is IOC, unfilled slippage is not consumed. The -buffer is intentionally wide to maximize fill probability across volatile conditions. - -### Client order ID encoding - -dYdX requires `u32` client IDs on-chain, but Nautilus uses string-based `ClientOrderId` values -(e.g., `O-20260220-031943-001-000-51`). The adapter encodes these bidirectionally so that orders -can be reconciled across restarts without persisted state. - -For the standard O-format (`O-YYYYMMDD-HHMMSS-TTT-SSS-CCC`), the encoding is deterministic: - -| dYdX field | Bits | Contents | -|-------------------|------|----------------------------------------------------| -| `client_id` | 32 | `[trader:10][strategy:10][count:12]` (unique key). | -| `client_metadata` | 32 | Seconds since 2020-01-01 UTC (timestamp). | - -Because the encoding is deterministic, the adapter can decode any reconciled order back to its -original `ClientOrderId` string without needing a database or mapping file. - -Non-standard `ClientOrderId` formats (custom strings, plain numbers) fall back to sequential -allocation with an in-memory reverse map. These IDs can only be decoded within the same session. - -#### Restart collision prevention - -On restart, Nautilus resets the internal order counter based on the number of reconciled orders, -which may be lower than the highest counter value used in the previous session (e.g., if some -orders have expired from the API response). This can cause a new order to produce the same -`client_id` as a previous session's order, resulting in a duplicate venue order UUID. - -The adapter prevents this by registering every `client_id` seen during reconciliation. If a new -O-format encoding produces a `client_id` that was already used, the encoder logs a warning and -falls back to sequential allocation. Sequential allocation also skips any registered values. - -:::note -This protection is automatic and requires no user configuration. The warning log -`[ENCODER] client_id ... collides with reconciled order` is informational. The order will -still be submitted successfully with an alternative ID. -::: - -## Broadcasting and retry strategy - -### Short-term broadcast - -Short-term orders use Good-Til-Block (GTB) for replay protection. The chain's `ClobDecorator` -ante handler skips Cosmos SDK sequence checking for short-term messages, so: - -- **No semaphore**: broadcasts are fully concurrent -- **Cached sequence**: no increment or allocation needed -- **No retry**: if the broadcast fails, it fails immediately -- Benign cancel errors are treated as success (see below) - -### Long-term broadcast - -Long-term and conditional orders require proper Cosmos SDK sequence management: - -- **Semaphore** with 1 permit serializes all long-term broadcasts -- **Exponential backoff**: 500ms → 1s → 2s → 4s (max 5 retries) -- **10-second total budget** prevents indefinite retry loops -- On sequence mismatch, the sequence is **resynced from chain** before retry - -### Sequence mismatch detection - -| Error code | Source | Meaning | -|------------|----------------------|--------------------------------------------------| -| `code=32` | Cosmos SDK | Account sequence mismatch | -| `code=104` | dYdX authenticator | Signature verification failed (sequence-related) | - -Both trigger automatic resync + retry via the `RetryManager`. - -### Benign cancel errors - -These errors during short-term cancel operations are treated as **success**: - -| Error code | Meaning | -|-------------|----------------------------------------------------------------| -| `code=19` | Transaction already in mempool cache (duplicate tx) | -| `code=9` | Cancel already exists in memclob with >= GoodTilBlock | -| `code=3006` | Order to cancel does not exist (already filled/expired/cancelled) | - -### Batch cancel partitioning - -When cancelling multiple orders, the adapter partitions them by lifetime: - -1. **Short-term orders** → Single `MsgBatchCancel` via `broadcast_short_term()` -2. **Long-term orders** → Batched `MsgCancelOrder` messages via `broadcast_with_retry()` - -This ensures each group uses the appropriate broadcast strategy. - -## Funding rates - -dYdX perpetual futures use a fixed 1-hour funding interval. The adapter sets `interval` -to `60` (minutes) on all `FundingRateUpdate` objects for both WebSocket and historical -funding data. - -## Rate limiting - -### gRPC rate limiting - -The adapter rate-limits gRPC `broadcast_tx` calls to prevent `ResourceExhausted` (429) errors -from validator nodes. - -| Setting | Default | Description | -|-------------------------------|---------|-------------------------------------------| -| `grpc_rate_limit_per_second` | `4` | Maximum gRPC broadcast requests per second. Set to `None` to disable. | - -### Provider limits - -Known rate limits for public gRPC providers: - -| Provider | Limit | Notes | -|------------|--------------------|-----------------| -| Polkachu | 300 req/min (~5/s) | | -| KingNodes | 250 req/min (~4.2/s) | | -| AutoStake | 4 req/s | | - -The default of 4 req/s is conservative and works across all public providers. - -### Multiple gRPC URL fallback - -The adapter supports configuring multiple gRPC URLs for failover: - -```python -exec_config = DydxExecClientConfig( - base_url_grpc="https://primary-grpc.example.com:443,https://fallback-grpc.example.com:443", - # ... -) -``` - -## Price and size quantization - -dYdX uses integer-based quantization for prices and sizes. The adapter handles all conversions -automatically via `OrderMessageBuilder`, but understanding the parameters helps with debugging. - -### Market parameters - -| Parameter | Description | -|--------------------------------|----------------------------------------------------------| -| `atomic_resolution` | Exponent for converting human-readable size to quantums | -| `quantum_conversion_exponent` | Exponent for converting quantums to tokens | -| `step_base_quantums` | Minimum order size step in quantums | -| `subticks_per_tick` | Price granularity within each tick | - -### Market order pricing - -Market orders use the oracle price with a 5% slippage buffer (the "pay-through price"): - -- **Buy**: `oracle_price × 1.05` -- **Sell**: `oracle_price × 0.95` - -The oracle price is cached from the Indexer and refreshed periodically. - -### Automatic handling - -All price and size quantization is handled automatically by `OrderMessageBuilder`. -No manual conversion is needed when submitting orders through Nautilus. - -## Data subscriptions - -The v4 adapter supports the following data subscriptions: - -| Data type | Subscription | Historical request | Notes | -|---------------------|--------------|--------------------|-------------------------------------------| -| Trade ticks | ✓ | ✓ | | -| Quote ticks | ✓ | - | Synthesized from order book top-of-book. | -| Order book deltas | ✓ | ✓ | L2 depth only. Snapshot via HTTP request. | -| Bars | ✓ | ✓ | See supported resolutions below. | -| Mark prices | ✓ | - | Via markets channel. | -| Index prices | ✓ | - | Via markets channel. | -| Funding rates | ✓ | - | Via markets channel. | -| Instrument status | ✓ | - | Via markets channel. | - -### Supported bar resolutions - -| Resolution | dYdX candle | -|------------|-------------| -| 1-MINUTE | `1MIN` | -| 5-MINUTE | `5MINS` | -| 15-MINUTE | `15MINS` | -| 30-MINUTE | `30MINS` | -| 1-HOUR | `1HOUR` | -| 4-HOUR | `4HOURS` | -| 1-DAY | `1DAY` | - -## Subaccounts - -dYdX supports multiple subaccounts per wallet address, allowing segregation of trading strategies -and risk management within a single wallet. - -### Key concepts - -- Each wallet address can have multiple numbered subaccounts (0, 1, 2, ..., 127). -- Subaccount 0 is the **default** and is automatically created on first deposit. -- Each subaccount maintains its own: - - Positions - - Open orders - - Collateral balance - - Margin requirements - -### Configuration - -Specify the subaccount number in the execution client config: - -```python -config = TradingNodeConfig( - exec_clients={ - "DYDX": DydxExecClientConfig( - subaccount=0, # Default subaccount - ), - }, -) -``` - -:::note -Most users will use subaccount `0` (the default). Advanced users can configure multiple execution -clients for different subaccounts to implement strategy segregation or risk isolation. -::: - -## Testnet setup - -The dYdX testnet (`dydx-testnet-4`) is a full replica of mainnet for testing strategies -without risking real funds. All default testnet endpoints are resolved automatically when -`is_testnet=True`. - -### 1. Create a testnet wallet - -**Option A: Via the dYdX testnet web app (easiest)** - -1. Go to [v4.testnet.dydx.exchange](https://v4.testnet.dydx.exchange) -2. Connect with MetaMask, Keplr, Phantom, or WalletConnect -3. A dYdX account is generated automatically -4. Export your secret phrase: click your address (top-right) and select "Export secret phrase" - -**Option B: Use an existing secp256k1 private key** - -Any 32-byte hex-encoded secp256k1 private key will work. The adapter derives the `dydx1...` -address from the key automatically using Cosmos bech32 encoding. - -### 2. Fund the testnet account - -A subaccount must be funded before the adapter can connect (see [First-time account activation](#architecture)). - -**Via the testnet web app:** - -Click the deposit/recharge button on [v4.testnet.dydx.exchange](https://v4.testnet.dydx.exchange) -to receive testnet USDC automatically. - -**Via the faucet API directly:** - -```bash -# Fund subaccount 0 with 2000 USDC -curl -X POST https://faucet.v4testnet.dydx.exchange/faucet/tokens \ - -H "Content-Type: application/json" \ - -d '{"address": "dydx1...", "subaccountNumber": 0, "amount": 2000}' - -# Fund native tokens (for gas fees) -curl -X POST https://faucet.v4testnet.dydx.exchange/faucet/native-token \ - -H "Content-Type: application/json" \ - -d '{"address": "dydx1..."}' -``` - -### 3. Set environment variables - -```bash -export DYDX_TESTNET_WALLET_ADDRESS="dydx1..." -export DYDX_TESTNET_PRIVATE_KEY="0x..." # hex-encoded, 0x prefix optional -``` - -### 4. Configure the trading node - -Set `is_testnet=True` on both data and execution clients: - -```python -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - DYDX: DydxDataClientConfig( - wallet_address=None, # Falls back to DYDX_TESTNET_WALLET_ADDRESS env var - instrument_provider=InstrumentProviderConfig(load_all=True), - is_testnet=True, - ), - }, - exec_clients={ - DYDX: DydxExecClientConfig( - wallet_address=None, # Falls back to DYDX_TESTNET_WALLET_ADDRESS env var - private_key=None, # Falls back to DYDX_TESTNET_PRIVATE_KEY env var - subaccount=0, - instrument_provider=InstrumentProviderConfig(load_all=True), - is_testnet=True, - ), - }, -) -``` - -### Testnet endpoints - -Default testnet endpoints are used automatically. Override with `base_url_*` config options if needed. - -| Service | Default URL | -|-----------|------------------------------------------------------| -| HTTP | `https://indexer.v4testnet.dydx.exchange` | -| WebSocket | `wss://indexer.v4testnet.dydx.exchange/v4/ws` | -| gRPC | `https://test-dydx-grpc.kingnodes.com:443` (primary) | -| Faucet | `https://faucet.v4testnet.dydx.exchange` | -| Web app | `https://v4.testnet.dydx.exchange` | - -### Mainnet endpoints - -Default mainnet endpoints are used automatically. Override with `base_url_*` -config options if needed. - -| Service | Default URL | -|-----------|-----------------------------------------------------| -| HTTP | `https://indexer.dydx.trade` | -| WebSocket | `wss://indexer.dydx.trade/v4/ws` | -| gRPC | `https://dydx-ops-grpc.kingnodes.com:443` (primary) | - -## Configuration - -Configure the dYdX adapter through the trading node configuration. Both data and execution -clients support environment variable fallbacks for credentials and network-specific settings. - -### Data client configuration options - -| Option | Default | Description | -|---------------------------|---------|------------------------------------------------------------------------------------------| -| `wallet_address` | `None` | dYdX wallet address. Falls back to `DYDX_WALLET_ADDRESS` / `DYDX_TESTNET_WALLET_ADDRESS` env var. | -| `is_testnet` | `False` | Connect to dYdX testnet when `True`. | -| `bars_timestamp_on_close` | `True` | Use bar close time for `ts_event` timestamps. Set `False` to use venue-native open time. | -| `base_url_http` | `None` | HTTP API endpoint override. | -| `base_url_ws` | `None` | WebSocket endpoint override. | -| `max_retries` | `3` | Maximum retry attempts for REST/WebSocket recovery. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | - -### Execution client configuration options - -| Option | Default | Description | -|--------------------------------|---------|----------------------------------------------------------------------------------------------------| -| `wallet_address` | `None` | dYdX wallet address. Falls back to `DYDX_WALLET_ADDRESS` / `DYDX_TESTNET_WALLET_ADDRESS` env var. | -| `subaccount` | `0` | Subaccount number (0-127). Subaccount 0 is the default. | -| `private_key` | `None` | Hex-encoded private key for signing. Falls back to `DYDX_PRIVATE_KEY` / `DYDX_TESTNET_PRIVATE_KEY` env var. | -| `authenticator_ids` | `None` | List of authenticator IDs for permissioned key trading (institutional setups). | -| `is_testnet` | `False` | Connect to dYdX testnet when `True`. | -| `base_url_http` | `None` | HTTP client custom endpoint override. | -| `base_url_ws` | `None` | WebSocket client custom endpoint override. | -| `base_url_grpc` | `None` | gRPC client custom endpoint override. Supports fallback with multiple URLs. | -| `max_retries` | `3` | Maximum retry attempts for order operations. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `10,000` | Maximum delay (milliseconds) between retries. | -| `grpc_rate_limit_per_second` | `4` | Maximum gRPC requests per second. Set to `None` to disable. | - -### Basic setup - -Configure a live `TradingNode` to include dYdX data and execution clients: - -```python -from nautilus_trader.adapters.dydx import DydxDataClientConfig -from nautilus_trader.adapters.dydx import DydxExecClientConfig -from nautilus_trader.adapters.dydx.constants import DYDX -from nautilus_trader.config import InstrumentProviderConfig -from nautilus_trader.config import TradingNodeConfig - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - DYDX: DydxDataClientConfig( - wallet_address=None, # Falls back to env var - instrument_provider=InstrumentProviderConfig(load_all=True), - is_testnet=False, - ), - }, - exec_clients={ - DYDX: DydxExecClientConfig( - wallet_address=None, # Falls back to env var - private_key=None, # Falls back to env var - subaccount=0, - instrument_provider=InstrumentProviderConfig(load_all=True), - is_testnet=False, - ), - }, -) -``` - -Then, create a `TradingNode` and register the client factories: - -```python -from nautilus_trader.adapters.dydx import DydxLiveDataClientFactory -from nautilus_trader.adapters.dydx import DydxLiveExecClientFactory -from nautilus_trader.adapters.dydx.constants import DYDX -from nautilus_trader.live.node import TradingNode - -node = TradingNode(config=config) - -node.add_data_client_factory(DYDX, DydxLiveDataClientFactory) -node.add_exec_client_factory(DYDX, DydxLiveExecClientFactory) - -node.build() -``` - -### API credentials - -Credentials can be passed directly via the Python config (`wallet_address`, `private_key`) or -resolved automatically from environment variables based on the `is_testnet` setting. - -#### Environment variables - -| Variable | Network | Description | -|---------------------------------|----------|------------------------------------------------| -| `DYDX_WALLET_ADDRESS` | Mainnet | Bech32-encoded wallet address (`dydx1...`). | -| `DYDX_PRIVATE_KEY` | Mainnet | Hex-encoded secp256k1 private key for signing. | -| `DYDX_TESTNET_WALLET_ADDRESS` | Testnet | Testnet wallet address (`dydx1...`). | -| `DYDX_TESTNET_PRIVATE_KEY` | Testnet | Testnet private key. | - -#### Resolution priority - -1. Value passed in the Python config (if non-empty) -2. Environment variable (selected by `is_testnet` flag) - -### Permissioned key trading - -#### What are API Trading Keys - -API Trading Keys let you delegate trading to a separate signing key without sharing your main -wallet's seed phrase. The API key can place trades using all available margin in the owner's -cross-margin account, but cannot withdraw funds or transfer assets. - -#### Creating an API key - -1. In the dYdX web app, navigate to **More → API Trading Keys** -2. Click **Generate New API Key** -3. Save the **API Wallet Address** and **Private Key** (shown once, not stored by dYdX) -4. Click **Authorize API Key** (this registers the key on-chain as an authenticator) -5. The key is now active and can be used for trading - -See the [dYdX API Trading Keys guide](https://docs.dydx.xyz/concepts/trading/api-trading-keys) for full details on creating and managing API keys. - -#### Adapter configuration - -There are two ways to configure the adapter for API Trading Key usage: - -**Auto-resolution (recommended):** Set the API key's private key as `DYDX_PRIVATE_KEY` and the -owner's wallet address as `DYDX_WALLET_ADDRESS`. The adapter detects the mismatch during connect -and automatically queries the chain for matching authenticator IDs. No manual ID configuration -needed. - -```python -config = DydxExecClientConfig( - wallet_address="dydx1owner...", # Owner account (holds margin) - private_key="0xapikey...", # API Trading Key private key - # authenticator_ids resolved automatically -) -``` - -**Manual override:** If you know the authenticator IDs (e.g., from the dYdX TypeScript client), -pass them directly to skip auto-resolution: - -```python -config = DydxExecClientConfig( - wallet_address="dydx1owner...", - private_key="0xapikey...", - authenticator_ids=[1, 2], # Skip auto-resolution -) -``` - -:::note -API Trading Keys only work with **cross-margin** accounts and cross markets. Isolated margin -is not supported. -::: - -## Order books - -Order books can be maintained at full depth or top-of-book quotes depending on the subscription. -The venue does not provide quotes directly. Instead, the adapter subscribes to order book deltas -and synthesizes quotes for the `DataEngine` when there is a top-of-book price or size change. -Only L2 (MBP) book type is supported. - -## Contributing - -:::info -For additional features or to contribute to the dYdX adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/hyperliquid.md b/nautilus-docs/docs/integrations/hyperliquid.md deleted file mode 100644 index eebaa95..0000000 --- a/nautilus-docs/docs/integrations/hyperliquid.md +++ /dev/null @@ -1,494 +0,0 @@ -# Hyperliquid - -[Hyperliquid](https://hyperliquid.gitbook.io/hyperliquid-docs) is a decentralized perpetual futures -and spot exchange built on the Hyperliquid L1, a purpose-built blockchain optimized for trading. -HyperCore provides a fully on-chain order book and matching engine. This integration supports -live market data ingest and order execution on Hyperliquid. - -## Overview - -This adapter is implemented in Rust with Python bindings. It provides direct integration -with Hyperliquid's REST and WebSocket APIs without requiring external client libraries. - -The Hyperliquid adapter includes multiple components: - -- `HyperliquidHttpClient`: Low-level HTTP API connectivity. -- `HyperliquidWebSocketClient`: Low-level WebSocket API connectivity. -- `HyperliquidInstrumentProvider`: Instrument parsing and loading functionality. -- `HyperliquidDataClient`: Market data feed manager. -- `HyperliquidExecutionClient`: Account management and trade execution gateway. -- `HyperliquidLiveDataClientFactory`: Factory for Hyperliquid data clients (used by the trading node builder). -- `HyperliquidLiveExecClientFactory`: Factory for Hyperliquid execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as shown below) -and won't need to work directly with these lower-level components. -::: - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/hyperliquid/). - -## Revoking builder code approval - -Previous versions of NautilusTrader required users to approve a builder code fee before trading. -**This is no longer required.** If you previously approved the builder fee and wish to revoke it, -you can run the revoke script. - -The script reads your private key from environment variables (`HYPERLIQUID_PK` or `HYPERLIQUID_TESTNET_PK`). - -```bash -python nautilus_trader/adapters/hyperliquid/scripts/builder_fee_revoke.py -``` - -Testnet: - -```bash -HYPERLIQUID_TESTNET=true python nautilus_trader/adapters/hyperliquid/scripts/builder_fee_revoke.py -``` - -Alternatively, from Rust: - -```bash -cargo run --bin hyperliquid-builder-fee-revoke -``` - -## Testnet setup - -Hyperliquid provides a testnet environment for testing strategies with mock funds. - -:::info -**Mainnet account required.** Hyperliquid's testnet faucet only works for wallets that have -previously deposited on mainnet. You must fund a mainnet account first before you can obtain -testnet USDC. -::: - -### Getting testnet funds - -To receive testnet USDC, you must first have deposited on **mainnet** using the same wallet address: - -1. Visit the [Hyperliquid mainnet portal](https://app.hyperliquid.xyz/) and make a deposit with your wallet. -2. Visit the [testnet faucet](https://app.hyperliquid-testnet.xyz/drip) using the same wallet. -3. Claim 1,000 mock USDC from the faucet. - -:::note -**Email wallet users**: Email login generates different addresses for mainnet vs testnet. -To use the faucet, export your email wallet from mainnet, import it into MetaMask or Rabby, -then connect the extension to testnet. -::: - -### Creating a testnet account - -1. Visit the [Hyperliquid testnet portal](https://app.hyperliquid-testnet.xyz/). -2. Connect your wallet (MetaMask, WalletConnect, or email). -3. The testnet automatically creates an account for your wallet address. - -### Exporting your private key - -To use your testnet account with NautilusTrader, you need to export your wallet's private key: - -**MetaMask:** - -1. Click the three dots menu next to your account. -2. Select "Account details". -3. Click "Show private key". -4. Enter your password and copy the private key. - -:::warning -**Never share your private keys.** -Store private keys securely using environment variables; never commit them to version control. -::: - -### Setting environment variables - -Set your testnet credentials as environment variables: - -```bash -export HYPERLIQUID_TESTNET_PK="your_private_key_here" -# Optional: for vault trading -export HYPERLIQUID_TESTNET_VAULT="vault_address_here" -``` - -The adapter automatically loads these when `testnet=True` in the configuration. - -## Product support - -Hyperliquid offers linear perpetual futures and native spot markets. - -| Product Type | Data Feed | Trading | Notes | -|-------------------|-----------|---------|----------------------------| -| Perpetual Futures | ✓ | ✓ | USDC-settled linear perps. | -| Spot | ✓ | ✓ | Native spot markets. | - -:::note -Perpetual futures on Hyperliquid are settled in USDC. Spot markets are standard currency pairs. -::: - -## Symbology - -Hyperliquid uses a specific symbol format for instruments: - -### Perpetual futures - -Format: `{Base}-USD-PERP` - -Examples: - -- `BTC-USD-PERP` - Bitcoin perpetual futures -- `ETH-USD-PERP` - Ethereum perpetual futures -- `SOL-USD-PERP` - Solana perpetual futures - -To subscribe in your strategy: - -```python -InstrumentId.from_str("BTC-USD-PERP.HYPERLIQUID") -InstrumentId.from_str("ETH-USD-PERP.HYPERLIQUID") -``` - -### Spot markets - -Format: `{Base}-{Quote}-SPOT` - -Examples: - -- `PURR-USDC-SPOT` - PURR/USDC spot pair -- `HYPE-USDC-SPOT` - HYPE/USDC spot pair - -To subscribe in your strategy: - -```python -InstrumentId.from_str("PURR-USDC-SPOT.HYPERLIQUID") -``` - -:::note -Spot instruments may include vault tokens (prefixed with `vntls:`). These are automatically -handled by the instrument provider. -::: - -## Instrument provider - -The instrument provider supports filtering when loading instruments via -`InstrumentProviderConfig(filters=...)`: - -| Filter key | Type | Description | -|-----------------------------|-------------|---------------------------------------------| -| `market_types` (or `kinds`) | `list[str]` | `"perp"` or `"spot"`. | -| `bases` | `list[str]` | Base currency codes, e.g. `["BTC", "ETH"]`. | -| `quotes` | `list[str]` | Quote currency codes, e.g. `["USDC"]`. | -| `symbols` | `list[str]` | Full symbols, e.g. `["BTC-USD-PERP"]`. | - -Example loading only perpetual instruments: - -```python -instrument_provider=InstrumentProviderConfig( - load_all=True, - filters={"market_types": ["perp"]}, -) -``` - -## Data subscriptions - -The adapter supports the following data subscriptions: - -| Data type | Subscription | Historical | Nautilus type | Notes | -|-------------------|--------------|------------|--------------------|--------------------------------------------| -| Trade ticks | ✓ | - | `TradeTick` | Via WebSocket trades channel. | -| Quote ticks | ✓ | - | `QuoteTick` | Best bid/offer from WebSocket. | -| Order book deltas | ✓ | - | `OrderBookDelta` | L2 depth. Each message is a full snapshot. | -| Bars | ✓ | ✓ | `Bar` | See supported intervals below. | -| Mark prices | ✓ | - | `MarkPriceUpdate` | Perpetual mark price ticks. | -| Index prices | ✓ | - | `IndexPriceUpdate` | Underlying index reference prices. | -| Funding rates | ✓ | - | `FundingRate` | Perpetual funding rate updates. | - -:::note -Historical quote tick and trade tick requests are not yet supported by this adapter. -::: - -### Supported bar intervals - -| Resolution | Hyperliquid candle | -|------------|--------------------| -| 1-MINUTE | `1m` | -| 3-MINUTE | `3m` | -| 5-MINUTE | `5m` | -| 15-MINUTE | `15m` | -| 30-MINUTE | `30m` | -| 1-HOUR | `1h` | -| 2-HOUR | `2h` | -| 4-HOUR | `4h` | -| 8-HOUR | `8h` | -| 12-HOUR | `12h` | -| 1-DAY | `1d` | -| 3-DAY | `3d` | -| 1-WEEK | `1w` | -| 1-MONTH | `1M` | - -## Orders capability - -Hyperliquid supports a full set of order types and execution options. - -### Order types - -| Order Type | Perpetuals | Spot | Notes | -|---------------------|------------|------|-------------------------------------------| -| `MARKET` | ✓ | ✓ | IOC limit at 0.5% slippage from best BBO. | -| `LIMIT` | ✓ | ✓ | | -| `STOP_MARKET` | ✓ | ✓ | Stop loss orders. | -| `STOP_LIMIT` | ✓ | ✓ | Stop loss with limit execution. | -| `MARKET_IF_TOUCHED` | ✓ | ✓ | Take profit at market. | -| `LIMIT_IF_TOUCHED` | ✓ | ✓ | Take profit with limit execution. | - -:::info -Conditional orders (stop and if-touched) are implemented using Hyperliquid's native trigger -order functionality with automatic TP/SL mode detection. All trigger orders are evaluated -against the [mark price](https://hyperliquid.gitbook.io/hyperliquid-docs/trading/robust-price-indices). -::: - -:::note -Market orders require cached quote data. The adapter uses the best ask (for buys) or best bid -(for sells) with 0.5% slippage. Prices are rounded to 5 significant figures, which is a -Hyperliquid API requirement for all limit prices. Ensure you subscribe to quotes for any -instrument you intend to trade with market orders. -::: - -:::note -`STOP_MARKET` and `MARKET_IF_TOUCHED` orders do not carry a limit price. The adapter derives -one from the trigger price with 0.5% slippage, rounds to 5 significant figures, and clamps to -the instrument's price precision (ceiling for buys, floor for sells). This guarantees -Hyperliquid's `limit_px >= trigger_px` (buys) / `limit_px <= trigger_px` (sells) constraint. -::: - -:::warning -**Price normalization is enabled by default.** Hyperliquid enforces a maximum of 5 significant -figures on all order prices. This is a dynamic constraint that depends on the price magnitude -and cannot be fully encoded in the static instrument price precision. For example, if ETH is -trading at $2,600 (4 integer digits), only 1 decimal place is allowed despite the instrument -having `price_precision=2`. - -By default, the adapter normalizes all outgoing limit and trigger prices to 5 significant -figures to prevent order rejections. This means your submitted prices may shift slightly. -To disable this and take full control of price formatting, set `normalize_prices=False` -in your `HyperliquidExecClientConfig`. - -If you disable normalization, you can apply the same rounding in your strategy: - -```python -from decimal import Decimal, ROUND_DOWN - -def round_to_sig_figs(price: Decimal, sig_figs: int = 5) -> Decimal: - if price == 0: - return Decimal(0) - shift = sig_figs - int(price.adjusted()) - 1 - if shift <= 0: - factor = Decimal(10) ** (-shift) - return (price / factor).to_integral_value() * factor - return round(price, shift) -``` - -::: - -### Time in force - -| Time in force | Perpetuals | Spot | Notes | -|---------------|------------|------|----------------------| -| `GTC` | ✓ | ✓ | Good Till Canceled. | -| `IOC` | ✓ | ✓ | Immediate or Cancel. | -| `FOK` | - | - | *Not supported*. | -| `GTD` | - | - | *Not supported*. | - -### Execution instructions - -| Instruction | Perpetuals | Spot | Notes | -|---------------|------------|------|----------------------------------| -| `post_only` | ✓ | ✓ | Equivalent to ALO time in force. | -| `reduce_only` | ✓ | ✓ | Close-only orders. | - -:::info -Post-only orders that would immediately match are rejected by Hyperliquid. The adapter detects -this and generates an `OrderRejected` event. Post-only orders are routed through Hyperliquid's -ALO (Add-Liquidity-Only) lane. -::: - -### Order operations - -| Operation | Perpetuals | Spot | Notes | -|-------------------|------------|------|-------------------------------------------------| -| Submit order | ✓ | ✓ | Single order submission. | -| Submit order list | ✓ | ✓ | Batch order submission (single API call). | -| Modify order | ✓ | ✓ | Requires venue order ID. | -| Cancel order | ✓ | ✓ | Cancel by client order ID. | -| Cancel all orders | ✓ | ✓ | Iterates cached open orders by instrument/side. | -| Batch cancel | ✓ | ✓ | Iterates provided cancel list. | - -:::warning -Cancel all and batch cancel issue individual cancel requests per order. -::: - -:::info -Orders placed outside NautilusTrader (e.g. via the Hyperliquid web UI or another client) -are detected and tracked as external orders. They appear in order status reports and position -reconciliation. -::: - -## Order books - -Order books are maintained via L2 WebSocket subscription. Each message delivers a full-depth -snapshot (clear + rebuild), not incremental deltas. - -:::note -There is a limitation of one order book per instrument per trader instance. -::: - -## Account and position management - -The adapter uses cross-margin mode and reports account state with USDC balances and margin -usage. On connect, the execution client performs a full reconciliation of orders, fills, and -positions against Hyperliquid's clearinghouse state. This ensures the local cache is -consistent even after restarts or disconnections. - -:::note -Leverage is managed directly through the Hyperliquid web UI or API, not through the adapter. -Set your desired leverage per instrument on Hyperliquid before trading. -::: - -## Connection management - -The adapter automatically reconnects on WebSocket disconnection using exponential backoff -(starting at 250ms, up to 5s). On reconnect, all active subscriptions are resubscribed -automatically, and order book snapshots are rebuilt. No manual intervention is required. - -A heartbeat ping is sent every 30 seconds to keep the connection alive (Hyperliquid closes -idle connections after 60 seconds). - -## API credentials - -There are two options for supplying your credentials to the Hyperliquid clients. -Either pass the corresponding values to the configuration objects, or -set the following environment variables: - -For Hyperliquid mainnet clients, you can set: - -- `HYPERLIQUID_PK` -- `HYPERLIQUID_VAULT` (optional, for vault trading) - -For Hyperliquid testnet clients, you can set: - -- `HYPERLIQUID_TESTNET_PK` -- `HYPERLIQUID_TESTNET_VAULT` (optional, for vault trading) - -:::tip -We recommend using environment variables to manage your credentials. -::: - -## Vault trading - -Hyperliquid supports [vault trading](https://hyperliquid.gitbook.io/hyperliquid-docs/trading/vaults), -where a wallet operates on behalf of a vault (sub-account). Orders are signed with the -wallet's private key but include the vault address in the signature payload. - -To trade via a vault, set the `vault_address` in your execution client config (or set the -`HYPERLIQUID_VAULT` / `HYPERLIQUID_TESTNET_VAULT` environment variable). - -:::warning -When vault trading is enabled, WebSocket subscriptions for order and fill updates automatically -use the vault address instead of the wallet address. This is required to receive the vault's -order and fill events. -::: - -## Funding rates - -Hyperliquid perpetual futures use a fixed 1-hour funding interval. The adapter sets -`interval` to `60` (minutes) on all `FundingRateUpdate` objects. - -## Rate limiting - -The adapter implements a token bucket rate limiter for Hyperliquid's REST API with a capacity -of 1200 weight per minute. HTTP info requests are automatically retried with exponential -backoff (full jitter) on rate limit (429) and server error (5xx) responses. - -## Configuration - -### Data client configuration options - -| Option | Default | Description | -|---------------------|---------|-------------------------------------------------| -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `testnet` | `False` | Connect to the Hyperliquid testnet when `True`. | -| `http_timeout_secs` | `10` | Timeout (seconds) applied to REST calls. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Reserved; WebSocket proxy not yet implemented. | - -### Execution client configuration options - -| Option | Default | Description | -|----------------------------|---------|-------------------------------------------------------------------------------------------| -| `private_key` | `None` | EVM private key; loaded from `HYPERLIQUID_PK` or `HYPERLIQUID_TESTNET_PK` when omitted. | -| `vault_address` | `None` | Vault address; loaded from `HYPERLIQUID_VAULT` or `HYPERLIQUID_TESTNET_VAULT` if omitted. | -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `testnet` | `False` | Connect to the Hyperliquid testnet when `True`. | -| `max_retries` | `None` | Maximum retry attempts for submit, cancel, or modify order requests. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | -| `http_timeout_secs` | `10` | Timeout (seconds) applied to REST calls. | -| `normalize_prices` | `True` | Normalize order prices to 5 significant figures before submission. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Reserved; WebSocket proxy not yet implemented. | - -### Configuration example - -```python -from nautilus_trader.adapters.hyperliquid import HYPERLIQUID -from nautilus_trader.adapters.hyperliquid import HyperliquidDataClientConfig -from nautilus_trader.adapters.hyperliquid import HyperliquidExecClientConfig -from nautilus_trader.config import InstrumentProviderConfig -from nautilus_trader.config import TradingNodeConfig - -config = TradingNodeConfig( - data_clients={ - HYPERLIQUID: HyperliquidDataClientConfig( - instrument_provider=InstrumentProviderConfig(load_all=True), - testnet=True, # Use testnet - ), - }, - exec_clients={ - HYPERLIQUID: HyperliquidExecClientConfig( - private_key=None, # Loads from HYPERLIQUID_TESTNET_PK env var - vault_address=None, # Optional: loads from HYPERLIQUID_TESTNET_VAULT - instrument_provider=InstrumentProviderConfig(load_all=True), - testnet=True, # Use testnet - normalize_prices=True, # Rounds prices to 5 significant figures - ), - }, -) -``` - -:::note -When `testnet=True`, the adapter automatically uses testnet environment variables -(`HYPERLIQUID_TESTNET_PK` and `HYPERLIQUID_TESTNET_VAULT`) instead of mainnet variables. -::: - -Then, create a `TradingNode` and add the client factories: - -```python -from nautilus_trader.adapters.hyperliquid import HYPERLIQUID -from nautilus_trader.adapters.hyperliquid import HyperliquidLiveDataClientFactory -from nautilus_trader.adapters.hyperliquid import HyperliquidLiveExecClientFactory -from nautilus_trader.live.node import TradingNode - -# Instantiate the live trading node with a configuration -node = TradingNode(config=config) - -# Register the client factories with the node -node.add_data_client_factory(HYPERLIQUID, HyperliquidLiveDataClientFactory) -node.add_exec_client_factory(HYPERLIQUID, HyperliquidLiveExecClientFactory) - -# Finally build the node -node.build() -``` - -## Contributing - -:::info -For additional features or to contribute to the Hyperliquid adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/ib.md b/nautilus-docs/docs/integrations/ib.md deleted file mode 100644 index c745ea7..0000000 --- a/nautilus-docs/docs/integrations/ib.md +++ /dev/null @@ -1,2058 +0,0 @@ -# Interactive Brokers - -Interactive Brokers (IB) is a trading platform providing market access across a wide range of financial instruments, including stocks, options, futures, currencies, bonds, funds, and cryptocurrencies. NautilusTrader offers an adapter to integrate with IB using their [Trader Workstation (TWS) API](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/) through their Python library, [ibapi](https://github.com/nautechsystems/ibapi). - -The TWS API is an interface to IB's standalone trading applications: TWS and IB Gateway. Both can be downloaded from the IB website. If you haven't installed TWS or IB Gateway yet, refer to the [Initial Setup](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/#tws-download) guide. In NautilusTrader, you'll establish a connection to one of these applications via the `InteractiveBrokersClient`. - -Alternatively, you can start with a [dockerized version](https://github.com/gnzsnz/ib-gateway-docker) of the IB Gateway, which is particularly useful when deploying trading strategies on a hosted cloud platform. This requires having [Docker](https://www.docker.com/) installed on your machine, along with the [docker](https://pypi.org/project/docker/) Python package, which NautilusTrader conveniently includes as an extra package. - -:::note -The standalone TWS and IB Gateway applications require manually inputting username, password, and trading mode (live or paper) at startup. The dockerized version of the IB Gateway handles these steps programmatically. -::: - -## Installation - -To install NautilusTrader with Interactive Brokers (and Docker) support: - -```bash -uv pip install "nautilus_trader[ib,docker]" -``` - -To build from source with all extras (including IB and Docker): - -```bash -uv sync --all-extras -``` - -:::note -Because IB does not provide wheels for `ibapi`, NautilusTrader [repackages](https://pypi.org/project/nautilus-ibapi/) it for release on PyPI. -::: - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/interactive_brokers/). - -## Getting started - -Before implementing your trading strategies, make sure that either TWS (Trader Workstation) or IB Gateway is running. You can log in to one of these standalone applications with your credentials, or connect programmatically via `DockerizedIBGateway`. - -### Connection methods - -There are two primary ways to connect to Interactive Brokers: - -1. **Connect to an existing TWS or IB Gateway instance** -2. **Use the dockerized IB Gateway (recommended for automated deployments)** - -### Default ports - -Interactive Brokers uses different default ports depending on the application and trading mode: - -| Application | Paper Trading | Live Trading | -|-------------|---------------|--------------| -| TWS | 7497 | 7496 | -| IB Gateway | 4002 | 4001 | - -### Establish connection to an existing gateway or TWS - -When connecting to a pre-existing gateway or TWS, specify the `ibg_host` and `ibg_port` parameters in both the `InteractiveBrokersDataClientConfig` and `InteractiveBrokersExecClientConfig`: - -```python -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig - -# Example for TWS paper trading (default port 7497) -data_config = InteractiveBrokersDataClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=1, -) - -exec_config = InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=1, - account_id="DU123456", # Your paper trading account ID -) -``` - -### Establish connection to Dockerized IB Gateway - -For automated deployments, the dockerized gateway is recommended. Supply `dockerized_gateway` with an instance of `DockerizedIBGatewayConfig` in both client configurations. The `ibg_host` and `ibg_port` parameters are not needed as they're managed automatically. - -```python -from nautilus_trader.adapters.interactive_brokers.config import DockerizedIBGatewayConfig -from nautilus_trader.adapters.interactive_brokers.gateway import DockerizedIBGateway - -gateway_config = DockerizedIBGatewayConfig( - username="your_username", # Or set TWS_USERNAME env var - password="your_password", # Or set TWS_PASSWORD env var - trading_mode="paper", # "paper" or "live" - read_only_api=True, # Set to False to allow order execution - timeout=300, # Startup timeout in seconds -) - -# This may take a short while to start up, especially the first time -gateway = DockerizedIBGateway(config=gateway_config) -gateway.start() - -# Confirm you are logged in -print(gateway.is_logged_in(gateway.container)) - -# Inspect the logs -print(gateway.container.logs()) -``` - -### Environment variables - -To supply credentials to the Interactive Brokers Gateway, either pass the `username` and `password` to the `DockerizedIBGatewayConfig`, or set the following environment variables: - -- `TWS_USERNAME`: Your IB account username. -- `TWS_PASSWORD`: Your IB account password. -- `TWS_ACCOUNT`: Your IB account ID (used as the fallback for `account_id`). - -### Connection management - -The adapter includes connection management features: - -- **Automatic reconnection**: Configure retries with the `IB_MAX_CONNECTION_ATTEMPTS` environment variable. -- **Connection timeout**: Adjust the timeout with the `connection_timeout` parameter (default: 300 seconds). -- **Connection watchdog**: Monitor connection health and trigger reconnection automatically when required. -- **Graceful error handling**: Handle diverse connection scenarios with error classification. - -## Overview - -The Interactive Brokers adapter provides an integration with IB's TWS API. The adapter includes several major components: - -### Core components - -- **`InteractiveBrokersClient`**: The central client that executes TWS API requests using `ibapi`. Manages connections, handles errors, and coordinates all API interactions. -- **`InteractiveBrokersDataClient`**: Connects to the Gateway for streaming market data including quotes, trades, and bars. -- **`InteractiveBrokersExecutionClient`**: Handles account information, order management, and trade execution. -- **`InteractiveBrokersInstrumentProvider`**: Retrieves and manages instrument definitions, including support for options and futures chains. -- **`HistoricInteractiveBrokersClient`**: Provides methods for retrieving instruments and historical data, useful for backtesting and research. - -### Supporting components - -- **`DockerizedIBGateway`**: Manages dockerized IB Gateway instances for automated deployments. -- **Configuration classes**: Provide configuration options for all components. -- **Factory classes**: Create and configure client instances with the necessary dependencies. - -### Supported asset classes - -The adapter supports trading across all major asset classes available through Interactive Brokers: - -- **Equities**: Stocks, ETFs, and equity options. -- **Fixed income**: Bonds and bond funds. -- **Derivatives**: Futures, options, and warrants. -- **Foreign exchange**: Spot FX and FX forwards. -- **Cryptocurrencies**: Bitcoin, Ethereum, and other digital assets. -- **Commodities**: Physical commodities and commodity futures. -- **Indices**: Index products and index options. - -## The Interactive Brokers client - -The `InteractiveBrokersClient` is the central component of the IB adapter, overseeing a range of functions. These include establishing and maintaining connections, handling API errors, executing trades, and gathering various types of data such as market data, contract/instrument data, and account details. - -The `InteractiveBrokersClient` is divided into specialized mixin classes, each handling a specific responsibility. - -### Client architecture - -The client uses a mixin-based architecture where each mixin handles a specific aspect of the IB API: - -#### Connection management (`InteractiveBrokersClientConnectionMixin`) - -- Establishes and maintains socket connections to TWS/Gateway. -- Handles connection timeouts and reconnection logic. -- Manages connection state and health monitoring. -- Supports configurable reconnection attempts via `IB_MAX_CONNECTION_ATTEMPTS` environment variable. - -#### Error handling (`InteractiveBrokersClientErrorMixin`) - -- Processes all API errors and warnings. -- Categorizes errors by type (client errors, connectivity issues, request errors). -- Handles subscription and request-specific error scenarios. -- Provides error logging and debugging information. - -#### Account management (`InteractiveBrokersClientAccountMixin`) - -- Retrieves account information and balances. -- Manages position data and portfolio updates. -- Handles multi-account scenarios. -- Processes account-related notifications. - -#### Contract/instrument management (`InteractiveBrokersClientContractMixin`) - -- Retrieves contract details and specifications. -- Handles instrument searches and lookups. -- Manages contract validation and verification. -- Supports complex instrument types (options chains, futures chains). - -#### Market data management (`InteractiveBrokersClientMarketDataMixin`) - -- Handles real-time and historical market data subscriptions. -- Processes quotes, trades, and bar data. -- Manages market data type settings (real-time, delayed, frozen). -- Handles tick-by-tick data and market depth. - -#### Order management (`InteractiveBrokersClientOrderMixin`) - -- Processes order placement, modification, and cancellation. -- Handles order status updates and execution reports. -- Manages order validation and error handling. -- Supports complex order types and conditions. - -### Key features - -- **Asynchronous operation**: All operations are fully asynchronous using Python's asyncio. -- **Error handling**: Error categorization and handling. -- **Connection resilience**: Automatic reconnection with configurable retry logic. -- **Message processing**: Efficient message queue processing for high-throughput scenarios. -- **State management**: Proper state tracking for connections, subscriptions, and requests. - -:::tip -To troubleshoot TWS API incoming message issues, consider starting at the `InteractiveBrokersClient._process_message` method, which acts as the primary gateway for processing all messages received from the API. -::: - -## Symbology - -The `InteractiveBrokersInstrumentProvider` supports three methods for constructing `InstrumentId` instances, which can be configured via the `symbology_method` enum in `InteractiveBrokersInstrumentProviderConfig`. - -### Symbology methods - -#### 1. Simplified symbology (`IB_SIMPLIFIED`) - default - -When `symbology_method` is set to `IB_SIMPLIFIED` (the default setting), the system uses intuitive, human-readable symbology rules: - -**Format Rules by Asset Class:** - -- **Forex**: `{symbol}/{currency}.{exchange}` - - Example: `EUR/USD.IDEALPRO` -- **Stocks**: `{localSymbol}.{primaryExchange}` - - Spaces in localSymbol are replaced with hyphens - - Example: `BF-B.NYSE`, `SPY.ARCA` -- **Futures**: `{localSymbol}.{exchange}` - - Individual contracts use single digit years - - Example: `ESM4.CME`, `CLZ7.NYMEX` -- **Continuous Futures**: `{symbol}.{exchange}` - - Represents front month, automatically rolling - - Example: `ES.CME`, `CL.NYMEX` -- **Options on Futures (FOP)**: `{localSymbol}.{exchange}` - - Format: `{symbol}{month}{year} {right}{strike}` - - Example: `ESM4 C4200.CME` -- **Options**: `{localSymbol}.{exchange}` - - All spaces removed from localSymbol - - Example: `AAPL230217P00155000.SMART` -- **Indices**: `^{localSymbol}.{exchange}` - - Example: `^SPX.CBOE`, `^NDX.NASDAQ` -- **Bonds**: `{localSymbol}.{exchange}` - - Example: `912828XE8.SMART` -- **Cryptocurrencies**: `{symbol}/{currency}.{exchange}` - - Example: `BTC/USD.PAXOS`, `ETH/USD.PAXOS` - -#### 2. Raw symbology (`IB_RAW`) - -Setting `symbology_method` to `IB_RAW` enforces stricter parsing rules that align directly with the fields defined in the IB API. This method provides maximum compatibility across all regions and instrument types: - -**Format Rules:** - -- **CFDs**: `{localSymbol}={secType}.IBCFD` -- **Commodities**: `{localSymbol}={secType}.IBCMDTY` -- **Default for Other Types**: `{localSymbol}={secType}.{exchange}` - -**Examples:** - -- `IBUS30=CFD.IBCFD` -- `XAUUSD=CMDTY.IBCMDTY` -- `AAPL=STK.SMART` - -This configuration ensures explicit instrument identification and supports instruments from any region, especially those with non-standard symbology where simplified parsing may fail. - -### MIC venue conversion - -The adapter supports converting Interactive Brokers exchange codes to Market Identifier Codes (MIC) for standardized venue identification: - -#### `convert_exchange_to_mic_venue` - -When set to `True`, the adapter automatically converts IB exchange codes to their corresponding MIC codes: - -```python -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - convert_exchange_to_mic_venue=True, # Enable MIC conversion - symbology_method=SymbologyMethod.IB_SIMPLIFIED, -) -``` - -**Examples of MIC Conversion:** - -- `CME` → `XCME` (Chicago Mercantile Exchange) -- `NASDAQ` → `XNAS` (Nasdaq Stock Market) -- `NYSE` → `XNYS` (New York Stock Exchange) -- `LSE` → `XLON` (London Stock Exchange) - -#### `symbol_to_mic_venue` - -For custom venue mapping, use the `symbol_to_mic_venue` dictionary to override default conversions: - -```python -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - convert_exchange_to_mic_venue=True, - symbol_to_mic_venue={ - "ES": "XCME", # All ES futures/options use CME MIC - "SPY": "ARCX", # SPY specifically uses ARCA - }, -) -``` - -### Supported instrument formats - -The adapter supports various instrument formats based on Interactive Brokers' contract specifications: - -#### Futures month codes - -- **F** = January, **G** = February, **H** = March, **J** = April -- **K** = May, **M** = June, **N** = July, **Q** = August -- **U** = September, **V** = October, **X** = November, **Z** = December - -#### Supported exchanges by asset class - -**Futures Exchanges:** - -- `CME`, `CBOT`, `NYMEX`, `COMEX`, `KCBT`, `MGE`, `NYBOT`, `SNFE` - -**Options Exchanges:** - -- `SMART` (IB's smart routing) - -**Forex Exchanges:** - -- `IDEALPRO` (IB's forex platform) - -**Cryptocurrency Exchanges:** - -- `PAXOS` (IB's crypto platform) - -**CFD/Commodity Exchanges:** - -- `IBCFD`, `IBCMDTY` (IB's internal routing) - -### Choosing the right symbology method - -- **Use `IB_SIMPLIFIED`** (default) for most use cases - provides clean, readable instrument IDs -- **Use `IB_RAW`** when dealing with complex international instruments or when simplified parsing fails -- **Enable `convert_exchange_to_mic_venue`** when you need standardized MIC venue codes for compliance or data consistency - -## Instruments and contracts - -In Interactive Brokers, a NautilusTrader `Instrument` corresponds to an IB [Contract](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/#contracts). The adapter handles two types of contract representations: - -### Contract types - -#### Basic contract (`IBContract`) - -- Contains essential contract identification fields -- Used for contract searches and basic operations -- Cannot be directly converted to a NautilusTrader `Instrument` - -#### Contract details (`IBContractDetails`) - -- Contains contract information including: - - Order types supported - - Trading hours and calendar - - Margin requirements - - Price increments and multipliers - - Market data permissions -- Can be converted to a NautilusTrader `Instrument` -- Required for trading operations - -### Contract discovery - -To search for contract information, use the [IB Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/). - -### Loading instruments - -There are two primary methods for loading instruments: - -#### 1. Using `load_ids` (recommended) - -Use `symbology_method=SymbologyMethod.IB_SIMPLIFIED` (default) with `load_ids` for clean, intuitive instrument identification: - -```python -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig -from nautilus_trader.adapters.interactive_brokers.config import SymbologyMethod - -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - symbology_method=SymbologyMethod.IB_SIMPLIFIED, - load_ids=frozenset([ - "EUR/USD.IDEALPRO", # Forex - "SPY.ARCA", # Stock - "ESM24.CME", # Future - "BTC/USD.PAXOS", # Crypto - "^SPX.CBOE", # Index - ]), -) -``` - -#### 2. Using `load_contracts` (for complex instruments) - -Use `load_contracts` with `IBContract` instances for complex scenarios like options/futures chains: - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBContract - -# Load options chain for specific expiry -options_chain_expiry = IBContract( - secType="IND", - symbol="SPX", - exchange="CBOE", - build_options_chain=True, - lastTradeDateOrContractMonth='20240718', -) - -# Load options chain for date range -options_chain_range = IBContract( - secType="IND", - symbol="SPX", - exchange="CBOE", - build_options_chain=True, - min_expiry_days=0, - max_expiry_days=30, -) - -# Load futures chain -futures_chain = IBContract( - secType="CONTFUT", - exchange="CME", - symbol="ES", - build_futures_chain=True, -) - -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - load_contracts=frozenset([ - options_chain_expiry, - options_chain_range, - futures_chain, - ]), -) -``` - -### IBContract examples by asset class - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBContract - -# Stocks -IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY') -IBContract(secType='STK', exchange='SMART', primaryExchange='NASDAQ', symbol='AAPL') - -# Bonds -IBContract(secType='BOND', secIdType='ISIN', secId='US03076KAA60') -IBContract(secType='BOND', secIdType='CUSIP', secId='912828XE8') - -# Individual Options -IBContract(secType='OPT', exchange='SMART', symbol='SPY', - lastTradeDateOrContractMonth='20251219', strike=500, right='C') - -# Options Chain (loads all strikes/expirations) -IBContract(secType='STK', exchange='SMART', primaryExchange='ARCA', symbol='SPY', - build_options_chain=True, min_expiry_days=10, max_expiry_days=60) - -# CFDs -IBContract(secType='CFD', symbol='IBUS30') -IBContract(secType='CFD', symbol='DE40EUR', exchange='SMART') - -# Individual Futures -IBContract(secType='FUT', exchange='CME', symbol='ES', - lastTradeDateOrContractMonth='20240315') - -# Futures Chain (loads all expirations) -IBContract(secType='CONTFUT', exchange='CME', symbol='ES', build_futures_chain=True) - -# Options on Futures (FOP) - Individual -IBContract(secType='FOP', exchange='CME', symbol='ES', - lastTradeDateOrContractMonth='20240315', strike=4200, right='C') - -# Options on Futures Chain (loads all strikes/expirations) -IBContract(secType='CONTFUT', exchange='CME', symbol='ES', - build_options_chain=True, min_expiry_days=7, max_expiry_days=60) - -# Forex -IBContract(secType='CASH', exchange='IDEALPRO', symbol='EUR', currency='USD') -IBContract(secType='CASH', exchange='IDEALPRO', symbol='GBP', currency='JPY') - -# Cryptocurrencies -IBContract(secType='CRYPTO', symbol='BTC', exchange='PAXOS', currency='USD') -IBContract(secType='CRYPTO', symbol='ETH', exchange='PAXOS', currency='USD') - -# Indices -IBContract(secType='IND', symbol='SPX', exchange='CBOE') -IBContract(secType='IND', symbol='NDX', exchange='NASDAQ') - -# Commodities -IBContract(secType='CMDTY', symbol='XAUUSD', exchange='SMART') -``` - -### Advanced configuration options - -```python -# Options chain with custom exchange -IBContract( - secType="STK", - symbol="AAPL", - exchange="SMART", - primaryExchange="NASDAQ", - build_options_chain=True, - options_chain_exchange="CBOE", # Use CBOE for options instead of SMART - min_expiry_days=7, - max_expiry_days=45, -) - -# Futures chain with specific months -IBContract( - secType="CONTFUT", - exchange="NYMEX", - symbol="CL", # Crude Oil - build_futures_chain=True, - min_expiry_days=30, - max_expiry_days=180, -) -``` - -### Continuous futures - -For continuous futures contracts (using `secType='CONTFUT'`), the adapter creates instrument IDs using just the symbol and venue: - -```python -# Continuous futures examples -IBContract(secType='CONTFUT', exchange='CME', symbol='ES') # → ES.CME -IBContract(secType='CONTFUT', exchange='NYMEX', symbol='CL') # → CL.NYMEX - -# With MIC venue conversion enabled -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - convert_exchange_to_mic_venue=True, -) -# Results in: -# ES.XCME (instead of ES.CME) -# CL.XNYM (instead of CL.NYMEX) -``` - -**Continuous Futures vs Individual Futures:** - -- **Continuous**: `ES.CME` - Represents the front month contract, automatically rolls -- **Individual**: `ESM4.CME` - Specific March 2024 contract - -:::note -When using `build_options_chain=True` or `build_futures_chain=True`, the `secType` and `symbol` should be specified for the underlying contract. The adapter will automatically discover and load all related derivative contracts within the specified expiry range. -::: - -## Option spreads - -Interactive Brokers supports option spreads through BAG contracts, which combine multiple option legs into a single tradeable instrument. NautilusTrader provides support for creating, loading, and trading option spreads. - -### Creating option spread instrument IDs - -Option spreads are created using the `new_generic_spread_id()` function, which combines individual option legs with their respective ratios: - -```python -from nautilus_trader.model.identifiers import InstrumentId, new_generic_spread_id - -# Create individual option instrument IDs -call_leg = InstrumentId.from_str("SPY C400.SMART") -put_leg = InstrumentId.from_str("SPY P390.SMART") - -# Create a 1:1 call spread (long call, short call) -call_spread_id = new_generic_spread_id([ - (call_leg, 1), # Long 1 contract - (put_leg, -1), # Short 1 contract -]) - -# Create a 1:2 ratio spread -ratio_spread_id = new_generic_spread_id([ - (call_leg, 1), # Long 1 contract - (put_leg, 2), # Long 2 contracts -]) -``` - -### Dynamic spread loading - -Option spreads must be requested before they can be traded or subscribed to for market data. Use the `request_instrument()` method to dynamically load spread instruments: - -```python -# In your strategy's on_start method -def on_start(self): - # Request the spread instrument - self.request_instrument(spread_id) - -def on_instrument(self, instrument): - # Handle the loaded spread instrument - self.log.info(f"Loaded spread: {instrument.id}") - - # Now you can subscribe to market data - self.subscribe_quote_ticks(instrument.id) - - # And place orders - order = self.order_factory.market( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(1), - time_in_force=TimeInForce.DAY, - ) - self.submit_order(order) -``` - -### Spread trading requirements - -1. **Load individual legs first**: Ensure the individual option legs are available before creating spreads. -2. **Request the spread instrument**: Use `request_instrument()` to load the spread before trading. -3. **Subscribe to market data**: Request quote ticks after the spread is loaded. -4. **Place orders**: Any order type can be used once the spread is available. - -## Historical data and backtesting - -The `HistoricInteractiveBrokersClient` provides methods for retrieving historical data from Interactive Brokers for backtesting and research purposes. - -### Supported data types - -- **Bar data**: OHLCV bars with time, tick, and volume aggregations. -- **Tick data**: Trade ticks and quote ticks with microsecond precision. -- **Instrument data**: Complete contract specifications and trading rules. - -### Historical data client - -```python -from nautilus_trader.adapters.interactive_brokers.historical.client import HistoricInteractiveBrokersClient -from ibapi.common import MarketDataTypeEnum - -# Initialize the client -client = HistoricInteractiveBrokersClient( - host="127.0.0.1", - port=7497, - client_id=1, - market_data_type=MarketDataTypeEnum.DELAYED_FROZEN, # Use delayed data if no subscription - log_level="INFO" -) - -# Connect to TWS/Gateway -await client.connect() -``` - -### Retrieving instruments - -#### Basic instrument retrieval - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBContract - -# Define contracts -contracts = [ - IBContract(secType="STK", symbol="AAPL", exchange="SMART", primaryExchange="NASDAQ"), - IBContract(secType="STK", symbol="MSFT", exchange="SMART", primaryExchange="NASDAQ"), - IBContract(secType="CASH", symbol="EUR", currency="USD", exchange="IDEALPRO"), -] - -# Request instrument definitions -instruments = await client.request_instruments(contracts=contracts) -``` - -#### Option chain retrieval with catalog storage - -You can download entire option chains using `request_instruments` in your strategy, with the added benefit of saving the data to the catalog using `update_catalog=True`: - -```python -# In your strategy's on_start method -def on_start(self): - self.request_instruments( - venue=IB_VENUE, - update_catalog=True, - params={ - "ib_contracts": ( - # SPY options - { - "secType": "STK", - "symbol": "SPY", - "exchange": "SMART", - "primaryExchange": "ARCA", - "build_options_chain": True, - "min_expiry_days": 7, - "max_expiry_days": 30, - }, - # QQQ options - { - "secType": "STK", - "symbol": "QQQ", - "exchange": "SMART", - "primaryExchange": "NASDAQ", - "build_options_chain": True, - "min_expiry_days": 7, - "max_expiry_days": 30, - }, - # ES futures options - { - "secType": "CONTFUT", - "exchange": "CME", - "symbol": "ES", - "build_options_chain": True, - "min_expiry_days": 0, - "max_expiry_days": 60, - }, - # SPX index options - { - "secType": "IND", - "symbol": "SPX", - "exchange": "CBOE", - "build_options_chain": True, - "min_expiry_days": 0, - "max_expiry_days": 5, - }, - # ES futures chain and futures options - { - "secType": "CONTFUT", - "exchange": "CME", - "symbol": "ES", - "build_futures_chain": True, - "build_options_chain": True, - "min_expiry_days": 0, - "max_expiry_days": 2, - }, - # ESTX50 index options (Eurex) - { - "secType": "IND", - "exchange": "EUREX", - "symbol": "ESTX50", - "build_options_chain": True, - "min_expiry_days": 0, - "max_expiry_days": 2, - }, - ), - }, - ) -``` - -### Retrieving historical bars - -```python -import datetime - -# Request historical bars -bars = await client.request_bars( - bar_specifications=[ - "1-MINUTE-LAST", # 1-minute bars using last price - "5-MINUTE-MID", # 5-minute bars using midpoint - "1-HOUR-LAST", # 1-hour bars using last price - "1-DAY-LAST", # Daily bars using last price - ], - start_date_time=datetime.datetime(2023, 11, 1, 9, 30), - end_date_time=datetime.datetime(2023, 11, 6, 16, 30), - tz_name="America/New_York", - contracts=contracts, - use_rth=True, # Regular Trading Hours only - timeout=120, # Request timeout in seconds -) -``` - -### Retrieving historical ticks - -```python -# Request historical tick data (use tick_type="TRADES" or "BID_ASK" for quote ticks) -ticks = await client.request_ticks( - tick_type="TRADES", - start_date_time=datetime.datetime(2023, 11, 6, 9, 30), - end_date_time=datetime.datetime(2023, 11, 6, 16, 30), - tz_name="America/New_York", - contracts=contracts, - use_rth=True, - timeout=120, -) -``` - -### Bar specifications - -The adapter supports various bar specifications: - -#### Time-based bars - -- `"1-SECOND-LAST"`, `"5-SECOND-LAST"`, `"10-SECOND-LAST"`, `"15-SECOND-LAST"`, `"30-SECOND-LAST"` -- `"1-MINUTE-LAST"`, `"2-MINUTE-LAST"`, `"3-MINUTE-LAST"`, `"5-MINUTE-LAST"`, `"10-MINUTE-LAST"`, `"15-MINUTE-LAST"`, `"20-MINUTE-LAST"`, `"30-MINUTE-LAST"` -- `"1-HOUR-LAST"`, `"2-HOUR-LAST"`, `"3-HOUR-LAST"`, `"4-HOUR-LAST"`, `"8-HOUR-LAST"` -- `"1-DAY-LAST"`, `"1-WEEK-LAST"`, `"1-MONTH-LAST"` - -#### Price types - -- `LAST` - Last traded price -- `MID` - Midpoint of bid/ask -- `BID` - Bid price -- `ASK` - Ask price - -### Complete example - -```python -import asyncio -import datetime -from nautilus_trader.adapters.interactive_brokers.common import IBContract -from nautilus_trader.adapters.interactive_brokers.historical.client import HistoricInteractiveBrokersClient -from nautilus_trader.persistence.catalog import ParquetDataCatalog - - -async def download_historical_data(): - # Initialize client - client = HistoricInteractiveBrokersClient( - host="127.0.0.1", - port=7497, - client_id=5, - ) - - # Connect - await client.connect() - await asyncio.sleep(2) # Allow connection to stabilize - - # Define contracts - contracts = [ - IBContract(secType="STK", symbol="AAPL", exchange="SMART", primaryExchange="NASDAQ"), - IBContract(secType="CASH", symbol="EUR", currency="USD", exchange="IDEALPRO"), - ] - - # Request instruments - instruments = await client.request_instruments(contracts=contracts) - - # Request historical bars - bars = await client.request_bars( - bar_specifications=["1-HOUR-LAST", "1-DAY-LAST"], - start_date_time=datetime.datetime(2023, 11, 1, 9, 30), - end_date_time=datetime.datetime(2023, 11, 6, 16, 30), - tz_name="America/New_York", - contracts=contracts, - use_rth=True, - ) - - # Request tick data - ticks = await client.request_ticks( - tick_type="TRADES", - start_date_time=datetime.datetime(2023, 11, 6, 14, 0), - end_date_time=datetime.datetime(2023, 11, 6, 15, 0), - tz_name="America/New_York", - contracts=contracts, - ) - - # Save to catalog - catalog = ParquetDataCatalog("./catalog") - catalog.write_data(instruments) - catalog.write_data(bars) - catalog.write_data(ticks) - - print(f"Downloaded {len(instruments)} instruments") - print(f"Downloaded {len(bars)} bars") - print(f"Downloaded {len(ticks)} ticks") - - # Disconnect - await client.disconnect() - -# Run the example -if __name__ == "__main__": - asyncio.run(download_historical_data()) -``` - -### Data limitations - -Be aware of Interactive Brokers' historical data limitations: - -- **Rate Limits**: IB enforces rate limits on historical data requests -- **Data Availability**: Historical data availability varies by instrument and subscription level -- **Market Data Permissions**: Some data requires specific market data subscriptions -- **Time Ranges**: Maximum lookback periods vary by bar size and instrument type - -### Best practices - -1. **Use Delayed Data**: For backtesting, `MarketDataTypeEnum.DELAYED_FROZEN` is often sufficient -2. **Batch Requests**: Group multiple instruments in single requests when possible -3. **Handle Timeouts**: Set appropriate timeout values for large data requests -4. **Respect Rate Limits**: Add delays between requests to avoid hitting rate limits -5. **Validate Data**: Always check data quality and completeness before backtesting - -:::warning -Interactive Brokers enforces pacing limits; excessive historical-data or order requests trigger pacing violations and IB can disable the API session for several minutes. -::: - -## Live trading - -Live trading with Interactive Brokers requires setting up a `TradingNode` that incorporates both `InteractiveBrokersDataClient` and `InteractiveBrokersExecutionClient`. These clients depend on the `InteractiveBrokersInstrumentProvider` for instrument management. - -### Architecture overview - -The live trading setup consists of three main components: - -1. **InstrumentProvider**: Manages instrument definitions and contract details -2. **DataClient**: Handles real-time market data subscriptions -3. **ExecutionClient**: Manages orders, positions, and account information - -### InstrumentProvider configuration - -The `InteractiveBrokersInstrumentProvider` provides access to financial instrument data from IB. It supports loading individual instruments, options chains, and futures chains. - -#### Basic configuration - -```python -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig -from nautilus_trader.adapters.interactive_brokers.config import SymbologyMethod -from nautilus_trader.adapters.interactive_brokers.common import IBContract - -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - symbology_method=SymbologyMethod.IB_SIMPLIFIED, - build_futures_chain=False, # Set to True if fetching futures chains - build_options_chain=False, # Set to True if fetching options chains - min_expiry_days=10, # Minimum days to expiry for derivatives - max_expiry_days=60, # Maximum days to expiry for derivatives - convert_exchange_to_mic_venue=False, # Use MIC codes for venue mapping - cache_validity_days=1, # Cache instrument data for 1 day - load_ids=frozenset([ - # Individual instruments using simplified symbology - "EUR/USD.IDEALPRO", # Forex - "BTC/USD.PAXOS", # Cryptocurrency - "SPY.ARCA", # Stock ETF - "V.NYSE", # Individual stock - "ESM4.CME", # Future contract (single digit year) - "^SPX.CBOE", # Index - ]), - load_contracts=frozenset([ - # Complex instruments using IBContract - IBContract(secType='STK', symbol='AAPL', exchange='SMART', primaryExchange='NASDAQ'), - IBContract(secType='CASH', symbol='GBP', currency='USD', exchange='IDEALPRO'), - ]), -) -``` - -#### Advanced configuration for derivatives - -```python -# Configuration for options and futures chains -advanced_config = InteractiveBrokersInstrumentProviderConfig( - symbology_method=SymbologyMethod.IB_SIMPLIFIED, - build_futures_chain=True, # Enable futures chain loading - build_options_chain=True, # Enable options chain loading - min_expiry_days=7, # Load contracts expiring in 7+ days - max_expiry_days=90, # Load contracts expiring within 90 days - load_contracts=frozenset([ - # Load SPY options chain - IBContract( - secType='STK', - symbol='SPY', - exchange='SMART', - primaryExchange='ARCA', - build_options_chain=True, - ), - # Load ES futures chain - IBContract( - secType='CONTFUT', - exchange='CME', - symbol='ES', - build_futures_chain=True, - ), - ]), -) -``` - -#### Filtering security types - -Use `filter_sec_types` to ignore specific IB `secType` values. Any contract whose `secType` matches an entry in this frozenset is skipped with a warning (for example unsupported types such as `WAR` or `IOPT`): - -```python -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - load_ids=frozenset(["SPY.ARCA"]), - filter_sec_types=frozenset({"WAR", "IOPT"}), # Opt out from unsupported asset types -) -``` - -### Integration with external data providers - -The Interactive Brokers adapter can be used alongside other data providers for enhanced market data coverage. When using multiple data sources: - -- Use consistent symbology methods across providers -- Consider using `convert_exchange_to_mic_venue=True` for standardized venue identification -- Ensure instrument cache management is handled properly to avoid conflicts - -### Data client configuration - -The `InteractiveBrokersDataClient` interfaces with IB for streaming and retrieving real-time market data. Upon connection, it configures the [market data type](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/#delayed-market-data) and loads instruments based on the `InteractiveBrokersInstrumentProviderConfig` settings. - -#### Supported data types - -- **Quote Ticks**: Real-time bid/ask prices and sizes -- **Trade Ticks**: Real-time trade prices and volumes -- **Bar Data**: Real-time OHLCV bars (1-second to 1-day intervals) -- **Market Depth**: Level 2 order book data (where available) - -#### Market data types - -Interactive Brokers supports several market data types: - -- `REALTIME`: Live market data (requires market data subscriptions) -- `DELAYED`: 15-20 minute delayed data (free for most markets) -- `DELAYED_FROZEN`: Delayed data that doesn't update (useful for testing) -- `FROZEN`: Last known real-time data (when market is closed) - -#### Basic data client configuration - -```python -from nautilus_trader.adapters.interactive_brokers.config import IBMarketDataTypeEnum -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig - -data_client_config = InteractiveBrokersDataClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, # TWS paper trading port - ibg_client_id=1, - use_regular_trading_hours=True, # RTH only for stocks - market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN, # Use delayed data - ignore_quote_tick_size_updates=False, # Include size-only updates - instrument_provider=instrument_provider_config, - connection_timeout=300, # 5 minutes - request_timeout_secs=60, # 1 minute -) -``` - -#### Advanced data client configuration - -```python -# Configuration for production with real-time data -production_data_config = InteractiveBrokersDataClientConfig( - ibg_host="127.0.0.1", - ibg_port=4001, # IB Gateway live trading port - ibg_client_id=1, - use_regular_trading_hours=False, # Include extended hours - market_data_type=IBMarketDataTypeEnum.REALTIME, # Real-time data - ignore_quote_tick_size_updates=True, # Reduce tick volume - handle_revised_bars=True, # Handle bar revisions - instrument_provider=instrument_provider_config, - dockerized_gateway=dockerized_gateway_config, # If using Docker - connection_timeout=300, - request_timeout_secs=60, -) -``` - -### Data client configuration options - -| Option | Default | Description | -|---------------------------------|-------------------------------------------------|-------------| -| `instrument_provider` | `InteractiveBrokersInstrumentProviderConfig()` | Instrument provider settings controlling which contracts load at startup. | -| `ibg_host` | `127.0.0.1` | Hostname or IP for TWS/IB Gateway. | -| `ibg_port` | `None` | Port for TWS/IB Gateway (`7497`/`7496` for TWS, `4002`/`4001` for IBG). | -| `ibg_client_id` | `1` | Unique client identifier used when connecting to TWS/IB Gateway. | -| `use_regular_trading_hours` | `True` | Request bars limited to regular trading hours when `True`. | -| `market_data_type` | `REALTIME` | Market data feed type (`REALTIME`, `DELAYED`, `DELAYED_FROZEN`, etc.). | -| `ignore_quote_tick_size_updates`| `False` | Suppress quote ticks where only size changes when `True`. | -| `handle_revised_bars` | `False` | When `True`, processes bar revisions from IB (bars can be updated after initial publication). | -| `dockerized_gateway` | `None` | Optional `DockerizedIBGatewayConfig` for containerized setups. | -| `connection_timeout` | `300` | Seconds to wait for the initial API connection. | -| `request_timeout_secs` | `60` | Seconds to wait for historical data requests before timing out. | - -#### Notes - -- **`use_regular_trading_hours`**: When `True`, only requests data during regular trading hours. Primarily affects bar data for stocks. -- **`ignore_quote_tick_size_updates`**: When `True`, filters out quote ticks where only the size changed (not price), reducing data volume. -- **`handle_revised_bars`**: When `True`, processes bar revisions from IB (bars can be updated after initial publication). -- **`connection_timeout`**: Maximum time to wait for initial connection establishment. -- **`request_timeout_secs`**: Maximum time to wait for historical data requests. - -### Execution client configuration options - -| Option | Default | Description | -|-----------------------------------------|-------------------------------------------------|-------------| -| `instrument_provider` | `InteractiveBrokersInstrumentProviderConfig()` | Instrument provider settings controlling which contracts load at startup. | -| `ibg_host` | `127.0.0.1` | Hostname or IP for TWS/IB Gateway. | -| `ibg_port` | `None` | Port for TWS/IB Gateway (`7497`/`7496` for TWS, `4002`/`4001` for IBG). | -| `ibg_client_id` | `1` | Unique client identifier used when connecting to TWS/IB Gateway. | -| `account_id` | `None` | Interactive Brokers account identifier (falls back to `TWS_ACCOUNT` env var). | -| `dockerized_gateway` | `None` | Optional `DockerizedIBGatewayConfig` for containerized setups. | -| `connection_timeout` | `300` | Seconds to wait for the initial API connection. | -| `request_timeout_secs` | `60` | Seconds to wait for request responses (contract details, etc.). | -| `fetch_all_open_orders` | `False` | When `True`, pulls open orders for every API client ID (not just this session). | -| `track_option_exercise_from_position_update` | `False` | Subscribe to real-time position updates to detect option exercises when `True`. | - -### Execution client configuration - -The `InteractiveBrokersExecutionClient` handles trade execution, order management, account information, and position tracking. It provides order lifecycle management and real-time account updates. - -#### Supported functionality - -- **Order Management**: Place, modify, and cancel orders -- **Order Types**: Market, limit, stop, stop-limit, trailing stop, and more -- **Account Information**: Real-time balance and margin updates -- **Position Tracking**: Real-time position updates and P&L -- **Trade Reporting**: Execution reports and fill notifications -- **Risk Management**: Pre-trade risk checks and position limits - -#### Supported order types - -The adapter supports most Interactive Brokers order types: - -- **Market Orders**: `OrderType.MARKET` -- **Limit Orders**: `OrderType.LIMIT` -- **Stop Orders**: `OrderType.STOP_MARKET` -- **Stop-Limit Orders**: `OrderType.STOP_LIMIT` -- **Market-If-Touched**: `OrderType.MARKET_IF_TOUCHED` -- **Limit-If-Touched**: `OrderType.LIMIT_IF_TOUCHED` -- **Trailing Stop Market**: `OrderType.TRAILING_STOP_MARKET` -- **Trailing Stop Limit**: `OrderType.TRAILING_STOP_LIMIT` -- **Market-on-Close**: `OrderType.MARKET` with `TimeInForce.AT_THE_CLOSE` -- **Limit-on-Close**: `OrderType.LIMIT` with `TimeInForce.AT_THE_CLOSE` - -#### Time in force options - -- **Day Orders**: `TimeInForce.DAY` -- **Good-Till-Canceled**: `TimeInForce.GTC` -- **Immediate-or-Cancel**: `TimeInForce.IOC` -- **Fill-or-Kill**: `TimeInForce.FOK` -- **Good-Till-Date**: `TimeInForce.GTD` -- **At-the-Open**: `TimeInForce.AT_THE_OPEN` -- **At-the-Close**: `TimeInForce.AT_THE_CLOSE` - -#### Batch operations - -| Operation | Supported | Notes | -|--------------------|-----------|----------------------------------------------| -| Batch Submit | ✓ | Submit multiple orders in single request. | -| Batch Modify | ✓ | Modify multiple orders in single request. | -| Batch Cancel | ✓ | Cancel multiple orders in single request. | - -#### Position management - -| Feature | Supported | Notes | -|--------------------|-----------|----------------------------------------------| -| Query positions | ✓ | Real-time position updates. | -| Position mode | ✓ | Net vs separate long/short positions. | -| Leverage control | ✓ | Account-level margin requirements. | -| Margin mode | ✓ | Portfolio vs individual margin. | - -#### Order querying - -| Feature | Supported | Notes | -|--------------------|-----------|----------------------------------------------| -| Query open orders | ✓ | List all active orders. | -| Query order history | ✓ | Historical order data. | -| Order status updates| ✓ | Real-time order state changes. | -| Trade history | ✓ | Execution and fill reports. | - -#### Contingent orders - -| Feature | Supported | Notes | -|--------------------|-----------|----------------------------------------------| -| Order lists | ✓ | Atomic multi-order submission. | -| OCO orders | ✓ | One-Cancels-Other with customizable OCA types (1, 2, 3). | -| Bracket orders | ✓ | Parent-child order relationships. | -| Conditional orders | ✓ | Advanced order conditions and triggers. | - -#### Basic execution client configuration - -```python -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig -from nautilus_trader.config import RoutingConfig - -exec_client_config = InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, # TWS paper trading port - ibg_client_id=1, - account_id="DU123456", # Your IB account ID (paper or live) - instrument_provider=instrument_provider_config, - connection_timeout=300, - routing=RoutingConfig(default=True), # Route all orders through this client -) -``` - -#### Advanced execution client configuration - -```python -# Production configuration with dockerized gateway -production_exec_config = InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=4001, # IB Gateway live trading port - ibg_client_id=1, - account_id=None, # Will use TWS_ACCOUNT environment variable - instrument_provider=instrument_provider_config, - dockerized_gateway=dockerized_gateway_config, - connection_timeout=300, - routing=RoutingConfig(default=True), -) -``` - -#### Account ID configuration - -The `account_id` parameter is crucial and must match the account logged into TWS/Gateway: - -```python -# Option 1: Specify directly in config -exec_config = InteractiveBrokersExecClientConfig( - account_id="DU123456", # Paper trading account - # ... other parameters -) - -# Option 2: Use environment variable -import os -os.environ["TWS_ACCOUNT"] = "DU123456" -exec_config = InteractiveBrokersExecClientConfig( - account_id=None, # Will use TWS_ACCOUNT env var - # ... other parameters -) -``` - -#### Order tags and advanced features - -The adapter supports IB-specific order parameters through order tags: - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags - -# Create order with IB-specific parameters -order_tags = IBOrderTags( - allOrNone=True, # All-or-none order - ocaGroup="MyGroup1", # One-cancels-all group - ocaType=1, # Cancel with block - activeStartTime="20240315 09:30:00 EST", # GTC activation time - activeStopTime="20240315 16:00:00 EST", # GTC deactivation time - goodAfterTime="20240315 09:35:00 EST", # Good after time -) - -# Apply tags to an order -order = order_factory.limit( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(100), - price=instrument.make_price(100.0), - tags=[order_tags.value], -) -``` - -#### OCA (one-cancels-all) orders - -The adapter provides support for OCA orders through explicit configuration using `IBOrderTags`: - -### Basic OCA configuration - -All OCA functionality must be explicitly configured using `IBOrderTags`: - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags - -# Create OCA configuration -oca_tags = IBOrderTags( - ocaGroup="MY_OCA_GROUP", - ocaType=1, # Type 1: Cancel All with Block (recommended) -) - -# Apply to bracket orders -bracket_order = order_factory.bracket( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(100), - tp_price=instrument.make_price(110.0), - sl_trigger_price=instrument.make_price(90.0), - tp_tags=[oca_tags.value], # Must explicitly add OCA tags - sl_tags=[oca_tags.value], # Must explicitly add OCA tags -) -``` - -### Advanced OCA configuration - -You can specify different OCA types and behaviors using `IBOrderTags`: - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags - -# Create custom OCA configuration -custom_oca_tags = IBOrderTags( - ocaGroup="MY_CUSTOM_GROUP", - ocaType=2, # Use Type 2: Reduce with Block -) - -# Apply to individual orders -order = order_factory.limit( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(100), - price=instrument.make_price(100.0), - tags=[custom_oca_tags.value], -) -``` - -### OCA types - -Interactive Brokers supports three OCA types: - -| Type | Name | Behavior | Use Case | -|------|------|----------|----------| -| **1** | Cancel All with Block | Cancel all remaining orders with block protection | **Default** - Safest option, prevents overfills | -| **2** | Reduce with Block | Proportionally reduce remaining orders with block protection | Partial fills with overfill protection | -| **3** | Reduce without Block | Proportionally reduce remaining orders without block protection | Fastest execution, higher overfill risk | - -#### Multiple orders in same OCA group - -```python -# Create multiple orders with the same OCA group -oca_tags = IBOrderTags( - ocaGroup="MULTI_ORDER_GROUP", - ocaType=3, # Use Type 3: Reduce without Block -) - -order1 = order_factory.limit( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(50), - price=instrument.make_price(99.0), - tags=[oca_tags.value], -) - -order2 = order_factory.limit( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(50), - price=instrument.make_price(101.0), - tags=[oca_tags.value], -) -``` - -### OCA configuration requirements - -OCA functionality is **only** available through explicit configuration: - -1. **IBOrderTags Required** - OCA settings must be explicitly specified in order tags -2. **No Automatic Detection** - `ContingencyType.OCO` and `ContingencyType.OUO` do not automatically create OCA groups -3. **Manual Configuration** - All OCA groups and types must be manually specified - -### Conditional orders - -The adapter supports Interactive Brokers conditional orders through the `conditions` parameter in `IBOrderTags`. Conditional orders allow you to specify criteria that must be met before an order is transmitted or cancelled. - -#### Supported condition types - -- **Price Conditions**: Trigger based on price movements of a specific instrument -- **Time Conditions**: Trigger at a specific date and time -- **Volume Conditions**: Trigger based on trading volume thresholds -- **Execution Conditions**: Trigger when trades occur for a specific instrument -- **Margin Conditions**: Trigger based on account margin levels -- **Percent Change Conditions**: Trigger based on percentage price changes - -#### Basic conditional order example - -```python -from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags - -# Create a price condition: trigger when SPY goes above $250 -price_condition = { - "type": "price", - "conId": 265598, # SPY contract ID - "exchange": "SMART", - "isMore": True, # Trigger when price is greater than threshold - "price": 250.00, - "triggerMethod": 0, # Default trigger method - "conjunction": "and", -} - -# Create order tags with condition -order_tags = IBOrderTags( - conditions=[price_condition], - conditionsCancelOrder=False, # Transmit order when condition is met -) - -# Apply to order -order = order_factory.limit( - instrument_id=instrument.id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(100), - price=instrument.make_price(251.00), - tags=[order_tags.value], -) -``` - -#### Multiple conditions with logic - -```python -# Create multiple conditions with AND/OR logic -conditions = [ - { - "type": "price", - "conId": 265598, - "exchange": "SMART", - "isMore": True, - "price": 250.00, - "triggerMethod": 0, - "conjunction": "and", # AND with next condition - }, - { - "type": "time", - "time": "20250315-09:30:00", - "isMore": True, - "conjunction": "or", # OR with next condition - }, - { - "type": "volume", - "conId": 265598, - "exchange": "SMART", - "isMore": True, - "volume": 10000000, - "conjunction": "and", - }, -] - -order_tags = IBOrderTags( - conditions=conditions, - conditionsCancelOrder=False, -) -``` - -#### Condition parameters - -**Price Condition:** - -- `conId`: Contract ID of the instrument to monitor -- `exchange`: Exchange to monitor (e.g., "SMART", "NASDAQ") -- `isMore`: True for >=, False for <= -- `price`: Price threshold -- `triggerMethod`: 0=Default, 1=DoubleBidAsk, 2=Last, 3=DoubleLast, 4=BidAsk, 7=LastBidAsk, 8=MidPoint - -**Time Condition:** - -- `time`: Time string in UTC format "YYYYMMDD-HH:MM:SS" (e.g., "20250315-09:30:00") -- `isMore`: True for after time, False for before time - -**Volume Condition:** - -- `conId`: Contract ID of the instrument to monitor -- `exchange`: Exchange to monitor -- `isMore`: True for >=, False for <= -- `volume`: Volume threshold - -**Execution Condition:** - -- `symbol`: Symbol to monitor for trades -- `secType`: Security type (e.g., "STK", "OPT", "FUT") -- `exchange`: Exchange to monitor - -**Margin Condition:** - -- `percent`: Margin cushion percentage threshold -- `isMore`: True for >=, False for <= - -**Percent Change Condition:** - -- `conId`: Contract ID of the instrument to monitor -- `exchange`: Exchange to monitor -- `isMore`: True for >=, False for <= -- `changePercent`: Percentage change threshold - -#### Complete example: all condition types - -```python -# Example showing all 6 supported condition types -from nautilus_trader.adapters.interactive_brokers.common import IBOrderTags - -# 1. Price Condition - trigger when ES futures > 6000 -price_condition = { - "type": "price", - "conId": 495512563, # ES futures contract ID - "exchange": "CME", - "isMore": True, - "price": 6000.0, - "triggerMethod": 0, - "conjunction": "and", -} - -# 2. Time Condition - trigger at specific time -time_condition = { - "type": "time", - "time": "20250315-09:30:00", # UTC format - "isMore": True, - "conjunction": "and", -} - -# 3. Volume Condition - trigger when volume > 100,000 -volume_condition = { - "type": "volume", - "conId": 495512563, - "exchange": "CME", - "isMore": True, - "volume": 100000, - "conjunction": "and", -} - -# 4. Execution Condition - trigger when SPY trades -execution_condition = { - "type": "execution", - "symbol": "SPY", - "secType": "STK", - "exchange": "SMART", - "conjunction": "and", -} - -# 5. Margin Condition - trigger when margin cushion > 75% -margin_condition = { - "type": "margin", - "percent": 75, - "isMore": True, - "conjunction": "and", -} - -# 6. Percent Change Condition - trigger when price changes > 5% -percent_change_condition = { - "type": "percent_change", - "conId": 495512563, - "exchange": "CME", - "changePercent": 5.0, - "isMore": True, - "conjunction": "and", -} - -# Use any combination of conditions -order_tags = IBOrderTags( - conditions=[price_condition, time_condition], # Multiple conditions - conditionsCancelOrder=False, # Transmit when conditions met -) -``` - -#### Order behavior - -Set `conditionsCancelOrder` to control what happens when conditions are met: - -- `False`: Transmit the order when conditions are satisfied -- `True`: Cancel the order when conditions are satisfied - -#### Implementation notes - -- **All 6 condition types are fully supported** and tested with live Interactive Brokers orders -- **Price conditions** work correctly despite a known bug in the ibapi library where `PriceCondition.__str__` is incorrectly decorated as a property -- **Time conditions** use UTC format with dash separator (`YYYYMMDD-HH:MM:SS`) for reliable parsing -- **Conjunction logic** allows complex condition combinations using "and"/"or" operators - -### Complete trading node configuration - -Setting up a complete trading environment involves configuring a `TradingNodeConfig` with all necessary components. Here are examples for different scenarios. - -#### Paper trading configuration - -```python -import os -from nautilus_trader.adapters.interactive_brokers.common import IB -from nautilus_trader.adapters.interactive_brokers.common import IB_VENUE -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersDataClientConfig -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersExecClientConfig -from nautilus_trader.adapters.interactive_brokers.config import InteractiveBrokersInstrumentProviderConfig -from nautilus_trader.adapters.interactive_brokers.config import IBMarketDataTypeEnum -from nautilus_trader.adapters.interactive_brokers.config import SymbologyMethod -from nautilus_trader.adapters.interactive_brokers.factories import InteractiveBrokersLiveDataClientFactory -from nautilus_trader.adapters.interactive_brokers.factories import InteractiveBrokersLiveExecClientFactory -from nautilus_trader.config import LiveDataEngineConfig -from nautilus_trader.config import LoggingConfig -from nautilus_trader.config import RoutingConfig -from nautilus_trader.config import TradingNodeConfig -from nautilus_trader.live.node import TradingNode - -# Instrument provider configuration -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - symbology_method=SymbologyMethod.IB_SIMPLIFIED, - load_ids=frozenset([ - "EUR/USD.IDEALPRO", - "GBP/USD.IDEALPRO", - "SPY.ARCA", - "QQQ.NASDAQ", - "AAPL.NASDAQ", - "MSFT.NASDAQ", - ]), -) - -# Data client configuration -data_client_config = InteractiveBrokersDataClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, # TWS paper trading - ibg_client_id=1, - use_regular_trading_hours=True, - market_data_type=IBMarketDataTypeEnum.DELAYED_FROZEN, - instrument_provider=instrument_provider_config, -) - -# Execution client configuration -exec_client_config = InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, # TWS paper trading - ibg_client_id=1, - account_id="DU123456", # Your paper trading account - instrument_provider=instrument_provider_config, - routing=RoutingConfig(default=True), -) - -# Trading node configuration -config_node = TradingNodeConfig( - trader_id="PAPER-TRADER-001", - logging=LoggingConfig(log_level="INFO"), - data_clients={IB: data_client_config}, - exec_clients={IB: exec_client_config}, - data_engine=LiveDataEngineConfig( - time_bars_timestamp_on_close=False, # IB standard: use bar open time - validate_data_sequence=True, # Discard out-of-sequence bars - ), - timeout_connection=90.0, - timeout_reconciliation=5.0, - timeout_portfolio=5.0, - timeout_disconnection=5.0, - timeout_post_stop=2.0, -) - -# Create and configure the trading node -node = TradingNode(config=config_node) -node.add_data_client_factory(IB, InteractiveBrokersLiveDataClientFactory) -node.add_exec_client_factory(IB, InteractiveBrokersLiveExecClientFactory) -node.build() - -if __name__ == "__main__": - try: - node.run() - finally: - node.dispose() -``` - -## Live trading with Dockerized gateway - -```python -from nautilus_trader.adapters.interactive_brokers.config import DockerizedIBGatewayConfig - -# Dockerized gateway configuration -dockerized_gateway_config = DockerizedIBGatewayConfig( - username=os.environ.get("TWS_USERNAME"), - password=os.environ.get("TWS_PASSWORD"), - trading_mode="live", # "paper" or "live" - read_only_api=False, # Allow order execution - timeout=300, -) - -# Data client with dockerized gateway -data_client_config = InteractiveBrokersDataClientConfig( - ibg_client_id=1, - use_regular_trading_hours=False, # Include extended hours - market_data_type=IBMarketDataTypeEnum.REALTIME, - instrument_provider=instrument_provider_config, - dockerized_gateway=dockerized_gateway_config, -) - -# Execution client with dockerized gateway -exec_client_config = InteractiveBrokersExecClientConfig( - ibg_client_id=1, - account_id=os.environ.get("TWS_ACCOUNT"), # Live account ID - instrument_provider=instrument_provider_config, - dockerized_gateway=dockerized_gateway_config, - routing=RoutingConfig(default=True), -) - -# Live trading node configuration -config_node = TradingNodeConfig( - trader_id="LIVE-TRADER-001", - logging=LoggingConfig(log_level="INFO"), - data_clients={IB: data_client_config}, - exec_clients={IB: exec_client_config}, - data_engine=LiveDataEngineConfig( - time_bars_timestamp_on_close=False, - validate_data_sequence=True, - ), -) -``` - -### Multi-client configuration - -For advanced setups, you can configure multiple clients with different purposes: - -```python -# Separate data and execution clients with different client IDs -data_client_config = InteractiveBrokersDataClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=1, # Data client uses ID 1 - market_data_type=IBMarketDataTypeEnum.REALTIME, - instrument_provider=instrument_provider_config, -) - -exec_client_config = InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=2, # Execution client uses ID 2 - account_id="DU123456", - instrument_provider=instrument_provider_config, - routing=RoutingConfig(default=True), -) -``` - -### Multiple IB execution clients for different accounts - -NautilusTrader supports using multiple Interactive Brokers execution clients simultaneously, each connected to a different IB account. This is useful when you need to trade with multiple accounts, such as: - -- Separate accounts for different strategies -- Paper trading and live trading accounts running simultaneously -- Multiple managed accounts under the same IB login - -To configure multiple IB execution clients, provide multiple entries in the `exec_clients` dictionary with unique keys. Each entry specifies a different `account_id`: - -```python -from nautilus_trader.adapters.interactive_brokers.config import ( - InteractiveBrokersDataClientConfig, - InteractiveBrokersExecClientConfig, - InteractiveBrokersInstrumentProviderConfig, - SymbologyMethod, - IBMarketDataTypeEnum, -) -from nautilus_trader.live.config import TradingNodeConfig, RoutingConfig, LoggingConfig -from nautilus_trader.model.identifiers import AccountId, Venue, ClientId - -# Shared instrument provider configuration -instrument_provider_config = InteractiveBrokersInstrumentProviderConfig( - symbology_method=SymbologyMethod.IB_SIMPLIFIED, -) - -# Data client (shared across all accounts) -data_client_config = InteractiveBrokersDataClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=1, - market_data_type=IBMarketDataTypeEnum.REALTIME, - instrument_provider=instrument_provider_config, -) - -# Configuration for multiple IB execution clients -config_node = TradingNodeConfig( - trader_id="MULTI-ACCOUNT-001", - logging=LoggingConfig(log_level="INFO"), - - # Single data client shared across accounts - data_clients={ - "IB": data_client_config, - }, - - # Multiple execution clients, one per account - exec_clients={ - # First account: Paper trading account - "IB-PAPER": InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=2, # Unique IB API client ID - account_id="DU123456", # Paper trading account ID - instrument_provider=instrument_provider_config, - routing=RoutingConfig(default=False), # Not default - ), - - # Second account: Live trading account - "IB-LIVE": InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=3, # Unique IB API client ID - account_id="U987654", # Live account ID - instrument_provider=instrument_provider_config, - routing=RoutingConfig(default=True), # Set as default - ), - - # Third account: Another managed account - "IB-ACCOUNT3": InteractiveBrokersExecClientConfig( - ibg_host="127.0.0.1", - ibg_port=7497, - ibg_client_id=4, # Unique IB API client ID - account_id="U456789", # Another account ID - instrument_provider=instrument_provider_config, - routing=RoutingConfig(default=False), - ), - }, -) -``` - -**Key points for multiple IB execution clients:** - -1. **Unique keys**: Each entry in `exec_clients` must have a unique key (e.g., `"IB-PAPER"`, `"IB-LIVE"`). This key becomes the `account_issuer` for that client. - -2. **Unique client IDs**: Each execution client must use a different `ibg_client_id` (2, 3, 4, etc.). IB Gateway/TWS requires each API connection to use a unique client ID. - -3. **Account ID**: Each execution client must specify a different `account_id` matching the account logged into IB Gateway/TWS. - -4. **Account identifiers**: The system creates `AccountId` instances like: - - `AccountId("IB-PAPER-DU123456")` - - `AccountId("IB-LIVE-U987654")` - - `AccountId("IB-ACCOUNT3-U456789")` - -5. **Routing**: Orders and queries are automatically routed to the correct execution client based on: - - Explicit `client_id` in the command - - `account_id` issuer (for `QueryAccount` commands or orders with account_id set) - - Default client (if one is marked with `routing=RoutingConfig(default=True)`) - -6. **Portfolio queries**: When querying portfolio properties, you can specify either: - - `account_id` for account-specific queries: `portfolio.realized_pnls(account_id=AccountId("IB-PAPER-DU123456"))` - - `venue` for aggregated queries across all accounts with that venue: `portfolio.realized_pnls(venue=Venue("IB-PAPER"))` - -**Example: Using multiple IB execution clients in a strategy:** - -```python -from nautilus_trader.model.identifiers import AccountId, ClientId -from nautilus_trader.trading.strategy import Strategy - -class MultiAccountStrategy(Strategy): - """Example strategy using multiple IB accounts.""" - - def on_start(self): - # Define account IDs for easy reference - self.paper_account = AccountId("IB-PAPER-DU123456") - self.live_account = AccountId("IB-LIVE-U987654") - - # Query paper account balance - paper_account_state = self.cache.account(self.paper_account) - if paper_account_state: - self.log.info(f"Paper account balance: {paper_account_state.balance_total()}") - - # Query live account balance - live_account_state = self.cache.account(self.live_account) - if live_account_state: - self.log.info(f"Live account balance: {live_account_state.balance_total()}") - - def submit_order_to_paper(self, order): - """Submit order to paper trading account.""" - self.submit_order(order, client_id=ClientId("IB-PAPER")) - - def submit_order_to_live(self, order): - """Submit order to live trading account.""" - self.submit_order(order, client_id=ClientId("IB-LIVE")) - - def check_paper_pnl(self, instrument_id): - """Check realized PnL for paper account.""" - pnl = self.portfolio.realized_pnl( - instrument_id=instrument_id, - account_id=self.paper_account - ) - return pnl - - def check_live_pnl(self, instrument_id): - """Check realized PnL for live account.""" - pnl = self.portfolio.realized_pnl( - instrument_id=instrument_id, - account_id=self.live_account - ) - return pnl -``` - -**Example: Querying account information with multiple IB clients:** - -```python -from nautilus_trader.model.identifiers import AccountId - -# Query specific account -paper_account = cache.account(AccountId("IB-PAPER-DU123456")) -live_account = cache.account(AccountId("IB-LIVE-U987654")) - -# Query account using account_id (preferred method) -paper_account_by_id = cache.account(AccountId("IB-PAPER-DU123456")) - -# Alternative: Query account using account_id parameter (also works) -paper_account_via_account_id = cache.account_for_venue( - account_id=AccountId("IB-PAPER-DU123456") -) - -# Query portfolio properties by account -paper_realized_pnl = portfolio.realized_pnl( - instrument_id=instrument_id, - account_id=AccountId("IB-PAPER-DU123456") -) - -# Query portfolio properties aggregated across all IB accounts -# Note: This aggregates across all accounts with the same venue -all_ib_realized_pnl = portfolio.realized_pnls(venue=Venue("IB")) -``` - -### Running the trading node - -```python -def run_trading_node(): - """Run the trading node with proper error handling.""" - node = None - try: - # Create and build node - node = TradingNode(config=config_node) - node.add_data_client_factory(IB, InteractiveBrokersLiveDataClientFactory) - node.add_exec_client_factory(IB, InteractiveBrokersLiveExecClientFactory) - node.build() - - # Add your strategies here - # node.trader.add_strategy(YourStrategy()) - - # Run the node - node.run() - - except KeyboardInterrupt: - print("Shutting down...") - except Exception as e: - print(f"Error: {e}") - finally: - if node: - node.dispose() - -if __name__ == "__main__": - run_trading_node() -``` - -### Additional configuration options - -#### Environment variables - -Set these environment variables for easier configuration: - -```bash -export TWS_USERNAME="your_ib_username" -export TWS_PASSWORD="your_ib_password" -export TWS_ACCOUNT="your_account_id" -export IB_MAX_CONNECTION_ATTEMPTS="5" # Optional: limit reconnection attempts -``` - -#### Logging configuration - -```python -# Enhanced logging configuration -logging_config = LoggingConfig( - log_level="INFO", - log_level_file="DEBUG", - log_file_format="json", # JSON format for structured logging - log_component_levels={ - "InteractiveBrokersClient": "DEBUG", - "InteractiveBrokersDataClient": "INFO", - "InteractiveBrokersExecutionClient": "INFO", - }, -) -``` - -You can find additional examples here: - -## Troubleshooting - -### Common connection issues - -#### Connection refused - -- **Cause**: TWS/Gateway not running or wrong port -- **Solution**: Verify TWS/Gateway is running and check port configuration -- **Default Ports**: TWS (7497/7496), IB Gateway (4002/4001) - -#### Authentication errors - -- **Cause**: Incorrect credentials or account not logged in -- **Solution**: Verify username/password and ensure account is logged into TWS/Gateway - -#### Client ID conflicts - -- **Cause**: Multiple clients using the same client ID -- **Solution**: Use unique client IDs for each connection - -#### Market data permissions - -- **Cause**: Insufficient market data subscriptions -- **Solution**: Use `IBMarketDataTypeEnum.DELAYED_FROZEN` for testing or subscribe to required data feeds - -### Error codes - -Interactive Brokers uses specific error codes. Common ones include: - -- **200**: No security definition found -- **201**: Order rejected - reason follows -- **202**: Order cancelled -- **300**: Can't find EId with ticker ID -- **354**: Requested market data is not subscribed -- **2104**: Market data farm connection is OK -- **2106**: HMDS data farm connection is OK - -### Performance optimization - -#### Reduce data volume - -```python -# Reduce quote tick volume by ignoring size-only updates -data_config = InteractiveBrokersDataClientConfig( - ignore_quote_tick_size_updates=True, - # ... other config -) -``` - -#### Connection management - -```python -# Set reasonable timeouts -config = InteractiveBrokersDataClientConfig( - connection_timeout=300, # 5 minutes - request_timeout_secs=60, # 1 minute - # ... other config -) -``` - -#### Memory management - -- Use appropriate bar sizes for your strategy -- Limit the number of simultaneous subscriptions -- Consider using historical data for backtesting instead of live data - -### Best practices - -#### Security - -- Never hardcode credentials in source code -- Use environment variables for sensitive information -- Use paper trading for development and testing -- Set `read_only_api=True` for data-only applications - -#### Development workflow - -1. **Start with Paper Trading**: Always test with paper trading first -2. **Use Delayed Data**: Use `DELAYED_FROZEN` market data for development -3. **Implement Proper Error Handling**: Handle connection losses and API errors gracefully -4. **Monitor Logs**: Enable appropriate logging levels for debugging -5. **Test Reconnection**: Test your strategy's behavior during connection interruptions - -#### Production deployment - -- Use dockerized gateway for automated deployments -- Implement proper monitoring and alerting -- Set up log aggregation and analysis -- Use real-time data subscriptions only when necessary -- Implement circuit breakers and position limits - -#### Order management - -- Always validate orders before submission -- Implement proper position sizing -- Use appropriate order types for your strategy -- Monitor order status and handle rejections -- Implement timeout handling for order operations - -### Debugging tips - -#### Enable debug logging - -```python -logging_config = LoggingConfig( - log_level="DEBUG", - log_component_levels={ - "InteractiveBrokersClient": "DEBUG", - }, -) -``` - -#### Monitor connection status - -```python -# Check connection status in your strategy -if not self.data_client.is_connected: - self.log.warning("Data client disconnected") -``` - -#### Validate instruments - -```python -# Ensure instruments are loaded before trading -instruments = self.cache.instruments() -if not instruments: - self.log.error("No instruments loaded") -``` - -### Support and resources - -- **IB API Documentation**: [TWS API Guide](https://ibkrcampus.com/ibkr-api-page/trader-workstation-api/) -- **NautilusTrader Examples**: [GitHub Examples](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/interactive_brokers) -- **IB Contract Search**: [Contract Information Center](https://pennies.interactivebrokers.com/cstools/contract_info/) -- **Market Data Subscriptions**: [IB Market Data](https://www.interactivebrokers.com/en/pricing/market-data-pricing.php) - -## Contributing - -:::info -For additional features or to contribute to the Interactive Brokers adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/index.md b/nautilus-docs/docs/integrations/index.md deleted file mode 100644 index 6f89cb8..0000000 --- a/nautilus-docs/docs/integrations/index.md +++ /dev/null @@ -1,60 +0,0 @@ -# Integrations - -NautilusTrader uses modular *adapters* to connect to trading venues and data providers, translating raw APIs into a unified interface and normalized domain model. - -The following integrations are currently supported: - -| Name | ID | Type | Status | Docs | -| :--------------------------------------------------------------------------- | :-------------------- | :---------------------- | :------------------------------------------------------ | :------------------------ | -| [AX Exchange](https://architect.exchange) | `AX` | Perpetuals Exchange | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](architect_ax.md) | -| [Architect](https://architect.co) | `ARCHITECT` | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/planned-gray) | - | -| [Betfair](https://betfair.com) | `BETFAIR` | Sports Betting Exchange | ![status](https://img.shields.io/badge/stable-green) | [Guide](betfair.md) | -| [Binance](https://binance.com) | `BINANCE` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](binance.md) | -| [BitMEX](https://www.bitmex.com) | `BITMEX` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](bitmex.md) | -| [Bybit](https://www.bybit.com) | `BYBIT` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](bybit.md) | -| [Databento](https://databento.com) | `DATABENTO` | Data Provider | ![status](https://img.shields.io/badge/stable-green) | [Guide](databento.md) | -| [Deribit](https://www.deribit.com) | `DERIBIT` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](deribit.md) | -| [dYdX](https://dydx.exchange/) | `DYDX` | Crypto Exchange (DEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](dydx.md) | -| [Hyperliquid](https://hyperliquid.xyz) | `HYPERLIQUID` | Crypto Exchange (DEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](hyperliquid.md) | -| [Interactive Brokers](https://www.interactivebrokers.com) | `INTERACTIVE_BROKERS` | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/stable-green) | [Guide](ib.md) | -| [Kraken](https://kraken.com) | `KRAKEN` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/beta-yellow) | [Guide](kraken.md) | -| [OKX](https://okx.com) | `OKX` | Crypto Exchange (CEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](okx.md) | -| [Polymarket](https://polymarket.com) | `POLYMARKET` | Prediction Market (DEX) | ![status](https://img.shields.io/badge/stable-green) | [Guide](polymarket.md) | -| [Tardis](https://tardis.dev) | `TARDIS` | Crypto Data Provider | ![status](https://img.shields.io/badge/stable-green) | [Guide](tardis.md) | - -- **ID**: The default client ID for the integrations adapter clients. -- **Type**: The type of integration (often the venue type). - -## Status - -- `planned`: Planned for future development. -- `building`: Under construction and likely not in a usable state. -- `beta`: Completed to a minimally working state and in a 'beta' testing phase. -- `stable`: Stabilized feature set and API, the integration has been tested by both developers and users to a reasonable level (some bugs may still remain). - -## Implementation goals - -The primary goal of NautilusTrader is to provide a unified trading system for -use with a variety of integrations. To support the widest range of trading -strategies, priority will be given to *standard* functionality: - -- Requesting historical market data. -- Streaming live market data. -- Reconciling execution state. -- Submitting standard order types with standard execution instructions. -- Modifying existing orders (if possible on an exchange). -- Canceling orders. - -The implementation of each integration aims to meet the following criteria: - -- Low-level client components should match the exchange API as closely as possible. -- The full range of an exchange's functionality (where applicable to NautilusTrader) should *eventually* be supported. -- Exchange specific data types will be added to support the functionality and return types which are reasonably expected by a user. -- Actions unsupported by an exchange or NautilusTrader will be logged as a warning or error when invoked. - -## API unification - -All integrations must conform to NautilusTrader’s system API, requiring normalization and standardization: - -- Symbols should use the venue’s native symbol format unless disambiguation is required (e.g., Binance Spot vs. Binance Futures). -- Timestamps must use UNIX epoch nanoseconds. If milliseconds are used, field/property names should explicitly end with `_ms`. diff --git a/nautilus-docs/docs/integrations/kraken.md b/nautilus-docs/docs/integrations/kraken.md deleted file mode 100644 index 8d55c10..0000000 --- a/nautilus-docs/docs/integrations/kraken.md +++ /dev/null @@ -1,555 +0,0 @@ -# Kraken - -Founded in 2011, Kraken is one of the most established cryptocurrency exchanges -globally and the largest exchange in Europe by euro trading volume. The platform -offers spot and derivatives trading across a wide range of digital assets. This -integration connects to Kraken Pro and supports live market data ingest and order -execution for both Kraken Spot and Kraken Derivatives (Futures) markets. - -## Overview - -This adapter is implemented in Rust with Python bindings for ease of use in -Python-based workflows. It does not require external Kraken client libraries; the -core components are compiled as a static library and linked automatically during -the build. - -This guide assumes a trader is setting up for both live market data feeds and -trade execution. The Kraken adapter includes multiple components, which can be -used together or separately depending on the use case. - -- `KrakenRawHttpClient`: Low-level HTTP API connectivity for Spot and Futures. -- `KrakenHttpClient`: Higher-level HTTP client with instrument caching and reconciliation support. -- `KrakenInstrumentProvider`: Instrument parsing and loading functionality. -- `KrakenDataClient`: Market data feed manager. -- `KrakenExecutionClient`: Account management and trade execution gateway. -- `KrakenLiveDataClientFactory`: Factory for Kraken data clients (used by the - trading node builder). -- `KrakenLiveExecClientFactory`: Factory for Kraken execution clients (used by - the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), and -won't need to work directly with these lower-level components. -::: - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/kraken/). - -## Kraken documentation - -Kraken provides extensive documentation for users: - -- [Kraken API Documentation](https://docs.kraken.com/api/) -- [Kraken Spot REST API](https://docs.kraken.com/api/docs/guides/spot-rest-intro) -- [Kraken Futures REST API](https://docs.kraken.com/api/docs/futures-api) - -Refer to the Kraken documentation in conjunction with this NautilusTrader -integration guide. - -## Products - -Kraken supports two primary product categories: - -| Product Type | Supported | Notes | -|--------------------------|-----------|-----------------------------------------------------------| -| Spot | ✓ | Standard cryptocurrency pairs with margin support. | -| Futures (Perpetual) | ✓ | Inverse (`PI_`) and USD-margined (`PF_`) perpetual swaps. | -| Futures (Dated/Flex) | ✓ | Fixed maturity (`FI_`) and flex (`FF_`) contracts. | - -:::note -**Dual-product deployments**: When both `SPOT` and `FUTURES` product types are -configured, the adapter queries both APIs and merges the account states. This -ensures the execution engine has visibility into collateral across both markets. -::: - -## Bar streaming - -### Supported intervals - -The Kraken adapter supports real-time bar (OHLC) streaming for Spot markets via -WebSocket. The following intervals are available: - -| Interval | BarType specification | -|------------|-----------------------| -| 1 minute | `1-MINUTE-LAST` | -| 5 minutes | `5-MINUTE-LAST` | -| 15 minutes | `15-MINUTE-LAST` | -| 30 minutes | `30-MINUTE-LAST` | -| 1 hour | `1-HOUR-LAST` | -| 4 hours | `4-HOUR-LAST` | -| 1 day | `1-DAY-LAST` | -| 1 week | `1-WEEK-LAST` | -| 15 days | `15-DAY-LAST` | - -:::note -**Futures limitation**: Kraken Futures does not support bar streaming via -WebSocket. Use `request_bars()` for historical bar data instead. -::: - -### Bar emission latency - -Kraken's WebSocket OHLC channel pushes updates for the *current* (incomplete) -bar on every trade. Unlike some exchanges (e.g., Binance), Kraken does not -provide an "is_closed" indicator to signal when a bar is complete. - -To avoid emitting partial/incomplete bars, the adapter buffers the current bar -and only emits it when the next bar period begins (i.e., when a message with a -new `interval_begin` timestamp arrives). This means: - -- Bars are emitted with a delay of up to one bar period. -- For 1-minute bars, the maximum delay is ~1 minute. -- The emitted bar data is complete and final. - -We chose this approach over timer-based emission because: - -- Timer-based emission could miss the final update before the bar closes. -- Kraken's updates are not guaranteed to arrive at exact interval boundaries. -- Buffering ensures data integrity at the cost of latency. - -:::warning -If bar latency matters for your strategy, consider using trade tick data -and aggregating bars locally with `BarAggregator`. -::: - -:::tip -For most use cases, we recommend using `INTERNAL` bar aggregation (subscribing to -trades and aggregating bars locally) rather than `EXTERNAL` exchange-provided bars: - -- Bars are emitted immediately when complete, with no buffering delay. -- Consistent behavior across all exchanges, simplifying multi-venue strategies. - -::: - -## Symbology - -### Bitcoin symbol format (BTC vs XBT) - -Kraken uses different Bitcoin symbol conventions across their APIs: - -| Market | Symbol Format | Example | Notes | -|---------|---------------|--------------------|---------------------------------------------| -| Spot | `BTC` | `BTC/USD.KRAKEN` | Adapter normalizes XBT → BTC at load time. | -| Futures | `XBT` | `PI_XBTUSD.KRAKEN` | Uses Kraken's native XBT format. | - -:::note -Kraken's REST API returns `XBT` for Bitcoin (following ISO 4217 conventions for -supranational currencies), but their WebSocket v2 API requires the `BTC` format. -The adapter automatically normalizes spot symbols to `BTC` when loading instruments, -whether XBT appears as the base currency (e.g., `XBT/USD` → `BTC/USD`) or quote -currency (e.g., `ETH/XBT` → `ETH/BTC`). Futures retain Kraken's native `XBT` format. -::: - -### Spot markets - -NautilusTrader uses ISO 4217-A3 format for Kraken Spot instrument symbols, -which provides a standardized representation across exchanges. The adapter -handles translation to Kraken's native format internally. - -**Instrument ID format:** - -```python -InstrumentId.from_str("BTC/USD.KRAKEN") # Spot BTC/USD -InstrumentId.from_str("ETH/USD.KRAKEN") # Spot ETH/USD -InstrumentId.from_str("SOL/USD.KRAKEN") # Spot SOL/USD -InstrumentId.from_str("BTC/USDT.KRAKEN") # Spot BTC/USDT -InstrumentId.from_str("ETH/BTC.KRAKEN") # Spot ETH/BTC (normalized from ETH/XBT) -``` - -### Futures markets - -Kraken Futures instruments use a specific naming convention with prefixes: - -- `PI_` - Perpetual Inverse contracts (e.g., `PI_XBTUSD`) -- `PF_` - Perpetual Fixed-margin contracts (e.g., `PF_XBTUSD`) -- `FI_` - Fixed maturity Inverse contracts (e.g., `FI_XBTUSD_230929`) -- `FF_` - Flex futures contracts - -**Instrument ID format:** - -```python -InstrumentId.from_str("PI_XBTUSD.KRAKEN") # Perpetual inverse BTC -InstrumentId.from_str("PI_ETHUSD.KRAKEN") # Perpetual inverse ETH -InstrumentId.from_str("PF_XBTUSD.KRAKEN") # Perpetual fixed-margin BTC -``` - -## Orders capability - -### Order types - -| Order Type | Spot | Futures | Notes | -|------------------------|------|---------|--------------------------------------------| -| `MARKET` | ✓ | ✓ | Immediate execution at market price. | -| `LIMIT` | ✓ | ✓ | Execution at specified price or better. | -| `STOP_MARKET` | ✓ | ✓ | Conditional market order (stop-loss). | -| `MARKET_IF_TOUCHED` | ✓ | ✓ | Conditional market order (take-profit). | -| `STOP_LIMIT` | ✓ | ✓ | Conditional limit order (stop-loss-limit). | -| `LIMIT_IF_TOUCHED` | ✓ | - | *Futures: not yet implemented*. | - -### Time in force - -| Time in Force | Spot | Futures | Notes | -|---------------|------|---------|-----------------------------------------------------| -| `GTC` | ✓ | ✓ | Good Till Canceled. | -| `GTD` | ✓ | - | Good Till Date (Spot only, requires `expire_time`). | -| `IOC` | ✓ | ✓ | Immediate or Cancel. | -| `FOK` | - | - | *Not supported by Kraken*. | - -:::note -**Market orders** are inherently immediate and do not support time-in-force. -`IOC` only applies to limit-type orders. -::: - -### Execution instructions - -| Instruction | Spot | Futures | Notes | -|---------------|------|---------|---------------------------------------------| -| `post_only` | ✓ | ✓ | Available for limit orders. | -| `reduce_only` | - | ✓ | Futures only. Reduces position, no reversal.| - -### Batch operations - -| Operation | Spot | Futures | Notes | -|--------------------|------|---------|----------------------------------------------| -| Batch Submit | - | - | *Not yet implemented*. | -| Batch Modify | - | - | *Not yet implemented* (Futures only). | -| Batch Cancel | ✓ | ✓ | Auto-chunks into batches of 50. | - -:::note -**Cancel all orders**: - -- Order side filtering is not supported; all orders are canceled regardless of side. -- Spot: Cancels all open orders across all symbols. -- Futures: Requires an `instrument_id`; cancels orders for that symbol only. - -::: - -### Position management - -| Feature | Spot | Futures | Notes | -|-------------------|------|---------|-----------------------------------------------------------| -| Query positions | ✓* | ✓ | *Spot: opt-in via `use_spot_position_reports`. See below. | -| Position mode | - | - | Single position per instrument. | -| Leverage control | - | ✓ | Configured per account tier. | -| Margin mode | - | ✓ | Cross margin for Futures. | - -### Order querying - -| Feature | Spot | Futures | Notes | -|----------------------|------|---------|----------------------------------------------| -| Query open orders | ✓ | ✓ | List all active orders. | -| Query order history | ✓ | ✓ | Historical order data with pagination. | -| Order status updates | ✓ | ✓ | Real-time order state changes via WebSocket. | -| Trade history | ✓ | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Spot | Futures | Notes | -|---------------------|------|---------|------------------------------------------| -| Order lists | - | - | *Not supported*. | -| OCO orders | - | - | *Not supported*. | -| Bracket orders | - | - | *Not supported*. | -| Conditional orders | ✓ | ✓ | Stop and take-profit orders. | - -## Reconciliation - -The Kraken adapter provides reconciliation capabilities for both -Spot and Futures markets, allowing traders to synchronize their local state with -the exchange state at startup or during operation. - -### Spot reconciliation - -**Order status reports:** - -- Open orders: Fetches all currently active orders. -- Closed orders: Fetches historical orders with pagination support. -- Time-bounded queries: Supports filtering by start/end timestamps. - -**Fill reports:** - -- Trade history: Fetches execution history with pagination. -- Time-bounded queries: Supports filtering by start/end timestamps. -- All fill types: Market, limit, and conditional order fills. - -### Futures reconciliation - -**Order status reports:** - -- Open orders: Fetches all currently active futures orders. -- Historical orders: Fetches closed and filled orders when `open_only=False`. -- Order events: Full order lifecycle history via `/api/history/v2/orders` - endpoint. - -**Fill reports:** - -- Fill history: Fetches all execution reports. -- Time filtering: Client-side filtering by start/end timestamps (parses - RFC3339 timestamps). -- All fill types: Maker and taker fills with fee information. - -**Position status reports:** - -- Open positions: Fetches all active futures positions. -- Real-time data: Includes unrealized funding, average price, and position size. - -:::note -**Futures time filtering**: The Kraken Futures fills endpoint does not support -server-side time range filtering. The adapter implements client-side filtering -by parsing `fillTime` fields and comparing against requested start/end -timestamps. -::: - -### Spot position reports - -The Kraken adapter can optionally report wallet balances as position status -reports for spot instruments. This feature is disabled by default and must be -explicitly enabled via configuration. - -**How it works:** - -- When enabled, wallet balances are converted to `PositionStatusReport` objects. -- Positive balances are reported as `LONG` positions. -- Only instruments matching the configured quote currency are reported (default: `USDT`). -- This prevents duplicate reports when the same asset is available with multiple quote currencies (e.g., BTC/USD, BTC/USDT, BTC/EUR). - -**Configuration:** - -```python -exec_clients={ - KRAKEN: { - "use_spot_position_reports": True, - "spot_positions_quote_currency": "USDT", # Default - }, -} -``` - -:::warning -**Use with caution**: Enabling spot position reports may lead to unintended -behavior if your strategy is not designed to handle spot positions. For example, -a strategy that expects to close positions may attempt to sell your wallet -holdings. -::: - -## Funding rates - -The adapter receives funding rate data from the -[Ticker](https://docs.kraken.com/api/docs/futures-api/websocket/ticker) -WebSocket feed, which provides `relative_funding_rate` and `next_funding_rate_time` for -perpetual futures. - -The `interval` field on `FundingRateUpdate` is `None` for Kraken because the ticker feed -does not include a funding interval field and the Kraken API documentation does not -specify a fixed funding period. - -## Rate limiting - -The adapter implements automatic rate limiting to comply with Kraken's API requirements. - -| Endpoint Type | Limit (requests/sec) | Notes | -|-----------------------|----------------------|--------------------------------------| -| Spot REST (global) | 5 | Global rate limit for Spot API. | -| Futures REST (global) | 5 | Global rate limit for Futures API. | - -:::info -Kraken uses a counter-based rate limiting system with tier-dependent limits: - -- **Starter tier**: 15 max counter, -0.33/sec decay -- **Intermediate tier**: 20 max counter, -0.5/sec decay -- **Pro tier**: 20 max counter, -1/sec decay - -Ledger/trade history calls add +2 to the counter; other calls add +1. -::: - -:::warning -Kraken may temporarily block IP addresses that exceed rate limits. The adapter -automatically queues requests when limits are approached. -::: - -## Configuration - -The product types for each client must be specified in the configurations. - -### Data client configuration options - -| Option | Default | Description | -|---------------------------------|-----------|-------------------------------------------------------------------------| -| `api_key` | `None` | API key; loaded from environment variables (see below) when omitted. | -| `api_secret` | `None` | API secret; loaded from environment variables (see below) when omitted. | -| `environment` | `mainnet` | Trading environment (`mainnet` or `demo`); demo only for Futures. | -| `product_types` | `(SPOT,)` | Product types tuple (e.g., `(KrakenProductType.SPOT,)`). | -| `base_url_http_spot` | `None` | Override for Kraken Spot REST base URL. | -| `base_url_http_futures` | `None` | Override for Kraken Futures REST base URL. | -| `base_url_ws_spot` | `None` | Override for Kraken Spot WebSocket URL. | -| `base_url_ws_futures` | `None` | Override for Kraken Futures WebSocket URL. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | WebSocket proxy URL (*not yet implemented*). | -| `update_instruments_interval_mins` | `60` | Interval (minutes) to reload instruments; `None` to disable. | -| `max_retries` | `None` | Maximum retry attempts for REST requests. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | -| `http_timeout_secs` | `None` | HTTP request timeout in seconds. | -| `ws_heartbeat_secs` | `30` | WebSocket heartbeat interval in seconds. | -| `max_requests_per_second` | `None` | Override rate limit (default 5 req/s); for higher tier accounts. | - -### Execution client configuration options - -| Option | Default | Description | -|---------------------------------|-----------|-------------------------------------------------------------------------| -| `api_key` | `None` | API key; loaded from environment variables (see below) when omitted. | -| `api_secret` | `None` | API secret; loaded from environment variables (see below) when omitted. | -| `environment` | `mainnet` | Trading environment (`mainnet` or `demo`); demo only for Futures. | -| `product_types` | `(SPOT,)` | Product types tuple; `SPOT` uses CASH, `FUTURES` uses MARGIN account. | -| `base_url_http_spot` | `None` | Override for Kraken Spot REST base URL. | -| `base_url_http_futures` | `None` | Override for Kraken Futures REST base URL. | -| `base_url_ws_spot` | `None` | Override for Kraken Spot WebSocket URL. | -| `base_url_ws_futures` | `None` | Override for Kraken Futures WebSocket URL. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | WebSocket proxy URL (*not yet implemented*). | -| `max_retries` | `None` | Maximum retry attempts for order submission/cancel calls. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | -| `http_timeout_secs` | `None` | HTTP request timeout in seconds. | -| `ws_heartbeat_secs` | `30` | WebSocket heartbeat interval in seconds. | -| `max_requests_per_second` | `None` | Override rate limit (default 5 req/s); for higher tier accounts. | -| `use_spot_position_reports` | `False` | Report wallet balances as positions (see below). | -| `spot_positions_quote_currency` | `"USDT"` | Quote currency filter for spot position reports. | - -### Demo environment setup - -To test with Kraken Futures demo (paper trading): - -1. Sign up at [https://demo-futures.kraken.com](https://demo-futures.kraken.com) - and generate API credentials. -2. Set environment variables with your demo credentials: - - `KRAKEN_FUTURES_DEMO_API_KEY` - - `KRAKEN_FUTURES_DEMO_API_SECRET` -3. Configure the adapter with `environment=KrakenEnvironment.DEMO` and - `product_types=(KrakenProductType.FUTURES,)`. - -```python -from nautilus_trader.adapters.kraken import KRAKEN -from nautilus_trader.adapters.kraken import KrakenEnvironment -from nautilus_trader.adapters.kraken import KrakenProductType - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - KRAKEN: { - "environment": KrakenEnvironment.DEMO, - "product_types": (KrakenProductType.FUTURES,), - }, - }, - exec_clients={ - KRAKEN: { - "environment": KrakenEnvironment.DEMO, - "product_types": (KrakenProductType.FUTURES,), - }, - }, -) -``` - -### Production configuration - -The most common use case is to configure a live `TradingNode` to include Kraken -data and execution clients. Add a `KRAKEN` section to your client -configuration(s): - -```python -from nautilus_trader.adapters.kraken import KRAKEN -from nautilus_trader.adapters.kraken import KrakenEnvironment -from nautilus_trader.adapters.kraken import KrakenProductType -from nautilus_trader.live.node import TradingNode - -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - KRAKEN: { - "environment": KrakenEnvironment.MAINNET, - "product_types": (KrakenProductType.SPOT,), - }, - }, - exec_clients={ - KRAKEN: { - "environment": KrakenEnvironment.MAINNET, - "product_types": (KrakenProductType.SPOT,), - }, - }, -) -``` - -### Dual-product configuration (Spot + Futures) - -When trading both Spot and Futures markets, include both product types: - -```python -config = TradingNodeConfig( - ..., # Omitted - data_clients={ - KRAKEN: { - "environment": KrakenEnvironment.MAINNET, - "product_types": (KrakenProductType.SPOT, KrakenProductType.FUTURES), - }, - }, - exec_clients={ - KRAKEN: { - "environment": KrakenEnvironment.MAINNET, - "product_types": (KrakenProductType.SPOT, KrakenProductType.FUTURES), - }, - }, -) -``` - -Then, create a `TradingNode` and add the client factories: - -```python -from nautilus_trader.adapters.kraken import KRAKEN -from nautilus_trader.adapters.kraken import KrakenLiveDataClientFactory -from nautilus_trader.adapters.kraken import KrakenLiveExecClientFactory -from nautilus_trader.live.node import TradingNode - -# Instantiate the live trading node with a configuration -node = TradingNode(config=config) - -# Register the client factories with the node -node.add_data_client_factory(KRAKEN, KrakenLiveDataClientFactory) -node.add_exec_client_factory(KRAKEN, KrakenLiveExecClientFactory) - -# Finally build the node -node.build() -``` - -### API credentials - -There are two options for supplying your credentials to the Kraken clients. -Either pass the corresponding `api_key` and `api_secret` values to the -configuration objects, or set the following environment variables: - -| Environment Variable | Description | -|----------------------------------|------------------------------------------| -| `KRAKEN_SPOT_API_KEY` | API key for Kraken Spot (mainnet). | -| `KRAKEN_SPOT_API_SECRET` | API secret for Kraken Spot (mainnet). | -| `KRAKEN_FUTURES_API_KEY` | API key for Kraken Futures (mainnet). | -| `KRAKEN_FUTURES_API_SECRET` | API secret for Kraken Futures (mainnet). | -| `KRAKEN_FUTURES_DEMO_API_KEY` | API key for Kraken Futures (demo). | -| `KRAKEN_FUTURES_DEMO_API_SECRET` | API secret for Kraken Futures (demo). | - -:::note -**Demo environment**: Only Kraken Futures offers a demo environment -(`https://demo-futures.kraken.com`) for testing without real funds. Kraken Spot -does not have a testnet - the `environment` setting only affects Futures -connections. -::: - -:::tip -We recommend using environment variables to manage your credentials. -::: - -When starting the trading node, you'll receive immediate confirmation of whether -your credentials are valid and have trading permissions. - -## Contributing - -:::info -For additional features or to contribute to the Kraken adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/okx.md b/nautilus-docs/docs/integrations/okx.md deleted file mode 100644 index 3782bc1..0000000 --- a/nautilus-docs/docs/integrations/okx.md +++ /dev/null @@ -1,675 +0,0 @@ -# OKX - -Founded in 2017, OKX is a leading cryptocurrency exchange offering spot, perpetual swap, -futures, and options trading. This integration supports live market data ingest and order -execution on OKX. - -## Overview - -This adapter is implemented in Rust, with optional Python bindings for ease of use in Python-based workflows. -It does not require external OKX client libraries; the core components are compiled as a static library and linked automatically during the build. - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/okx/). - -### Product support - -| Product Type | Data Feed | Trading | Notes | -|-------------------|-----------|---------|--------------------------------------------------| -| Spot | ✓ | ✓ | Use for index prices. | -| Perpetual Swaps | ✓ | ✓ | Linear and inverse contracts. | -| Futures | ✓ | ✓ | Specific expiration dates. | -| Margin | ✓ | ✓ | Spot trading with margin/leverage (spot margin). | -| Options | ✓ | - | *Data feed supported, trading coming soon*. | - -:::note -**Options support**: While you can subscribe to options market data and receive price updates, order execution for options is not yet implemented. You can use the symbology format shown above to subscribe to options data feeds. -::: - -:::info -**Instrument multipliers**: For derivatives (SWAP, FUTURES, OPTIONS), instrument multipliers are calculated as the product of OKX's `ctMult` (contract multiplier) and `ctVal` (contract value) fields. This ensures position sizing accurately reflects both the contract size and value. -::: - -The OKX adapter includes multiple components, which can be used separately or together depending on your use case. - -- `OKXHttpClient`: Low-level HTTP API connectivity. -- `OKXWebSocketClient`: Low-level WebSocket API connectivity. -- `OKXInstrumentProvider`: Instrument parsing and loading functionality. -- `OKXDataClient`: Market data feed manager. -- `OKXExecutionClient`: Account management and trade execution gateway. -- `OKXLiveDataClientFactory`: Factory for OKX data clients (used by the trading node builder). -- `OKXLiveExecClientFactory`: Factory for OKX execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as shown below), -and won’t need to work directly with these lower-level components. -::: - -## Symbology - -OKX uses specific symbol conventions for different instrument types. All instrument IDs should include the `.OKX` suffix when referencing them (e.g., `BTC-USDT.OKX` for spot Bitcoin). - -### Symbol format by instrument type - -#### SPOT - -Format: `{BaseCurrency}-{QuoteCurrency}` - -Examples: - -- `BTC-USDT` - Bitcoin against USDT (Tether) -- `BTC-USDC` - Bitcoin against USDC -- `ETH-USDT` - Ethereum against USDT -- `SOL-USDT` - Solana against USDT - -To subscribe to spot Bitcoin USD in your strategy: - -```python -InstrumentId.from_str("BTC-USDT.OKX") # For USDT-quoted spot -InstrumentId.from_str("BTC-USDC.OKX") # For USDC-quoted spot -``` - -#### SWAP (Perpetual Futures) - -Format: `{BaseCurrency}-{QuoteCurrency}-SWAP` - -Examples: - -- `BTC-USDT-SWAP` - Bitcoin perpetual swap (linear, USDT-margined) -- `BTC-USD-SWAP` - Bitcoin perpetual swap (inverse, coin-margined) -- `ETH-USDT-SWAP` - Ethereum perpetual swap (linear) -- `ETH-USD-SWAP` - Ethereum perpetual swap (inverse) - -Linear vs Inverse contracts: - -- **Linear** (USDT-margined): Uses stablecoins like USDT as margin. -- **Inverse** (coin-margined): Uses the base cryptocurrency as margin. - -#### FUTURES (Dated Futures) - -Format: `{BaseCurrency}-{QuoteCurrency}-{YYMMDD}` - -Examples: - -- `BTC-USD-251226` - Bitcoin futures expiring December 26, 2025 -- `ETH-USD-251226` - Ethereum futures expiring December 26, 2025 -- `BTC-USD-250328` - Bitcoin futures expiring March 28, 2025 - -Note: Futures are typically inverse contracts (coin-margined). - -#### OPTIONS - -Format: `{BaseCurrency}-{QuoteCurrency}-{YYMMDD}-{Strike}-{Type}` - -Examples: - -- `BTC-USD-251226-100000-C` - Bitcoin call option, $100,000 strike, expiring December 26, 2025 -- `BTC-USD-251226-100000-P` - Bitcoin put option, $100,000 strike, expiring December 26, 2025 -- `ETH-USD-251226-4000-C` - Ethereum call option, $4,000 strike, expiring December 26, 2025 - -Where: - -- `C` = Call option -- `P` = Put option - -### Common questions - -**Q: How do I subscribe to spot Bitcoin USD?** -A: Use `BTC-USDT.OKX` for USDT-margined spot or `BTC-USDC.OKX` for USDC-margined spot. - -**Q: What's the difference between BTC-USDT-SWAP and BTC-USD-SWAP?** -A: `BTC-USDT-SWAP` is a linear perpetual (USDT-margined), while `BTC-USD-SWAP` is an inverse perpetual (BTC-margined). - -**Q: How do I know which contract type to use?** -A: Check the `contract_types` parameter in the configuration: - -- For linear contracts: `OKXContractType.LINEAR`. -- For inverse contracts: `OKXContractType.INVERSE`. - -## Orders capability - -Below are the order types, execution instructions, and time-in-force options supported -for linear perpetual swap products on OKX. - -### Client order ID requirements - -:::note -OKX has specific requirements for client order IDs: - -- **No hyphens allowed**: OKX does not accept hyphens (`-`) in client order IDs. -- Maximum length: 32 characters. -- Allowed characters: alphanumeric characters and underscores only. - -When configuring your strategy, ensure you set: - -```python -use_hyphens_in_client_order_ids=False -``` - -::: - -### Order types - -| Order Type | Linear Perpetual Swap | Notes | -|---------------------|-----------------------|---------------------------------------------------------------| -| `MARKET` | ✓ | Immediate execution at market price. Supports quote quantity. | -| `MARKET_TO_LIMIT` | ✓ | Market order converted to IOC limit. | -| `LIMIT` | ✓ | Execution at specified price or better. | -| `STOP_MARKET` | ✓ | Conditional market order (OKX algo order). | -| `STOP_LIMIT` | ✓ | Conditional limit order (OKX algo order). | -| `MARKET_IF_TOUCHED` | ✓ | Conditional market order (OKX algo order). | -| `LIMIT_IF_TOUCHED` | ✓ | Conditional limit order (OKX algo order). | -| `TRAILING_STOP_MARKET` | ✓ | Trailing stop market order (OKX advance algo order). | - -:::info -**Conditional orders**: `STOP_MARKET`, `STOP_LIMIT`, `MARKET_IF_TOUCHED`, `LIMIT_IF_TOUCHED`, and `TRAILING_STOP_MARKET` are implemented as OKX algo orders, providing advanced trigger capabilities with multiple price sources. `TRAILING_STOP_MARKET` uses OKX's advance algo order API (`move_order_stop`) and requires the separate `cancel-advance-algos` endpoint for cancellation. -::: - -### Quantity semantics for spot margin trading - -When using spot margin trading (`use_spot_margin=True`), OKX interprets order quantities differently depending on the order side: - -- **Limit** orders interpret `quantity` as the number of base currency units. -- **Market SELL** orders also use base-unit quantities. -- **Market BUY** orders interpret `quantity` as quote notional (e.g., USDT). - -:::warning -**When submitting spot margin market BUY orders, you must**: - -1. Set `quote_quantity=True` on the order (or pre-compute the quote-denominated amount). -2. Configure the execution engine with `convert_quote_qty_to_base=False` so the quote amount reaches the adapter unchanged. - -The OKX execution client will deny base-denominated market buy orders for spot margin to prevent unintended fills. - -**On the first fill**, the order quantity will be automatically updated from the quote quantity to the actual base quantity received, -reflecting the executed trade. -::: - -```python -from nautilus_trader.execution.config import ExecEngineConfig -from nautilus_trader.execution.engine import ExecutionEngine - -# Disable automatic conversion for quote quantities -config = ExecEngineConfig(convert_quote_qty_to_base=False) -engine = ExecutionEngine(msgbus=msgbus, cache=cache, clock=clock, config=config) - -# Correct: Spot margin market BUY with quote quantity (spend $100 USDT) -order = strategy.order_factory.market( - instrument_id=instrument_id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(100.0), - quote_quantity=True, # Interpret as USDT notional -) -strategy.submit_order(order) -``` - -### Execution instructions - -| Instruction | Linear Perpetual Swap | Notes | -|----------------|-----------------------|------------------------| -| `post_only` | ✓ | Only for LIMIT orders. | -| `reduce_only` | ✓ | Only for derivatives. | - -### Time in force - -| Time in force | Linear Perpetual Swap | Notes | -|---------------|-----------------------|---------------------------------------------------| -| `GTC` | ✓ | Good Till Canceled. | -| `FOK` | ✓ | Fill or Kill. | -| `IOC` | ✓ | Immediate or Cancel. | -| `GTD` | ✗ | *Not supported by OKX API.* | - -:::note -**GTD (Good Till Date) time in force**: OKX does not support native GTD functionality through their API. - -If you need GTD functionality, you must use Nautilus's strategy-managed GTD feature, which will handle the order expiration by canceling the order at the specified expiry time. -::: - -### Batch operations - -| Operation | Linear Perpetual Swap | Notes | -|--------------------|-----------------------|-------------------------------------------| -| Batch Submit | ✓ | Submit multiple orders in single request. | -| Batch Modify | ✓ | Modify multiple orders in single request. | -| Batch Cancel | ✓ | Cancel multiple orders in single request. | - -### Position management - -| Feature | Linear Perpetual Swap | Notes | -|-------------------|-----------------------|------------------------------------------------------| -| Query positions | ✓ | Real-time position updates. | -| Position mode | ✓ | Net vs Long/Short mode (see below). | -| Leverage control | ✓ | Dynamic leverage adjustment per instrument. | -| Margin mode | ✓ | Supports cash, isolated, and cross modes. | - -#### Position modes - -OKX supports two position modes for derivatives trading: - -- **Net mode** (Netting): Single position per instrument that can be positive (LONG) or negative (SHORT). Buy and sell orders net against each other. This is the default and recommended for most traders. -- **Long/Short mode** (Hedging): Separate long and short positions for the same instrument. Allows simultaneous long and short positions, useful for hedging strategies. - -:::note -Position mode must be configured via the OKX Web/App interface and applies account-wide. The adapter automatically detects the current position mode and handles position reporting accordingly. -::: - -### Trade modes and margin configuration - -OKX's unified account system supports different trade modes for spot and derivatives trading. The adapter automatically determines the correct trade mode based on your configuration and instrument type. - -:::note -**Important**: Account modes must be initially configured via the OKX Web/App interface. The API cannot set the account mode for the first time. -::: - -For more details on OKX's account modes and margin system, see the [OKX Account Mode documentation](https://www.okx.com/docs-v5/en/#overview-account-mode). - -#### Trade modes overview - -OKX supports four trade modes, which the adapter selects automatically based on your configuration: - -| Mode | Used For | Leverage | Borrowing | Configuration | -|---------------------|--------------------------------------------|----------|-----------|---------------| -| **`cash`** | Simple spot trading | - | - | `use_spot_margin=False` (default for SPOT) | -| **`isolated`** | Spot margin or derivatives (default) | ✓ | ✓ | `use_spot_margin=True` with `margin_mode=ISOLATED` (or unset) for SPOT; default for derivatives | -| **`cross`** | Spot margin or derivatives, shared pool | ✓ | ✓ | `use_spot_margin=True` with `margin_mode=CROSS` for SPOT; `margin_mode=CROSS` for derivatives | - -#### Configuration-based trade mode selection - -**The adapter automatically selects the correct trade mode** based on: - -1. **Instrument type** (SPOT vs derivatives) -2. **Configuration settings** (`use_spot_margin` for SPOT, `margin_mode` for derivatives) - -##### For SPOT trading - -```python -# Simple SPOT trading without leverage (uses 'cash' mode) -exec_clients={ - OKX: OKXExecClientConfig( - instrument_types=(OKXInstrumentType.SPOT,), - use_spot_margin=False, # Default - simple SPOT - # ... other config - ), -} - -# SPOT trading WITH margin/leverage (uses 'isolated' or 'cross' mode) -exec_clients={ - OKX: OKXExecClientConfig( - instrument_types=(OKXInstrumentType.SPOT,), - use_spot_margin=True, # Enable margin trading for SPOT - margin_mode=OKXMarginMode.ISOLATED, # Or CROSS for shared margin - # ... other config - ), -} -``` - -##### For derivatives trading (SWAP/FUTURES/OPTIONS) - -```python -# Derivatives with isolated margin (default - uses 'isolated' mode) -exec_clients={ - OKX: OKXExecClientConfig( - instrument_types=(OKXInstrumentType.SWAP,), - margin_mode=OKXMarginMode.ISOLATED, # Or omit - ISOLATED is default - # ... other config - ), -} - -# Derivatives with cross margin (uses 'cross' mode) -exec_clients={ - OKX: OKXExecClientConfig( - instrument_types=(OKXInstrumentType.SWAP,), - margin_mode=OKXMarginMode.CROSS, # Share margin across all positions - # ... other config - ), -} -``` - -##### For mixed SPOT and derivatives trading - -When trading both SPOT and derivatives instruments simultaneously, the adapter automatically determines the correct trade mode **per-order** based on the instrument being traded: - -```python -# Mixed SPOT + SWAP configuration -exec_clients={ - OKX: OKXExecClientConfig( - instrument_types=(OKXInstrumentType.SPOT, OKXInstrumentType.SWAP), - use_spot_margin=True, # Applies to SPOT orders only - margin_mode=OKXMarginMode.CROSS, # Applies to SWAP orders only - # ... other config - ), -} -``` - -**How it works:** - -- **SPOT orders** → Uses `cross` mode (because `use_spot_margin=True` and `margin_mode=CROSS`) -- **SWAP orders** → Uses `cross` mode (because `margin_mode=CROSS`) -- Each order automatically gets the correct `tdMode` based on its instrument type -- No manual intervention required - -This enables strategies that trade across multiple instrument types with different margin configurations, such as: - -- Spot-futures arbitrage strategies -- Delta-neutral strategies combining spot and perpetual swaps -- Market making across spot and derivatives markets - -:::warning -**Manual trade mode override**: While you can still manually override the trade mode per order using `params={"td_mode": "..."}`, this is **not recommended** as it bypasses automatic mode selection and can lead to order rejection if the wrong mode is specified for the instrument type (e.g., using `isolated` for SPOT instruments). - -Only use manual override if you have specific requirements that cannot be met through configuration. -::: - -#### Benefits of configuration-based approach - -- **Type-safe**: Configuration is validated at startup before placing any orders. -- **Automatic**: System chooses correct mode based on instrument type and intent. -- **Clear**: Field names explain purpose (`use_spot_margin` vs obscure `td_mode` parameter). -- **Safe**: Impossible to use incompatible combinations (e.g., `isolated` mode for SPOT). -- **Backwards compatible**: Default values maintain existing behavior. - -### Order querying - -| Feature | Linear Perpetual Swap | Notes | -|----------------------|-----------------------|-------------------------------------------| -| Query open orders | ✓ | List all active orders. | -| Query order history | ✓ | Historical order data. | -| Order status updates | ✓ | Real-time order state changes. | -| Trade history | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Linear Perpetual Swap | Notes | -|---------------------|-----------------------|--------------------------------------------| -| Order lists | - | *Not supported*. | -| OCO orders | ✓ | One-Cancels-Other orders. | -| Bracket orders | ✓ | Stop loss + take profit combinations. | -| Conditional orders | ✓ | Stop and limit-if-touched orders. | - -#### Conditional order architecture - -Conditional orders (OKX algo orders) use a hybrid architecture for optimal performance and reliability: - -- **Submission**: Via HTTP REST API (`/api/v5/trade/order-algo`) -- **Status updates**: Via WebSocket business endpoint (`/ws/v5/business`) on the `orders-algo` channel -- **Cancellation**: Via HTTP REST API using algo order ID tracking - -This design ensures: - -- Immediate submission acknowledgment through HTTP. -- Real-time status updates through WebSocket. -- Proper order lifecycle management with algo order ID mapping. - -#### Supported conditional order types - -| Order Type | Trigger Types | Notes | -|---------------------|------------------------|-------------------------------------------| -| `STOP_MARKET` | Last, Mark, Index | Market execution when triggered. | -| `STOP_LIMIT` | Last, Mark, Index | Limit order placement when triggered. | -| `MARKET_IF_TOUCHED` | Last, Mark, Index | Market execution when price touched. | -| `LIMIT_IF_TOUCHED` | Last, Mark, Index | Limit order placement when price touched. | -| `TRAILING_STOP_MARKET` | Last, Mark, Index | Trailing stop with callback ratio. | - -#### Trigger price types - -Conditional orders support different trigger price sources: - -- **Last Price** (`TriggerType.LAST_PRICE`): Uses the last traded price (default). -- **Mark Price** (`TriggerType.MARK_PRICE`): Uses the mark price (recommended for derivatives). -- **Index Price** (`TriggerType.INDEX_PRICE`): Uses the underlying index price. - -```python -# Example: Stop loss using mark price trigger -stop_order = order_factory.stop_market( - instrument_id=instrument_id, - order_side=OrderSide.SELL, - quantity=Quantity.from_str("0.1"), - trigger_price=Price.from_str("45000.0"), - trigger_type=TriggerType.MARK_PRICE, # Use mark price for trigger -) -strategy.submit_order(stop_order) -``` - -## Risk management - -### Liquidation and ADL event handling - -The OKX adapter automatically detects and handles exchange-initiated risk management events: - -- **Liquidation orders**: When a position is liquidated by the exchange (full or partial), the adapter detects the liquidation category and logs warnings with order details. These orders are processed normally through the order and fill pipeline. -- **Auto-Deleveraging (ADL)**: When your position is closed by the exchange to offset a counterparty's liquidation, the adapter detects and logs the ADL event with position details. - -:::info -**Liquidation and ADL events are logged at WARNING level** with details including order ID, instrument, and state. Monitor your logs for these events as part of your risk management process. - -The adapter handles these exchange-generated orders, generating appropriate `OrderFilled` events and updating positions accordingly. No special handling is required in your strategy code. -::: - -## Authentication - -To use the OKX adapter, you'll need to create API credentials in your OKX account: - -1. Log into your OKX account and navigate to the API management page. -2. Create a new API key with the required permissions for trading and data access. -3. Note down your API key, secret key, and passphrase. - -You can provide these credentials through environment variables: - -```bash -export OKX_API_KEY="your_api_key" -export OKX_API_SECRET="your_api_secret" -export OKX_API_PASSPHRASE="your_passphrase" -``` - -Or pass them directly in the configuration (not recommended for production). - -## Demo trading - -OKX provides a demo trading environment for testing strategies without real funds. - -### Setting up a demo account - -1. Log into your OKX account at [okx.com](https://www.okx.com). -2. Navigate to **Trade** → **Demo Trading**. -3. Go to **Personal Center** within Demo Trading. -4. Select **Demo Trading API** and create a new API key. -5. Note down your demo API key, secret, and passphrase. - -You can provide demo credentials through environment variables: - -```bash -export OKX_API_KEY="your_demo_api_key" -export OKX_API_SECRET="your_demo_api_secret" -export OKX_API_PASSPHRASE="your_demo_passphrase" -``` - -### Configuration - -Set `is_demo=True` in your client configuration: - -```python -config = TradingNodeConfig( - data_clients={ - OKX: OKXDataClientConfig( - is_demo=True, # Enable demo mode - # ... other config - ), - }, - exec_clients={ - OKX: OKXExecClientConfig( - is_demo=True, # Enable demo mode - # ... other config - ), - }, -) -``` - -When demo mode is enabled: - -- REST API requests include the `x-simulated-trading: 1` header. -- WebSocket connections use demo endpoints (`wspap.okx.com`). - -:::note -Demo API keys are separate from production keys. You must create API keys specifically for demo trading through the Demo Trading interface. Production API keys will not work in demo mode. -::: - -## Funding rates - -The adapter receives funding rate data from the -[Funding Rate Channel](https://www.okx.com/docs-v5/en/#public-data-websocket-funding-rate-channel) -WebSocket stream. OKX provides both `fundingTime` and `nextFundingTime` in each message, -and the adapter computes `interval` as the difference between these two values. - -For historical funding rate requests, the adapter computes the interval from consecutive -funding timestamps returned by the -[Get Funding Rate History](https://www.okx.com/docs-v5/en/#public-data-rest-api-get-funding-rate-history) -endpoint. - -## Rate limiting - -The adapter enforces OKX’s per-endpoint quotas while keeping sensible defaults for both REST and WebSocket calls. - -### REST limits - -- Global cap: 250 requests per second (matches 500 requests / 2 seconds IP allowance). -- Endpoint-specific quotas appear in the table below and mirror OKX’s published limits where available. - -### WebSocket limits - -- Connection establishment: 3 requests per second (per IP). -- Subscription operations (subscribe/unsubscribe/login): 480 requests per hour per connection. -- Order actions (place/cancel/amend): 250 requests per second. - -:::warning -OKX enforces per-endpoint and per-account quotas; exceeding them leads to HTTP 429 responses and temporary throttling on that key. -::: - -| Key / Endpoint | Limit (req/sec) | Notes | -|----------------------------------|-----------------|---------------------------------------------------------| -| `okx:global` | 250 | Matches 500 req / 2 s IP allowance. | -| `/api/v5/public/instruments` | 10 | Matches OKX 20 req / 2 s docs. | -| `/api/v5/market/candles` | 50 | Higher allowance for streaming candles. | -| `/api/v5/market/history-candles` | 20 | Conservative quota for large historical pulls. | -| `/api/v5/market/history-trades` | 30 | Trade history pulls. | -| `/api/v5/account/balance` | 5 | OKX guidance: 10 req / 2 s. | -| `/api/v5/trade/order` | 30 | 60 requests / 2 seconds per-instrument limit. | -| `/api/v5/trade/orders-pending` | 20 | Open order fetch. | -| `/api/v5/trade/orders-history` | 20 | Historical orders. | -| `/api/v5/trade/fills` | 30 | Execution reports. | -| `/api/v5/trade/order-algo` | 10 | Algo placements (conditional orders). | -| `/api/v5/trade/cancel-algos` | 10 | Algo cancellation. | - -All keys automatically include the `okx:global` bucket. URLs are normalised (query strings removed) before rate limiting, so requests with different filters share the same quota. - -:::info -For more details on rate limiting, see the official documentation: . -::: - -## Configuration - -### Configuration options - -The OKX data client provides the following configuration options: - -#### Data client - -| Option | Default | Description | -|--------------------------------------|---------------------------------|-------------| -| `instrument_types` | `(OKXInstrumentType.SPOT,)` | Controls which OKX instrument families are loaded (spot, swap, futures, options). | -| `contract_types` | `None` | Restricts loading to specific contract styles when combined with `instrument_types`. | -| `instrument_families` | `None` | Instrument families to load (e.g., "BTC-USD", "ETH-USD"). Required for OPTIONS. Optional for FUTURES/SWAP. Not applicable for SPOT/MARGIN. | -| `base_url_http` | `None` | Override for the OKX REST endpoint; defaults to the production URL resolved at runtime. | -| `base_url_ws` | `None` | Override for the market data WebSocket endpoint. | -| `api_key` | `None` | Falls back to `OKX_API_KEY` environment variable when unset. | -| `api_secret` | `None` | Falls back to `OKX_API_SECRET` environment variable when unset. | -| `api_passphrase` | `None` | Falls back to `OKX_API_PASSPHRASE` environment variable when unset. | -| `is_demo` | `False` | Connects to the OKX demo environment when `True`. | -| `http_timeout_secs` | `60` | Request timeout (seconds) for REST market data calls. | -| `max_retries` | `3` | Maximum retry attempts for recoverable REST errors. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) before retrying a failed request. | -| `retry_delay_max_ms` | `10,000` | Upper bound for exponential backoff delay between retries. | -| `update_instruments_interval_mins` | `60` | Interval, in minutes, between background instrument refreshes. | -| `vip_level` | `None` | Enables higher-depth order book channels when set to the matching OKX VIP tier. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. | - -The OKX execution client provides the following configuration options: - -#### Execution client - -| Option | Default | Description | -|----------------------------|-------------|-------------| -| `instrument_types` | `(OKXInstrumentType.SPOT,)` | Instrument families that should be tradable for this client. | -| `contract_types` | `None` | Restricts tradable contracts (linear, inverse, options) when paired with `instrument_types`. | -| `instrument_families` | `None` | Instrument families to load (e.g., "BTC-USD", "ETH-USD"). Required for OPTIONS. Optional for FUTURES/SWAP. Not applicable for SPOT/MARGIN. | -| `base_url_http` | `None` | Override for the OKX trading REST endpoint. | -| `base_url_ws` | `None` | Override for the private WebSocket endpoint. | -| `api_key` | `None` | Falls back to `OKX_API_KEY` environment variable when unset. | -| `api_secret` | `None` | Falls back to `OKX_API_SECRET` environment variable when unset. | -| `api_passphrase` | `None` | Falls back to `OKX_API_PASSPHRASE` environment variable when unset. | -| `margin_mode` | `None` | Margin mode for derivatives trading (`ISOLATED` or `CROSS`). Only applies to SWAP/FUTURES/OPTIONS. Defaults to `ISOLATED` if not specified. | -| `use_spot_margin` | `False` | Enables margin/leverage for SPOT trading. When `True`, uses `isolated` or `cross` trade mode (determined by `margin_mode`). When `False`, uses `cash` trade mode (no leverage). Only applies to SPOT instruments. | -| `is_demo` | `False` | Connects to the OKX demo trading environment. | -| `http_timeout_secs` | `60` | Request timeout (seconds) for REST trading calls. | -| `use_fills_channel` | `False` | Subscribes to the dedicated fills channel (VIP5+ required) for lower-latency fill reports. | -| `use_mm_mass_cancel` | `False` | Uses the market-maker bulk cancel endpoint when available; otherwise falls back to per-order cancels. | -| `max_retries` | `3` | Maximum retry attempts for recoverable REST errors. | -| `retry_delay_initial_ms` | `1,000` | Initial delay (milliseconds) applied before retrying a failed request. | -| `retry_delay_max_ms` | `10,000` | Upper bound for the exponential backoff delay between retries. | -| `use_spot_cash_position_reports` | `False` | Generate position reports for SPOT CASH instruments based on wallet balances. | -| `http_proxy_url` | `None` | Optional HTTP proxy URL. | -| `ws_proxy_url` | `None` | Optional WebSocket proxy URL. | - -Below is an example configuration for a live trading node using OKX data and execution clients: - -```python -from nautilus_trader.adapters.okx import OKX -from nautilus_trader.adapters.okx import OKXDataClientConfig, OKXExecClientConfig -from nautilus_trader.adapters.okx.factories import OKXLiveDataClientFactory, OKXLiveExecClientFactory -from nautilus_trader.config import InstrumentProviderConfig, TradingNodeConfig -from nautilus_trader.core.nautilus_pyo3 import OKXContractType -from nautilus_trader.core.nautilus_pyo3 import OKXInstrumentType -from nautilus_trader.core.nautilus_pyo3 import OKXMarginMode -from nautilus_trader.live.node import TradingNode - -config = TradingNodeConfig( - ..., - data_clients={ - OKX: OKXDataClientConfig( - api_key=None, # Will use OKX_API_KEY env var - api_secret=None, # Will use OKX_API_SECRET env var - api_passphrase=None, # Will use OKX_API_PASSPHRASE env var - base_url_http=None, - instrument_provider=InstrumentProviderConfig(load_all=True), - instrument_types=(OKXInstrumentType.SWAP,), - contract_types=(OKXContractType.LINEAR,), - is_demo=False, - ), - }, - exec_clients={ - OKX: OKXExecClientConfig( - api_key=None, - api_secret=None, - api_passphrase=None, - base_url_http=None, - base_url_ws=None, - instrument_provider=InstrumentProviderConfig(load_all=True), - instrument_types=(OKXInstrumentType.SWAP,), - contract_types=(OKXContractType.LINEAR,), - is_demo=False, - ), - }, -) -node = TradingNode(config=config) -node.add_data_client_factory(OKX, OKXLiveDataClientFactory) -node.add_exec_client_factory(OKX, OKXLiveExecClientFactory) -node.build() -``` - -## Contributing - -:::info -For additional features or to contribute to the OKX adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/polymarket.md b/nautilus-docs/docs/integrations/polymarket.md deleted file mode 100644 index cca5294..0000000 --- a/nautilus-docs/docs/integrations/polymarket.md +++ /dev/null @@ -1,801 +0,0 @@ -# Polymarket - -Founded in 2020, Polymarket is the world’s largest decentralized prediction market platform, -enabling traders to speculate on the outcomes of world events by buying and selling binary option contracts using cryptocurrency. - -NautilusTrader provides a venue integration for data and execution via Polymarket's Central Limit Order Book (CLOB) API. -The integration uses the [official Python CLOB client library](https://github.com/Polymarket/py-clob-client) -to interact with the Polymarket platform. - -NautilusTrader supports multiple Polymarket signature types for order signing, providing flexibility for different wallet configurations. -This integration ensures that traders can execute orders securely and efficiently across various wallet types, -while NautilusTrader abstracts the complexity of signing and preparing orders for execution. - -## Installation - -To install NautilusTrader with Polymarket support: - -```bash -uv pip install "nautilus_trader[polymarket]" -``` - -To build from source with all extras (including Polymarket): - -```bash -uv sync --all-extras -``` - -## Examples - -You can find live example scripts [here](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/polymarket/). - -## Binary options - -A [binary option](https://en.wikipedia.org/wiki/Binary_option) is a type of financial exotic option contract in which traders bet on the outcome of a yes-or-no proposition. -If the prediction is correct, the trader receives a fixed payout; otherwise, they receive nothing. - -All assets traded on Polymarket are quoted and settled in **USDC.e (PoS)**, [see below](#usdce-pos) for more information. - -## Polymarket documentation - -Polymarket offers resources for different audiences: - -- [Polymarket Learn](https://learn.polymarket.com/): Educational content and guides for users to understand the platform and how to engage with it. -- [Polymarket CLOB API](https://docs.polymarket.com/#introduction): Technical documentation for developers interacting with the Polymarket CLOB API. - -## Overview - -This guide assumes a trader is setting up for both live market data feeds and trade execution. -The Polymarket integration adapter includes multiple components, which can be used together or -separately depending on the use case. - -- `PolymarketWebSocketClient`: Low-level WebSocket API connectivity (built on top of the Nautilus `WebSocketClient` written in Rust). -- `PolymarketInstrumentProvider`: Instrument parsing and loading functionality for `BinaryOption` instruments. -- `PolymarketDataClient`: A market data feed manager. -- `PolymarketExecutionClient`: A trade execution gateway. -- `PolymarketLiveDataClientFactory`: Factory for Polymarket data clients (used by the trading node builder). -- `PolymarketLiveExecClientFactory`: Factory for Polymarket execution clients (used by the trading node builder). - -:::note -Most users will define a configuration for a live trading node (as below), -and won't need to work with these lower-level components directly. -::: - -## USDC.e (PoS) - -**USDC.e** is a bridged version of USDC from Ethereum to the Polygon network, operating on Polygon's **Proof of Stake (PoS)** chain. -This enables faster, more cost-efficient transactions on Polygon while maintaining backing by USDC on Ethereum. - -The contract address is [0x2791bca1f2de4661ed88a30c99a7a9449aa84174](https://polygonscan.com/address/0x2791bca1f2de4661ed88a30c99a7a9449aa84174) on the Polygon blockchain. -More information can be found in this [blog](https://polygon.technology/blog/phase-one-of-native-usdc-migration-on-polygon-pos-is-underway). - -## Wallets and accounts - -To interact with Polymarket via NautilusTrader, you'll need a **Polygon**-compatible wallet (such as MetaMask). - -### Signature types - -Polymarket supports multiple signature types for order signing and verification: - -| Signature Type | Wallet Type | Description | Use Case | -|----------------|--------------------------------|-------------|----------| -| `0` | EOA (Externally Owned Account) | Standard EIP712 signatures from wallets with direct private key control. | **Default.** Direct wallet connections (MetaMask, hardware wallets, etc.). | -| `1` | Email/Magic Wallet Proxy | Smart contract wallet for email-based accounts (Magic Link). Only the email-associated address can execute functions. | Polymarket Proxy associated with Email/Magic accounts. Requires `funder` address. | -| `2` | Browser Wallet Proxy | Modified Gnosis Safe (1-of-1 multisig) for browser wallets. | Polymarket Proxy associated with browser wallets. Enables UI verification. Requires `funder` address. | - -:::note -See also: [Proxy wallet](https://docs.polymarket.com/developers/proxy-wallet) in the Polymarket documentation for more details about signature types and proxy wallet infrastructure. -::: - -NautilusTrader defaults to signature type 0 (EOA) but can be configured to use any of the supported signature types via the `signature_type` configuration parameter. - -A single wallet address is supported per trader instance when using environment variables, -or multiple wallets could be configured with multiple `PolymarketExecutionClient` instances. - -:::note -Ensure your wallet is funded with **USDC.e**, otherwise you will encounter the "not enough balance / allowance" API error when submitting orders. -::: - -### Setting allowances for Polymarket contracts - -Before you can start trading, you need to ensure that your wallet has allowances set for Polymarket's smart contracts. -You can do this by running the provided script located at `nautilus_trader/adapters/polymarket/scripts/set_allowances.py`. - -This script is adapted from a [gist](https://gist.github.com/poly-rodr/44313920481de58d5a3f6d1f8226bd5e) created by @poly-rodr. - -:::note -You only need to run this script **once** per EOA wallet that you intend to use for trading on Polymarket. -::: - -This script automates the process of approving the necessary allowances for the Polymarket contracts. -It sets approvals for the USDC token and Conditional Token Framework (CTF) contract to allow the -Polymarket CLOB Exchange to interact with your funds. - -Before running the script, ensure the following prerequisites are met: - -- Install the web3 Python package: `uv pip install "web3==7.12.1"`. -- Have a **Polygon**-compatible wallet funded with some POL (used for gas fees). -- Set the following environment variables in your shell: - - `POLYGON_PRIVATE_KEY`: Your private key for the **Polygon**-compatible wallet. - - `POLYGON_PUBLIC_KEY`: Your public key for the **Polygon**-compatible wallet. - -Once you have these in place, the script will: - -- Approve the maximum possible amount of USDC (using the `MAX_INT` value) for the Polymarket USDC token contract. -- Set the approval for the CTF contract, allowing it to interact with your account for trading purposes. - -:::note -You can also adjust the approval amount in the script instead of using `MAX_INT`, -with the amount specified in *fractional units* of **USDC.e**, though this has not been tested. -::: - -Ensure that your private key and public key are correctly stored in the environment variables before running the script. -Here's an example of how to set the variables in your terminal session: - -```bash -export POLYGON_PRIVATE_KEY="YOUR_PRIVATE_KEY" -export POLYGON_PUBLIC_KEY="YOUR_PUBLIC_KEY" -``` - -Run the script using: - -```bash -python nautilus_trader/adapters/polymarket/scripts/set_allowances.py -``` - -### Script breakdown - -The script performs the following actions: - -- Connects to the Polygon network via an RPC URL (). -- Signs and sends a transaction to approve the maximum USDC allowance for Polymarket contracts. -- Sets approval for the CTF contract to manage Conditional Tokens on your behalf. -- Repeats the approval process for specific addresses like the Polymarket CLOB Exchange and Neg Risk adapter. - -This allows Polymarket to interact with your funds when executing trades and ensures smooth integration with the CLOB Exchange. - -## API keys - -To trade with Polymarket, you'll need to generate API credentials. Follow these steps: - -1. Ensure the following environment variables are set: - - `POLYMARKET_PK`: Your private key for signing transactions. - - `POLYMARKET_FUNDER`: The wallet address (public key) on the **Polygon** network used for funding trades on Polymarket. - -2. Run the script using: - - ```bash - python nautilus_trader/adapters/polymarket/scripts/create_api_key.py - ``` - -The script will generate and print API credentials, which you should save to the following environment variables: - -- `POLYMARKET_API_KEY` -- `POLYMARKET_API_SECRET` -- `POLYMARKET_PASSPHRASE` - -These can then be used for Polymarket client configurations: - -- `PolymarketDataClientConfig` -- `PolymarketExecClientConfig` - -## Configuration - -When setting up NautilusTrader to work with Polymarket, it’s crucial to properly configure the necessary parameters, particularly the private key. - -**Key parameters**: - -- `private_key`: The private key for your wallet used to sign orders. The interpretation depends on your `signature_type` configuration. If not explicitly provided in the configuration, it will automatically source the `POLYMARKET_PK` environment variable. -- `funder`: The **USDC.e** funding wallet address used for funding trades. If not provided, will source the `POLYMARKET_FUNDER` environment variable. -- API credentials: You will need to provide the following API credentials to interact with the Polymarket CLOB: - - `api_key`: If not provided, will source the `POLYMARKET_API_KEY` environment variable. - - `api_secret`: If not provided, will source the `POLYMARKET_API_SECRET` environment variable. - - `passphrase`: If not provided, will source the `POLYMARKET_PASSPHRASE` environment variable. - -:::tip -We recommend using environment variables to manage your credentials. -::: - -## Orders capability - -Polymarket operates as a prediction market with a more limited set of order types and instructions compared to traditional exchanges. - -### Order types - -| Order Type | Binary Options | Notes | -|------------------------|----------------|-------------------------------------| -| `MARKET` | ✓ | **BUY orders require quote quantity**, SELL orders require base quantity. | -| `LIMIT` | ✓ | | -| `STOP_MARKET` | - | *Not supported by Polymarket*. | -| `STOP_LIMIT` | - | *Not supported by Polymarket*. | -| `MARKET_IF_TOUCHED` | - | *Not supported by Polymarket*. | -| `LIMIT_IF_TOUCHED` | - | *Not supported by Polymarket*. | -| `TRAILING_STOP_MARKET` | - | *Not supported by Polymarket*. | - -### Quantity semantics - -Polymarket interprets order quantities differently depending on the order type *and* side: - -- **Limit** orders interpret `quantity` as the number of conditional tokens (base units). -- **Market SELL** orders also use base-unit quantities. -- **Market BUY** orders interpret `quantity` as quote notional in **USDC.e**. - -As a result, a market buy order submitted with a base-denominated quantity will execute far more size than intended. - -:::warning -When submitting market BUY orders, set `quote_quantity=True` (or pre-compute the quote-denominated amount) -and configure the execution engine with `convert_quote_qty_to_base=False` so the quote amount reaches the adapter unchanged. -The Polymarket execution client denies base-denominated market buys to prevent unintended fills. - -**NautilusTrader now forwards market orders to Polymarket's native market-order endpoint, so the -quote amount you specify for a BUY is executed directly (no more synthetic max-price limits).** -::: - -```python -from nautilus_trader.execution.config import ExecEngineConfig -from nautilus_trader.execution.engine import ExecutionEngine - -# Temporary: disable automatic conversion until the behaviour is fully removed in a future release -config = ExecEngineConfig(convert_quote_qty_to_base=False) -engine = ExecutionEngine(msgbus=msgbus, cache=cache, clock=clock, config=config) - -# Correct: Market BUY with quote quantity (spend $10 USDC) -order = strategy.order_factory.market( - instrument_id=instrument_id, - order_side=OrderSide.BUY, - quantity=instrument.make_qty(10.0), - quote_quantity=True, # Interpret as USDC.e notional -) -strategy.submit_order(order) -``` - -### Execution instructions - -| Instruction | Binary Options | Notes | -|---------------|----------------|------------------------------------------| -| `post_only` | - | *Not supported by Polymarket*. | -| `reduce_only` | - | *Not supported by Polymarket*. | - -### Time-in-force options - -| Time in force | Binary Options | Notes | -|---------------|----------------|------------------------------------------| -| `GTC` | ✓ | Good Till Canceled. | -| `GTD` | ✓ | Good Till Date. | -| `FOK` | ✓ | Fill or Kill. | -| `IOC` | ✓ | Immediate or Cancel (maps to FAK). | - -:::note -FAK (Fill and Kill) is Polymarket's terminology for Immediate or Cancel (IOC) semantics. -::: - -### Advanced order features - -| Feature | Binary Options | Notes | -|--------------------|----------------|------------------------------------| -| Order modification | - | Cancellation functionality only. | -| Bracket/OCO orders | - | *Not supported by Polymarket.* | -| Iceberg orders | - | *Not supported by Polymarket.* | - -### Batch operations - -| Operation | Binary Options | Notes | -|--------------------|----------------|-------------------------------------| -| Batch Submit | - | *Not supported by Polymarket*. | -| Batch Modify | - | *Not supported by Polymarket*. | -| Batch Cancel | - | *Not supported by Polymarket*. | - -### Position management - -| Feature | Binary Options | Notes | -|------------------|----------------|-----------------------------------| -| Query positions | ✓ | Contract balance-based positions. | -| Position mode | - | Binary outcome positions only. | -| Leverage control | - | No leverage available. | -| Margin mode | - | No margin trading. | - -### Order querying - -| Feature | Binary Options | Notes | -|----------------------|----------------|--------------------------------| -| Query open orders | ✓ | Active orders only. | -| Query order history | ✓ | Limited historical data. | -| Order status updates | ✓ | Real-time order state changes. | -| Trade history | ✓ | Execution and fill reports. | - -### Contingent orders - -| Feature | Binary Options | Notes | -|--------------------|----------------|-------------------------------------| -| Order lists | - | *Not supported by Polymarket*. | -| OCO orders | - | *Not supported by Polymarket*. | -| Bracket orders | - | *Not supported by Polymarket*. | -| Conditional orders | - | *Not supported by Polymarket*. | - -### Precision limits - -Polymarket enforces different precision constraints based on tick size and order type. - -**Binary Option instruments** typically support up to 6 decimal places for amounts (with 0.0001 tick size), but **market orders have stricter precision requirements**: - -- **FOK (Fill-or-Kill) market orders:** - - Sell orders: maker amount limited to **2 decimal places**. - - Taker amount: limited to **4 decimal places**. - - The product `size × price` must not exceed **2 decimal places**. - -- **Regular GTC orders:** More flexible precision based on market tick size. - -### Tick size precision hierarchy - -| Tick Size | Price Decimals | Size Decimals | Amount Decimals | -|-----------|----------------|---------------|-----------------| -| 0.1 | 1 | 2 | 3 | -| 0.01 | 2 | 2 | 4 | -| 0.001 | 3 | 2 | 5 | -| 0.0001 | 4 | 2 | 6 | - -:::note - -- The tick size precision hierarchy is defined in the [`py-clob-client` `ROUNDING_CONFIG`](https://github.com/Polymarket/py-clob-client/blob/main/py_clob_client/order_builder/builder.py). -- FOK market order precision limits (2 decimals for maker amount) are based on API error responses documented in [issue #121](https://github.com/Polymarket/py-clob-client/issues/121). -- Tick sizes can change dynamically during market conditions, particularly when markets become one-sided. - -::: - -## Trades - -Trades on Polymarket can have the following statuses: - -- `MATCHED`: Trade has been matched and sent to the executor service by the operator. The executor service submits the trade as a transaction to the Exchange contract. -- `MINED`: Trade is observed to be mined into the chain, and no finality threshold is established. -- `CONFIRMED`: Trade has achieved strong probabilistic finality and was successful. -- `RETRYING`: Trade transaction has failed (revert or reorg) and is being retried/resubmitted by the operator. -- `FAILED`: Trade has failed and is not being retried. - -Once a trade is initially matched, subsequent trade status updates will be received via the WebSocket. -NautilusTrader records the initial trade details in the `info` field of the `OrderFilled` event, -with additional trade events stored in the cache as JSON under a custom key to retain this information. - -## Fees - -Polymarket charges **zero fees** on most markets. The exception is **15-minute crypto markets**: - -| Trade Type | Fee Deducted In | Rate Range | -|------------|-----------------|-------------| -| Buy | Tokens | 0.2% - 1.6% | -| Sell | USDC | 0.8% - 3.7% | - -Fees are rounded to 4 decimal places (0.0001 USDC minimum). Market makers receive daily rebates from collected taker fees. - -:::note -For the latest rates, see Polymarket's [Fees](https://docs.polymarket.com/polymarket-learn/trading/fees) and [Maker Rebates](https://docs.polymarket.com/polymarket-learn/trading/maker-rebates-program) documentation. -::: - -## Reconciliation - -The Polymarket API returns either all **active** (open) orders or specific orders when queried by the -Polymarket order ID (`venue_order_id`). The execution reconciliation procedure for Polymarket is as follows: - -- Generate order reports for all instruments with active (open) orders, as reported by Polymarket. -- Generate position reports from contract balances reported by Polymarket, for instruments available in the cache. -- Compare these reports with Nautilus execution state. -- Generate missing orders to bring Nautilus execution state in line with positions reported by Polymarket. - -**Note**: Polymarket does not directly provide data for orders which are no longer active. - -:::warning -An optional execution client configuration, `generate_order_history_from_trades`, is currently under development. -It is not recommended for production use at this time. -::: - -## WebSockets - -The `PolymarketWebSocketClient` is built on top of the high-performance Nautilus `WebSocketClient` base class, written in Rust. - -### Data - -The main data WebSocket handles all `market` channel subscriptions received during the initial -connection sequence, up to `ws_connection_delay_secs`. For any additional subscriptions, a new `PolymarketWebSocketClient` is -created for each new instrument (asset). - -### Execution - -The main execution WebSocket manages all `user` channel subscriptions based on the Polymarket instruments -available in the cache during the initial connection sequence. When trading commands are issued for additional instruments, -a separate `PolymarketWebSocketClient` is created for each new instrument (asset). - -:::note -Polymarket does not support unsubscribing from channel streams once subscribed. -::: - -### Subscription limits - -Polymarket enforces a **maximum of 500 instruments per WebSocket connection** (undocumented limitation). - -When you attempt to subscribe to 501 or more instruments on a single WebSocket connection: - -- You will **not** receive the initial order book snapshot for each instrument. -- You will only receive subsequent order book updates. - -NautilusTrader automatically manages WebSocket connections to handle this limitation: - -- The adapter defaults to **200 instrument subscriptions per connection** (configurable via `ws_max_subscriptions_per_connection`). -- When the subscription count exceeds this limit, additional WebSocket connections are created automatically. -- This ensures you receive complete order book data (including initial snapshots) for all subscribed instruments. - -:::tip -If you need to subscribe to a large number of instruments (e.g., 5000+), the adapter will automatically distribute these subscriptions across multiple WebSocket connections. You can tune the per-connection limit up to 500 via `ws_max_subscriptions_per_connection`. -::: - -## Rate limiting - -Polymarket enforces rate limits via Cloudflare throttling. When limits are exceeded, the API returns -HTTP 429 responses and requests are throttled rather than immediately rejected. - -### REST limits - -| Endpoint Type | Limit | Notes | -|----------------------|--------------------|---------------------------------------------| -| Public endpoints | 100 req/min per IP | Unauthenticated market data requests. | -| Authenticated reads | 300 req/min per key| Authenticated data queries. | -| Trading endpoints | 60 orders/min | Order placement and cancellation. | -| Order endpoint | 3,000 req/10 min | Rolling or fixed window (unconfirmed). | - -### WebSocket limits - -- **Subscriptions per connection**: 500 instruments maximum (adapter defaults to 200, configurable via `ws_max_subscriptions_per_connection`). -- **Connections**: 20 subscriptions per connection for some endpoints. - -:::warning -Exceeding Polymarket rate limits returns HTTP 429 and may result in temporary blocking. -::: - -### Data loader rate limiting - -The `PolymarketDataLoader` includes built-in rate limiting when using the default HTTP client. -Requests are automatically throttled to 100 requests per minute, matching Polymarket's -public endpoint limit. - -When fetching large date ranges across multiple markets: - -- Multiple loaders sharing the same `http_client` instance will coordinate rate limiting automatically. -- For higher throughput with authenticated requests, pass a custom `http_client` with adjusted quotas. -- The loader does not implement automatic retry on 429 errors, so implement backoff if needed. - -:::info -For the latest rate limit details, see the official Polymarket documentation: - -::: - -## Limitations and considerations - -The following limitations are currently known: - -- Order signing via the Polymarket Python client is slow, taking around one second. -- Post-only orders are not supported. -- Reduce-only orders are not supported. - -## Configuration - -### Data client configuration options - -| Option | Default | Description | -|------------------------------------|--------------|-------------| -| `venue` | `POLYMARKET` | Venue identifier registered for the data client. | -| `private_key` | `None` | Wallet private key; sourced from `POLYMARKET_PK` when omitted. | -| `signature_type` | `0` | Signature scheme (0 = EOA, 1 = email proxy, 2 = browser wallet proxy). | -| `funder` | `None` | USDC.e funding wallet; sourced from `POLYMARKET_FUNDER` when omitted. | -| `api_key` | `None` | API key; sourced from `POLYMARKET_API_KEY` when omitted. | -| `api_secret` | `None` | API secret; sourced from `POLYMARKET_API_SECRET` when omitted. | -| `passphrase` | `None` | API passphrase; sourced from `POLYMARKET_PASSPHRASE` when omitted. | -| `base_url_http` | `None` | Override for the REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `ws_connection_initial_delay_secs` | `5` | Delay (seconds) before the first WebSocket connection to buffer subscriptions. | -| `ws_connection_delay_secs` | `0.1` | Delay (seconds) between subsequent WebSocket connection attempts. | -| `update_instruments_interval_mins` | `60` | Interval (minutes) between instrument catalogue refreshes. | -| `compute_effective_deltas` | `False` | Compute effective order book deltas for bandwidth savings. | -| `drop_quotes_missing_side` | `True` | Drop quotes with missing bid/ask prices instead of substituting boundary values. | -| `instrument_config` | `None` | Optional `PolymarketInstrumentProviderConfig` for instrument loading. | -| `ws_max_subscriptions_per_connection` | `200` | Maximum instrument subscriptions per WebSocket connection (Polymarket limit is 500). | - -### Execution client configuration options - -| Option | Default | Description | -|--------------------------------------|--------------|-------------| -| `venue` | `POLYMARKET` | Venue identifier registered for the execution client. | -| `private_key` | `None` | Wallet private key; sourced from `POLYMARKET_PK` when omitted. | -| `signature_type` | `0` | Signature scheme (0 = EOA, 1 = email proxy, 2 = browser wallet proxy). | -| `funder` | `None` | USDC.e funding wallet; sourced from `POLYMARKET_FUNDER` when omitted. | -| `api_key` | `None` | API key; sourced from `POLYMARKET_API_KEY` when omitted. | -| `api_secret` | `None` | API secret; sourced from `POLYMARKET_API_SECRET` when omitted. | -| `passphrase` | `None` | API passphrase; sourced from `POLYMARKET_PASSPHRASE` when omitted. | -| `base_url_http` | `None` | Override for the REST base URL. | -| `base_url_ws` | `None` | Override for the WebSocket base URL. | -| `max_retries` | `None` | Maximum retry attempts for submit/cancel requests. | -| `retry_delay_initial_ms` | `None` | Initial delay (milliseconds) between retries. | -| `retry_delay_max_ms` | `None` | Maximum delay (milliseconds) between retries. | -| `ack_timeout_secs` | `5.0` | Timeout (seconds) to wait for order/trade acknowledgment from cache. | -| `generate_order_history_from_trades` | `False` | Generate synthetic order history from trade reports when `True` (experimental). | -| `log_raw_ws_messages` | `False` | Log raw WebSocket payloads at INFO level when `True`. | -| `instrument_config` | `None` | Optional `PolymarketInstrumentProviderConfig` for instrument loading. | -| `ws_max_subscriptions_per_connection` | `200` | Maximum instrument subscriptions per WebSocket connection (Polymarket limit is 500). | -| `use_data_api` | `False` | Use the Data API instead of CLOB API for fetching user positions (experimental). | - -### Instrument provider configuration options - -The instrument provider config is passed via the `instrument_config` parameter on the data client config. - -| Option | Default | Description | -|----------------------|---------|---------------------------------------------------------------------------------------------------| -| `load_all` | `False` | Load all venue instruments on start. Auto-set to `True` when `event_slug_builder` is provided. | -| `event_slug_builder` | `None` | Fully qualified path to a callable returning event slugs (e.g., `"mymodule:build_slugs"`). | - -#### Event slug builder - -The `event_slug_builder` feature enables efficient loading of niche markets without downloading -all 151k+ instruments. Instead of loading everything, you provide a function that returns -event slugs for the specific markets you need. - -```python -from nautilus_trader.adapters.polymarket.providers import PolymarketInstrumentProviderConfig - -# Configure with a slug builder function -instrument_config = PolymarketInstrumentProviderConfig( - event_slug_builder="myproject.slugs:build_temperature_slugs", -) -``` - -The callable must have signature `() -> list[str]` and return a list of event slugs: - -```python -# myproject/slugs.py -from datetime import UTC, datetime, timedelta - -def build_temperature_slugs() -> list[str]: - """Build slugs for NYC temperature markets.""" - slugs = [] - today = datetime.now(tz=UTC).date() - - for i in range(7): - date = today + timedelta(days=i) - slug = f"highest-temperature-in-nyc-on-{date.strftime('%B-%d').lower()}" - slugs.append(slug) - - return slugs -``` - -See `examples/live/polymarket/slug_builders.py` for more examples including crypto UpDown markets. - -## Historical data loading - -The `PolymarketDataLoader` provides methods for fetching and parsing historical market data -for research and backtesting purposes. The loader integrates with multiple Polymarket APIs to provide the required data. - -:::note -All data fetching methods are **asynchronous** and must be called with `await`. The loader can optionally accept an `http_client` parameter for dependency injection (useful for testing). -::: - -### Data sources - -The loader fetches data from two primary sources: - -1. **Polymarket Gamma API** - Market metadata, instrument details, and active market listings. -2. **Polymarket CLOB API** - Price/trade history timeseries and order book history snapshots. - -### Method naming conventions - -The loader provides two ways to access the Polymarket APIs: - -| Prefix | Type | Use case | -|-----------|------------------|------------------------------------------------------------------------| -| `query_*` | Static methods | API exploration without an instrument. No loader instance needed. | -| `fetch_*` | Instance methods | Data fetching with a configured loader. Uses the loader's HTTP client. | - -**Use `query_*` when** you want to explore markets, discover events, or fetch metadata -before committing to a specific instrument: - -```python -# No loader needed - just query the API directly -market = await PolymarketDataLoader.query_market_by_slug("some-market") -event = await PolymarketDataLoader.query_event_by_slug("some-event") -``` - -**Use `fetch_*` when** you have a loader instance and want to fetch data using its -configured HTTP client (for coordinated rate limiting across multiple calls): - -```python -loader = await PolymarketDataLoader.from_market_slug("some-market") - -# All fetch calls share the loader's HTTP client -markets = await loader.fetch_markets(active=True, limit=100) -events = await loader.fetch_events(active=True) -details = await loader.fetch_market_details(condition_id) -``` - -### Finding markets - -Use the provided utility scripts to discover active markets: - -```bash -# List all active markets -python nautilus_trader/adapters/polymarket/scripts/active_markets.py - -# List BTC and ETH UpDown markets specifically -python nautilus_trader/adapters/polymarket/scripts/list_updown_markets.py -``` - -### Basic usage - -The recommended way to create a loader is using the factory classmethods, which handle -all the API calls and instrument creation automatically: - -```python -import asyncio - -from nautilus_trader.adapters.polymarket import PolymarketDataLoader - -async def main(): - # Create loader from market slug (recommended) - loader = await PolymarketDataLoader.from_market_slug("gta-vi-released-before-june-2026") - - # Loader is ready to use with instrument and token_id set - print(loader.instrument) - print(loader.token_id) - -asyncio.run(main()) -``` - -For events with multiple markets (e.g., temperature buckets), use `from_event_slug`: - -```python -# Returns a list of loaders, one per market in the event -loaders = await PolymarketDataLoader.from_event_slug("highest-temperature-in-nyc-on-january-26") -``` - -### Discovering markets and events - -Use `fetch_markets()` and `fetch_events()` to discover available markets programmatically: - -```python -loader = await PolymarketDataLoader.from_market_slug("any-market") - -# List active markets -markets = await loader.fetch_markets(active=True, closed=False, limit=100) -for market in markets: - print(f"{market['slug']}: {market['question']}") - -# List active events -events = await loader.fetch_events(active=True, limit=50) -for event in events: - print(f"{event['slug']}: {event['title']}") - -# Get all markets within a specific event -event_markets = await loader.get_event_markets("highest-temperature-in-nyc-on-january-26") -``` - -For quick exploration without creating a loader, use the static `query_*` methods -(see [Method naming conventions](#method-naming-conventions) above). - -### Fetching trade history - -The `load_trades()` convenience method fetches and parses historical trades in one step: - -```python -import pandas as pd - -# Load all available trades -trades = await loader.load_trades() - -# Or filter by time range (client-side filtering) -end = pd.Timestamp.now(tz="UTC") -start = end - pd.Timedelta(hours=24) - -trades = await loader.load_trades( - start=start, - end=end, -) -``` - -Alternatively, you can fetch and parse separately using the lower-level methods: - -```python -condition_id = loader.condition_id - -# Fetch raw trades from the Polymarket Data API -raw_trades = await loader.fetch_trades(condition_id=condition_id) - -# Parse to NautilusTrader TradeTicks -trades = loader.parse_trades(raw_trades) -``` - -Trade data is sourced from the [Polymarket Data API](https://data-api.polymarket.com/trades), -which provides real execution data including price, size, side, and on-chain transaction hash. - -### Complete backtest example - -A complete working example is available at `examples/backtest/polymarket_simple_quoter.py`: - -```python -import asyncio -from decimal import Decimal - -from nautilus_trader.adapters.polymarket import POLYMARKET_VENUE -from nautilus_trader.adapters.polymarket import PolymarketDataLoader -from nautilus_trader.backtest.config import BacktestEngineConfig -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.examples.strategies.ema_cross_long_only import EMACrossLongOnly -from nautilus_trader.examples.strategies.ema_cross_long_only import EMACrossLongOnlyConfig -from nautilus_trader.model.currencies import USDC_POS -from nautilus_trader.model.data import BarType -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.model.identifiers import TraderId -from nautilus_trader.model.objects import Money - -async def run_backtest(): - # Initialize loader and fetch market data - loader = await PolymarketDataLoader.from_market_slug("gta-vi-released-before-june-2026") - instrument = loader.instrument - - # Load historical trades from the Polymarket Data API - trades = await loader.load_trades() - - # Configure and run backtest - config = BacktestEngineConfig(trader_id=TraderId("BACKTESTER-001")) - engine = BacktestEngine(config=config) - - engine.add_venue( - venue=POLYMARKET_VENUE, - oms_type=OmsType.NETTING, - account_type=AccountType.CASH, - base_currency=USDC_POS, - starting_balances=[Money(10_000, USDC_POS)], - ) - - engine.add_instrument(instrument) - engine.add_data(trades) - - bar_type = BarType.from_str(f"{instrument.id}-100-TICK-LAST-INTERNAL") - strategy_config = EMACrossLongOnlyConfig( - instrument_id=instrument.id, - bar_type=bar_type, - trade_size=Decimal("20"), - ) - - strategy = EMACrossLongOnly(config=strategy_config) - engine.add_strategy(strategy=strategy) - engine.run() - - # Display results - print(engine.trader.generate_account_report(POLYMARKET_VENUE)) - -# Run the backtest -asyncio.run(run_backtest()) -``` - -**Run the complete example**: - -```bash -python examples/backtest/polymarket_simple_quoter.py -``` - -### Helper functions - -The adapter provides utility functions for working with Polymarket identifiers: - -```python -from nautilus_trader.adapters.polymarket import get_polymarket_instrument_id - -# Create NautilusTrader InstrumentId from Polymarket identifiers -instrument_id = get_polymarket_instrument_id( - condition_id="0xcccb7e7613a087c132b69cbf3a02bece3fdcb824c1da54ae79acc8d4a562d902", - token_id="8441400852834915183759801017793514978104486628517653995211751018945988243154" -) -``` - -## Contributing - -:::info -For additional features or to contribute to the Polymarket adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/integrations/tardis.md b/nautilus-docs/docs/integrations/tardis.md deleted file mode 100644 index 0a1fc91..0000000 --- a/nautilus-docs/docs/integrations/tardis.md +++ /dev/null @@ -1,695 +0,0 @@ -# Tardis - -Tardis provides granular data for cryptocurrency markets including tick-by-tick order book snapshots & updates, -trades, open interest, funding rates, options chains and liquidations data for leading crypto exchanges. - -NautilusTrader provides an integration with the Tardis API and data formats, enabling access. -The capabilities of this adapter include: - -- `TardisCSVDataLoader`: Reads Tardis-format CSV files and converts them into Nautilus data, with support for both bulk loading and memory-efficient streaming. -- `TardisMachineClient`: Supports live streaming and historical replay of data from the Tardis Machine WebSocket server - converting messages into Nautilus data. -- `TardisHttpClient`: Requests instrument definition metadata from the Tardis HTTP API, parsing it into Nautilus instrument definitions. -- `TardisDataClient`: Provides a live data client for subscribing to data streams from a Tardis Machine WebSocket server. -- `TardisInstrumentProvider`: Provides instrument definitions from Tardis through the HTTP instrument metadata API. -- **Data pipeline functions**: Enables replay of historical data from Tardis Machine and writes it to the Nautilus Parquet format, including direct catalog integration for data management (see below). - -:::info -A Tardis API key is required for the adapter to operate correctly. See also [environment variables](#environment-variables). -::: - -## Overview - -This adapter is implemented in Rust, with optional Python bindings for ease of use in Python-based workflows. -It does not require any external Tardis client library dependencies. - -:::info -There is **no** need for additional installation steps for `tardis`. -The core components of the adapter are compiled as static libraries and automatically linked during the build process. -::: - -## Tardis documentation - -Tardis provides extensive user [documentation](https://docs.tardis.dev/). -We recommend also referring to the Tardis documentation in conjunction with this NautilusTrader integration guide. - -## Supported formats - -Tardis provides *normalized* market data, a unified format consistent across all supported exchanges. -This normalization is highly valuable because it allows a single parser to handle data from any [Tardis-supported exchange](#venues), reducing development time and complexity. -As a result, NautilusTrader will not support exchange-native market data formats, as it would be inefficient to implement separate parsers for each exchange at this stage. - -The following normalized Tardis formats are supported by NautilusTrader: - -| Tardis format | Nautilus data type | -|:----------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------| -| [book_change](https://docs.tardis.dev/api/tardis-machine#book_change) | `OrderBookDelta` | -| [book_snapshot_*](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) | `OrderBookDepth10` or `OrderBookDeltas` (see [book snapshot output](#book-snapshot-output)) | -| [quote](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) | `QuoteTick` | -| [quote_10s](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) | `QuoteTick` | -| [trade](https://docs.tardis.dev/api/tardis-machine#trade) | `Trade` | -| [trade_bar_*](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) | `Bar` | -| [instrument](https://docs.tardis.dev/api/instruments-metadata-api) | `CurrencyPair`, `CryptoFuture`, `CryptoPerpetual`, `OptionContract` | -| [derivative_ticker](https://docs.tardis.dev/api/tardis-machine#derivative_ticker) | `FundingRateUpdate` | -| [disconnect](https://docs.tardis.dev/api/tardis-machine#disconnect) | *Not applicable* | - -**Notes:** - -- [quote](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) is an alias for [book_snapshot_1_0ms](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit). -- [quote_10s](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit) is an alias for [book_snapshot_1_10s](https://docs.tardis.dev/api/tardis-machine#book_snapshot_-number_of_levels-_-snapshot_interval-time_unit). -- Both quote, quote\_10s, and one-level snapshots are parsed as `QuoteTick`. - -:::info -See also the Tardis [normalized market data APIs](https://docs.tardis.dev/api/tardis-machine#normalized-market-data-apis). -::: - -## Bars - -The adapter will automatically convert [Tardis trade bar interval and suffix](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) to Nautilus `BarType`s. -This includes the following: - -| Tardis suffix | Nautilus bar aggregation | -|:-------------------------------------------------------------------------------------------------------------|:----------------------------| -| [ms](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - milliseconds | `MILLISECOND` | -| [s](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - seconds | `SECOND` | -| [m](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - minutes | `MINUTE` | -| [ticks](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - number of ticks | `TICK` | -| [vol](https://docs.tardis.dev/api/tardis-machine#trade_bar_-aggregation_interval-suffix) - volume size | `VOLUME` | - -## Symbology and normalization - -The Tardis integration ensures compatibility with NautilusTrader’s crypto exchange adapters -by consistently normalizing symbols. Typically, NautilusTrader uses the native exchange naming conventions -provided by Tardis. However, for certain exchanges, raw symbols are adjusted to adhere to the Nautilus symbology normalization, as outlined below: - -### Common rules - -- All symbols are converted to uppercase. -- Market type suffixes are appended with a hyphen for some exchanges (see [exchange-specific normalizations](#exchange-specific-normalizations)). -- Original exchange symbols are preserved in the Nautilus instrument definitions `raw_symbol` field. - -### Exchange-specific normalizations - -- **Binance**: Nautilus appends the suffix `-PERP` to all perpetual symbols. -- **Bybit**: Nautilus uses specific product category suffixes, including `-SPOT`, `-LINEAR`, `-INVERSE`, `-OPTION`. -- **dYdX**: Nautilus appends the suffix `-PERP` to all perpetual symbols. -- **Gate.io**: Nautilus appends the suffix `-PERP` to all perpetual symbols. - -For detailed symbology documentation per exchange: - -- [Binance symbology](./binance.md#symbology) -- [Bybit symbology](./bybit.md#symbology) -- [dYdX symbology](./dydx.md#symbology) - -## Venues - -Some exchanges on Tardis are partitioned into multiple venues. -The table below outlines the mappings between Nautilus venues and corresponding Tardis exchanges, as well as the exchanges that Tardis supports: - -| Nautilus venue | Tardis exchange(s) | -|:------------------------|:------------------------------------------------------| -| `ASCENDEX` | `ascendex` | -| `BINANCE` | `binance`, `binance-dex`, `binance-european-options`, `binance-futures`, `binance-jersey`, `binance-options` | -| `BINANCE_DELIVERY` | `binance-delivery` (*COIN-margined contracts*) | -| `BINANCE_US` | `binance-us` | -| `BITFINEX` | `bitfinex`, `bitfinex-derivatives` | -| `BITFLYER` | `bitflyer` | -| `BITGET` | `bitget`, `bitget-futures` | -| `BITMEX` | `bitmex` | -| `BITNOMIAL` | `bitnomial` | -| `BITSTAMP` | `bitstamp` | -| `BLOCKCHAIN_COM` | `blockchain-com` | -| `BYBIT` | `bybit`, `bybit-options`, `bybit-spot` | -| `COINBASE` | `coinbase` | -| `COINBASE_INTX` | `coinbase-international` | -| `COINFLEX` | `coinflex` (*for historical research*) | -| `CRYPTO_COM` | `crypto-com`, `crypto-com-derivatives` | -| `CRYPTOFACILITIES` | `cryptofacilities` | -| `DELTA` | `delta` | -| `DERIBIT` | `deribit` | -| `DYDX` | `dydx` | -| `DYDX_V4` | `dydx-v4` | -| `FTX` | `ftx`, `ftx-us` (*historical research*) | -| `GATE_IO` | `gate-io`, `gate-io-futures` | -| `GEMINI` | `gemini` | -| `HITBTC` | `hitbtc` | -| `HUOBI` | `huobi`, `huobi-dm`, `huobi-dm-linear-swap`, `huobi-dm-options` | -| `HUOBI_DELIVERY` | `huobi-dm-swap` | -| `HYPERLIQUID` | `hyperliquid` | -| `KRAKEN` | `kraken` | -| `KUCOIN` | `kucoin`, `kucoin-futures` | -| `MANGO` | `mango` | -| `OKCOIN` | `okcoin` | -| `OKEX` | `okex`, `okex-futures`, `okex-options`, `okex-spreads`, `okex-swap` | -| `PHEMEX` | `phemex` | -| `POLONIEX` | `poloniex` | -| `SERUM` | `serum` (*historical research*) | -| `STAR_ATLAS` | `star-atlas` | -| `UPBIT` | `upbit` | -| `WOO_X` | `woo-x` | - -## Environment variables - -The following environment variables are used by Tardis and NautilusTrader. - -- `TM_API_KEY`: API key for the Tardis Machine. -- `TARDIS_API_KEY`: API key for NautilusTrader Tardis clients. -- `TARDIS_MACHINE_WS_URL` (optional): WebSocket URL for the `TardisMachineClient` in NautilusTrader. -- `TARDIS_BASE_URL` (optional): Base URL for the `TardisHttpClient` in NautilusTrader. -- `NAUTILUS_PATH` (optional): Parent directory containing the `catalog/` subdirectory for writing replay data in the Nautilus catalog format. - -## Running Tardis Machine historical replays - -The [Tardis Machine Server](https://docs.tardis.dev/api/tardis-machine) is a locally runnable server -with built-in data caching, providing both tick-level historical and consolidated real-time cryptocurrency market data through HTTP and WebSocket APIs. - -You can perform complete Tardis Machine WebSocket replays of historical data and output the results -in Nautilus Parquet format, using either Python or Rust. Since the function is implemented in Rust, -performance is consistent whether run from Python or Rust, letting you choose based on your preferred workflow. - -The end-to-end `run_tardis_machine_replay` data pipeline function uses a specified [configuration](#configuration) to execute the following steps: - -- Connect to the Tardis Machine server. -- Request and parse all necessary instrument definitions from the [Tardis instruments metadata](https://docs.tardis.dev/api/instruments-metadata-api) HTTP API. -- Stream all requested instruments and data types for the specified time ranges from the Tardis Machine server. -- For each instrument, data type and date (UTC), generate a `.parquet` file in the catalog-compatible format. -- Disconnect from the Tardis Machine server, and terminate the program. - -**File Naming Convention** - -Files are written one per day, per instrument, using ISO 8601 timestamp ranges that clearly indicate the exact time span of data: - -- **Format**: `{start_timestamp}_{end_timestamp}.parquet` -- **Example**: `2023-10-01T00-00-00-000000000Z_2023-10-01T23-59-59-999999999Z.parquet` -- **Structure**: `data/{data_type}/{instrument_id}/{filename}` - -This format is fully compatible with the Nautilus data catalog, enabling querying, consolidation, and data management operations. - -:::note -You can request data for the first day of each month without an API key. For all other dates, a Tardis Machine API key is required. -::: - -This process is optimized for direct output to a Nautilus Parquet data catalog. -Ensure that the `NAUTILUS_PATH` environment variable is set to the parent directory containing the `catalog/` subdirectory. -Parquet files will then be organized under `/catalog/data/` in the expected subdirectories corresponding to data type and instrument. - -If no `output_path` is specified in the configuration file and the `NAUTILUS_PATH` environment variable is unset, the system will default to the current working directory. - -### Procedure - -First, ensure the `tardis-machine` docker container is running. Use the following command: - -```bash -docker run -p 8000:8000 -p 8001:8001 -e "TM_API_KEY=YOUR_API_KEY" -d tardisdev/tardis-machine -``` - -This command starts the `tardis-machine` server without a persistent local cache, which may affect performance. -For improved performance, consider running the server with a persistent volume. Refer to the [Tardis Docker documentation](https://docs.tardis.dev/api/tardis-machine#docker) for details. - -### Configuration - -Next, ensure you have a configuration JSON file available. - -**Configuration JSON format** - -| Field | Type | Description | Default | -|:------------------------|:------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------| -| `tardis_ws_url` | string (optional) | The Tardis Machine WebSocket URL. | If `null` then will use the `TARDIS_MACHINE_WS_URL` env var. | -| `normalize_symbols` | bool (optional) | If Nautilus [symbol normalization](#symbology-and-normalization) should be applied. | If `null` then will default to `true`. | -| `output_path` | string (optional) | The output directory path to write Nautilus Parquet data to. | If `null` then will use the `NAUTILUS_PATH` env var, otherwise the current working directory. | -| `book_snapshot_output` | string (optional) | Output format for `book_snapshot_*` data: `"deltas"` or `"depth10"`. See [book snapshot output](#book-snapshot-output). | If `null` then will default to `"deltas"`. | -| `ws_proxy_url` | string (optional) | Optional WebSocket proxy URL. | If `null` then no proxy is used. | -| `options` | JSON[] | An array of [ReplayNormalizedRequestOptions](https://docs.tardis.dev/api/tardis-machine#replay-normalized-options) objects. | - -An example configuration file, `example_config.json`, is available [here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_config.json): - -```json -{ - "tardis_ws_url": "ws://localhost:8001", - "output_path": null, - "options": [ - { - "exchange": "bitmex", - "symbols": [ - "xbtusd", - "ethusd" - ], - "data_types": [ - "trade" - ], - "from": "2019-10-01", - "to": "2019-10-02" - } - ] -} -``` - -### Book snapshot output - -The `book_snapshot_output` configuration option controls how Tardis `book_snapshot_*` messages (e.g., `book_snapshot_5_100ms`, `book_snapshot_10_1s`) are converted and stored. - -| Value | Nautilus Type | Output Directory | Description | -|:----------|:--------------------|:----------------------|:------------------------------------------------------------| -| `deltas` | `OrderBookDeltas` | `order_book_deltas/` | Individual price level updates with snapshot flag set (default) | -| `depth10` | `OrderBookDepth10` | `order_book_depths/` | Periodic depth snapshots with up to 10 price levels | - -**When to use each format:** - -- **`deltas` (default)**: Best when you need to reconstruct the full order book state or when working with `book_change` data. Each price level becomes a separate delta record. -- **`depth10`**: Best for strategies that need periodic order book snapshots. More memory-efficient as each snapshot is a single record containing all levels. Snapshots with more than 10 levels will have only the first 10 preserved. - -**Avoiding file overwrites:** - -When downloading both `book_snapshot_*` and `book_change` data for the same instrument and date range, using `depth10` format ensures they are written to separate directories (`order_book_depths/` vs `order_book_deltas/`), preventing file overwrites. - -Example configuration with explicit format: - -```json -{ - "tardis_ws_url": "ws://localhost:8001", - "book_snapshot_output": "depth10", - "options": [ - { - "exchange": "binance-futures", - "symbols": ["btcusdt"], - "data_types": ["book_snapshot_5_100ms", "book_change"], - "from": "2024-01-01", - "to": "2024-01-02" - } - ] -} -``` - -### Python Replays - -To run a replay in Python, create a script similar to the following: - -```python -import asyncio - -from nautilus_trader.core import nautilus_pyo3 - - -async def run(): - config_filepath = Path("YOUR_CONFIG_FILEPATH") - await nautilus_pyo3.run_tardis_machine_replay(str(config_filepath.resolve())) - - -if __name__ == "__main__": - asyncio.run(run()) -``` - -### Rust Replays - -To run a replay in Rust, create a binary similar to the following: - -```rust -use std::path::PathBuf; - -use nautilus_adapters::tardis::replay::run_tardis_machine_replay_from_config; - -#[tokio::main] -async fn main() { - nautilus_common::logging::ensure_logging_initialized(); - - let config_filepath = PathBuf::from("YOUR_CONFIG_FILEPATH"); - run_tardis_machine_replay_from_config(&config_filepath).await; -} -``` - -Logging defaults to INFO level. To enable debug logging, export the following environment variable: - -```bash -export NAUTILUS_LOG=debug -``` - -A working example binary can be found [here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_replay.rs). - -This can also be run using cargo: - -```bash -cargo run --bin tardis-replay -``` - -## Loading Tardis CSV data - -Tardis-format CSV data can be loaded using either Python or Rust. The loader reads the CSV text data -from disk and parses it into Nautilus data. Since the loader is implemented in Rust, performance remains -consistent regardless of whether you run it from Python or Rust, allowing you to choose based on your preferred workflow. - -You can also optionally specify a `limit` parameter for the `load_*` functions/methods to control the maximum number of rows loaded. - -:::note -Loading mixed-instrument CSV files is challenging due to precision requirements and is not recommended. Use single-instrument CSV files instead (see below). -::: - -### Loading CSV Data in Python - -You can load Tardis-format CSV data in Python using the `TardisCSVDataLoader`. -When loading data, you can optionally specify the instrument ID but must specify both the price precision, and size precision. -Providing the instrument ID improves loading performance, while specifying the precisions is required, as they cannot be inferred from the text data alone. - -To load the data, create a script similar to the following: - -```python -from nautilus_trader.adapters.tardis import TardisCSVDataLoader -from nautilus_trader.model import InstrumentId - - -instrument_id = InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") -loader = TardisCSVDataLoader( - price_precision=1, - size_precision=0, - instrument_id=instrument_id, -) - -filepath = Path("YOUR_CSV_DATA_PATH") -limit = None - -deltas = loader.load_deltas(filepath, limit) -``` - -### Loading CSV Data in Rust - -You can load Tardis-format CSV data in Rust using the loading functions found [here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/src/csv/mod.rs). -When loading data, you can optionally specify the instrument ID but must specify both the price precision and size precision. -Providing the instrument ID improves loading performance, while specifying the precisions is required, as they cannot be inferred from the text data alone. - -For a complete example, see the [example binary here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_csv.rs). - -To load the data, you can use code similar to the following: - -```rust -use std::path::Path; - -use nautilus_adapters::tardis; -use nautilus_model::identifiers::InstrumentId; - -#[tokio::main] -async fn main() { - // You must specify precisions and the CSV filepath - let price_precision = 1; - let size_precision = 0; - let filepath = Path::new("YOUR_CSV_DATA_PATH"); - - // Optionally specify an instrument ID and/or limit - let instrument_id = InstrumentId::from("BTC-PERPETUAL.DERIBIT"); - let limit = None; - - // Consider propagating any parsing error depending on your workflow - let _deltas = tardis::csv::load_deltas( - filepath, - price_precision, - size_precision, - Some(instrument_id), - limit, - ) - .unwrap(); -} -``` - -## Streaming Tardis CSV Data - -For memory-efficient processing of large CSV files, the Tardis integration provides streaming capabilities that load and process data in configurable chunks rather than loading entire files into memory at once. This is particularly useful for processing multi-gigabyte CSV files without exhausting system memory. - -The streaming functionality is available for all supported Tardis data types: - -- Order book deltas (`stream_deltas`). -- Quote ticks (`stream_quotes`). -- Trade ticks (`stream_trades`). -- Order book depth snapshots (`stream_depth10`). - -### Streaming CSV Data in Python - -The `TardisCSVDataLoader` provides streaming methods that yield chunks of data as iterators. Each method accepts a `chunk_size` parameter that controls how many records are read from the CSV file per chunk: - -```python -from nautilus_trader.adapters.tardis import TardisCSVDataLoader -from nautilus_trader.model import InstrumentId - -instrument_id = InstrumentId.from_str("BTC-PERPETUAL.DERIBIT") -loader = TardisCSVDataLoader( - price_precision=1, - size_precision=0, - instrument_id=instrument_id, -) - -filepath = Path("large_trades_file.csv") -chunk_size = 100_000 # Process 100,000 records per chunk (default) - -# Stream trade ticks in chunks -for chunk in loader.stream_trades(filepath, chunk_size): - print(f"Processing chunk with {len(chunk)} trades") - # Process each chunk - only this chunk is in memory - for trade in chunk: - # Your processing logic here - pass -``` - -### Streaming Order Book Data - -For order book data, streaming is available for both deltas and depth snapshots: - -```python -# Stream order book deltas -for chunk in loader.stream_deltas(filepath): - print(f"Processing {len(chunk)} deltas") - # Process delta chunk - -# Stream depth10 snapshots (specify levels: 5 or 25) -for chunk in loader.stream_depth10(filepath, levels=5): - print(f"Processing {len(chunk)} depth snapshots") - # Process depth chunk -``` - -### Streaming Quote Data - -Quote data can be streamed similarly: - -```python -# Stream quote ticks -for chunk in loader.stream_quotes(filepath): - print(f"Processing {len(chunk)} quotes") - # Process quote chunk -``` - -### Memory Efficiency Benefits - -The streaming approach provides significant memory efficiency advantages: - -- **Controlled Memory Usage**: Only one chunk is loaded in memory at a time. -- **Scalable Processing**: Can process files larger than available RAM. -- **Configurable Chunk Sizes**: Tune `chunk_size` based on your system's memory and performance requirements (default 100,000). - -:::warning -When using streaming with precision inference (not providing explicit precisions), the inferred precision may differ from bulk loading the entire file. -This is because precision inference works within chunk boundaries, and different chunks may contain values with different precision requirements. -For deterministic precision behavior, provide explicit `price_precision` and `size_precision` parameters when calling streaming methods. -::: - -### Streaming CSV Data in Rust - -The underlying streaming functionality is implemented in Rust and can be used directly: - -```rust -use std::path::Path; -use nautilus_adapters::tardis::csv::{stream_trades, stream_deltas}; -use nautilus_model::identifiers::InstrumentId; - -#[tokio::main] -async fn main() { - let filepath = Path::new("large_trades_file.csv"); - let chunk_size = 100_000; - let price_precision = Some(1); - let size_precision = Some(0); - let instrument_id = Some(InstrumentId::from("BTC-PERPETUAL.DERIBIT")); - - // Stream trades in chunks - let stream = stream_trades( - filepath, - chunk_size, - price_precision, - size_precision, - instrument_id, - ).unwrap(); - - for chunk_result in stream { - match chunk_result { - Ok(chunk) => { - println!("Processing chunk with {} trades", chunk.len()); - // Process chunk - } - Err(e) => { - eprintln!("Error processing chunk: {}", e); - break; - } - } - } -} -``` - -## Requesting instrument definitions - -You can request instrument definitions in both Python and Rust using the `TardisHttpClient`. -This client interacts with the [Tardis instruments metadata API](https://docs.tardis.dev/api/instruments-metadata-api) to request and parse instrument metadata into Nautilus instruments. - -The `TardisHttpClient` constructor accepts optional parameters for `api_key`, `base_url`, and `timeout_secs` (default is 60 seconds). - -The client provides methods to retrieve either a specific `instrument`, or all `instruments` available on a particular exchange. -Ensure that you use Tardis’s lower-kebab casing convention when referring to a [Tardis-supported exchange](https://api.tardis.dev/v1/exchanges). - -:::note -A Tardis API key is required to access the instruments metadata API. -::: - -### Requesting Instruments in Python - -To request instrument definitions in Python, create a script similar to the following: - -```python -import asyncio - -from nautilus_trader.core import nautilus_pyo3 - - -async def run(): - http_client = nautilus_pyo3.TardisHttpClient() - - instrument = await http_client.instrument("bitmex", "xbtusd") - print(f"Received: {instrument}") - - instruments = await http_client.instruments("bitmex") - print(f"Received: {len(instruments)} instruments") - - -if __name__ == "__main__": - asyncio.run(run()) -``` - -### Requesting Instruments in Rust - -To request instrument definitions in Rust, use code similar to the following. -For a complete example, see the [example binary here](https://github.com/nautechsystems/nautilus_trader/blob/develop/crates/adapters/tardis/bin/example_http.rs). - -```rust -use nautilus_tardis::{ - enums::TardisExchange, - http::client::TardisHttpClient, -}; - -#[tokio::main] -async fn main() { - nautilus_common::logging::ensure_logging_initialized(); - - let client = TardisHttpClient::new(None, None, None, true).unwrap(); - - // Tardis instrument definitions - let resp = client - .instruments_info(TardisExchange::Bitmex, Some("XBTUSD"), None) - .await; - println!("Received: {resp:?}"); - - // Nautilus instrument definitions - let resp = client - .instruments(TardisExchange::Bitmex, Some("XBTUSD"), None, None, None, None, None, None) - .await; - println!("Received: {resp:?}"); -} -``` - -## Instrument provider - -The `TardisInstrumentProvider` requests and parses instrument definitions from Tardis through the HTTP instrument metadata API. -Since there are multiple [Tardis-supported exchanges](#venues), when loading all instruments, -you must filter for the desired venues using an `InstrumentProviderConfig`: - -```python -from nautilus_trader.config import InstrumentProviderConfig - -# See supported venues https://nautilustrader.io/docs/nightly/integrations/tardis#venues -venues = {"BINANCE", "BYBIT"} -filters = {"venues": frozenset(venues)} -instrument_provider_config = InstrumentProviderConfig(load_all=True, filters=filters) -``` - -You can also load specific instrument definitions in the usual way: - -```python -from nautilus_trader.config import InstrumentProviderConfig - -instrument_ids = [ - InstrumentId.from_str("BTCUSDT-PERP.BINANCE"), # Will use the 'binance-futures' exchange - InstrumentId.from_str("BTCUSDT.BINANCE"), # Will use the 'binance' exchange -] -instrument_provider_config = InstrumentProviderConfig(load_ids=instrument_ids) -``` - -### Option exchange filtering - -The instrument provider automatically filters out option-specific exchanges (such as `binance-options`, `binance-european-options`, `bybit-options`, `okex-options`, and `huobi-dm-options`) when the `instrument_type` filter is not provided or does not include `"option"`. - -To explicitly load option instruments, include `"option"` in the `instrument_type` filter: - -```python -from nautilus_trader.config import InstrumentProviderConfig - -venues = {"BINANCE", "BYBIT"} -filters = { - "venues": frozenset(venues), - "instrument_type": {"option"}, # Explicitly request options -} -instrument_provider_config = InstrumentProviderConfig(load_all=True, filters=filters) -``` - -This filtering mechanism prevents unnecessary API calls to option exchanges when they are not needed, improving performance and reducing API usage. - -:::note -Instruments must be available in the cache for all subscriptions. -For simplicity, it's recommended to load all instruments for the venues you intend to subscribe to. -::: - -## Live data client - -The `TardisDataClient` enables integration of a Tardis Machine with a running NautilusTrader system. -It supports subscriptions to the following data types: - -- `OrderBookDelta` (L2 granularity from Tardis, includes all changes or full-depth snapshots) -- `OrderBookDepth10` (L2 granularity from Tardis, provides snapshots up to 10 levels) -- `QuoteTick` -- `TradeTick` -- `Bar` (trade bars with [Tardis-supported bar aggregations](#bars)) -- `FundingRateUpdate` (from derivative_ticker messages) - -### Data WebSockets - -The main `TardisMachineClient` data WebSocket manages all stream subscriptions received during the initial connection phase, -up to the duration specified by `ws_connection_delay_secs`. For any additional subscriptions made -after this period, a new `TardisMachineClient` is created. This approach optimizes performance by -allowing the main WebSocket to handle potentially hundreds of subscriptions in a single stream if -they are provided at startup. - -When an initial subscription delay is set with `ws_connection_delay_secs`, unsubscribing from any -of these streams will not actually remove the subscription from the Tardis Machine stream, as selective -unsubscription is not supported by Tardis. However, the component will still unsubscribe from message -bus publishing as expected. - -All subscriptions made after any initial delay will behave normally, fully unsubscribing from the -Tardis Machine stream when requested. - -:::tip -If you anticipate frequent subscription and unsubscription of data, it is recommended to set -`ws_connection_delay_secs` to zero. This will create a new client for each initial subscription, -allowing them to be later closed individually upon unsubscription. -::: - -## Limitations and considerations - -The following limitations and considerations are currently known: - -- Historical data requests are not supported, as each would require a minimum one-day replay from the Tardis Machine, potentially with a filter. This approach is neither practical nor efficient. - -## Contributing - -:::info -For additional features or to contribute to the Tardis adapter, please see our -[contributing guide](https://github.com/nautechsystems/nautilus_trader/blob/develop/CONTRIBUTING.md). -::: diff --git a/nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md b/nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md deleted file mode 100644 index 9e276d0..0000000 --- a/nautilus-docs/docs/tutorials/ax_fx_mean_reversion.md +++ /dev/null @@ -1,279 +0,0 @@ -# AX Exchange - FX Perpetual Mean Reversion - -This tutorial walks through backtesting a **Bollinger Band mean reversion** strategy on -**EURUSD-PERP** (EUR/USD perpetual) using [AX Exchange](https://architect.exchange) instrument -definitions and [TrueFX](https://www.truefx.com) spot FX data as a proxy. - -## Introduction - -Mean reversion strategies assume that prices tend to return to a statistical average after -deviating from it. **Bollinger Bands** provide a volatility-adaptive envelope around a moving -average: the upper and lower bands expand in volatile markets and contract in quiet ones. -When price touches a band, it may be overextended relative to recent history. - -This strategy adds a **Relative Strength Index (RSI)** filter as confirmation. A touch of the -lower band alone is not sufficient to buy - RSI must also indicate oversold conditions. This -two-indicator approach reduces whipsaws in trending markets. - -For demonstration purposes, NautilusTrader ships with a `BBMeanReversion` example strategy -that is intentionally simple (no alpha advantage). - -### Why proxy data? - -AX Exchange is a new venue and is not yet covered by most historical data vendors. -[TrueFX](https://www.truefx.com) provides free institutional-grade spot FX tick data sourced -from Integral and Jefferies liquidity pools. EUR/USD spot data serves as a representative -proxy for backtesting an AX EURUSD-PERP strategy. - -## Prerequisites - -- **NautilusTrader** installed (see the [installation guide](../getting_started/installation.md)). -- **TrueFX account** (free): Sign up at [truefx.com](https://www.truefx.com) to access - historical tick data downloads. - -## Data preparation - -### Download TrueFX EUR/USD tick data - -1. Go to the [TrueFX historical downloads page](https://www.truefx.com/truefx-historical-downloads/). -2. Select **EUR/USD** and your desired month (e.g., December 2025). -3. Download and extract the CSV file (e.g., `EURUSD-2025-12.csv`). - -The raw TrueFX format has **no headers**. Columns are: `pair, timestamp, bid, ask`. - -### Load and prepare the data - -Use pandas to load the CSV and parse timestamps, then process through -`QuoteTickDataWrangler` which auto-renames `bid`/`ask` columns: - -```python -from pathlib import Path - -import pandas as pd - -from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler - -data_path = Path("EURUSD-2025-12.csv") - -df = pd.read_csv( - data_path, - header=None, - names=["pair", "timestamp", "bid", "ask"], -) -df["timestamp"] = pd.to_datetime(df["timestamp"], format="%Y%m%d %H:%M:%S.%f") -df = df.set_index("timestamp") -df = df[["bid", "ask"]] - -wrangler = QuoteTickDataWrangler(instrument=EURUSD_PERP) # defined below -ticks = wrangler.process(df) -``` - -The wrangler produces `QuoteTick` objects tagged with the instrument ID. These ticks drive -bar aggregation internally - 1-minute MID bars will be built from the quote tick stream. - -## Instrument definition - -Since we are using proxy data, we define the EURUSD-PERP instrument manually as a -`PerpetualContract`. The multiplier of 1000 means each contract represents 1000 EUR notional: - -```python -from decimal import Decimal - -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AssetClass -from nautilus_trader.model.identifiers import InstrumentId -from nautilus_trader.model.identifiers import Symbol -from nautilus_trader.model.instruments import PerpetualContract -from nautilus_trader.model.objects import Price -from nautilus_trader.model.objects import Quantity - -instrument_id = InstrumentId.from_str("EURUSD-PERP.AX") - -EURUSD_PERP = PerpetualContract( - instrument_id=instrument_id, - raw_symbol=Symbol("EURUSD-PERP"), - underlying="EUR", - asset_class=AssetClass.FX, - quote_currency=USD, - settlement_currency=USD, - is_inverse=False, - price_precision=5, - size_precision=0, - price_increment=Price.from_str("0.00001"), - size_increment=Quantity.from_int(1), - multiplier=Quantity.from_int(1000), - lot_size=Quantity.from_int(1), - margin_init=Decimal("0.05"), - margin_maint=Decimal("0.025"), - maker_fee=Decimal("0.0002"), - taker_fee=Decimal("0.0005"), - ts_event=0, - ts_init=0, -) -``` - -Fees are explicit backtest assumptions and should be set deliberately. Check the -[AX Exchange documentation](https://docs.architect.exchange/) for current rates. - -## Strategy overview - -The `BBMeanReversion` strategy works as follows: - -1. **Wait for warm-up**: Both indicators must be initialized before trading. -2. **Exit check (first)**: If long and close >= BB middle band → close position (mean - reversion target reached). If short and close <= BB middle band → close position. -3. **Entry signals**: If close <= BB lower band AND RSI < buy threshold → buy. If - close >= BB upper band AND RSI > sell threshold → sell. Existing positions in the - opposite direction are closed before entering. - -### Configuration - -| Parameter | Value | Description | -| -------------------- | ------ | ---------------------------------------------------- | -| `bb_period` | `20` | 20-bar lookback for Bollinger Bands. | -| `bb_std` | `2.0` | 2 standard deviations for band width. | -| `rsi_period` | `14` | 14-bar lookback for RSI. | -| `rsi_buy_threshold` | `0.30` | RSI below 0.30 confirms oversold (range 0-1). | -| `rsi_sell_threshold` | `0.70` | RSI above 0.70 confirms overbought (range 0-1). | -| `trade_size` | `1` | 1 contract per trade (1000 EUR notional). | - -:::tip -NautilusTrader RSI outputs values in the range [0.0, 1.0], not [0, 100]. Set thresholds -accordingly - 0.30 corresponds to the traditional RSI level of 30. -::: - -## Backtest setup - -### Configure the engine - -```python -from nautilus_trader.backtest.config import BacktestEngineConfig -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.config import LoggingConfig -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.model.identifiers import TraderId -from nautilus_trader.model.identifiers import Venue -from nautilus_trader.model.objects import Money - -config = BacktestEngineConfig( - trader_id=TraderId("BACKTESTER-001"), - logging=LoggingConfig(log_level="INFO"), -) - -engine = BacktestEngine(config=config) -``` - -### Add the venue - -AX Exchange uses margin accounts with netting position management: - -```python -AX = Venue("AX") - -engine.add_venue( - venue=AX, - oms_type=OmsType.NETTING, - account_type=AccountType.MARGIN, - base_currency=USD, - starting_balances=[Money(100_000, USD)], -) -``` - -### Add instrument, data, and strategy - -The bar type `EURUSD-PERP.AX-1-MINUTE-MID-INTERNAL` tells the engine to build 1-minute -bars from the mid-price of the quote tick stream: - -```python -from nautilus_trader.examples.strategies.bb_mean_reversion import BBMeanReversion -from nautilus_trader.examples.strategies.bb_mean_reversion import BBMeanReversionConfig -from nautilus_trader.model.data import BarType - -bar_type = BarType.from_str("EURUSD-PERP.AX-1-MINUTE-MID-INTERNAL") - -strategy_config = BBMeanReversionConfig( - instrument_id=instrument_id, - bar_type=bar_type, - trade_size=Decimal("1"), - bb_period=20, - bb_std=2.0, - rsi_period=14, - rsi_buy_threshold=0.30, - rsi_sell_threshold=0.70, -) - -strategy = BBMeanReversion(config=strategy_config) - -engine.add_instrument(EURUSD_PERP) -engine.add_data(ticks) -engine.add_strategy(strategy) -``` - -### Run the backtest - -```python -engine.run() -``` - -## Results - -After the run completes, generate reports to analyze performance: - -```python -import pandas as pd - -with pd.option_context( - "display.max_rows", 100, - "display.max_columns", None, - "display.width", 300, -): - print(engine.trader.generate_account_report(AX)) - print(engine.trader.generate_order_fills_report()) - print(engine.trader.generate_positions_report()) -``` - -Clean up when done: - -```python -engine.reset() -engine.dispose() -``` - -## Complete script - -The complete script is available as -[`architect_ax_mean_reversion.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_mean_reversion.py) -in the examples directory. - -## Next steps - -- **Tune parameters**: Experiment with `bb_period`, `bb_std`, and RSI thresholds to - understand their effect on trade frequency and PnL. -- **Try different pairs**: Download GBP/USD or USD/JPY data from TrueFX and define the - corresponding perpetual contract. -- **Add stop losses**: Extend the strategy with stop-loss orders to limit downside on - positions that move against you. -- **Go live on AX sandbox**: Once you are satisfied with backtest results, connect to the - AX sandbox environment for paper trading. See the - [AX Exchange integration guide](../integrations/architect_ax.md) for setup instructions. - -## Running live - -The same strategy used in this backtest can be run live with no code changes - only a -launch script is needed. NautilusTrader's architecture means your strategy is -venue-agnostic: switching from backtest to live is a configuration change, not a rewrite. - -See the complete live example: -[`ax_mean_reversion.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/architect_ax/ax_mean_reversion.py) - -For connection setup and API key configuration, refer to the -[AX Exchange integration guide](../integrations/architect_ax.md). - -## Further reading - -- [AX Exchange mean reversion backtest example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_mean_reversion.py) -- [`BBMeanReversion` strategy source](https://github.com/nautechsystems/nautilus_trader/tree/develop/nautilus_trader/examples/strategies/bb_mean_reversion.py) -- [AX Exchange gold book imbalance tutorial](ax_gold_book_imbalance.md) -- [Architect Exchange documentation](https://docs.architect.exchange/) diff --git a/nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md b/nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md deleted file mode 100644 index 015da31..0000000 --- a/nautilus-docs/docs/tutorials/ax_gold_book_imbalance.md +++ /dev/null @@ -1,306 +0,0 @@ -# AX Exchange - Gold Perpetual Book Imbalance - -This tutorial walks through backtesting an **order book imbalance** strategy on -**XAU-PERP** (gold perpetual) using [AX Exchange](https://architect.exchange) instrument -definitions and [Databento](https://databento.com) CME gold futures data as a proxy. - -## Introduction - -Order book imbalance is a canonical microstructure signal used in high-frequency and -short-term trading. When there is significantly more volume resting on one side of the -book than the other, this can signal informed flow and near-term price movement in that -direction. For a deeper dive into the statistical foundations, -see Databento's [blog post on HFT signals with sklearn](https://databento.com/blog/hft-sklearn-python) -which demonstrates the predictive power of book imbalance features. - -For demonstration purposes, NautilusTrader ships with an `OrderBookImbalance` example -strategy that is intentionally simple (no alpha advantage). -The strategy monitors the ratio of the smaller to larger side at the top of book, and when -this ratio drops below a configurable threshold it fires a fill-or-kill (FOK) limit order. Because -it only needs top-of-book data, it works with Databento `mbp-1` (market by price best bid/ask) quotes -rather than full depth-of-book, which is significantly cheaper to source. - -### Why proxy data? - -AX Exchange is a new venue and is not yet covered specifically by data vendors like Databento. -CME gold futures (GC) are the most liquid gold derivatives market globally, and -provide representative price action for backtesting gold strategies. We download CME GC -quote data from Databento and replay it through a NautilusTrader backtest with an AX-style -`PerpetualContract` instrument definition. - -## Prerequisites - -- **NautilusTrader** installed (see the [installation guide](../getting_started/installation.md)). -- **Databento API key**: Sign up at [databento.com](https://databento.com) and set the - environment variable: - -```bash -export DATABENTO_API_KEY="your-api-key" -``` - -- **Databento Python client**: Install with `pip install databento`. - -## Data preparation - -### Download CME gold futures quotes - -We use Databento's `mbp-1` schema (top-of-book best bid/ask), which maps directly to -NautilusTrader `QuoteTick` objects. This is simpler and cheaper than downloading full -depth-of-book data. - -We use a Databento **continuous contract** (`GC.v.0`) rather than a specific expiration like -`GCZ4`. Continuous contracts stitch together successive contracts based on a roll rule. -`v.0` tracks the highest-volume contract, which closely mirrors how a perpetual follows -liquidity. The `stype_in="continuous"` parameter tells Databento to resolve the symbol -through its continuous contract mapping. - -```python -import databento as db -from pathlib import Path - -data_path = Path("gc_gold_mbp1.dbn.zst") - -if not data_path.exists(): - client = db.Historical() - data = client.timeseries.get_range( - dataset="GLBX.MDP3", - symbols=["GC.v.0"], - stype_in="continuous", - schema="mbp-1", - start="2024-11-15", - end="2024-11-16", - ) - data.to_file(data_path) -``` - -This downloads one day of top-of-book data for the front-month gold contract. The -file is written once and reused on subsequent runs. The `instrument_id` override in the -loading step below is safe because the continuous contract resolves to a single instrument -at any point in time. - -### Load the data - -Use the `DatabentoDataLoader` to parse the DBN file into Nautilus quote ticks. -We pass `instrument_id` to override the Databento symbology with our AX instrument ID, -so all quote ticks appear to come from XAU-PERP.AX: - -```python -from nautilus_trader.adapters.databento import DatabentoDataLoader -from nautilus_trader.model.identifiers import InstrumentId - -instrument_id = InstrumentId.from_str("XAU-PERP.AX") - -loader = DatabentoDataLoader() -quotes = loader.from_dbn_file( - path=data_path, - instrument_id=instrument_id, -) -``` - -## Instrument definition - -Since we are using proxy data, we define the XAU-PERP instrument manually as a -`PerpetualContract`. The price precision and tick size are set to match the CME source -data, while margin and fee parameters reflect AX Exchange conditions: - -```python -from decimal import Decimal - -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AssetClass -from nautilus_trader.model.identifiers import Symbol -from nautilus_trader.model.instruments import PerpetualContract -from nautilus_trader.model.objects import Price -from nautilus_trader.model.objects import Quantity - -XAU_PERP = PerpetualContract( - instrument_id=instrument_id, - raw_symbol=Symbol("XAU-PERP"), - underlying="XAU", - asset_class=AssetClass.COMMODITY, - quote_currency=USD, - settlement_currency=USD, - is_inverse=False, - price_precision=2, - size_precision=0, - price_increment=Price.from_str("0.01"), - size_increment=Quantity.from_int(1), - multiplier=Quantity.from_int(1), - lot_size=Quantity.from_int(1), - margin_init=Decimal("0.08"), - margin_maint=Decimal("0.04"), - maker_fee=Decimal("0.0002"), - taker_fee=Decimal("0.0005"), - ts_event=0, - ts_init=0, -) -``` - -Fees are explicit backtest assumptions and should be set deliberately. Check the -[AX Exchange documentation](https://docs.architect.exchange/) for current rates. - -## Strategy overview - -The `OrderBookImbalance` strategy works as follows: - -1. **Monitor top of book**: On each quote tick update, compute the ratio of the - smaller side to the larger side (`smaller / larger`). -2. **Check trigger conditions**: If the larger side exceeds `trigger_min_size` and - the ratio falls below `trigger_imbalance_ratio`, a trigger fires. -3. **Determine direction**: If bid size > ask size, the strategy buys at the ask - (anticipating upward pressure). If ask size > bid size, it sells at the bid. -4. **Submit FOK order**: A fill-or-kill limit order is submitted at the opposing - best price, sized to the lesser of the opposing level size and `max_trade_size`. -5. **Cooldown**: A configurable minimum time between triggers prevents overtrading. - -### Configuration - -We use `use_quote_ticks=True` with `book_type="L1_MBP"` since our data is top-of-book -quotes rather than order book deltas: - -```python -from nautilus_trader.examples.strategies.orderbook_imbalance import OrderBookImbalance -from nautilus_trader.examples.strategies.orderbook_imbalance import OrderBookImbalanceConfig - -strategy_config = OrderBookImbalanceConfig( - instrument_id=instrument_id, - max_trade_size=Decimal("10"), - trigger_min_size=1.0, - trigger_imbalance_ratio=0.10, - min_seconds_between_triggers=5.0, - book_type="L1_MBP", - use_quote_ticks=True, -) - -strategy = OrderBookImbalance(config=strategy_config) -``` - -| Parameter | Value | Description | -| ------------------------------ | -------- | --------------------------------------------- | -| `max_trade_size` | `10` | Maximum 10 contracts per order. | -| `trigger_min_size` | `1.0` | Minimum 1 contract on the larger side. | -| `trigger_imbalance_ratio` | `0.10` | Trigger when ratio drops below 10%. | -| `min_seconds_between_triggers` | `5.0` | 5-second cooldown between consecutive trades. | -| `book_type` | `L1_MBP` | Top-of-book data only. | -| `use_quote_ticks` | `True` | Drive the strategy from quote ticks. | - -:::tip -Start with conservative parameters (higher ratio, longer cooldown) and tighten them -as you study the backtest results. A `trigger_imbalance_ratio` of 0.10 means the smaller -side must be less than 10% of the larger side to trigger. -::: - -## Backtest setup - -### Configure the engine - -```python -from nautilus_trader.backtest.config import BacktestEngineConfig -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.config import LoggingConfig -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.model.identifiers import TraderId -from nautilus_trader.model.identifiers import Venue -from nautilus_trader.model.objects import Money - -config = BacktestEngineConfig( - trader_id=TraderId("BACKTESTER-001"), - logging=LoggingConfig(log_level="INFO"), -) - -engine = BacktestEngine(config=config) -``` - -### Add the venue - -AX Exchange uses margin accounts with netting position management: - -```python -AX = Venue("AX") - -engine.add_venue( - venue=AX, - oms_type=OmsType.NETTING, - account_type=AccountType.MARGIN, - base_currency=USD, - starting_balances=[Money(100_000, USD)], -) -``` - -### Add instrument, data, and strategy - -```python -engine.add_instrument(XAU_PERP) -engine.add_data(quotes) -engine.add_strategy(strategy) -``` - -### Run the backtest - -```python -engine.run() -``` - -## Results - -After the run completes, generate reports to analyze performance: - -```python -import pandas as pd - -with pd.option_context( - "display.max_rows", 100, - "display.max_columns", None, - "display.width", 300, -): - print(engine.trader.generate_account_report(AX)) - print(engine.trader.generate_order_fills_report()) - print(engine.trader.generate_positions_report()) -``` - -Clean up when done: - -```python -engine.reset() -engine.dispose() -``` - -## Complete script - -The complete script is available as -[`architect_ax_book_imbalance.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_book_imbalance.py) -in the examples directory. - -## Next steps - -- **Tune parameters**: Experiment with `trigger_imbalance_ratio`, `min_seconds_between_triggers`, - and `max_trade_size` to understand their effect on fill rates and PnL. -- **Try different data**: Download different time periods or different front-month contracts - to see how the strategy performs across varying market conditions. -- **Go live on AX sandbox**: Once you are satisfied with backtest results, connect to the - AX sandbox environment for paper trading. See the - [AX Exchange integration guide](../integrations/architect_ax.md) for setup instructions. -- **Explore other instruments**: AX offers perpetuals on FX pairs (GBPUSD-PERP, EURUSD-PERP), - metals (XAG-PERP), and more. Adapt this tutorial by downloading the corresponding CME - futures data from Databento. - -## Running live - -The same strategy used in this backtest can be run live with no code changes - only a -launch script is needed. NautilusTrader's architecture means your strategy is -venue-agnostic: switching from backtest to live is a configuration change, not a rewrite. - -See the complete live example: -[`ax_book_imbalance.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/live/architect_ax/ax_book_imbalance.py) - -For connection setup and API key configuration, refer to the -[AX Exchange integration guide](../integrations/architect_ax.md). - -## Further reading - -- [AX Exchange book imbalance backtest example](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/architect_ax_book_imbalance.py) -- [`OrderBookImbalance` strategy source](https://github.com/nautechsystems/nautilus_trader/tree/develop/nautilus_trader/examples/strategies/orderbook_imbalance.py) -- [Architect Exchange documentation](https://docs.architect.exchange/) -- [Databento: HFT signals with sklearn](https://databento.com/blog/hft-sklearn-python) diff --git a/nautilus-docs/docs/tutorials/backtest_binance_orderbook.py b/nautilus-docs/docs/tutorials/backtest_binance_orderbook.py deleted file mode 100644 index f770a02..0000000 --- a/nautilus-docs/docs/tutorials/backtest_binance_orderbook.py +++ /dev/null @@ -1,190 +0,0 @@ -# %% [markdown] -# # Backtest: Binance OrderBook data -# -# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/backtest_binance_orderbook.py). - -# %% [markdown] -# ## Overview -# -# This tutorial sets up the data catalog and a `BacktestNode` to backtest an `OrderBookImbalance` strategy on order book data. You need to bring your own Binance order book data. - -# %% [markdown] -# ## Prerequisites -# -# - Python 3.12+ installed -# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) - -# %% [markdown] -# ## Imports -# -# We'll start with all of our imports for the remainder of this tutorial: - -# %% -import shutil -from decimal import Decimal -from pathlib import Path - -import pandas as pd - -from nautilus_trader.adapters.binance.loaders import BinanceOrderBookDeltaDataLoader -from nautilus_trader.backtest.node import BacktestDataConfig -from nautilus_trader.backtest.node import BacktestEngineConfig -from nautilus_trader.backtest.node import BacktestNode -from nautilus_trader.backtest.node import BacktestRunConfig -from nautilus_trader.backtest.node import BacktestVenueConfig -from nautilus_trader.config import ImportableStrategyConfig -from nautilus_trader.config import LoggingConfig -from nautilus_trader.core.datetime import dt_to_unix_nanos -from nautilus_trader.model import OrderBookDelta -from nautilus_trader.persistence.catalog import ParquetDataCatalog -from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# %% [markdown] -# ## Loading data - -# %% -# Path to your data directory, using user /Downloads as an example -DATA_DIR = "~/Downloads" - -# %% -data_path = Path(DATA_DIR).expanduser() / "Data" / "Binance" -raw_files = [f for f in data_path.iterdir() if f.is_file()] -assert raw_files, f"Unable to find any data files in directory {data_path}" -raw_files - -# %% -# First we'll load the initial order book snapshot -path_snap = data_path / "BTCUSDT_T_DEPTH_2022-11-01_depth_snap.csv" -df_snap = BinanceOrderBookDeltaDataLoader.load(path_snap) -df_snap.head() - -# %% -# Load the order book updates, limiting to 1 million rows -path_update = data_path / "BTCUSDT_T_DEPTH_2022-11-01_depth_update.csv" -nrows = 1_000_000 -df_update = BinanceOrderBookDeltaDataLoader.load(path_update, nrows=nrows) -df_update.head() - -# %% [markdown] -# ### Process deltas using a wrangler - -# %% -BTCUSDT_BINANCE = TestInstrumentProvider.btcusdt_binance() -wrangler = OrderBookDeltaDataWrangler(BTCUSDT_BINANCE) - -deltas = wrangler.process(df_snap) -deltas += wrangler.process(df_update) -deltas.sort(key=lambda x: x.ts_init) # Ensure data is non-decreasing by `ts_init` -deltas[:10] - -# %% [markdown] -# ### Set up data catalog - -# %% -CATALOG_PATH = Path.cwd() / "catalog" - -# Clear if it already exists, then create fresh -if CATALOG_PATH.exists(): - shutil.rmtree(CATALOG_PATH) -CATALOG_PATH.mkdir() - -catalog = ParquetDataCatalog(CATALOG_PATH) - -# %% -# Write instrument and ticks to catalog -catalog.write_data([BTCUSDT_BINANCE]) -catalog.write_data(deltas) - -# %% -# Confirm the instrument was written -catalog.instruments() - -# %% -# Explore the available data in the catalog -start = dt_to_unix_nanos(pd.Timestamp("2022-11-01", tz="UTC")) -end = dt_to_unix_nanos(pd.Timestamp("2022-11-04", tz="UTC")) - -deltas = catalog.order_book_deltas(start=start, end=end) -print(len(deltas)) -deltas[:10] - -# %% [markdown] -# ## Configure backtest - -# %% -instrument = catalog.instruments()[0] -book_type = "L2_MBP" # Data book type must match venue book type - -data_configs = [ - BacktestDataConfig( - catalog_path=CATALOG_PATH, - data_cls=OrderBookDelta, - instrument_id=instrument.id, - # start_time=start, # Run across all data - # end_time=end, # Run across all data - ), -] - -venues_configs = [ - BacktestVenueConfig( - name="BINANCE", - oms_type="NETTING", - account_type="CASH", - base_currency=None, - starting_balances=["20 BTC", "100000 USDT"], - book_type=book_type, # <-- Venues book type - ), -] - -strategies = [ - ImportableStrategyConfig( - strategy_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalance", - config_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalanceConfig", - config={ - "instrument_id": instrument.id, - "book_type": book_type, - "max_trade_size": Decimal("1.000"), - "min_seconds_between_triggers": 1.0, - }, - ), -] - -config = BacktestRunConfig( - engine=BacktestEngineConfig( - strategies=strategies, - logging=LoggingConfig(log_level="ERROR"), - ), - data=data_configs, - venues=venues_configs, -) - -config - -# %% [markdown] -# ## Run the backtest - -# %% -node = BacktestNode(configs=[config]) - -result = node.run() - -# %% -result - -# %% -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.model import Venue - - -engine: BacktestEngine = node.get_engine(config.id) - -engine.trader.generate_order_fills_report() - -# %% -engine.trader.generate_positions_report() - -# %% -engine.trader.generate_account_report(Venue("BINANCE")) diff --git a/nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py b/nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py deleted file mode 100644 index 6f70f75..0000000 --- a/nautilus-docs/docs/tutorials/backtest_bybit_orderbook.py +++ /dev/null @@ -1,184 +0,0 @@ -# %% [markdown] -# # Backtest: Bybit OrderBook data -# -# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/backtest_bybit_orderbook.py). - -# %% [markdown] -# ## Overview -# -# This tutorial sets up the data catalog and a `BacktestNode` to backtest an `OrderBookImbalance` strategy on order book data. This example requires order book depth data from Bybit. -# - -# %% [markdown] -# ## Prerequisites -# -# - Python 3.12+ installed -# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) - -# %% [markdown] -# ## Imports -# -# We'll start with all of our imports for the remainder of this tutorial: - -# %% -import shutil -from decimal import Decimal -from pathlib import Path - -import pandas as pd - -from nautilus_trader.adapters.bybit.loaders import BybitOrderBookDeltaDataLoader -from nautilus_trader.backtest.node import BacktestDataConfig -from nautilus_trader.backtest.node import BacktestEngineConfig -from nautilus_trader.backtest.node import BacktestNode -from nautilus_trader.backtest.node import BacktestRunConfig -from nautilus_trader.backtest.node import BacktestVenueConfig -from nautilus_trader.config import ImportableStrategyConfig -from nautilus_trader.config import LoggingConfig -from nautilus_trader.core.datetime import dt_to_unix_nanos -from nautilus_trader.model import OrderBookDelta -from nautilus_trader.persistence.catalog import ParquetDataCatalog -from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# %% [markdown] -# ## Loading data - -# %% -# Path to your data directory, using user /Downloads as an example -DATA_DIR = "~/Downloads" - -# %% -data_path = Path(DATA_DIR).expanduser() / "Data" / "Bybit" -raw_files = [f for f in data_path.iterdir() if f.is_file()] -assert raw_files, f"Unable to find any data files in directory {data_path}" -raw_files - -# %% -# We'll use orderbook depth 500 data provided by Bybit with limit of 1000000 rows -path_update = data_path / "2024-12-01_XRPUSDT_ob500.data.zip" -nrows = 1_000_000 -df_raw = BybitOrderBookDeltaDataLoader.load(path_update, nrows=nrows) -df_raw.head() - -# %% [markdown] -# ### Process deltas using a wrangler - -# %% -XRPUSDT_BYBIT = TestInstrumentProvider.xrpusdt_linear_bybit() -wrangler = OrderBookDeltaDataWrangler(XRPUSDT_BYBIT) - -deltas = wrangler.process(df_raw) -deltas.sort(key=lambda x: x.ts_init) # Ensure data is non-decreasing by `ts_init` -deltas[:10] - -# %% [markdown] -# ### Set up data catalog - -# %% -CATALOG_PATH = Path.cwd() / "catalog" - -# Clear if it already exists, then create fresh -if CATALOG_PATH.exists(): - shutil.rmtree(CATALOG_PATH) -CATALOG_PATH.mkdir() - -catalog = ParquetDataCatalog(CATALOG_PATH) - -# %% -# Write instrument and ticks to catalog -catalog.write_data([XRPUSDT_BYBIT]) -catalog.write_data(deltas) - -# %% -# Confirm the instrument was written -catalog.instruments() - -# %% -# Explore the available data in the catalog -start = dt_to_unix_nanos(pd.Timestamp("2022-11-01", tz="UTC")) -end = dt_to_unix_nanos(pd.Timestamp("2022-11-04", tz="UTC")) - -deltas = catalog.order_book_deltas(start=start, end=end) -print(len(deltas)) -deltas[:10] - -# %% [markdown] -# ## Configure backtest - -# %% -instrument = catalog.instruments()[0] -book_type = "L2_MBP" # Data book type must match venue book type - -data_configs = [ - BacktestDataConfig( - catalog_path=CATALOG_PATH, - data_cls=OrderBookDelta, - instrument_id=instrument.id, - # start_time=start, # Run across all data - # end_time=end, # Run across all data - ), -] - -venues_configs = [ - BacktestVenueConfig( - name="BYBIT", - oms_type="NETTING", - account_type="CASH", - base_currency=None, - starting_balances=["200000 XRP", "100000 USDT"], - book_type=book_type, # <-- Venues book type - ), -] - -strategies = [ - ImportableStrategyConfig( - strategy_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalance", - config_path="nautilus_trader.examples.strategies.orderbook_imbalance:OrderBookImbalanceConfig", - config={ - "instrument_id": instrument.id, - "book_type": book_type, - "max_trade_size": Decimal("1.000"), - "min_seconds_between_triggers": 1.0, - }, - ), -] - -config = BacktestRunConfig( - engine=BacktestEngineConfig( - strategies=strategies, - logging=LoggingConfig(log_level="ERROR"), - ), - data=data_configs, - venues=venues_configs, -) - -config - -# %% [markdown] -# ## Run the backtest - -# %% -node = BacktestNode(configs=[config]) - -result = node.run() - -# %% -result - -# %% -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.model import Venue - - -engine: BacktestEngine = node.get_engine(config.id) - -engine.trader.generate_order_fills_report() - -# %% -engine.trader.generate_positions_report() - -# %% -engine.trader.generate_account_report(Venue("BYBIT")) diff --git a/nautilus-docs/docs/tutorials/backtest_fx_bars.py b/nautilus-docs/docs/tutorials/backtest_fx_bars.py deleted file mode 100644 index 74f766f..0000000 --- a/nautilus-docs/docs/tutorials/backtest_fx_bars.py +++ /dev/null @@ -1,164 +0,0 @@ -# %% [markdown] -# # Backtest: FX bar data -# -# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/backtest_fx_bars.py). - -# %% [markdown] -# ## Overview -# -# This tutorial runs through how to set up a `BacktestEngine` (low-level API) for a single 'one-shot' backtest run using FX bar data. - -# %% [markdown] -# ## Prerequisites -# -# - Python 3.12+ installed -# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) - -# %% [markdown] -# ## Imports -# -# We'll start with all of our imports for the remainder of this tutorial. - -# %% -from decimal import Decimal - -from nautilus_trader.backtest.config import BacktestEngineConfig -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.backtest.models import FillModel -from nautilus_trader.backtest.modules import FXRolloverInterestConfig -from nautilus_trader.backtest.modules import FXRolloverInterestModule -from nautilus_trader.config import LoggingConfig -from nautilus_trader.config import RiskEngineConfig -from nautilus_trader.examples.strategies.ema_cross import EMACross -from nautilus_trader.examples.strategies.ema_cross import EMACrossConfig -from nautilus_trader.model import BarType -from nautilus_trader.model import Money -from nautilus_trader.model import Venue -from nautilus_trader.model.currencies import JPY -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler -from nautilus_trader.test_kit.providers import TestDataProvider -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# %% [markdown] -# ## Set up backtest engine - -# %% -# Initialize a backtest configuration -config = BacktestEngineConfig( - trader_id="BACKTESTER-001", - logging=LoggingConfig(log_level="ERROR"), - risk_engine=RiskEngineConfig( - bypass=True, # Example of bypassing pre-trade risk checks for backtests - ), -) - -# Build backtest engine -engine = BacktestEngine(config=config) - -# %% [markdown] -# ## Add simulation module -# -# We can optionally plug in a module to simulate rollover interest. The data is available from pre-packaged test data. - -# %% -provider = TestDataProvider() -interest_rate_data = provider.read_csv("short-term-interest.csv") -config = FXRolloverInterestConfig(interest_rate_data) -fx_rollover_interest = FXRolloverInterestModule(config=config) - -# %% [markdown] -# ## Add fill model - -# %% [markdown] -# For this backtest we'll use a simple probabilistic fill model. - -# %% -fill_model = FillModel( - prob_fill_on_limit=0.2, - prob_slippage=0.5, - random_seed=42, -) - -# %% [markdown] -# ## Add venue - -# %% [markdown] -# For this backtest we need a single trading venue which will be a simulated FX ECN. - -# %% -SIM = Venue("SIM") -engine.add_venue( - venue=SIM, - oms_type=OmsType.HEDGING, # Venue will generate position IDs - account_type=AccountType.MARGIN, - base_currency=None, # Multi-currency account - starting_balances=[Money(1_000_000, USD), Money(10_000_000, JPY)], - fill_model=fill_model, - modules=[fx_rollover_interest], -) - -# %% [markdown] -# ## Add instruments and data - -# %% [markdown] -# Now we can add instruments and data. For this backtest we'll pre-process bid and ask side bar data into quotes using a `QuoteTickDataWrangler`. - -# %% -# Add instruments -USDJPY_SIM = TestInstrumentProvider.default_fx_ccy("USD/JPY", SIM) -engine.add_instrument(USDJPY_SIM) - -# Add data -wrangler = QuoteTickDataWrangler(instrument=USDJPY_SIM) -ticks = wrangler.process_bar_data( - bid_data=provider.read_csv_bars("fxcm/usdjpy-m1-bid-2013.csv"), - ask_data=provider.read_csv_bars("fxcm/usdjpy-m1-ask-2013.csv"), -) -engine.add_data(ticks) - -# %% [markdown] -# ## Configure strategy - -# %% [markdown] -# Next we'll configure and initialize a simple `EMACross` strategy we'll use for the backtest. - -# %% -# Configure your strategy -config = EMACrossConfig( - instrument_id=USDJPY_SIM.id, - bar_type=BarType.from_str("USD/JPY.SIM-5-MINUTE-BID-INTERNAL"), - fast_ema_period=10, - slow_ema_period=20, - trade_size=Decimal(1_000_000), -) - -# Instantiate and add your strategy -strategy = EMACross(config=config) -engine.add_strategy(strategy=strategy) - -# %% [markdown] -# ## Run backtest -# -# Run the backtest. The engine logs a post-analysis report when it finishes processing all data. - -# %% -engine.run() - -# %% [markdown] -# ## Generating reports -# -# Produce reports for post-run analysis. - -# %% -engine.trader.generate_account_report(SIM) - -# %% -engine.trader.generate_order_fills_report() - -# %% -engine.trader.generate_positions_report() diff --git a/nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md b/nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md deleted file mode 100644 index 8fa4e35..0000000 --- a/nautilus-docs/docs/tutorials/bitmex_grid_market_maker.md +++ /dev/null @@ -1,541 +0,0 @@ -# BitMEX - Grid Market Making with Deadman's Switch - -This tutorial walks through backtesting a grid market making strategy on BitMEX using free -historical quote tick data from [Tardis.dev](https://tardis.dev), then running it live -using the Rust-native `LiveNode`. The key differentiator covered here is BitMEX's -**deadman's switch**, a server-side safety mechanism that automatically cancels all open -orders if your client loses connectivity. - -## Introduction - -### Why BitMEX for grid market making? - -BitMEX is one of the deepest and most liquid Bitcoin derivatives venues, with a live order -book going back to 2014. The `XBTUSD` inverse perpetual swap is among the most-traded -instruments in crypto derivatives. Its thick order book and predictable spread behaviour -make it a natural venue for grid market making. - -Two features make BitMEX particularly well-suited for automated market making: - -1. **Deadman's switch** (`cancelAllAfter`): BitMEX maintains a server-side countdown timer. - Your client refreshes it periodically. If the connection drops and the timer expires, - BitMEX cancels all open orders on your behalf, protecting you from stranded quotes. - -2. **Submit/cancel broadcaster**: The adapter can fan out order submissions and - cancellations across multiple independent HTTP connections simultaneously, with the first - successful response short-circuiting the rest. This provides redundancy against transient - network failures. - -### Deadman's switch mechanics - -When `deadmans_switch_timeout_secs` is set in the execution client config, a background -task runs continuously: - -``` -timeout = 60s → refresh interval = timeout / 4 = 15s - - t=0s Strategy starts, cancelAllAfter(60000ms) sent - t=15s Refresh: cancelAllAfter(60000ms) sent (resets timer) - t=30s Refresh: cancelAllAfter(60000ms) sent - t=45s Refresh: cancelAllAfter(60000ms) sent - ↓ - Connectivity lost at t=50s (last refresh was at t=45s) - ↓ - t=105s Server timer fires → BitMEX cancels all open orders -``` - -For market making specifically, stranded quotes are a serious risk: if your software crashes -while holding grid orders around the mid-price, price can move against those orders before -they are cancelled. The deadman's switch caps the window of exposure at `timeout` seconds, -regardless of why connectivity was lost. - -## Prerequisites - -- **NautilusTrader** installed (see the [installation guide](../getting_started/installation.md)). -- **Rust toolchain** (`cargo`) for the live example. Install from [rustup.rs](https://rustup.rs/). -- **BitMEX account**: sign up at [bitmex.com](https://www.bitmex.com/) and generate an API key - with order management permissions. For testing, use the - [BitMEX testnet](https://testnet.bitmex.com/). - -### Environment variables - -```bash -# Mainnet -export BITMEX_API_KEY="your-api-key" -export BITMEX_API_SECRET="your-api-secret" - -# Testnet -export BITMEX_TESTNET_API_KEY="your-testnet-api-key" -export BITMEX_TESTNET_API_SECRET="your-testnet-api-secret" -``` - -Alternatively, place these in a `.env` file in the project root (loaded automatically via `dotenvy`). - -## Backtesting with Tardis free quote data - -BitMEX does not offer historical market data via its own API beyond recent trade history. -[Tardis.dev](https://tardis.dev) captures and archives tick-level BitMEX data from March 2019 -onward in its native WebSocket format. The **first day of each month is freely downloadable** -without an API key (enough for a representative backtest run). - -The grid market maker subscribes to best-bid/ask quotes, so the `quotes` dataset is the -right source: it records every change to the top of book. - -### Download the data - -```bash -curl -LO https://datasets.tardis.dev/v1/bitmex/quotes/2024/01/01/XBTUSD.csv.gz -``` - -This downloads January 1 2024 XBTUSD quote data. No API key required. - -:::tip -Full historical data (all dates) requires a paid Tardis API key. Use the -[Tardis download utility](https://docs.tardis.dev/downloadable-csv-files) for bulk fetches. -::: - -### Load the data - -`TardisCSVDataLoader` parses the `.csv.gz` file directly (no decompression needed) and -returns a list of `QuoteTick` objects: - -```python -from nautilus_trader.adapters.tardis.loaders import TardisCSVDataLoader -from nautilus_trader.model.identifiers import InstrumentId - -instrument_id = InstrumentId.from_str("XBTUSD.BITMEX") - -loader = TardisCSVDataLoader(instrument_id=instrument_id) -quotes = loader.load_quotes("XBTUSD.csv.gz") -``` - -The `instrument_id` override ensures every tick is tagged `XBTUSD.BITMEX` regardless of -what appears in the CSV. - -### Instrument definition - -Since we are loading the data directly (not through the live BitMEX adapter), we define the -`XBTUSD` instrument manually. XBTUSD is an **inverse perpetual**: prices are quoted in USD -but the contract is margined and settled in BTC. One contract equals 1 USD of notional -exposure. - -```python -from decimal import Decimal - -from nautilus_trader.model.currencies import BTC -from nautilus_trader.model.currencies import USD -from nautilus_trader.model.enums import AssetClass -from nautilus_trader.model.identifiers import Symbol -from nautilus_trader.model.instruments import PerpetualContract -from nautilus_trader.model.objects import Price -from nautilus_trader.model.objects import Quantity - -XBTUSD = PerpetualContract( - instrument_id=instrument_id, - raw_symbol=Symbol("XBTUSD"), - underlying="XBT", - asset_class=AssetClass.CRYPTOCURRENCY, - base_currency=BTC, - quote_currency=USD, - settlement_currency=BTC, - is_inverse=True, - price_precision=1, # $0.5 tick → one decimal place - size_precision=0, # integer contracts - price_increment=Price.from_str("0.5"), - size_increment=Quantity.from_int(1), - multiplier=Quantity.from_int(1), # 1 USD per contract - lot_size=Quantity.from_int(1), - margin_init=Decimal("0.01"), # 1% initial margin = 100x max leverage - margin_maint=Decimal("0.005"), - maker_fee=Decimal("-0.00025"), # maker rebate - taker_fee=Decimal("0.00075"), - ts_event=0, - ts_init=0, -) -``` - -Fee rates are explicit backtest assumptions. Check -[bitmex.com/app/fees](https://www.bitmex.com/app/fees) for current rates. - -### Backtest engine setup - -XBTUSD is BTC-margined, so the starting balance is in BTC: - -```python -from nautilus_trader.backtest.config import BacktestEngineConfig -from nautilus_trader.backtest.engine import BacktestEngine -from nautilus_trader.config import LoggingConfig -from nautilus_trader.model.enums import AccountType -from nautilus_trader.model.enums import OmsType -from nautilus_trader.model.identifiers import TraderId -from nautilus_trader.model.identifiers import Venue -from nautilus_trader.model.objects import Money - -config = BacktestEngineConfig( - trader_id=TraderId("BACKTESTER-001"), - logging=LoggingConfig(log_level="INFO"), -) -engine = BacktestEngine(config=config) - -BITMEX = Venue("BITMEX") -engine.add_venue( - venue=BITMEX, - oms_type=OmsType.NETTING, - account_type=AccountType.MARGIN, - base_currency=BTC, - starting_balances=[Money(1, BTC)], -) - -engine.add_instrument(XBTUSD) -engine.add_data(quotes) -``` - -### Strategy configuration - -```python -from nautilus_trader.examples.strategies.grid_market_maker import GridMarketMaker -from nautilus_trader.examples.strategies.grid_market_maker import GridMarketMakerConfig - -strategy_config = GridMarketMakerConfig( - instrument_id=instrument_id, - max_position=Quantity.from_int(300), # 300 USD contracts max exposure - trade_size=Quantity.from_int(100), # 100 USD contracts per level - num_levels=3, - grid_step_bps=100, # 1% between levels - skew_factor=0.5, - requote_threshold_bps=10, -) -strategy = GridMarketMaker(config=strategy_config) -engine.add_strategy(strategy) -``` - -### Run and review results - -```python -import pandas as pd - -engine.run() - -with pd.option_context("display.max_rows", 100, "display.max_columns", None, "display.width", 300): - print(engine.trader.generate_account_report(BITMEX)) - print(engine.trader.generate_order_fills_report()) - print(engine.trader.generate_positions_report()) - -engine.reset() -engine.dispose() -``` - -The complete script is available as -[`bitmex_grid_market_maker.py`](https://github.com/nautechsystems/nautilus_trader/tree/develop/examples/backtest/bitmex_grid_market_maker.py) -in the examples directory. - -## Live trading: GridMarketMaker with deadman's switch - -Once you have validated data loading and strategy mechanics in the backtest, the same -configuration runs live using the Rust `LiveNode`. The `GridMarketMaker` strategy is -implemented natively in Rust for maximum throughput. - -### Environment setup - -Credentials are loaded automatically from environment variables when not set explicitly in -the config: - -```bash -# Testnet (recommended for initial setup) -export BITMEX_TESTNET_API_KEY="your-key" -export BITMEX_TESTNET_API_SECRET="your-secret" -``` - -```ini -# Or use a .env file at the project root -BITMEX_TESTNET_API_KEY=your-key -BITMEX_TESTNET_API_SECRET=your-secret -``` - -### Code walkthrough - -The complete `main()` function from -[`node_grid_mm.rs`](https://github.com/nautechsystems/nautilus_trader/tree/develop/crates/adapters/bitmex/examples/node_grid_mm.rs): - -```rust -#[tokio::main] -async fn main() -> Result<(), Box> { - dotenvy::dotenv().ok(); // Load .env file if present - - let use_testnet = true; // Set false for mainnet - - let environment = Environment::Live; - let trader_id = TraderId::from("TESTER-001"); - let instrument_id = InstrumentId::from("XBTUSD.BITMEX"); - - // Minimal data client: just selects testnet or mainnet endpoints - let data_config = BitmexDataClientConfig { - use_testnet, - ..Default::default() - }; - - // Execution client with deadman's switch enabled - let exec_config = BitmexExecFactoryConfig::new( - trader_id, - BitmexExecClientConfig { - use_testnet, - deadmans_switch_timeout_secs: Some(60), // Cancels all orders after 60s without refresh - ..Default::default() - }, - ); - - let data_factory = BitmexDataClientFactory::new(); - let exec_factory = BitmexExecutionClientFactory::new(); - - let log_config = LoggerConfig { - stdout_level: LevelFilter::Info, - ..Default::default() - }; - - // Builder wires up logging, data and execution clients, and node options - let mut node = LiveNode::builder(trader_id, environment)? - .with_logging(log_config) - .add_data_client(None, Box::new(data_factory), Box::new(data_config))? - .add_exec_client(None, Box::new(exec_factory), Box::new(exec_config))? - .with_reconciliation(true) // Resume state across restarts - .with_reconciliation_lookback_mins(2880) // Look back 2 days (2880 min) - .with_delay_post_stop_secs(5) // Grace period for pending cancel/close events - .build()?; - - // Grid configuration: XBTUSD, 300 USD max position, 3 levels at 100 bps each - let config = GridMarketMakerConfig::new(instrument_id, Quantity::from("300")) - .with_num_levels(3) - .with_grid_step_bps(100) // 1% between levels - .with_skew_factor(0.5) // shift grid 0.5 price units per unit of inventory - .with_requote_threshold_bps(10); // requote when mid moves more than 0.1% - let strategy = GridMarketMaker::new(config); - - node.add_strategy(strategy)?; - node.run().await?; - - Ok(()) -} -``` - -Configuration points: - -- **`deadmans_switch_timeout_secs: Some(60)`**: enables the deadman's switch with a 60-second - timeout. The background task refreshes every 15 seconds (`timeout / 4`). -- **`with_reconciliation(true)`**: reconciles open orders and positions on startup by - querying the BitMEX REST API, allowing the strategy to resume correctly after a restart. -- **`with_reconciliation_lookback_mins(2880)`**: looks back 2 days when reconciling order history. -- **`with_delay_post_stop_secs(5)`**: allows 5 seconds after strategy stop for pending - cancel/fill events to arrive before the node exits. - -### Deadman's switch in context - -During normal operation the deadman's switch is invisible: the background task silently -refreshes the server-side timer. Its value becomes apparent in failure scenarios: - -``` -Normal operation: - ┌─────────────────────────────────────────────────────────┐ - │ Strategy running │ - │ t=0s cancelAllAfter(60_000ms) ──────────────► BitMEX │ - │ t=15s cancelAllAfter(60_000ms) ──────────────► BitMEX │ - │ t=30s cancelAllAfter(60_000ms) ──────────────► BitMEX │ - └─────────────────────────────────────────────────────────┘ - -Connectivity loss: - ┌──────────────────────────────────────────────────────────┐ - │ t=40s Network failure, no more refreshes sent │ - │ t=100s BitMEX timer fires → all open orders cancelled │ - │ (60s after the last successful refresh at t=40s) │ - └──────────────────────────────────────────────────────────┘ -``` - -Unlike dYdX where short-term order expiry provides a similar automatic cleanup, BitMEX -uses GTC orders (no expiry). Without the deadman's switch, a crashed client could leave -grid orders resting in the book indefinitely. - -### BitMEX-specific considerations - -#### GTC orders and post-only - -BitMEX grid orders use `GTC` (Good-Till-Cancelled) time-in-force combined with -`ParticipateDoNotInitiate` (post-only). Post-only ensures every order enters the book as -a maker order; if a grid price has moved through the book by the time it reaches the -matching engine, the order is rejected rather than filling as a taker. - -This differs from the dYdX setup where short-term orders provide automatic expiry every -~8 seconds. On BitMEX, the requote cycle is driven entirely by mid-price movement -(`requote_threshold_bps`) rather than order expiry. - -#### Order quantization - -All price and size quantization for BitMEX instruments is handled automatically by the -adapter. No manual rounding or conversion is needed in strategy code. - -#### Inverse perpetual accounting - -Because XBTUSD is inverse (BTC-margined), PnL is in BTC. A grid that captures a $1 spread -on a $42,000 BTC price earns approximately 1/42,000 BTC per fill. Account for this when -sizing `max_position` and `trade_size`. - -### Run the example - -```bash -cargo run --example bitmex-grid-mm --package nautilus-bitmex -``` - -### Graceful shutdown - -Press **Ctrl+C** to stop the node. The shutdown sequence: - -1. SIGINT received, trader stops, `on_stop()` fires. -2. Strategy cancels all orders and closes positions. -3. 5-second grace period (`delay_post_stop_secs`) processes residual events. -4. Deadman's switch background task stops. -5. Clients disconnect, node exits. - -## Configuration - -### GridMarketMaker parameters - -| Parameter | Type | Default | Description | -| ----------------------- | -------------- | ---------- | ------------------------------------------------------------------------ | -| `instrument_id` | `InstrumentId` | *required* | Instrument to trade (e.g., `XBTUSD.BITMEX`). | -| `max_position` | `Quantity` | *required* | Maximum net exposure in contracts (long or short). | -| `trade_size` | `Quantity` | `None` | Size per grid level. If `None`, uses instrument's `min_quantity` or 1.0. | -| `num_levels` | `usize` | `3` | Number of buy and sell levels. | -| `grid_step_bps` | `u32` | `10` | Grid spacing in basis points (100 = 1%). | -| `skew_factor` | `f64` | `0.0` | How aggressively to shift the grid based on net inventory. | -| `requote_threshold_bps` | `u32` | `5` | Minimum mid-price move (bps) before re-quoting. | -| `expire_time_secs` | `Option` | `None` | Order expiry in seconds. Use `None` for GTC on BitMEX. | -| `on_cancel_resubmit` | `bool` | `false` | Resubmit grid on next quote after an unexpected cancel. | - -### Deadman's switch parameter - -| Parameter | Type | Description | -| ----------------------------- | ----------------- | ------------------------------------------------------------------------------------------------- | -| `deadmans_switch_timeout_secs`| `Option` | Server-side cancel timer in seconds. Refresh interval = `timeout / 4` (minimum 1s). `None` disables the feature. | - -**Recommended value**: `60`. This gives a 15-second refresh interval and a 60-second window -before BitMEX fires the timer. Lower values reduce the exposure window but increase API -call frequency; higher values reduce overhead but extend the window. - -### Choosing grid parameters - -**`grid_step_bps`**: XBTUSD has tight spreads. Start wider (50–100 bps) to ensure fills -before tightening. Each level captures half the step as spread (buy fills $1 below mid, -sells $1 above on a 200 bps total spread). - -**`skew_factor`**: Start at `0.0` (no skew). A value of `0.5` shifts the grid by 0.5 price -units per unit of net position. For XBTUSD, this is 0.5 USD per contract; with max_position -of 300, full skew is ±150 price units. - -**`requote_threshold_bps`**: 10 bps (0.1%) is a reasonable starting point for XBTUSD. -Too low causes excessive cancel/replace churn; too high leaves orders stale during fast moves. - -## Event flow - -``` -LiveNode starts - │ - ├── connect() → REST: load instruments; WebSocket: subscribe channels - │ - ├── deadman's switch task starts - │ └── cancelAllAfter(timeout_ms) sent every timeout/4 seconds - │ - ├── on_start() - │ └── subscribe_quotes(XBTUSD.BITMEX) - │ - ├── on_quote() [repeated] - │ ├── Calculate mid-price - │ ├── Check should_requote(): skip if within threshold - │ ├── cancel_all_orders(): record IDs in pending_self_cancels - │ ├── Compute grid with inventory skew - │ └── Submit GTC post-only limit orders - │ - ├── on_order_filled() - │ └── Remove from pending_self_cancels; position/skew update - │ - ├── on_order_canceled() - │ ├── Self-cancel? → no action - │ └── Unexpected cancel? → reset last_quoted_mid (triggers requote) - │ - └── on_stop() - ├── cancel_all_orders() - ├── close_all_positions() - ├── unsubscribe_quotes() - └── deadman's switch task stops -``` - -## Monitoring and understanding output - -### Key log messages - -| Log message | Meaning | -| ----------------------------------------------------------------------- | --------------------------------------------------------- | -| `Requoting grid: mid=X, last_mid=Y` | Mid-price moved beyond threshold, refreshing grid. | -| `Starting dead man's switch: timeout=60s, refresh_interval=15s` | Deadman's switch armed at node start. | -| `Dead man's switch heartbeat failed: ...` | Transient network issue; switch will retry next interval. | -| `Disarming dead man's switch` | Switch stopped cleanly during shutdown. | -| `benign cancel error, treating as success` | Cancel for an already-filled or cancelled order (normal). | -| `Reconciling orders from last 2880 minutes` | Startup reconciliation loading prior state. | - -### Expected behaviour patterns - -1. **Startup**: Instruments load, reconciliation queries prior orders, WebSocket connects, - first quote triggers initial grid. -2. **Steady state**: Grid persists across ticks; requotes only when mid moves beyond threshold. -3. **Fills**: Position updates, skew adjusts on next requote. -4. **Shutdown**: All orders cancelled, positions closed, deadman's switch stops. -5. **Restart**: Reconciliation restores open order state; strategy resumes from prior grid. - -## Customization tips - -### High vs low volatility - -| Condition | Adjustment | -| --------------- | -------------------------------------------------------------------------- | -| High volatility | Wider `grid_step_bps` (100–200), fewer `num_levels`, lower `skew_factor`. | -| Low volatility | Tighter `grid_step_bps` (20–50), more `num_levels`, higher `skew_factor`. | -| Thin liquidity | Increase `requote_threshold_bps` to reduce cancel frequency. | - -### Enabling the submit broadcaster - -For production deployments, enable the submit broadcaster to provide redundant order -submission across multiple HTTP connections: - -```rust -let exec_config = BitmexExecFactoryConfig::new( - trader_id, - BitmexExecClientConfig { - use_testnet: false, - deadmans_switch_timeout_secs: Some(60), - submitter_pool_size: Some(2), // two parallel submission paths - canceller_pool_size: Some(2), // two parallel cancel paths - ..Default::default() - }, -); -``` - -With `submitter_pool_size=2`, each order submission fans out to two HTTP clients in -parallel; the first successful response wins. This reduces the probability of a missed -submission due to a transient network failure on a single path. - -### Mainnet toggle - -Change a single flag to switch networks: - -```rust -let use_testnet = false; // true for testnet -``` - -All endpoints and credential environment variables are resolved automatically. - -## Further reading - -- [BitMEX integration guide](../integrations/bitmex.md): full adapter reference. -- [dYdX grid market maker tutorial](./dydx_grid_market_maker.md): comparison with - short-term order expiry as an alternative to the deadman's switch. -- [Tardis downloadable CSV files](https://docs.tardis.dev/downloadable-csv-files): full - schema documentation for `incremental_book_L2` and other data types. -- [BitMEX API documentation](https://www.bitmex.com/app/apiOverview): `cancelAllAfter` - endpoint and order management reference. diff --git a/nautilus-docs/docs/tutorials/databento_data_catalog.py b/nautilus-docs/docs/tutorials/databento_data_catalog.py deleted file mode 100644 index 157bf1e..0000000 --- a/nautilus-docs/docs/tutorials/databento_data_catalog.py +++ /dev/null @@ -1,221 +0,0 @@ -# %% [markdown] -# # Databento data catalog -# -# Tutorial for [NautilusTrader](https://nautilustrader.io/docs/latest/) a high-performance algorithmic trading platform and event-driven backtester. -# -# [View source on GitHub](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/tutorials/databento_data_catalog.py). - -# %% [markdown] -# ## Overview -# -# This tutorial walks through setting up a Nautilus Parquet data catalog with various Databento schemas. - -# %% [markdown] -# ## Prerequisites -# -# - Python 3.12+ installed -# - [NautilusTrader](https://pypi.org/project/nautilus_trader/) latest release installed (`uv pip install nautilus_trader`) -# - [databento](https://pypi.org/project/databento/) Python client library installed to make data requests (`uv pip install databento`) -# - [Databento](https://databento.com) account - -# %% [markdown] -# ## Requesting data - -# %% [markdown] -# We'll use a Databento historical client for the rest of this tutorial. You can either initialize one by passing your Databento API key to the constructor, or implicitly use the `DATABENTO_API_KEY` environment variable (as shown). - -# %% -import databento as db - - -client = db.Historical() # Uses the DATABENTO_API_KEY environment variable - -# %% [markdown] -# **Every historical streaming request from `timeseries.get_range` incurs a cost (even for the same data), so**: -# - Check the cost before making a request -# - Avoid requesting the same data twice -# - Write responses to disk as zstd compressed DBN files - -# %% [markdown] -# Use the metadata [get_cost endpoint](https://databento.com/docs/api-reference-historical/metadata/metadata-get-cost?historical=python&live=python) to quote the cost before each request. Only request data that does not already exist on disk. -# -# The response is in USD, displayed as fractional cents. - -# %% [markdown] -# The following request is for a small amount of data (as used in this Medium article [Building high-frequency trading signals in Python with Databento and sklearn](https://databento.com/blog/hft-sklearn-python)) to demonstrate the workflow. - -# %% -from pathlib import Path - -from databento import DBNStore - -# %% [markdown] -# We'll prepare a directory for the raw Databento DBN format data, which we'll use for the rest of the tutorial. - -# %% -DATABENTO_DATA_DIR = Path("databento") -DATABENTO_DATA_DIR.mkdir(exist_ok=True) - -# %% -# Request cost quote (USD) - this endpoint is 'free' -client.metadata.get_cost( - dataset="GLBX.MDP3", - symbols=["ES.n.0"], - stype_in="continuous", - schema="mbp-10", - start="2023-12-06T14:30:00", - end="2023-12-06T20:30:00", -) - -# %% [markdown] -# Use the historical API to request the data used in the Medium article. - -# %% -path = DATABENTO_DATA_DIR / "es-front-glbx-mbp10.dbn.zst" - -if not path.exists(): - # Request data - client.timeseries.get_range( - dataset="GLBX.MDP3", - symbols=["ES.n.0"], - stype_in="continuous", - schema="mbp-10", - start="2023-12-06T14:30:00", - end="2023-12-06T20:30:00", - path=path, # <-- Passing a `path` writes the data to disk - ) - -# %% [markdown] -# Read the data from disk and convert to a pandas.DataFrame - -# %% -data = DBNStore.from_file(path) - -df = data.to_df() -df - -# %% [markdown] -# ## Write to data catalog - -# %% -import shutil -from pathlib import Path - -from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader -from nautilus_trader.model import InstrumentId -from nautilus_trader.persistence.catalog import ParquetDataCatalog - -# %% -CATALOG_PATH = Path.cwd() / "catalog" - -# Clear if it already exists -if CATALOG_PATH.exists(): - shutil.rmtree(CATALOG_PATH) -CATALOG_PATH.mkdir() - -# Create a catalog instance -catalog = ParquetDataCatalog(CATALOG_PATH) - -# %% [markdown] -# Use a `DatabentoDataLoader` to decode and load the data into Nautilus objects. - -# %% -loader = DatabentoDataLoader() - -# %% [markdown] -# Load Rust pyo3 objects by setting `as_legacy_cython=False` (more efficient than legacy Cython objects). -# -# Passing an `instrument_id` is optional but speeds up loading by skipping symbology mapping. If provided, use the Nautilus `symbol.venue` format (e.g., "ES.GLBX"). - -# %% -path = DATABENTO_DATA_DIR / "es-front-glbx-mbp10.dbn.zst" - -# Option 1 (recommended): Let the loader infer the instrument ID from DBN metadata -depth10 = loader.from_dbn_file( - path=path, - as_legacy_cython=False, -) - -# Option 2: Explicitly specify a valid Nautilus instrument ID (symbol.venue format) -# instrument_id = InstrumentId.from_str("ESZ3.GLBX") # E-mini S&P December 2023 futures on Globex -# depth10 = loader.from_dbn_file( -# path=path, -# instrument_id=instrument_id, -# as_legacy_cython=False, -# ) - -# %% -# Write data to catalog (this takes ~20 seconds or ~250,000/second for writing MBP-10 at the moment) -catalog.write_data(depth10) - -# %% -# Test reading from catalog -depths = catalog.order_book_depth10() -len(depths) - -# %% [markdown] -# ## Preparing a month of AAPL trades - -# %% [markdown] -# Now we'll expand on this workflow by preparing a month of AAPL trades on the Nasdaq exchange using the Databento `trade` schema, which will translate to Nautilus `TradeTick` objects. - -# %% -# Request cost quote (USD) - this endpoint is 'free' -client.metadata.get_cost( - dataset="XNAS.ITCH", - symbols=["AAPL"], - schema="trades", - start="2024-01", -) - -# %% [markdown] -# Pass a `path` parameter when requesting historical data to write it to disk. - -# %% -path = DATABENTO_DATA_DIR / "aapl-xnas-202401.trades.dbn.zst" - -if not path.exists(): - # Request data - client.timeseries.get_range( - dataset="XNAS.ITCH", - symbols=["AAPL"], - schema="trades", - start="2024-01", - path=path, # <-- Passing a `path` parameter - ) - -# %% [markdown] -# Read the data from disk and convert to a pandas.DataFrame - -# %% -data = DBNStore.from_file(path) - -df = data.to_df() -df - -# %% [markdown] -# We'll use an `InstrumentId` of `"AAPL.XNAS"`, where XNAS is the ISO 10383 MIC (Market Identifier Code) for the Nasdaq venue. -# -# Passing an `instrument_id` speeds up loading by skipping symbology mapping. Setting `as_legacy_cython=False` is more efficient when writing to the catalog. - -# %% -instrument_id = InstrumentId.from_str("AAPL.XNAS") - -trades = loader.from_dbn_file( - path=path, - instrument_id=instrument_id, - as_legacy_cython=False, -) - -# %% [markdown] -# Here we organize data as one file per month. A file per day works equally well. - -# %% -# Write data to catalog -catalog.write_data(trades) - -# %% -trades = catalog.trade_ticks([instrument_id]) - -# %% -len(trades) diff --git a/nautilus-docs/docs/tutorials/dydx_grid_market_maker.md b/nautilus-docs/docs/tutorials/dydx_grid_market_maker.md deleted file mode 100644 index 2b1e6c0..0000000 --- a/nautilus-docs/docs/tutorials/dydx_grid_market_maker.md +++ /dev/null @@ -1,597 +0,0 @@ -# dYdX - Grid Market Making - -This tutorial walks through running a grid market making strategy on dYdX v4 using the -Rust-native `LiveNode`. By the end, you will have a working grid quoter that places symmetric -limit orders around the mid-price, skews the grid to manage inventory, and continuously -requotes as short-term orders cycle through expiry. - -## Introduction - -### What is grid market making? - -A grid market maker maintains a ladder of resting buy and sell limit orders at fixed price -intervals around the current mid-price. When an order fills, the strategy profits from the -spread between buy and sell levels. The approach is conceptually simple but requires careful -inventory management to avoid accumulating a large directional position. - -### Inventory skewing (Avellaneda-Stoikov inspired) - -The strategy implements inventory-based skewing: when the position grows long, the entire grid -shifts downward (cheaper buys, cheaper sells) to encourage selling. When the position grows -short, the grid shifts upward. This is inspired by the Avellaneda-Stoikov framework for -optimal market making, adapted to a discrete grid. - -### Why dYdX v4? - -dYdX v4 is well-suited for market making strategies because: - -- **Short-term orders** (~20s expiry) provide low-latency placement without on-chain storage. -- **~0.5s block times** give fast confirmation cycles. -- **No gas fees for cancellations**: short-term order cancels are free (GTB replay protection). -- **On-chain order book**: deterministic matching within each block. -- **Batch cancel**: cancel all short-term orders in a single `MsgBatchCancel` call. - -## Prerequisites - -### Funded dYdX account - -You need a dYdX account with USDC collateral. See the -[Testnet setup](../integrations/dydx.md#testnet-setup) section in the integration guide -for instructions on creating and funding a testnet account. - -### Environment variables - -```bash -# For mainnet -export DYDX_PRIVATE_KEY="0x..." -export DYDX_WALLET_ADDRESS="dydx1..." - -# For testnet -export DYDX_TESTNET_PRIVATE_KEY="0x..." -export DYDX_TESTNET_WALLET_ADDRESS="dydx1..." -``` - -## Strategy overview - -### Geometric grid pricing - -The strategy uses a **geometric grid** where each level is a fixed percentage (basis points) -away from the mid-price: - -``` -Buy level N: mid × (1 - bps/10000)^N - skew -Sell level N: mid × (1 + bps/10000)^N - skew -``` - -Where `skew = skew_factor × net_position`. - -For a 3-level grid with `grid_step_bps=100` (1%) around a mid of 1000.00: - -``` - Sell 3: 1030.30 - Sell 2: 1020.10 - Sell 1: 1010.00 - ─── Mid: 1000.00 ─── - Buy 1: 990.00 - Buy 2: 980.10 - Buy 3: 970.30 -``` - -With inventory skew (long 2 units, `skew_factor=1.0`), the entire grid shifts down by 2.0: - -``` - Sell 3: 1028.30 - Sell 2: 1018.10 - Sell 1: 1008.00 - ─── Mid: 1000.00 ─── - Buy 1: 988.00 - Buy 2: 978.10 - Buy 3: 968.30 -``` - -### Inventory management - -The strategy enforces position limits through two mechanisms: - -1. **`max_position`**: Hard cap on net exposure (long or short). When the projected exposure - from adding another grid level would exceed this cap, that level is skipped. -2. **Projected exposure tracking**: Before placing each level, the strategy tracks the - worst-case per-side exposure (current position + all pending orders) to prevent over-committing. - -Because `cancel_all_orders()` is asynchronous, pending orders may still fill between the cancel -request and acknowledgement. The strategy accounts for this by tracking worst-case per-side -exposure (current position + all pending buy/sell orders) before placing new grid levels. This -prevents momentary over-exposure during cancel-requote transitions. - -### Requote threshold - -The `requote_threshold_bps` parameter controls how much the mid-price must move (in basis points) -before the strategy cancels all existing orders and places a fresh grid. This creates a -trade-off: - -- **Lower threshold** (e.g., 5 bps): More responsive to price moves, but generates more - cancel/place transactions. -- **Higher threshold** (e.g., 50 bps): Fewer transactions, but orders may sit further from - the current price. - -## Configuration - -| Parameter | Type | Default | Description | -| ----------------------- | -------------- | ---------- | ------------------------------------------------------------------------ | -| `instrument_id` | `InstrumentId` | *required* | Instrument to trade (e.g., `ETH-USD-PERP.DYDX`). | -| `max_position` | `Quantity` | *required* | Maximum net exposure (long or short). | -| `trade_size` | `Quantity` | `None` | Size per grid level. If `None`, uses instrument's `min_quantity` or 1.0. | -| `num_levels` | `usize` | `3` | Number of buy and sell levels. | -| `grid_step_bps` | `u32` | `10` | Grid spacing in basis points (10 = 0.1%). | -| `skew_factor` | `f64` | `0.0` | How aggressively to shift the grid based on inventory. | -| `requote_threshold_bps` | `u32` | `5` | Minimum mid-price move in bps before re-quoting. | -| `expire_time_secs` | `Option` | `None` | Order expiry in seconds. Uses GTD when set, GTC otherwise. | -| `on_cancel_resubmit` | `bool` | `false` | Resubmit grid on next quote after an unexpected cancel. | - -### Choosing parameters - -**`grid_step_bps`**: Start wider (50-100 bps) in volatile markets, tighter (5-20 bps) in calm -conditions. Wider grids capture more spread per fill but fill less frequently. - -**`skew_factor`**: Start at `0.0` (no skew) and increase gradually. A value of `0.5` means -each unit of position shifts the grid by 0.5 price units. Too aggressive a skew can cause -the grid to move entirely above or below the mid-price. - -**`expire_time_secs`**: For dYdX short-term orders, set to `8` seconds. This fits within -the 40-block (~20s) short-term window, giving the orders time to rest while keeping them -in the fast short-term path. When `None`, orders use GTC (long-term path). - -**`on_cancel_resubmit`**: Resubmits the grid on the next quote tick after any unexpected -cancel (e.g. self-trade prevention, risk limits). Note that short-term order expiry is -silent and does not generate cancel events, so the grid refreshes naturally via continuous -requoting, not through this flag. - -## dYdX-specific considerations - -### Short-term order expiry - -When `expire_time_secs=8`, orders are classified as short-term by the adapter: - -1. The adapter checks: `8 seconds < max_short_term_secs (40 blocks × ~0.5s = ~20s)`. -2. Since it fits, the order is submitted as short-term with `GoodTilBlock = current_height + N`. -3. The order expires silently after ~8 seconds if not filled. - -This is the recommended configuration for market making because: - -- Short-term orders have lower latency. -- No gas fees for expiry (GTB replay protection handles it). -- Continuous requoting naturally replaces expired orders. - -See the [Order classification](../integrations/dydx.md#order-classification) section -in the integration guide for full details. - -### Unexpected cancels and `on_cancel_resubmit` - -The `pending_self_cancels` set distinguishes between self-initiated and unexpected cancels: - -1. When the strategy calls `cancel_all_orders()`, it records all open order IDs in - `pending_self_cancels`. -2. When `on_order_canceled` fires: - - If the order ID is in `pending_self_cancels`, it's a self-cancel and no action is needed. - - If not, it's an unexpected cancel (e.g. self-trade prevention or risk limits). - Reset `last_quoted_mid` to trigger a full grid resubmission on the next quote. - -This prevents the strategy from re-quoting unnecessarily during its own cancel waves while -still responding to unexpected cancels. - -`on_order_filled` also removes the order from `pending_self_cancels`. If an order fills -before the cancel acknowledgement arrives, this prevents stale entries from accumulating -in the set. - -### Order quantization - -All price and size quantization for dYdX markets is handled automatically by the adapter's -`OrderMessageBuilder`. No manual rounding or conversion is needed. See -[Price and size quantization](../integrations/dydx.md#price-and-size-quantization) for details. - -### Post-only orders - -All grid orders are submitted with `post_only=true`. This ensures every order enters the -book as a maker order (never crosses the spread). If a grid price has moved through the -book by the time it reaches the matching engine, the order is rejected rather than filling -as a taker. This guarantees maker fee rates and prevents unintended crossing during -requote transitions. - -## Running and stopping - -### Environment setup - -Credentials can be set via environment variables or a `.env` file in the project root -(loaded automatically via `dotenvy`): - -```bash -# Export directly -export DYDX_PRIVATE_KEY="0x..." -export DYDX_WALLET_ADDRESS="dydx1..." -``` - -```bash -# Or use a .env file (alternative to shell exports) -DYDX_PRIVATE_KEY=0x... -DYDX_WALLET_ADDRESS=dydx1... -``` - -### Run the example - -```bash -cargo run --example dydx-grid-mm --package nautilus-dydx -``` - -### Graceful shutdown - -Press **Ctrl+C** to stop the node. The shutdown sequence: - -1. SIGINT received, trader stops, `on_stop()` fires. -2. Strategy cancels all orders and closes positions. -3. 5-second grace period (`delay_post_stop_secs`) processes residual events. -4. Clients disconnect, node exits. - -## Code walkthrough - -### Node setup - -The complete `main()` function from the example (`node_grid_mm.rs`): - -```rust -#[tokio::main] -async fn main() -> Result<(), Box> { - dotenvy::dotenv().ok(); // Load .env file if present - - // Configuration - let is_testnet = false; - let network = if is_testnet { - DydxNetwork::Testnet - } else { - DydxNetwork::Mainnet - }; - - let environment = Environment::Live; - let trader_id = TraderId::from("TESTER-001"); - let account_id = AccountId::from("DYDX-001"); - let node_name = "DYDX-GRID-MM-001".to_string(); - let instrument_id = InstrumentId::from("ETH-USD-PERP.DYDX"); - - // Load credentials from environment (testnet/mainnet-aware) - let private_key_env = if is_testnet { - "DYDX_TESTNET_PRIVATE_KEY" - } else { - "DYDX_PRIVATE_KEY" - }; - let private_key = get_env_option(private_key_env); - let wallet_env = if is_testnet { - "DYDX_TESTNET_WALLET_ADDRESS" - } else { - "DYDX_WALLET_ADDRESS" - }; - let wallet_address = get_env_option(wallet_env); - - if private_key.is_none() && wallet_address.is_none() { - return Err( - format!("Set {private_key_env} or {wallet_env} environment variable").into(), - ); - } - - // Minimal data client config: is_testnet selects the correct endpoints - let data_config = DydxDataClientConfig { - is_testnet, - ..Default::default() - }; - - // Execution client with trader ID, network, credentials, and rate limiting - let exec_config = DYDXExecClientConfig { - trader_id, - account_id, - network, - private_key, - wallet_address, - subaccount_number: 0, - grpc_endpoint: None, - grpc_urls: vec![], - ws_endpoint: None, - http_endpoint: None, - authenticator_ids: vec![], - http_timeout_secs: Some(30), - max_retries: Some(3), - retry_delay_initial_ms: Some(1000), - retry_delay_max_ms: Some(10000), - grpc_rate_limit_per_second: Some(4), // Conservative for public providers - }; - - let data_factory = DydxDataClientFactory::new(); - let exec_factory = DydxExecutionClientFactory::new(); - - let log_config = LoggerConfig { - stdout_level: LevelFilter::Info, - ..Default::default() - }; - - // Builder pattern wires up logging, data/execution clients, and node options - let mut node = LiveNode::builder(trader_id, environment)? - .with_name(node_name) - .with_logging(log_config) - .add_data_client(None, Box::new(data_factory), Box::new(data_config))? - .add_exec_client(None, Box::new(exec_factory), Box::new(exec_config))? - .with_reconciliation(false) // Disabled for simplicity; enable in production - // to resume state across restarts - .with_delay_post_stop_secs(5) // Grace period for pending cancel/close events - .build()?; - - // Strategy configuration and registration - let config = GridMarketMakerConfig::new(instrument_id, Quantity::from("0.10")) - .with_num_levels(3) - .with_grid_step_bps(100) - .with_skew_factor(0.5) - .with_requote_threshold_bps(10) - .with_expire_time_secs(8) - .with_on_cancel_resubmit(true); - let strategy = GridMarketMaker::new(config); - - node.add_strategy(strategy)?; - node.run().await?; - - Ok(()) -} -``` - -Key configuration points: - -- **`dotenvy::dotenv().ok()`**: loads a `.env` file from the project root (if present). -- **`with_reconciliation(false)`**: disabled for simplicity; enable in production to resume - state across restarts. -- **`with_delay_post_stop_secs(5)`**: grace period for pending cancel/close events to finalize - during shutdown. - -### Event flow - -``` -LiveNode starts - │ - ├── connect() → HTTP: load instruments, WebSocket: subscribe channels - │ - ├── on_start() - │ └── subscribe_quotes(ETH-USD-PERP.DYDX) - │ - ├── on_quote() [repeated] - │ ├── Calculate mid-price - │ ├── Check should_requote(): skip if within threshold - │ ├── cancel_all_orders(): record IDs in pending_self_cancels - │ ├── Compute grid with inventory skew - │ └── Submit limit orders (GTD, expire in 8s) - │ - ├── on_order_filled() - │ └── Remove from pending_self_cancels - │ - ├── on_order_canceled() - │ ├── Self-cancel? → no action - │ └── Protocol cancel? → reset last_quoted_mid (triggers requote) - │ - └── on_stop() - ├── cancel_all_orders() - ├── close_all_positions() - └── unsubscribe_quotes() -``` - -## Strategy internals - -This section shows the key Rust code from `grid_mm.rs` so you can see exactly how the -strategy works without reading the full source. - -### Trade size resolution (`on_start`) - -When the strategy starts, it resolves the trade size from the instrument cache. The fallback -chain is: config value → instrument `min_quantity` → `1.0`: - -```rust -fn on_start(&mut self) -> anyhow::Result<()> { - let instrument_id = self.config.instrument_id; - let (price_precision, size_precision, min_quantity) = { - let cache = self.cache(); - let instrument = cache - .instrument(&instrument_id) - .expect("Instrument should be in cache"); - ( - instrument.price_precision(), - instrument.size_precision(), - instrument.min_quantity(), - ) - }; - self.price_precision = price_precision; - - // Resolve trade_size from instrument when not explicitly provided - if self.trade_size.is_none() { - self.trade_size = - Some(min_quantity.unwrap_or_else(|| Quantity::new(1.0, size_precision))); - } - - self.subscribe_quotes(instrument_id, None, None); - Ok(()) -} -``` - -### Quote handler (`on_quote`, abbreviated) - -This is the heart of the strategy. On each quote tick it computes the mid-price, -checks whether a requote is needed, cancels stale orders, computes worst-case exposure, -and places new grid orders with GTD + post_only: - -```rust -fn on_quote(&mut self, quote: &QuoteTick) -> anyhow::Result<()> { - let mid_f64 = (quote.bid_price.as_f64() + quote.ask_price.as_f64()) / 2.0; - let mid = Price::new(mid_f64, self.price_precision); - - if !self.should_requote(mid) { - return Ok(()); // Mid hasn't moved enough, keep existing grid - } - - // ... record open order IDs in pending_self_cancels (for on_cancel_resubmit) ... - - self.cancel_all_orders(instrument_id, None, None)?; - - // Compute worst-case per-side exposure (position + all pending orders) - // since cancels are async and pending orders may still fill - let (net_position, worst_long, worst_short) = { /* ... */ }; - - let grid = self.grid_orders(mid, net_position, worst_long, worst_short); - - if grid.is_empty() { - return Ok(()); // Don't advance requote anchor when fully constrained - } - - // Compute time-in-force from config - let (tif, expire_time) = match self.config.expire_time_secs { - Some(secs) => { - let now_ns = self.core.clock().timestamp_ns(); - let expire_ns = now_ns + secs * 1_000_000_000; - (Some(TimeInForce::Gtd), Some(expire_ns)) - } - None => (None, None), - }; - - for (side, price) in grid { - let order = self.core.order_factory().limit( - instrument_id, - side, - trade_size, - price, - tif, - expire_time, - Some(true), // post_only - // ... remaining None fields ... - ); - self.submit_order(order, None, None)?; - } - - self.last_quoted_mid = Some(mid); - Ok(()) -} -``` - -### Grid pricing (`grid_orders`) - -Computes geometric grid prices and enforces max_position per-level. This is the function -behind the ASCII diagrams in the [Strategy overview](#geometric-grid-pricing) section: - -```rust -fn grid_orders( - &self, - mid: Price, - net_position: f64, - worst_long: Decimal, - worst_short: Decimal, -) -> Vec<(OrderSide, Price)> { - let mid_f64 = mid.as_f64(); - let skew_f64 = self.config.skew_factor * net_position; - let pct = self.config.grid_step_bps as f64 / 10_000.0; - let trade_size = self.trade_size - .expect("trade_size should be resolved in on_start") - .as_decimal(); - let max_pos = self.config.max_position.as_decimal(); - let mut projected_long = worst_long; - let mut projected_short = worst_short; - let mut orders = Vec::new(); - - for level in 1..=self.config.num_levels { - let buy_price = Price::new( - mid_f64 * (1.0 - pct).powi(level as i32) - skew_f64, - precision, - ); - let sell_price = Price::new( - mid_f64 * (1.0 + pct).powi(level as i32) - skew_f64, - precision, - ); - - // Only place buy if projected long exposure stays within max_position - if projected_long + trade_size <= max_pos { - orders.push((OrderSide::Buy, buy_price)); - projected_long += trade_size; - } - - // Only place sell if projected short exposure stays within max_position - if projected_short - trade_size >= -max_pos { - orders.push((OrderSide::Sell, sell_price)); - projected_short -= trade_size; - } - } - - orders -} -``` - -## Monitoring and understanding output - -### Key log messages - -| Log message | Meaning | -| --------------------------------------------------- | -------------------------------------------------- | -| `Requoting grid: mid=X, last_mid=Y` | Mid-price moved beyond threshold, refreshing grid. | -| `Submit short-term order N` | Order submitted via short-term broadcast path. | -| `BatchCancel N short-term orders` | Batch cancel executed for expired/stale orders. | -| `benign cancel error, treating as success` | Cancel for already-filled/expired order (normal). | -| `Sequence mismatch detected, will resync and retry` | Cosmos SDK sequence error, auto-recovering. | - -### Expected behavior patterns - -1. **Startup**: Instruments load, WebSocket connects, first quote triggers initial grid. -2. **Steady state**: Grid persists across ticks; requotes only on significant mid-price moves. -3. **Fills**: Position updates, skew adjusts, next requote shifts grid. -4. **Expiry**: Short-term orders expire silently after ~8s; grid naturally refreshes on the next requote. -5. **Shutdown**: All orders cancelled, positions closed, WebSocket disconnected. - -## Customization tips - -### High vs low volatility - -| Condition | Adjustment | -| --------------- | ------------------------------------------------------------------------- | -| High volatility | Wider `grid_step_bps` (100-200), fewer `num_levels`, lower `skew_factor`. | -| Low volatility | Tighter `grid_step_bps` (10-30), more `num_levels`, higher `skew_factor`. | -| Thin liquidity | Increase `requote_threshold_bps` to reduce cancel frequency. | - -### Multiple instruments - -Run separate `GridMarketMaker` instances for each instrument. Each instance manages its own -grid, position tracking, and cancel state independently: - -```rust -let btc_config = GridMarketMakerConfig::new( - InstrumentId::from("BTC-USD-PERP.DYDX"), - Quantity::from("0.001"), -) -.with_strategy_id(StrategyId::from("GRID_MM-BTC")) -.with_order_id_tag("BTC".to_string()) -.with_grid_step_bps(50); - -let eth_config = GridMarketMakerConfig::new( - InstrumentId::from("ETH-USD-PERP.DYDX"), - Quantity::from("0.10"), -) -.with_strategy_id(StrategyId::from("GRID_MM-ETH")) -.with_order_id_tag("ETH".to_string()) -.with_grid_step_bps(100); - -node.add_strategy(GridMarketMaker::new(btc_config))?; -node.add_strategy(GridMarketMaker::new(eth_config))?; -``` - -### Mainnet vs testnet toggle - -Change a single flag to switch networks: - -```rust -let is_testnet = true; // false for mainnet -let network = if is_testnet { DydxNetwork::Testnet } else { DydxNetwork::Mainnet }; -``` - -All endpoints, chain IDs, and credential environment variables are resolved automatically -based on this flag. - -## Further reading - -- [dYdX v4 Integration Guide](../integrations/dydx.md): full adapter reference. -- [dYdX Protocol Documentation](https://docs.dydx.xyz/): official protocol docs. -- [Order types](https://docs.dydx.xyz/concepts/trading/orders): protocol-level order mechanics. diff --git a/nautilus-docs/docs/tutorials/index.md b/nautilus-docs/docs/tutorials/index.md deleted file mode 100644 index 5f589d0..0000000 --- a/nautilus-docs/docs/tutorials/index.md +++ /dev/null @@ -1,15 +0,0 @@ -# Tutorials - -Step-by-step walkthroughs demonstrating specific features and workflows. - -:::info -Each tutorial is a Jupytext percent-format Python file in the docs [tutorials directory](https://github.com/nautechsystems/nautilus_trader/tree/develop/docs/tutorials). You can run them directly as scripts or open them as notebooks with Jupytext. -::: - -:::tip - -- Make sure you are using the tutorial docs that match your NautilusTrader version: -- **Latest**: These docs are built from the HEAD of the `master` branch and work with the latest stable release. See . -- **Nightly**: These docs are built from the HEAD of the `nightly` branch and work with bleeding-edge and experimental features. See . - -::: diff --git a/nautilus-docs/docs/tutorials/loading_external_data.py b/nautilus-docs/docs/tutorials/loading_external_data.py deleted file mode 100644 index 4c47559..0000000 --- a/nautilus-docs/docs/tutorials/loading_external_data.py +++ /dev/null @@ -1,123 +0,0 @@ -# %% [markdown] -# # Loading External Data -# -# This tutorial demonstrates how to load external data into the `ParquetDataCatalog`, and then use this to run a one-shot backtest using a `BacktestNode`. - -# %% -import shutil -from decimal import Decimal -from pathlib import Path - -import pandas as pd - -from nautilus_trader.backtest.node import BacktestDataConfig -from nautilus_trader.backtest.node import BacktestEngineConfig -from nautilus_trader.backtest.node import BacktestNode -from nautilus_trader.backtest.node import BacktestRunConfig -from nautilus_trader.backtest.node import BacktestVenueConfig -from nautilus_trader.config import ImportableStrategyConfig -from nautilus_trader.core.datetime import dt_to_unix_nanos -from nautilus_trader.model import BarType -from nautilus_trader.model import QuoteTick -from nautilus_trader.persistence.catalog import ParquetDataCatalog -from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler -from nautilus_trader.test_kit.providers import CSVTickDataLoader -from nautilus_trader.test_kit.providers import TestInstrumentProvider - -# %% -DATA_DIR = "~/Downloads/Data/" - -# %% -path = Path(DATA_DIR).expanduser() / "HISTDATA" -raw_files = [ - f for f in path.iterdir() if f.is_file() and (f.suffix == ".csv" or f.name.endswith(".csv.gz")) -] -assert raw_files, f"Unable to find any data files in directory {path}" -raw_files - -# %% -# Load the first data file into a pandas DataFrame -df = CSVTickDataLoader.load(raw_files[0], index_col=0, datetime_format="%Y%m%d %H%M%S%f") -df.columns = ["bid_price", "ask_price"] - -# Process quotes using a wrangler -EURUSD = TestInstrumentProvider.default_fx_ccy("EUR/USD") -wrangler = QuoteTickDataWrangler(EURUSD) - -ticks = wrangler.process(df) - -# %% -CATALOG_PATH = Path.cwd() / "catalog" - -# Clear if it already exists, then create fresh -if CATALOG_PATH.exists(): - shutil.rmtree(CATALOG_PATH) -CATALOG_PATH.mkdir() - -catalog = ParquetDataCatalog(CATALOG_PATH) - -# %% -catalog.write_data([EURUSD]) -catalog.write_data(ticks) - -# %% -# Verify instruments written to catalog -catalog.instruments() - -# %% -start = dt_to_unix_nanos(pd.Timestamp("2020-01-03", tz="UTC")) -end = dt_to_unix_nanos(pd.Timestamp("2020-01-04", tz="UTC")) - -ticks = catalog.quote_ticks(instrument_ids=[EURUSD.id.value], start=start, end=end) -ticks[:10] - -# %% -instrument = catalog.instruments()[0] - -venue_configs = [ - BacktestVenueConfig( - name="SIM", - oms_type="HEDGING", - account_type="MARGIN", - base_currency="USD", - starting_balances=["1000000 USD"], - ), -] - -data_configs = [ - BacktestDataConfig( - catalog_path=str(catalog.path), - data_cls=QuoteTick, - instrument_id=instrument.id, - start_time=start, - end_time=end, - ), -] - -strategies = [ - ImportableStrategyConfig( - strategy_path="nautilus_trader.examples.strategies.ema_cross:EMACross", - config_path="nautilus_trader.examples.strategies.ema_cross:EMACrossConfig", - config={ - "instrument_id": instrument.id, - "bar_type": BarType.from_str(f"{instrument.id.value}-15-MINUTE-BID-INTERNAL"), - "fast_ema_period": 10, - "slow_ema_period": 20, - "trade_size": Decimal(1_000_000), - }, - ), -] - -config = BacktestRunConfig( - engine=BacktestEngineConfig(strategies=strategies), - data=data_configs, - venues=venue_configs, -) - -# %% -node = BacktestNode(configs=[config]) - -[result] = node.run() - -# %% -result diff --git a/nautilus-docs/REBUILD.md b/nautilus-docs/references/REBUILD.md similarity index 96% rename from nautilus-docs/REBUILD.md rename to nautilus-docs/references/REBUILD.md index 58c5dcc..37bdbd5 100644 --- a/nautilus-docs/REBUILD.md +++ b/nautilus-docs/references/REBUILD.md @@ -12,21 +12,22 @@ Prompt and checklist for regenerating SKILL.md when NautilusTrader's API changes ## Step 1: Update the Docs +Docs are a git submodule with sparse checkout (`docs/` only): + ```bash -# Pull latest official docs from nautilus_trader repo -# Replace docs/ contents with the updated versions -# Keep the directory structure identical +cd nautilus-docs/references/upstream +git fetch origin develop --depth 1 +git diff HEAD..FETCH_HEAD -- docs/ # review changes before applying +git checkout FETCH_HEAD -- docs/ +cd ../../.. +git add nautilus-docs/references/upstream ``` -Diff against previous version to identify: +Check for: - New files (new concepts, new adapters, new tutorials) - Removed files (deprecated features) - Changed files (API changes, renamed types) -```bash -diff -rq docs/ /path/to/new/docs/ | head -50 -``` - ## Step 2: Rebuild the Doc Navigator For every new `.md` or `.py` file in `docs/`, add a row to the appropriate navigator table in SKILL.md (Concepts, Venue Integrations, Dev/Setup, or Tutorials). diff --git a/nautilus-docs/battle_tested.md b/nautilus-docs/references/battle_tested.md similarity index 100% rename from nautilus-docs/battle_tested.md rename to nautilus-docs/references/battle_tested.md diff --git a/nautilus-docs/references/upstream b/nautilus-docs/references/upstream new file mode 160000 index 0000000..01572a4 --- /dev/null +++ b/nautilus-docs/references/upstream @@ -0,0 +1 @@ +Subproject commit 01572a44fad76a2a9d0b72269a1191d1f8202364 From 40ac0c024842e7d0818a1ee96b492358ba6f4df3 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 01:17:37 +0200 Subject: [PATCH 04/25] rust notes --- nautilus-docs/SKILL.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index c5597d2..92384c8 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -4,7 +4,6 @@ description: > NautilusTrader trading platform — strategies, backtesting, live deployment, order management, market data, Rust native binaries. Use when writing, reviewing, or debugging NautilusTrader code (Python or Rust), configuring venue adapters, or answering questions about the platform. -argument-hint: "[topic or question]" --- # NautilusTrader v1.224.0 @@ -150,6 +149,16 @@ All follow `nautilus-{venue}` naming: | With Parquet catalog | add `nautilus-persistence` | | `high-precision` (128-bit) | Default on most crates — `default-features = false` disables it, re-add explicitly | +### Rust vs Python + +- **No `load_ids` / `InstrumentProviderConfig` in Rust** — adapters auto-discover all instruments at connect +- **Logging**: Nautilus owns the `log` backend. Use `log::info!()`. Output goes to stdout. No `RUST_LOG` env var, no `env_logger` +- **First build**: minutes (430+ crates). Incremental: ~2s. Use `cargo build` not `--release` for iteration +- **`#[derive(Debug)]` on all actor structs** — Nautilus requires `Debug` trait; missing it = compile error (zero-cost, fine for prod) +- **Trade fields**: `trade.price.as_f64()`, `trade.size.as_f64()`, `trade.aggressor_side` (`AggressorSide::Buyer`/`Seller`) +- **`node.run().await`** blocks the event loop. Callbacks fire during run. `on_stop` fires on shutdown signal +- **Adapter configs** are all struct literals with `..Default::default()`, never builders. Pattern: `{Venue}DataClientConfig { environment, api_key, api_secret, ..Default::default() }`. Some add venue-specific fields (e.g. `product_types`, `is_testnet`). Check the venue's integration doc for exact fields + ### DataActor pattern (the only way to build custom Rust actors) ```rust From 1a7a294eec4871c3c3e1788887912b7e30ffb8a3 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 03:03:26 +0200 Subject: [PATCH 05/25] not works properly --- nautilus-docs/SKILL.md | 37 +++++------ nautilus-docs/references/battle_tested.md | 76 +++++++++++++++++++++++ 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 92384c8..d6fd091 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -84,14 +84,12 @@ Read the relevant doc before generating code or answering questions. Docs are a ## Supporting Files -- **[battle_tested.md](references/battle_tested.md)** — Non-obvious patterns verified against live exchanges. Load when: writing on_start() ordering, market making, signal pipelines, backtest config, venue-specific gotchas, performance optimization. +- **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern (orders/positions), credential issues, patching deps, on_start() ordering, market making, backtest config, venue gotchas. - **[REBUILD.md](references/REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. ## Rust Standalone Binary -External Rust binaries work via git dependencies. No need to clone the nautilus workspace. Requires Rust 1.94+, edition 2024. - -All crates: `{ git = "https://github.com/nautechsystems/nautilus_trader.git" }`. Add `default-features = false` if no Python headers. Naming: `nautilus-{name}` in Cargo.toml → `nautilus_{name}` in Rust. +Git dependencies, no workspace clone needed. All crates: `{ git = "https://github.com/nautechsystems/nautilus_trader.git" }`. Naming: `nautilus-{name}` (Cargo) → `nautilus_{name}` (Rust). ### Crate Map @@ -155,11 +153,13 @@ All follow `nautilus-{venue}` naming: - **Logging**: Nautilus owns the `log` backend. Use `log::info!()`. Output goes to stdout. No `RUST_LOG` env var, no `env_logger` - **First build**: minutes (430+ crates). Incremental: ~2s. Use `cargo build` not `--release` for iteration - **`#[derive(Debug)]` on all actor structs** — Nautilus requires `Debug` trait; missing it = compile error (zero-cost, fine for prod) +- **Callback naming**: Rust drops `_tick` suffix — `on_trade` not `on_trade_tick`, `on_quote` not `on_quote_tick`, `subscribe_trades` not `subscribe_trade_ticks` - **Trade fields**: `trade.price.as_f64()`, `trade.size.as_f64()`, `trade.aggressor_side` (`AggressorSide::Buyer`/`Seller`) - **`node.run().await`** blocks the event loop. Callbacks fire during run. `on_stop` fires on shutdown signal -- **Adapter configs** are all struct literals with `..Default::default()`, never builders. Pattern: `{Venue}DataClientConfig { environment, api_key, api_secret, ..Default::default() }`. Some add venue-specific fields (e.g. `product_types`, `is_testnet`). Check the venue's integration doc for exact fields +- **Adapter configs** are struct literals with `..Default::default()`. Full names: `{Venue}DataClientConfig`, `{Venue}ExecutionClientConfig`. Factory: `{Venue}DataClientFactory`, `{Venue}ExecutionClientFactory` +- **Ed25519 keys (v0.55.0)**: HTTP signature URL-encoding bug — HMAC keys may also be auto-detected as Ed25519 if valid base64. See battle_tested.md -### DataActor pattern (the only way to build custom Rust actors) +### DataActor pattern (Nautilus-specific — don't generalize Deref/DerefMut elsewhere) ```rust use std::ops::{Deref, DerefMut}; @@ -187,9 +187,7 @@ impl DataActor for MyActor { Ok(()) } fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { - let px = trade.price.as_f64(); - let qty = trade.size.as_f64(); - // logic here + log::info!("px={} qty={}", trade.price.as_f64(), trade.size.as_f64()); Ok(()) } fn on_stop(&mut self) -> anyhow::Result<()> { Ok(()) } // MUST implement — warns if missing @@ -233,12 +231,8 @@ node.add_actor(my_actor)?; node.run().await?; ``` -**Do NOT use `env_logger`** — Nautilus registers its own `log` backend. `env_logger::init()` will crash with "logger already initialized". Use `log::info!()` directly. - ## Common Hallucinations -These do NOT exist in v1.224.0: - | Hallucination | Reality | |--------------|---------| | `cache.position_for_instrument(id)` | `cache.positions_open(instrument_id=id)` returns list | @@ -294,10 +288,17 @@ These do NOT exist in v1.224.0: | `BinanceAccountType::UsdtFutures` (Rust) | `BinanceProductType::UsdM` from `nautilus_binance::common::enums` | | `nautilus_core::identifiers::InstrumentId` | `nautilus_model::identifiers::InstrumentId` | | `DataActor` in `nautilus_trading` | `DataActor` in `nautilus_common::actor` | -| `env_logger` alongside Nautilus | Crashes — Nautilus registers its own `log` backend. Use `log::info!()` directly | -| Nautilus crates from crates.io | Must use git source — `nautilus-persistence-macros` not on crates.io | -| `nautilus_trader_model` crate name | `nautilus_model` | +| `env_logger` alongside Nautilus | Crashes — Nautilus owns `log` backend. Use `log::info!()` | +| Nautilus crates from crates.io | Git source only | +| `nautilus_trader_model` | `nautilus_model` | | `on_book_delta` (singular) | `on_book_deltas` (plural — batch) | | Skip `on_stop` in DataActor | Warns "on_stop handler was called when not overridden" — always implement | -| `cancel_all_orders` in `on_stop` | Trading channel closed before `on_stop` — call from `on_trade` or other live callbacks | -| `node.stop()` exits cleanly | WebSocket lingers ~10s producing `channel closed` errors — add `std::process::exit(0)` after stop | +| `cancel_all_orders` in `on_stop` | Trading channel closed before `on_stop` — use earlier callbacks | +| `node.stop()` exits cleanly | WebSocket lingers ~10s. Use `tokio::time::timeout` on shutdown | +| `event.duration_ns` on PositionClosed | `event.duration` — no `_ns` suffix | +| `event.realized_pnl` is `Money` | `Option` — use `{:?}` format, not `{}` | +| `node.add_actor(strategy)` for Strategy | `node.add_strategy(strategy)?;` — Strategy ≠ DataActor | +| `Deref` for Strategy | `Deref` — StrategyCore auto-derefs through | +| `BinanceExecClientFactory` | `BinanceExecutionClientFactory` — full name, same for config | +| Patching a nautilus crate | Fork on GitHub + `[patch]` in Cargo.toml. See battle_tested.md | +| Ed25519 keys in Rust HTTP adapters (v0.55.0) | URL-encoding bug — base64 `+/=` not encoded. Needs fork | diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 7eef1fb..5b531ed 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -210,6 +210,82 @@ Pre-aggregated top 10 levels — lower overhead than full book subscription for - No `modify_order` support - Binary outcomes only — `YES`/`NO` tokens +## Rust Live Trading + +### Strategy pattern (for order management) + +DataActor is data-only. For submitting orders, use `Strategy` from `nautilus-trading`: + +```rust +use nautilus_common::actor::{DataActor, DataActorCore}; +use nautilus_trading::strategy::{Strategy, StrategyConfig, StrategyCore}; + +#[derive(Debug)] +struct MyStrategy { + core: StrategyCore, // wraps DataActorCore internally +} + +// CRITICAL: Deref target is DataActorCore, NOT StrategyCore. +// Strategy extends DataActor which requires Deref. +// StrategyCore itself implements Deref, so this chains through. +impl Deref for MyStrategy { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { &self.core } // auto-derefs StrategyCore → DataActorCore +} +impl DerefMut for MyStrategy { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.core } +} + +// DataActor impl — data callbacks + order submission +impl DataActor for MyStrategy { + fn on_start(&mut self) -> anyhow::Result<()> { Ok(()) } + fn on_stop(&mut self) -> anyhow::Result<()> { Ok(()) } +} + +// Strategy impl — empty, all logic lives in DataActor callbacks +impl Strategy for MyStrategy {} +``` + +Constructor: `StrategyCore::new(StrategyConfig { strategy_id: Some("MY-001".into()), ..Default::default() })`. + +Key event fields: +- `PositionOpened`: `entry` (OrderSide), `side` (PositionSide), `quantity`, `last_px`, `avg_px_open`, `currency` +- `PositionClosed`: `realized_pnl` (Option — use `{:?}` format), `duration` (not `duration_ns`), `avg_px_open`, `avg_px_close` +- `OrderFilled`: `last_qty`, `last_px`, `commission`, `liquidity_side`, `trade_id` + +Add to node: `node.add_strategy(my_strategy)?;` (not `add_actor`). + +### Credential handling + +Adapters using `SigningCredential` auto-detect key type: +1. Tries to strip PEM headers + base64 decode → Ed25519 if valid +2. Falls back to HMAC + +**Gotcha**: HMAC secrets that happen to be valid base64 (≥32 bytes) get misdetected as Ed25519. Symptoms: signature validation errors on execution client connect. This affects any adapter using `SigningCredential`. + +### Ed25519 HTTP signature encoding (v0.55.0) + +Ed25519 signatures are base64 containing `+`, `/`, `=`. Some adapter HTTP clients insert signatures into query strings without URL-encoding. HMAC signatures are hex (URL-safe) so they work fine. If you see signature errors only with Ed25519 keys, check whether the adapter's HTTP client URL-encodes the signature. Most have a `percent_encode()` helper already. + +### Patching Nautilus dependencies + +Fork on GitHub, push fix to a branch, use Cargo `[patch]`: + +```toml +[patch.'https://github.com/nautechsystems/nautilus_trader.git'] +nautilus-{venue} = { git = "https://github.com/YOUR-USER/nautilus_trader.git", branch = "fix/my-fix" } +``` + +Only list the crate(s) you modified. Cargo resolves transitive deps automatically. Submit upstream PR when the fix is general. + +### Rust adapter maturity gaps + +The Rust adapters are newer than Python. Expect: +- Some WebSocket event types unhandled (fall to `Unknown`, log warnings) — usually harmless if the critical event path works +- Some config options existing in Python but missing in Rust (e.g. `use_trade_lite`) +- HTTP signing edge cases with non-HMAC key types +- Check the venue's adapter crate README and compare with Python adapter for feature parity + ## Performance Checklist 1. Cache `self.instrument` in `on_start()` — avoid repeated `cache.instrument()` lookups in hot path From e0b9e4cc7f09bbe71a9cdbdb8221a74191ae273c Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 03:04:04 +0200 Subject: [PATCH 06/25] env ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d803c05..e8285a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .claude/ .tmp/ +.env \ No newline at end of file From f7de2094db11eb32aa3a14183b87a28f8ca3ffea Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 13:00:22 +0200 Subject: [PATCH 07/25] skill.md --- nautilus-docs/SKILL.md | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index d6fd091..ef010f3 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -8,7 +8,7 @@ description: > # NautilusTrader v1.224.0 -Read the relevant doc before generating code or answering questions. Docs are a git submodule (sparse checkout, `docs/` only) from [nautilus_trader](https://github.com/nautechsystems/nautilus_trader). Use `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for doc file paths. +**MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class/method/config name. Never guess signatures or constructors. ## Doc Navigator @@ -61,6 +61,7 @@ Read the relevant doc before generating code or answering questions. Docs are a |-------|-----| | Installation | [installation.md](references/upstream/docs/getting_started/installation.md) | | Quickstart | [quickstart.py](references/upstream/docs/getting_started/quickstart.py) | +| Backtest examples | [backtest_high_level.py](references/upstream/docs/getting_started/backtest_high_level.py), [backtest_low_level.py](references/upstream/docs/getting_started/backtest_low_level.py) | | Environment setup | [environment_setup.md](references/upstream/docs/developer_guide/environment_setup.md) | | Rust development | [rust.md](references/upstream/docs/developer_guide/rust.md) | | Testing | [testing.md](references/upstream/docs/developer_guide/testing.md) | @@ -82,6 +83,10 @@ Read the relevant doc before generating code or answering questions. Docs are a | Databento data catalog | [databento_data_catalog.py](references/upstream/docs/tutorials/databento_data_catalog.py) | | Loading external data | [loading_external_data.py](references/upstream/docs/tutorials/loading_external_data.py) | +### Search Strategy + +When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. Read BEFORE writing code — never invent an API call. + ## Supporting Files - **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern (orders/positions), credential issues, patching deps, on_start() ordering, market making, backtest config, venue gotchas. @@ -117,24 +122,7 @@ Git dependencies, no workspace clone needed. All crates: `{ git = "https://githu ### Adapter Crates -All follow `nautilus-{venue}` naming: - -| Crate | Markets | -|-------|---------| -| `nautilus-binance` | Spot, USDT-M, COIN-M | -| `nautilus-bybit` | Spot, Perpetuals | -| `nautilus-okx` | Spot, Futures, Options | -| `nautilus-kraken` | Spot, Futures | -| `nautilus-bitmex` | Perpetuals | -| `nautilus-deribit` | Options | -| `nautilus-dydx` | Perpetuals (DEX) | -| `nautilus-hyperliquid` | Perpetuals (DEX) | -| `nautilus-databento` | Historical data provider | -| `nautilus-tardis` | Historical data provider | -| `nautilus-betfair` | Betting exchange | -| `nautilus-polymarket` | Prediction markets | -| `nautilus-architect-ax` | FX, commodities | -| `nautilus-sandbox` | Simulated exchange | +All `nautilus-{venue}`: binance, bybit, okx, kraken, bitmex, deribit, dydx, hyperliquid, databento, tardis, betfair, polymarket, architect-ax, sandbox. See Venue Integrations docs for config details. ### Which crates for your task From e40594fdb2aca436618e69cf72d26e4daf8bb94b Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 16 Mar 2026 22:53:37 +0200 Subject: [PATCH 08/25] fix: address PR #40 review on nautilus-v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename VPIN → ImbalanceData in battle_tested.md (yk4r2: "never use vpin") - Fix Strategy trait impl: add required core()/core_mut() methods - Add on_order_filled ID reset pattern for market making - Add backtest → live migration checklist (10 items) - Add Rust hot path note: Price::new() vs Price::from(format!()) - Add Vec::with_capacity() performance note - Update battle_tested.md trigger in SKILL.md Co-Authored-By: Claude Opus 4.6 --- nautilus-docs/SKILL.md | 2 +- nautilus-docs/references/battle_tested.md | 55 +++++++++++++++++------ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index ef010f3..79f5d46 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -89,7 +89,7 @@ When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references ## Supporting Files -- **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern (orders/positions), credential issues, patching deps, on_start() ordering, market making, backtest config, venue gotchas. +- **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern (orders/positions), credential issues, patching deps, on_start() ordering, market making, backtest config, venue gotchas, backtest→live migration, hot path performance. - **[REBUILD.md](references/REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. ## Rust Standalone Binary diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 5b531ed..2ffcfd8 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -67,6 +67,18 @@ self.submit_order(tp) For Rust backtest: `support_contingent_orders = Some(true)` required in `add_venue`. +### Market maker: on_order_filled resets order IDs + +```python +def on_order_filled(self, event) -> None: + if event.client_order_id == self._bid_id: + self._bid_id = None + elif event.client_order_id == self._ask_id: + self._ask_id = None +``` + +Without this: stale `_bid_id`/`_ask_id` → `cache.order(stale_id)` returns filled order → `is_open` is False → submits duplicate instead of modifying. + ### Reconciliation delay `reconciliation_startup_delay_secs >= 10` — lower values cause false positive "missing order" resolutions on startup. @@ -108,9 +120,9 @@ def on_signal(self, signal) -> None: When signal values need to be richer than a single number: ```python -class VPINData(Data): - def __init__(self, vpin: float, volume: float, ts_event: int, ts_init: int): - self.vpin = vpin +class ImbalanceData(Data): + def __init__(self, imbalance: float, volume: float, ts_event: int, ts_init: int): + self.imbalance = imbalance self.volume = volume self._ts_event = ts_event self._ts_init = ts_init @@ -121,7 +133,7 @@ class VPINData(Data): def ts_init(self) -> int: return self._ts_init ``` -Actor: `self.publish_data(data_type=DataType(VPINData), data=vpin_data)` +Actor: `self.publish_data(data_type=DataType(ImbalanceData), data=data)` Strategy: subscribe in `on_start`, receive in `on_data` ## Backtest Configuration @@ -242,8 +254,10 @@ impl DataActor for MyStrategy { fn on_stop(&mut self) -> anyhow::Result<()> { Ok(()) } } -// Strategy impl — empty, all logic lives in DataActor callbacks -impl Strategy for MyStrategy {} +impl Strategy for MyStrategy { + fn core(&self) -> &StrategyCore { &self.core } + fn core_mut(&mut self) -> &mut StrategyCore { &mut self.core } +} ``` Constructor: `StrategyCore::new(StrategyConfig { strategy_id: Some("MY-001".into()), ..Default::default() })`. @@ -286,12 +300,27 @@ The Rust adapters are newer than Python. Expect: - HTTP signing edge cases with non-HMAC key types - Check the venue's adapter crate README and compare with Python adapter for feature parity +## Backtest → Live Checklist + +1. `InstrumentProviderConfig(load_ids=frozenset({"BTCUSDT-PERP.BINANCE"}))` — backtest uses `engine.add_instrument()`, live requires `load_ids` +2. Replace `TestInstrumentProvider` instruments with real venue instruments +3. Add `exec_clients` config (backtest doesn't need it) +4. Add `reconciliation=True`, `reconciliation_lookback_mins=1440` to `LiveExecEngineConfig` +5. Add `on_order_rejected` / `on_order_modify_rejected` handlers — backtest rarely rejects +6. Handle `modify_order` unsupported — fall back to cancel+new +7. Set realistic `LoggingConfig(log_level="INFO", log_directory="logs/")` +8. Expect `fills > orders` (partials), `frozen_account` semantics differ +9. Test on venue testnet first (`testnet=True` or `is_testnet=True` depending on venue) +10. WebSocket shutdown lingers ~10s — use `tokio::time::timeout` (Rust) or KeyboardInterrupt (Python) + ## Performance Checklist -1. Cache `self.instrument` in `on_start()` — avoid repeated `cache.instrument()` lookups in hot path -2. Keep `on_order_book_deltas` tight — fires at tick frequency -3. Never `time.sleep()` in callbacks — single-threaded event loop blocks everything -4. Use `modify_order` over cancel+replace where supported — single message, less latency -5. `sort=False` + `engine.sort_data()` for multi-instrument backtest loading -6. Order books are Rust-native — all operations execute as native code, no Python overhead -7. `indicators_initialized()` guard prevents acting on partial warmup values +1. **Rust hot path**: `Price::new(f64, u8)` — never `Price::from(format!("{:.prec$}", val, prec=p).as_str())` (heap alloc per tick) +2. Cache `self.instrument` in `on_start()` — avoid repeated `cache.instrument()` lookups in hot path +3. Keep `on_order_book_deltas` tight — fires at tick frequency +4. Never `time.sleep()` in callbacks — single-threaded event loop blocks everything +5. Use `modify_order` over cancel+replace where supported — single message, less latency +6. `sort=False` + `engine.sort_data()` for multi-instrument backtest loading +7. Order books are Rust-native — all operations execute as native code, no Python overhead +8. `indicators_initialized()` guard prevents acting on partial warmup values +9. `Vec::with_capacity(n)` for known-size collections in hot path — avoid reallocation From d58126556fc81619139ab1ccf8add9759d0b5e87 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Tue, 17 Mar 2026 12:16:24 +0200 Subject: [PATCH 09/25] feat: add state persistence, metrics, indicators, data pipeline to battle_tested.md Addresses yk4r2 PR #40 structural gaps: - State persistence (on_save/on_load + Redis) - Fill/latency/slippage metrics patterns - Log rotation guidance - Custom indicator development pattern - Data pipeline validation (gap detection, timestamp sanity) - Updated SKILL.md trigger line for new sections Co-Authored-By: Claude Opus 4.6 --- nautilus-docs/SKILL.md | 2 +- nautilus-docs/references/battle_tested.md | 113 ++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 79f5d46..f32a42a 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -89,7 +89,7 @@ When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references ## Supporting Files -- **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern (orders/positions), credential issues, patching deps, on_start() ordering, market making, backtest config, venue gotchas, backtest→live migration, hot path performance. +- **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern, credentials, patching deps, on_start() ordering, market making, backtest config, venue gotchas, backtest→live migration, hot path performance, state persistence, fill/latency metrics, log rotation, custom indicators, data pipeline validation. - **[REBUILD.md](references/REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. ## Rust Standalone Binary diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 2ffcfd8..13b8b04 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -313,6 +313,119 @@ The Rust adapters are newer than Python. Expect: 9. Test on venue testnet first (`testnet=True` or `is_testnet=True` depending on venue) 10. WebSocket shutdown lingers ~10s — use `tokio::time::timeout` (Rust) or KeyboardInterrupt (Python) +## State Persistence + +### on_save / on_load for strategy restart + +```python +def on_save(self) -> dict[str, bytes]: + return {"position_state": msgspec.json.encode(self._state)} + +def on_load(self, state: dict[str, bytes]) -> None: + if "position_state" in state: + self._state = msgspec.json.decode(state["position_state"]) +``` + +State is saved on graceful shutdown and restored on restart. Keys are strategy-scoped. + +### Redis for shared state + +```python +ExecEngineConfig( + cache_database=CacheDatabaseConfig(type="redis", host="localhost", port=6379), +) +``` + +Enables order/position state recovery across restarts. Without Redis, in-memory cache is lost on crash. + +## Metrics + +### Fill quality tracking + +```python +def on_order_filled(self, event) -> None: + expected = self._last_signal_price + actual = float(event.last_px) + slippage_bps = (actual - expected) / expected * 10_000 + self.log.info(f"Slippage: {slippage_bps:.1f} bps") +``` + +Track per-venue, per-side. Asymmetric slippage (buys worse than sells) indicates toxic flow detection. + +### Latency measurement + +```python +def on_order_filled(self, event) -> None: + round_trip_ns = event.ts_init - self._order_submitted_ns + self.log.info(f"Round-trip: {round_trip_ns / 1_000_000:.1f}ms") +``` + +`ts_init` on the fill event is when the exchange response arrived locally. Compare against submission timestamp for round-trip latency. + +## Log Rotation + +```python +LoggingConfig( + log_level="INFO", + log_directory="logs/", + log_file_format="{trader_id}_{instance_id}", + log_colors=False, +) +``` + +Nautilus creates one log file per run. For rotation, use external logrotate or cron. File names include trader_id for multi-strategy separation. + +## Custom Indicator Development + +```python +from nautilus_trader.indicators.base.indicator import Indicator + +class SpreadEMA(Indicator): + def __init__(self, period: int): + super().__init__([]) # no input params for registration + self.period = period + self._values = [] + self.value = 0.0 + + def handle_quote_tick(self, tick: QuoteTick) -> None: + spread = float(tick.ask_price - tick.bid_price) + self._values.append(spread) + if len(self._values) > self.period: + self._values.pop(0) + self.value = sum(self._values) / len(self._values) + self._set_initialized(len(self._values) >= self.period) + + def _set_initialized(self, value: bool) -> None: + self.initialized = value + + def reset(self) -> None: + self._values.clear() + self.value = 0.0 + self.initialized = False +``` + +Register: `self.register_indicator_for_quote_ticks(instrument_id, spread_ema)` in `on_start`. + +## Data Pipeline + +### Gap detection in tick data + +```python +def check_gaps(ticks, max_gap_ms=60_000): + for i in range(1, len(ticks)): + gap = (ticks[i].ts_event - ticks[i-1].ts_event) / 1_000_000 + if gap > max_gap_ms: + print(f"Gap: {gap:.0f}ms at index {i}") +``` + +Run before backtesting. Gaps > 1 minute during trading hours = suspect data. Gaps during maintenance windows (Binance: Tue 06:00-06:30 UTC) are normal. + +### Timestamp sanity + +- `ts_event` should increase monotonically +- `ts_init >= ts_event` always (init is local receipt time) +- `ts_init - ts_event > 60s` = suspect clock skew or stale data + ## Performance Checklist 1. **Rust hot path**: `Price::new(f64, u8)` — never `Price::from(format!("{:.prec$}", val, prec=p).as_str())` (heap alloc per tick) From 5fc99c64d43c42004fcbd548c9dddfc13dae58c7 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Tue, 17 Mar 2026 12:49:44 +0200 Subject: [PATCH 10/25] fix: rename vpin_rust to my_trading_system in REBUILD.md Last VPIN reference in nautilus-v2 branch. Co-Authored-By: Claude Opus 4.6 --- nautilus-docs/references/REBUILD.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nautilus-docs/references/REBUILD.md b/nautilus-docs/references/REBUILD.md index 37bdbd5..8b331c4 100644 --- a/nautilus-docs/references/REBUILD.md +++ b/nautilus-docs/references/REBUILD.md @@ -101,7 +101,7 @@ This is the highest-value part of the skill. Each row prevents a specific mistak 4. **Rust hallucinations** — rebuild by attempting to compile a standalone binary: ```bash - cd vpin_rust && cargo build 2>&1 | grep "error\[E" + cd my_trading_system && cargo build 2>&1 | grep "error\[E" ``` Every compilation error that comes from a wrong import path, missing trait, or wrong struct field is a hallucination row candidate. @@ -110,7 +110,7 @@ This is the highest-value part of the skill. Each row prevents a specific mistak The DataActor pattern and LiveNode wiring in SKILL.md must compile. Test: ```bash -cd vpin_rust && cargo build --release 2>&1 +cd my_trading_system && cargo build --release 2>&1 ``` If it fails, the Rust section needs updating. Common breakage: @@ -212,7 +212,7 @@ Steps: - For Rust: check actual struct/trait definitions in the crate source 5. Update the Rust crate map from workspace members 6. Update the DataActor pattern and LiveNode wiring from actual working examples -7. Test that the Rust example compiles: cd vpin_rust && cargo build +7. Test that the Rust example compiles: cd my_trading_system && cargo build 8. Verify word count is under 2,000 For anti-hallucination entries, try to write code for each of the "Fundamental Questions" From fdbef4a9104ac0f7d61ddcbff559ce174b71ac90 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Wed, 18 Mar 2026 22:12:18 +0200 Subject: [PATCH 11/25] update upstream + architecture must read --- nautilus-docs/SKILL.md | 53 ++++++++++--------------------- nautilus-docs/references/upstream | 2 +- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index f32a42a..f6c1b39 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -10,6 +10,8 @@ description: > **MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class/method/config name. Never guess signatures or constructors. +**MUST READ** before any NautilusTrader work: [architecture.md](references/upstream/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/upstream/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). + ## Doc Navigator ### Concepts @@ -73,19 +75,19 @@ description: > | Tutorial | Doc | |----------|-----| -| FX mean reversion (AX) | [ax_fx_mean_reversion.md](references/upstream/docs/tutorials/ax_fx_mean_reversion.md) | -| Gold book imbalance (AX) | [ax_gold_book_imbalance.md](references/upstream/docs/tutorials/ax_gold_book_imbalance.md) | -| dYdX grid market maker | [dydx_grid_market_maker.md](references/upstream/docs/tutorials/dydx_grid_market_maker.md) | -| BitMEX grid market maker | [bitmex_grid_market_maker.md](references/upstream/docs/tutorials/bitmex_grid_market_maker.md) | -| Backtest Binance orderbook | [backtest_binance_orderbook.py](references/upstream/docs/tutorials/backtest_binance_orderbook.py) | -| Backtest Bybit orderbook | [backtest_bybit_orderbook.py](references/upstream/docs/tutorials/backtest_bybit_orderbook.py) | +| FX mean reversion (AX) | [fx_mean_reversion_ax.md](references/upstream/docs/tutorials/fx_mean_reversion_ax.md) | +| Gold book imbalance (AX) | [gold_book_imbalance_ax.md](references/upstream/docs/tutorials/gold_book_imbalance_ax.md) | +| dYdX grid market maker | [grid_market_maker_dydx.md](references/upstream/docs/tutorials/grid_market_maker_dydx.md) | +| BitMEX grid market maker | [grid_market_maker_bitmex.md](references/upstream/docs/tutorials/grid_market_maker_bitmex.md) | +| Backtest Binance orderbook | [backtest_orderbook_binance.py](references/upstream/docs/tutorials/backtest_orderbook_binance.py) | +| Backtest Bybit orderbook | [backtest_orderbook_bybit.py](references/upstream/docs/tutorials/backtest_orderbook_bybit.py) | | Backtest FX bars | [backtest_fx_bars.py](references/upstream/docs/tutorials/backtest_fx_bars.py) | -| Databento data catalog | [databento_data_catalog.py](references/upstream/docs/tutorials/databento_data_catalog.py) | +| Databento data catalog | [data_catalog_databento.py](references/upstream/docs/tutorials/data_catalog_databento.py) | | Loading external data | [loading_external_data.py](references/upstream/docs/tutorials/loading_external_data.py) | ### Search Strategy -When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. Read BEFORE writing code — never invent an API call. +When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. **API reference docs** (`api_reference/`) have exact class/method signatures when concepts docs aren't enough. Read BEFORE writing code — never invent an API call. ## Supporting Files @@ -192,35 +194,10 @@ impl MyActor { } ``` -### LiveNode + Binance wiring - -```rust -use nautilus_binance::common::enums::{BinanceEnvironment, BinanceProductType}; -use nautilus_binance::config::BinanceDataClientConfig; -use nautilus_binance::factories::BinanceDataClientFactory; -use nautilus_common::enums::Environment; -use nautilus_live::node::LiveNode; -use nautilus_model::identifiers::TraderId; - -let data_config = BinanceDataClientConfig { - product_types: vec![BinanceProductType::UsdM], // or Spot, CoinM - environment: BinanceEnvironment::Mainnet, // or Testnet - api_key: Some(key), - api_secret: Some(secret), // Ed25519: wrap in PEM headers - ..Default::default() -}; - -let mut node = LiveNode::builder(TraderId::from("MY-001"), Environment::Live)? - .with_name("MyNode") - .add_data_client(None, Box::new(BinanceDataClientFactory::new()), Box::new(data_config))? - .build()?; - -node.add_actor(my_actor)?; -node.run().await?; -``` - ## Common Hallucinations +### Python + | Hallucination | Reality | |--------------|---------| | `cache.position_for_instrument(id)` | `cache.positions_open(instrument_id=id)` returns list | @@ -270,7 +247,11 @@ node.run().await?; | `testnet=True` for Deribit/dYdX | `is_testnet=True` | | `AccountType.CASH` for perps backtest | Use `AccountType.MARGIN` — CASH + perps = 0 fills silently | | `book.best_bid_price()` returns float | Returns `Price` object — cast with `float()` for arithmetic | -| **Rust-specific** | | + +### Rust + +| Hallucination | Reality | +|--------------|---------| | `LiveNode::new(config)` or `LiveNodeConfig::builder()` | `LiveNode::builder(trader_id, Environment::Live)?.build()?` | | `BinanceDataClientConfig::builder()` | Struct literal: `BinanceDataClientConfig { product_types: vec![...], ..Default::default() }` | | `BinanceAccountType::UsdtFutures` (Rust) | `BinanceProductType::UsdM` from `nautilus_binance::common::enums` | diff --git a/nautilus-docs/references/upstream b/nautilus-docs/references/upstream index 01572a4..e7c47fd 160000 --- a/nautilus-docs/references/upstream +++ b/nautilus-docs/references/upstream @@ -1 +1 @@ -Subproject commit 01572a44fad76a2a9d0b72269a1191d1f8202364 +Subproject commit e7c47fdaf476d799805cc8ed9b194ed2479021ef From b6219f2b3e34b95fa6d892acb81127200a465db4 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Thu, 19 Mar 2026 00:20:43 +0200 Subject: [PATCH 12/25] loading docs at install --- .gitignore | 3 +- .gitmodules | 4 - nautilus-docs/SKILL.md | 113 +++++++++++++++------------- nautilus-docs/references/REBUILD.md | 16 ++-- nautilus-docs/references/upstream | 1 - scripts/install.sh | 30 ++++++++ 6 files changed, 100 insertions(+), 67 deletions(-) delete mode 100644 .gitmodules delete mode 160000 nautilus-docs/references/upstream diff --git a/.gitignore b/.gitignore index e8285a7..31b173e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .claude/ .tmp/ -.env \ No newline at end of file +.env +nautilus-docs/references/docs/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 4d67b92..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "nautilus-docs/references/upstream"] - path = nautilus-docs/references/upstream - url = https://github.com/nautechsystems/nautilus_trader.git - shallow = true diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index f6c1b39..8aeb5a5 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -8,9 +8,9 @@ description: > # NautilusTrader v1.224.0 -**MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class/method/config name. Never guess signatures or constructors. +**MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `${CLAUDE_SKILL_DIR}/references/docs/` for the class/method/config name. Never guess signatures or constructors. -**MUST READ** before any NautilusTrader work: [architecture.md](references/upstream/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/upstream/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). +**MUST READ** before any NautilusTrader work: [architecture.md](references/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). ## Doc Navigator @@ -18,76 +18,81 @@ description: > | Topic | Doc | |-------|-----| -| Architecture / overview | [architecture.md](references/upstream/docs/concepts/architecture.md), [overview.md](references/upstream/docs/concepts/overview.md) | -| Actor development | [actors.md](references/upstream/docs/concepts/actors.md) | -| Strategy development | [strategies.md](references/upstream/docs/concepts/strategies.md) | -| Backtesting | [backtesting.md](references/upstream/docs/concepts/backtesting.md) | -| Live trading | [live.md](references/upstream/docs/concepts/live.md) | -| Order types & execution | [orders.md](references/upstream/docs/concepts/orders.md), [execution.md](references/upstream/docs/concepts/execution.md) | -| Order book | [order_book.md](references/upstream/docs/concepts/order_book.md) | -| Data types & custom data | [data.md](references/upstream/docs/concepts/data.md), [custom_data.md](references/upstream/docs/concepts/custom_data.md) | -| Instruments | [instruments.md](references/upstream/docs/concepts/instruments.md) | -| Value types (Price, Quantity, Money) | [value_types.md](references/upstream/docs/concepts/value_types.md) | -| Positions & PnL | [positions.md](references/upstream/docs/concepts/positions.md) | -| Cache | [cache.md](references/upstream/docs/concepts/cache.md) | -| MessageBus | [message_bus.md](references/upstream/docs/concepts/message_bus.md) | -| Portfolio | [portfolio.md](references/upstream/docs/concepts/portfolio.md) | -| Options & Greeks | [options.md](references/upstream/docs/concepts/options.md), [greeks.md](references/upstream/docs/concepts/greeks.md) | -| Logging | [logging.md](references/upstream/docs/concepts/logging.md) | -| Reports & analysis | [reports.md](references/upstream/docs/concepts/reports.md) | -| Visualization | [visualization.md](references/upstream/docs/concepts/visualization.md) | -| Adapter development | [concepts/adapters.md](references/upstream/docs/concepts/adapters.md), [developer_guide/adapters.md](references/upstream/docs/developer_guide/adapters.md) | +| Architecture / overview | [architecture.md](references/docs/concepts/architecture.md), [overview.md](references/docs/concepts/overview.md) | +| Actor development | [actors.md](references/docs/concepts/actors.md) | +| Strategy development | [strategies.md](references/docs/concepts/strategies.md) | +| Backtesting | [backtesting.md](references/docs/concepts/backtesting.md) | +| Live trading | [live.md](references/docs/concepts/live.md) | +| Order types & execution | [orders.md](references/docs/concepts/orders.md), [execution.md](references/docs/concepts/execution.md) | +| Order book | [order_book.md](references/docs/concepts/order_book.md) | +| Data types & custom data | [data.md](references/docs/concepts/data.md), [custom_data.md](references/docs/concepts/custom_data.md) | +| Instruments | [instruments.md](references/docs/concepts/instruments.md) | +| Value types (Price, Quantity, Money) | [value_types.md](references/docs/concepts/value_types.md) | +| Positions & PnL | [positions.md](references/docs/concepts/positions.md) | +| Cache | [cache.md](references/docs/concepts/cache.md) | +| MessageBus | [message_bus.md](references/docs/concepts/message_bus.md) | +| Portfolio | [portfolio.md](references/docs/concepts/portfolio.md) | +| Options & Greeks | [options.md](references/docs/concepts/options.md), [greeks.md](references/docs/concepts/greeks.md) | +| Logging | [logging.md](references/docs/concepts/logging.md) | +| Reports & analysis | [reports.md](references/docs/concepts/reports.md) | +| Visualization | [visualization.md](references/docs/concepts/visualization.md) | +| Adapter development | [concepts/adapters.md](references/docs/concepts/adapters.md), [developer_guide/adapters.md](references/docs/developer_guide/adapters.md) | ### Venue Integrations | Venue | Doc | |-------|-----| -| Binance | [binance.md](references/upstream/docs/integrations/binance.md) | -| Bybit | [bybit.md](references/upstream/docs/integrations/bybit.md) | -| OKX | [okx.md](references/upstream/docs/integrations/okx.md) | -| dYdX | [dydx.md](references/upstream/docs/integrations/dydx.md) | -| Deribit | [deribit.md](references/upstream/docs/integrations/deribit.md) | -| Hyperliquid | [hyperliquid.md](references/upstream/docs/integrations/hyperliquid.md) | -| Kraken | [kraken.md](references/upstream/docs/integrations/kraken.md) | -| Interactive Brokers | [ib.md](references/upstream/docs/integrations/ib.md) | -| Betfair | [betfair.md](references/upstream/docs/integrations/betfair.md) | -| Polymarket | [polymarket.md](references/upstream/docs/integrations/polymarket.md) | -| Databento | [databento.md](references/upstream/docs/integrations/databento.md) | -| Tardis | [tardis.md](references/upstream/docs/integrations/tardis.md) | -| BitMEX | [bitmex.md](references/upstream/docs/integrations/bitmex.md) | -| AX Exchange | [architect_ax.md](references/upstream/docs/integrations/architect_ax.md) | +| Binance | [binance.md](references/docs/integrations/binance.md) | +| Bybit | [bybit.md](references/docs/integrations/bybit.md) | +| OKX | [okx.md](references/docs/integrations/okx.md) | +| dYdX | [dydx.md](references/docs/integrations/dydx.md) | +| Deribit | [deribit.md](references/docs/integrations/deribit.md) | +| Hyperliquid | [hyperliquid.md](references/docs/integrations/hyperliquid.md) | +| Kraken | [kraken.md](references/docs/integrations/kraken.md) | +| Interactive Brokers | [ib.md](references/docs/integrations/ib.md) | +| Betfair | [betfair.md](references/docs/integrations/betfair.md) | +| Polymarket | [polymarket.md](references/docs/integrations/polymarket.md) | +| Databento | [databento.md](references/docs/integrations/databento.md) | +| Tardis | [tardis.md](references/docs/integrations/tardis.md) | +| BitMEX | [bitmex.md](references/docs/integrations/bitmex.md) | +| AX Exchange | [architect_ax.md](references/docs/integrations/architect_ax.md) | +| Blockchain / DeFi | [blockchain.md](references/docs/integrations/blockchain.md) | ### Dev / Setup | Topic | Doc | |-------|-----| -| Installation | [installation.md](references/upstream/docs/getting_started/installation.md) | -| Quickstart | [quickstart.py](references/upstream/docs/getting_started/quickstart.py) | -| Backtest examples | [backtest_high_level.py](references/upstream/docs/getting_started/backtest_high_level.py), [backtest_low_level.py](references/upstream/docs/getting_started/backtest_low_level.py) | -| Environment setup | [environment_setup.md](references/upstream/docs/developer_guide/environment_setup.md) | -| Rust development | [rust.md](references/upstream/docs/developer_guide/rust.md) | -| Testing | [testing.md](references/upstream/docs/developer_guide/testing.md) | -| Coding standards | [coding_standards.md](references/upstream/docs/developer_guide/coding_standards.md) | -| Benchmarking | [benchmarking.md](references/upstream/docs/developer_guide/benchmarking.md) | -| FFI | [ffi.md](references/upstream/docs/developer_guide/ffi.md) | +| Installation | [installation.md](references/docs/getting_started/installation.md) | +| Quickstart | [quickstart.py](references/docs/getting_started/quickstart.py) | +| Backtest examples | [backtest_high_level.py](references/docs/getting_started/backtest_high_level.py), [backtest_low_level.py](references/docs/getting_started/backtest_low_level.py) | +| Environment setup | [environment_setup.md](references/docs/developer_guide/environment_setup.md) | +| Rust development | [rust.md](references/docs/developer_guide/rust.md) | +| Python development | [python.md](references/docs/developer_guide/python.md) | +| Testing | [testing.md](references/docs/developer_guide/testing.md) | +| Data testing spec | [spec_data_testing.md](references/docs/developer_guide/spec_data_testing.md) | +| Execution testing spec | [spec_exec_testing.md](references/docs/developer_guide/spec_exec_testing.md) | +| Test datasets | [test_datasets.md](references/docs/developer_guide/test_datasets.md) | +| Coding standards | [coding_standards.md](references/docs/developer_guide/coding_standards.md) | +| Benchmarking | [benchmarking.md](references/docs/developer_guide/benchmarking.md), [criterion_template.rs](references/docs/dev_templates/criterion_template.rs), [iai_template.rs](references/docs/dev_templates/iai_template.rs) | +| FFI | [ffi.md](references/docs/developer_guide/ffi.md) | ### Tutorials | Tutorial | Doc | |----------|-----| -| FX mean reversion (AX) | [fx_mean_reversion_ax.md](references/upstream/docs/tutorials/fx_mean_reversion_ax.md) | -| Gold book imbalance (AX) | [gold_book_imbalance_ax.md](references/upstream/docs/tutorials/gold_book_imbalance_ax.md) | -| dYdX grid market maker | [grid_market_maker_dydx.md](references/upstream/docs/tutorials/grid_market_maker_dydx.md) | -| BitMEX grid market maker | [grid_market_maker_bitmex.md](references/upstream/docs/tutorials/grid_market_maker_bitmex.md) | -| Backtest Binance orderbook | [backtest_orderbook_binance.py](references/upstream/docs/tutorials/backtest_orderbook_binance.py) | -| Backtest Bybit orderbook | [backtest_orderbook_bybit.py](references/upstream/docs/tutorials/backtest_orderbook_bybit.py) | -| Backtest FX bars | [backtest_fx_bars.py](references/upstream/docs/tutorials/backtest_fx_bars.py) | -| Databento data catalog | [data_catalog_databento.py](references/upstream/docs/tutorials/data_catalog_databento.py) | -| Loading external data | [loading_external_data.py](references/upstream/docs/tutorials/loading_external_data.py) | +| FX mean reversion (AX) | [fx_mean_reversion_ax.md](references/docs/tutorials/fx_mean_reversion_ax.md) | +| Gold book imbalance (AX) | [gold_book_imbalance_ax.md](references/docs/tutorials/gold_book_imbalance_ax.md) | +| dYdX grid market maker | [grid_market_maker_dydx.md](references/docs/tutorials/grid_market_maker_dydx.md) | +| BitMEX grid market maker | [grid_market_maker_bitmex.md](references/docs/tutorials/grid_market_maker_bitmex.md) | +| Backtest Binance orderbook | [backtest_orderbook_binance.py](references/docs/tutorials/backtest_orderbook_binance.py) | +| Backtest Bybit orderbook | [backtest_orderbook_bybit.py](references/docs/tutorials/backtest_orderbook_bybit.py) | +| Backtest FX bars | [backtest_fx_bars.py](references/docs/tutorials/backtest_fx_bars.py) | +| Databento data catalog | [data_catalog_databento.py](references/docs/tutorials/data_catalog_databento.py) | +| Loading external data | [loading_external_data.py](references/docs/tutorials/loading_external_data.py) | ### Search Strategy -When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references/upstream/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. **API reference docs** (`api_reference/`) have exact class/method signatures when concepts docs aren't enough. Read BEFORE writing code — never invent an API call. +When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. Read BEFORE writing code — never invent an API call. ## Supporting Files diff --git a/nautilus-docs/references/REBUILD.md b/nautilus-docs/references/REBUILD.md index 8b331c4..9d0cd75 100644 --- a/nautilus-docs/references/REBUILD.md +++ b/nautilus-docs/references/REBUILD.md @@ -12,15 +12,17 @@ Prompt and checklist for regenerating SKILL.md when NautilusTrader's API changes ## Step 1: Update the Docs -Docs are a git submodule with sparse checkout (`docs/` only): +Docs are fetched on install into `nautilus-docs/references/docs/` (gitignored). To refresh: ```bash -cd nautilus-docs/references/upstream -git fetch origin develop --depth 1 -git diff HEAD..FETCH_HEAD -- docs/ # review changes before applying -git checkout FETCH_HEAD -- docs/ -cd ../../.. -git add nautilus-docs/references/upstream +rm -rf nautilus-docs/references/docs +# Re-run install.sh or manually: +temp=$(mktemp -d) +git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$temp" +git -C "$temp" sparse-checkout set docs/ +mv "$temp/docs" nautilus-docs/references/docs +rm -rf "$temp" ``` Check for: diff --git a/nautilus-docs/references/upstream b/nautilus-docs/references/upstream deleted file mode 160000 index e7c47fd..0000000 --- a/nautilus-docs/references/upstream +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e7c47fdaf476d799805cc8ed9b194ed2479021ef diff --git a/scripts/install.sh b/scripts/install.sh index 2b9415b..785304e 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -172,6 +172,36 @@ ensure_repo() { git -C "$REPO_DIR" fetch --tags --prune git -C "$REPO_DIR" checkout "$REPO_REF" fi + + fetch_nautilus_docs +} + +fetch_nautilus_docs() { + local docs_dir="$REPO_DIR/nautilus-docs/references/docs" + + if [[ -d "$docs_dir" ]]; then + return + fi + + # Skip if nautilus-docs skill doesn't exist in this checkout + if [[ ! -f "$REPO_DIR/nautilus-docs/SKILL.md" ]]; then + return + fi + + echo "Fetching NautilusTrader docs..." + + local temp + temp=$(mktemp -d) + trap "rm -rf '$temp'" RETURN + + git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$temp" + git -C "$temp" sparse-checkout set docs/ + + rm -rf "$temp/docs/api_reference" + + mkdir -p "$(dirname "$docs_dir")" + mv "$temp/docs" "$docs_dir" } collect_skills() { From 10e28a693dfed7c7b16ebe6769ac3d722623f5ec Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Thu, 19 Mar 2026 00:54:08 +0200 Subject: [PATCH 13/25] install sh --- scripts/install.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 785304e..bf7eae7 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3,7 +3,7 @@ set -euo pipefail usage() { cat <<'USAGE' -Install Codex skills and/or print Claude Code plugin install commands. +Install skills for Codex and/or Claude Code. Usage: install.sh [--platform codex|claude|both] @@ -31,14 +31,14 @@ USAGE } PLATFORM="codex" -REPO_URL="https://github.com/DeevsDeevs/agent-system.git" +REPO_URL="https://github.com/VladKochetov007/agent-system.git" CLONE_DIR="$HOME/src/agent-system" TARGET="${CODEX_HOME:-$HOME/.codex}/skills" MODE="symlink" SKILLS_CSV="" SKILLS_LIST=() REPO_DIR="" -REPO_REF="" +REPO_REF="nautilus-v2" UNINSTALL="false" NON_INTERACTIVE="false" @@ -286,11 +286,14 @@ uninstall_codex() { printf '\nDone. Restart Codex to reload skills.\n' } -print_claude_instructions() { - cat <<'EOC' +install_claude() { + ensure_repo + + cat < Date: Thu, 19 Mar 2026 00:57:20 +0200 Subject: [PATCH 14/25] original deevs link --- scripts/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index bf7eae7..c6b0b28 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -31,14 +31,14 @@ USAGE } PLATFORM="codex" -REPO_URL="https://github.com/VladKochetov007/agent-system.git" +REPO_URL="https://github.com/deevsDeevs/agent-system.git" CLONE_DIR="$HOME/src/agent-system" TARGET="${CODEX_HOME:-$HOME/.codex}/skills" MODE="symlink" SKILLS_CSV="" SKILLS_LIST=() REPO_DIR="" -REPO_REF="nautilus-v2" +REPO_REF="" UNINSTALL="false" NON_INTERACTIVE="false" From 18e192038bf0dae5186517d1c1356afd78675e84 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Thu, 19 Mar 2026 01:00:21 +0200 Subject: [PATCH 15/25] readme install guide --- README.md | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 51b0d0a..97ea266 100644 --- a/README.md +++ b/README.md @@ -4,45 +4,45 @@ Deevs' plugin marketplace for Claude Code with workflow chains, terminal control ## Installation +Use the installer for all platforms (Claude Code, Codex, or both): + ```bash -/plugin marketplace add git@github.com:DeevsDeevs/agent-system.git +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- ``` -## Codex Setup - -Codex loads repo-scoped skills from `.codex/skills`. This repo includes symlinks to each skill folder, so no extra install step is required. +The installer is interactive by default — it will ask for your platform and preferences. It clones the repo, fetches any external dependencies (e.g. NautilusTrader docs for the nautilus-docs skill), and sets everything up. -Usage: -- Invoke a skill with `$skill-name` or `/skills`. -- For chain operations: `$chain-system link `, `$chain-system load `, `$chain-system list`. -- For persona workflows: `$dev-experts `, `$bug-hunters `, `$research-experts `. +### Non-interactive -If Codex was already running, restart it to reload the skills. +```bash +# Claude Code +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --non-interactive --platform claude -Global (user-scoped) install for any repo: +# Codex +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --non-interactive --platform codex -```bash -git clone git@github.com:DeevsDeevs/agent-system.git ~/src/agent-system -mkdir -p ~/.codex/skills -for d in 97-dev anti-ai-slop datetime golang-pro polars-expertise arxiv-search chain-system dev-experts bug-hunters research-experts cost-status; do - ln -s ~/src/agent-system/$d ~/.codex/skills/$d -done +# Both +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --non-interactive --platform both ``` -Installer (interactive by default): +### Flags + +`--platform claude|codex|both`, `--repo-ref `, `--repo `, `--clone-dir `, `--target `, `--mode symlink|copy`, `--skills a,b,c`, `--non-interactive`, `--uninstall` + +### Uninstall ```bash -curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \\ - | bash -s -- +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --uninstall --platform both ``` -Installer flags: `--non-interactive`, `--platform claude|codex|both`, `--repo-ref `, `--uninstall`, `--skills a,b,c`. - -Uninstall: -- Codex: run the installer with `--uninstall`. -- Claude Code: use `/plugin uninstall @deevs-agent-system` inside Claude Code. +### Codex usage -Install a single skill by URL with `$skill-installer`. +Invoke a skill with `$skill-name` or `/skills`. If Codex was already running, restart it to reload skills. ## Plugins From 6cfda2265baf7ef35ff19a9255df50445d36969a Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Fri, 20 Mar 2026 21:26:23 +0200 Subject: [PATCH 16/25] install repo only if needed --- README.md | 29 +++++++++++++++++++++++------ scripts/install.sh | 18 +++++++++++------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 97ea266..1962bb5 100644 --- a/README.md +++ b/README.md @@ -4,34 +4,51 @@ Deevs' plugin marketplace for Claude Code with workflow chains, terminal control ## Installation -Use the installer for all platforms (Claude Code, Codex, or both): +Use the installer for all platforms (Claude Code, Codex, or both). Interactive by default: ```bash curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ | bash -s -- ``` -The installer is interactive by default — it will ask for your platform and preferences. It clones the repo, fetches any external dependencies (e.g. NautilusTrader docs for the nautilus-docs skill), and sets everything up. +The installer clones the repo, fetches any external dependencies (e.g. NautilusTrader docs — only when the nautilus-docs skill is selected), and sets everything up. ### Non-interactive ```bash -# Claude Code +# Claude Code — all skills curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ | bash -s -- --non-interactive --platform claude -# Codex +# Codex — all skills curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ | bash -s -- --non-interactive --platform codex -# Both +# Both platforms curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ | bash -s -- --non-interactive --platform both ``` +Install selected skills only: + +```bash +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --non-interactive --platform codex --skills dev-experts,bug-hunters +``` + ### Flags -`--platform claude|codex|both`, `--repo-ref `, `--repo `, `--clone-dir `, `--target `, `--mode symlink|copy`, `--skills a,b,c`, `--non-interactive`, `--uninstall` +| Flag | Description | +|------|-------------| +| `--platform claude\|codex\|both` | Target platform | +| `--skills a,b,c` | Install only these skills (default: all) | +| `--repo-ref ` | Checkout specific ref | +| `--repo ` | Custom repo URL | +| `--clone-dir ` | Where to clone (default: `~/src/agent-system`) | +| `--target ` | Codex skills dir (default: `~/.codex/skills`) | +| `--mode symlink\|copy` | Codex install mode | +| `--non-interactive` | Skip prompts | +| `--uninstall` | Remove installed skills | ### Uninstall diff --git a/scripts/install.sh b/scripts/install.sh index c6b0b28..d640da6 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -173,7 +173,6 @@ ensure_repo() { git -C "$REPO_DIR" checkout "$REPO_REF" fi - fetch_nautilus_docs } fetch_nautilus_docs() { @@ -183,11 +182,6 @@ fetch_nautilus_docs() { return fi - # Skip if nautilus-docs skill doesn't exist in this checkout - if [[ ! -f "$REPO_DIR/nautilus-docs/SKILL.md" ]]; then - return - fi - echo "Fetching NautilusTrader docs..." local temp @@ -204,6 +198,15 @@ fetch_nautilus_docs() { mv "$temp/docs" "$docs_dir" } +fetch_skill_deps() { + collect_skills + for skill in "${SKILLS_LIST[@]}"; do + if [[ "$skill" == "nautilus-docs" ]]; then + fetch_nautilus_docs + fi + done +} + collect_skills() { SKILLS_LIST=() if [[ -n "$SKILLS_CSV" ]]; then @@ -224,10 +227,10 @@ collect_skills() { install_codex() { ensure_repo + fetch_skill_deps mkdir -p "$TARGET" - collect_skills for skill in "${SKILLS_LIST[@]}"; do local src dst src="$REPO_DIR/$skill" @@ -288,6 +291,7 @@ uninstall_codex() { install_claude() { ensure_repo + fetch_skill_deps cat < Date: Sat, 21 Mar 2026 03:05:25 +0200 Subject: [PATCH 17/25] doc installation first --- nautilus-docs/SKILL.md | 6 ++++-- nautilus-docs/scripts/fetch-docs.sh | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100755 nautilus-docs/scripts/fetch-docs.sh diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 8aeb5a5..cd24426 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -8,7 +8,9 @@ description: > # NautilusTrader v1.224.0 -**MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `${CLAUDE_SKILL_DIR}/references/docs/` for the class/method/config name. Never guess signatures or constructors. +> **If `references/docs/` does not exist**, run `bash scripts/fetch-docs.sh` from this skill's directory before proceeding. + +**MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `references/docs/` for the class/method/config name. Never guess signatures or constructors. **MUST READ** before any NautilusTrader work: [architecture.md](references/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). @@ -92,7 +94,7 @@ description: > ### Search Strategy -When the navigator doesn't cover your need, grep `${CLAUDE_SKILL_DIR}/references/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. Read BEFORE writing code — never invent an API call. +When the navigator doesn't cover your need, grep `references/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. Read BEFORE writing code — never invent an API call. ## Supporting Files diff --git a/nautilus-docs/scripts/fetch-docs.sh b/nautilus-docs/scripts/fetch-docs.sh new file mode 100755 index 0000000..044b054 --- /dev/null +++ b/nautilus-docs/scripts/fetch-docs.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DOCS_DIR="$SCRIPT_DIR/../references/docs" + +if [ -d "$DOCS_DIR" ]; then + echo "Docs already present at $DOCS_DIR" + exit 0 +fi + +echo "Fetching NautilusTrader docs..." + +TEMP=$(mktemp -d) +trap 'rm -rf "$TEMP"' EXIT + +git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$TEMP" +git -C "$TEMP" sparse-checkout set docs/ + +rm -rf "$TEMP/docs/api_reference" +mv "$TEMP/docs" "$DOCS_DIR" + +echo "Done. Docs installed to $DOCS_DIR" From 9113f28d950d1359b7032c7863f80193686a623a Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Sat, 21 Mar 2026 03:43:35 +0200 Subject: [PATCH 18/25] user level skills -> marketplace (definitely need changes) --- README.md | 4 +- scripts/install.sh | 216 +++++++++++++++++++++++++++------------------ 2 files changed, 132 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 1962bb5..47132b2 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,12 @@ curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/script | Flag | Description | |------|-------------| | `--platform claude\|codex\|both` | Target platform | -| `--skills a,b,c` | Install only these skills (default: all) | +| `--skills a,b,c` | Install only these skills (default: all, codex only) | | `--repo-ref ` | Checkout specific ref | | `--repo ` | Custom repo URL | | `--clone-dir ` | Where to clone (default: `~/src/agent-system`) | | `--target ` | Codex skills dir (default: `~/.codex/skills`) | -| `--mode symlink\|copy` | Codex install mode | +| `--mode symlink\|copy` | Install mode (default: symlink) | | `--non-interactive` | Skip prompts | | `--uninstall` | Remove installed skills | diff --git a/scripts/install.sh b/scripts/install.sh index d640da6..b13c5ab 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -33,7 +33,7 @@ USAGE PLATFORM="codex" REPO_URL="https://github.com/deevsDeevs/agent-system.git" CLONE_DIR="$HOME/src/agent-system" -TARGET="${CODEX_HOME:-$HOME/.codex}/skills" +TARGET="" MODE="symlink" SKILLS_CSV="" SKILLS_LIST=() @@ -129,24 +129,25 @@ run_interactive() { fi if [[ "$PLATFORM" == "codex" || "$PLATFORM" == "both" ]]; then - prompt TARGET "Codex skills target dir" "$TARGET" prompt SKILLS_CSV "Skills (comma-separated, blank = all)" "$SKILLS_CSV" - local advanced="n" - ask_yn advanced "Show advanced options? (y/N)" "N" - if [[ "$advanced" == "y" || "$advanced" == "Y" ]]; then - prompt MODE "Mode (symlink|copy)" "$MODE" - - local repo_dir_input="" - prompt repo_dir_input "Local repo dir (blank to clone)" "$REPO_DIR" - if [[ -n "$repo_dir_input" ]]; then - REPO_DIR="$repo_dir_input" - else - prompt REPO_URL "Repo URL" "$REPO_URL" - prompt CLONE_DIR "Clone dir" "$CLONE_DIR" - fi - - prompt REPO_REF "Repo ref (tag/branch/commit, optional)" "$REPO_REF" + prompt TARGET "Codex skills target dir" "${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" + fi + + local advanced="n" + ask_yn advanced "Show advanced options? (y/N)" "N" + if [[ "$advanced" == "y" || "$advanced" == "Y" ]]; then + prompt MODE "Mode (symlink|copy)" "$MODE" + + local repo_dir_input="" + prompt repo_dir_input "Local repo dir (blank to clone)" "$REPO_DIR" + if [[ -n "$repo_dir_input" ]]; then + REPO_DIR="$repo_dir_input" + else + prompt REPO_URL "Repo URL" "$REPO_URL" + prompt CLONE_DIR "Clone dir" "$CLONE_DIR" fi + + prompt REPO_REF "Repo ref (tag/branch/commit, optional)" "$REPO_REF" fi } @@ -162,6 +163,7 @@ ensure_repo() { if [[ -z "$REPO_DIR" ]]; then if [[ -d "$CLONE_DIR/.git" ]]; then REPO_DIR="$CLONE_DIR" + git -C "$REPO_DIR" fetch --all --prune else git clone "$REPO_URL" "$CLONE_DIR" REPO_DIR="$CLONE_DIR" @@ -169,10 +171,16 @@ ensure_repo() { fi if [[ -n "$REPO_REF" ]]; then - git -C "$REPO_DIR" fetch --tags --prune git -C "$REPO_DIR" checkout "$REPO_REF" + git -C "$REPO_DIR" pull --ff-only 2>/dev/null || true fi +} +cleanup_repo() { + if [[ -d "$CLONE_DIR/.git" ]]; then + rm -rf "$CLONE_DIR" + echo "Removed cloned repo: $CLONE_DIR" + fi } fetch_nautilus_docs() { @@ -182,20 +190,21 @@ fetch_nautilus_docs() { return fi - echo "Fetching NautilusTrader docs..." - - local temp - temp=$(mktemp -d) - trap "rm -rf '$temp'" RETURN - - git clone --filter=blob:none --sparse --depth 1 \ - https://github.com/nautechsystems/nautilus_trader.git "$temp" - git -C "$temp" sparse-checkout set docs/ - - rm -rf "$temp/docs/api_reference" - - mkdir -p "$(dirname "$docs_dir")" - mv "$temp/docs" "$docs_dir" + local fetch_script="$REPO_DIR/nautilus-docs/scripts/fetch-docs.sh" + if [[ -x "$fetch_script" ]]; then + bash "$fetch_script" + else + echo "Fetching NautilusTrader docs..." + local temp + temp=$(mktemp -d) + trap "rm -rf '$temp'" RETURN + git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$temp" + git -C "$temp" sparse-checkout set docs/ + rm -rf "$temp/docs/api_reference" + mkdir -p "$(dirname "$docs_dir")" + mv "$temp/docs" "$docs_dir" + fi } fetch_skill_deps() { @@ -229,6 +238,7 @@ install_codex() { ensure_repo fetch_skill_deps + TARGET="${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" mkdir -p "$TARGET" for skill in "${SKILLS_LIST[@]}"; do @@ -250,79 +260,113 @@ install_codex() { printf '\nDone. Restart Codex to reload skills.\n' } -uninstall_codex() { - if [[ -z "$SKILLS_CSV" ]]; then - ensure_repo - elif [[ -n "$REPO_REF" ]]; then - echo "Warning: --repo-ref is ignored when --skills is provided for uninstall." >&2 +uninstall_from() { + local target_dir="$1" + + if [[ ! -d "$target_dir" ]]; then + echo "Nothing to uninstall — $target_dir does not exist." + return fi - collect_skills - for skill in "${SKILLS_LIST[@]}"; do - local dst link_target - dst="$TARGET/$skill" + local skills_to_remove=() + if [[ -n "$SKILLS_CSV" ]]; then + IFS=',' read -r -a skills_to_remove <<< "$SKILLS_CSV" + else + for entry in "$target_dir"/*/; do + [[ -f "${entry}SKILL.md" ]] && skills_to_remove+=("$(basename "$entry")") + done + fi + + for skill_name in "${skills_to_remove[@]}"; do + local dst="$target_dir/$skill_name" if [[ -L "$dst" ]]; then - if [[ -n "$REPO_DIR" ]]; then - link_target="$(readlink "$dst")" - if [[ "$link_target" == "$REPO_DIR"* ]]; then - rm -f "$dst" - echo "Removed $dst" - else - echo "Skipping $dst (symlink not pointing to repo)" >&2 - fi - else - rm -f "$dst" - echo "Removed $dst" - fi - elif [[ -d "$dst" ]]; then - if [[ -f "$dst/SKILL.md" ]]; then - rm -rf "$dst" - echo "Removed $dst" - else - echo "Skipping $dst (no SKILL.md)" >&2 - fi + rm -f "$dst" + echo "Removed $dst" + elif [[ -d "$dst" && -f "$dst/SKILL.md" ]]; then + rm -rf "$dst" + echo "Removed $dst" else echo "Not found: $dst" fi done +} +uninstall_codex() { + TARGET="${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" + uninstall_from "$TARGET" + cleanup_repo printf '\nDone. Restart Codex to reload skills.\n' } install_claude() { - ensure_repo - fetch_skill_deps + collect_skills + clean_plugin_cache + + local claude_target="$HOME/.claude/skills" + local plugin_skills=() + local local_skills=() - cat < $dst" + done + fi -/plugin marketplace add ${REPO_URL}${REPO_REF:+#${REPO_REF}} -/plugin install chain-system@deevs-agent-system -/plugin install dev-experts@deevs-agent-system -/plugin install bug-hunters@deevs-agent-system -/plugin install research-experts@deevs-agent-system -/plugin install cost-status@deevs-agent-system -/plugin install arxiv-search@deevs-agent-system + # Regular skills: plugin system handles them + if [[ ${#plugin_skills[@]} -gt 0 ]]; then + local ref_suffix="${REPO_REF:+#${REPO_REF}}" + printf '\nRun inside Claude Code:\n\n' + printf ' /plugin marketplace add %s%s\n\n' "$REPO_URL" "$ref_suffix" + for skill in "${plugin_skills[@]}"; do + printf ' /plugin install %s@deevs-agent-system\n' "$(basename "$skill")" + done + printf '\n' + fi -Note: These are Claude Code commands and will not run in a shell. -Repo cloned to: ${REPO_DIR} (docs fetched) -EOC + printf 'Done. Restart Claude Code to reload skills.\n' } uninstall_claude() { - cat <<'EOC' -Run the following commands inside Claude Code: - -/plugin uninstall chain-system@deevs-agent-system -/plugin uninstall dev-experts@deevs-agent-system -/plugin uninstall bug-hunters@deevs-agent-system -/plugin uninstall research-experts@deevs-agent-system -/plugin uninstall cost-status@deevs-agent-system -/plugin uninstall arxiv-search@deevs-agent-system - -Note: These are Claude Code commands and will not run in a shell. -EOC + uninstall_from "$HOME/.claude/skills" + clean_plugin_cache + cleanup_repo + printf '\nDone. Restart Claude Code to reload.\n' +} + +clean_plugin_cache() { + local plugins_dir="$HOME/.claude/plugins" + local marketplace="deevs-agent-system" + if [[ -d "$plugins_dir/cache/$marketplace" || -d "$plugins_dir/marketplaces/$marketplace" ]]; then + rm -rf "$plugins_dir/cache/$marketplace" "$plugins_dir/marketplaces/$marketplace" + if command -v jq &>/dev/null; then + [[ -f "$plugins_dir/installed_plugins.json" ]] && \ + jq ".plugins |= with_entries(select(.key | test(\"@${marketplace}\$\") | not))" \ + "$plugins_dir/installed_plugins.json" > "$plugins_dir/installed_plugins.json.tmp" && \ + mv "$plugins_dir/installed_plugins.json.tmp" "$plugins_dir/installed_plugins.json" + [[ -f "$plugins_dir/known_marketplaces.json" ]] && \ + jq "del(.\"$marketplace\")" \ + "$plugins_dir/known_marketplaces.json" > "$plugins_dir/known_marketplaces.json.tmp" && \ + mv "$plugins_dir/known_marketplaces.json.tmp" "$plugins_dir/known_marketplaces.json" + fi + echo "Cleaned old plugin cache" + fi } case "$PLATFORM" in From 4ac0eff0bb34183b6ef9690e5fe374c8db355fc4 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Sat, 21 Mar 2026 15:21:30 +0200 Subject: [PATCH 19/25] local plugin installation --- scripts/install.sh | 70 +++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 529b84a..26eba90 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -31,14 +31,14 @@ USAGE } PLATFORM="codex" -REPO_URL="https://github.com/deevsDeevs/agent-system.git" +REPO_URL="https://github.com/DeevsDeevs/agent-system.git" CLONE_DIR="$HOME/src/agent-system" TARGET="" MODE="symlink" SKILLS_CSV="" SKILLS_LIST=() REPO_DIR="" -REPO_REF="" +REPO_REF="nautilus-v2" UNINSTALL="false" NON_INTERACTIVE="false" @@ -128,8 +128,8 @@ run_interactive() { UNINSTALL="false" fi + prompt SKILLS_CSV "Skills (comma-separated, blank = all)" "$SKILLS_CSV" if [[ "$PLATFORM" == "codex" || "$PLATFORM" == "both" ]]; then - prompt SKILLS_CSV "Skills (comma-separated, blank = all)" "$SKILLS_CSV" prompt TARGET "Codex skills target dir" "${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" fi @@ -208,9 +208,8 @@ fetch_nautilus_docs() { } fetch_skill_deps() { - collect_skills for skill in "${SKILLS_LIST[@]}"; do - if [[ "$skill" == "nautilus-docs" ]]; then + if [[ "$(basename "$skill")" == "nautilus-docs" ]]; then fetch_nautilus_docs fi done @@ -238,6 +237,7 @@ collect_skills() { install_codex() { ensure_repo + collect_skills fetch_skill_deps TARGET="${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" @@ -301,60 +301,36 @@ uninstall_codex() { } install_claude() { + ensure_repo collect_skills - clean_plugin_cache + fetch_skill_deps - local claude_target="$HOME/.claude/skills" - local plugin_skills=() - local local_skills=() + local clone_abs marketplace + clone_abs="$(cd "$REPO_DIR" && pwd)" + marketplace="$(basename "$clone_abs")" + clean_plugin_cache "$marketplace" + + printf 'Run inside Claude Code:\n\n' + printf ' /plugin marketplace add %s\n\n' "$clone_abs" for skill in "${SKILLS_LIST[@]}"; do - if [[ "$skill" == "nautilus-docs" ]]; then - local_skills+=("$skill") - else - plugin_skills+=("$skill") - fi + printf ' /plugin install %s@%s\n' "$(basename "$skill")" "$marketplace" done - - # Skills with external deps: need the repo + docs fetch + symlink - if [[ ${#local_skills[@]} -gt 0 ]]; then - ensure_repo - fetch_nautilus_docs - mkdir -p "$claude_target" - for skill in "${local_skills[@]}"; do - local src skill_name dst - src="$REPO_DIR/$skill" - skill_name="$(basename "$skill")" - dst="$claude_target/$skill_name" - ln -sfn "$src" "$dst" - echo "Installed $skill_name -> $dst" - done - fi - - # Regular skills: plugin system handles them - if [[ ${#plugin_skills[@]} -gt 0 ]]; then - local ref_suffix="${REPO_REF:+#${REPO_REF}}" - printf '\nRun inside Claude Code:\n\n' - printf ' /plugin marketplace add %s%s\n\n' "$REPO_URL" "$ref_suffix" - for skill in "${plugin_skills[@]}"; do - printf ' /plugin install %s@deevs-agent-system\n' "$(basename "$skill")" - done - printf '\n' - fi - - printf 'Done. Restart Claude Code to reload skills.\n' + printf '\nDone. Restart Claude Code to reload skills.\n' } uninstall_claude() { - uninstall_from "$HOME/.claude/skills" - clean_plugin_cache + local marketplace + marketplace="$(basename "$CLONE_DIR")" + clean_plugin_cache "$marketplace" cleanup_repo - printf '\nDone. Restart Claude Code to reload.\n' + printf '\nDone. Remove any installed plugins inside Claude Code with:\n' + printf ' /plugin marketplace remove %s\n' "$marketplace" } clean_plugin_cache() { + local marketplace="$1" local plugins_dir="$HOME/.claude/plugins" - local marketplace="deevs-agent-system" if [[ -d "$plugins_dir/cache/$marketplace" || -d "$plugins_dir/marketplaces/$marketplace" ]]; then rm -rf "$plugins_dir/cache/$marketplace" "$plugins_dir/marketplaces/$marketplace" if command -v jq &>/dev/null; then @@ -367,7 +343,7 @@ clean_plugin_cache() { "$plugins_dir/known_marketplaces.json" > "$plugins_dir/known_marketplaces.json.tmp" && \ mv "$plugins_dir/known_marketplaces.json.tmp" "$plugins_dir/known_marketplaces.json" fi - echo "Cleaned old plugin cache" + echo "Cleaned plugin cache for $marketplace" fi } From ab0d143db9ad5f78832298176241070a7a5c47b1 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Sat, 21 Mar 2026 16:01:23 +0200 Subject: [PATCH 20/25] no unnecessary scripts --- nautilus-docs/SKILL.md | 2 -- nautilus-docs/scripts/fetch-docs.sh | 24 ------------------------ scripts/install.sh | 27 +++++++++++---------------- 3 files changed, 11 insertions(+), 42 deletions(-) delete mode 100755 nautilus-docs/scripts/fetch-docs.sh diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index cd24426..95617de 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -8,8 +8,6 @@ description: > # NautilusTrader v1.224.0 -> **If `references/docs/` does not exist**, run `bash scripts/fetch-docs.sh` from this skill's directory before proceeding. - **MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `references/docs/` for the class/method/config name. Never guess signatures or constructors. **MUST READ** before any NautilusTrader work: [architecture.md](references/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). diff --git a/nautilus-docs/scripts/fetch-docs.sh b/nautilus-docs/scripts/fetch-docs.sh deleted file mode 100755 index 044b054..0000000 --- a/nautilus-docs/scripts/fetch-docs.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -DOCS_DIR="$SCRIPT_DIR/../references/docs" - -if [ -d "$DOCS_DIR" ]; then - echo "Docs already present at $DOCS_DIR" - exit 0 -fi - -echo "Fetching NautilusTrader docs..." - -TEMP=$(mktemp -d) -trap 'rm -rf "$TEMP"' EXIT - -git clone --filter=blob:none --sparse --depth 1 \ - https://github.com/nautechsystems/nautilus_trader.git "$TEMP" -git -C "$TEMP" sparse-checkout set docs/ - -rm -rf "$TEMP/docs/api_reference" -mv "$TEMP/docs" "$DOCS_DIR" - -echo "Done. Docs installed to $DOCS_DIR" diff --git a/scripts/install.sh b/scripts/install.sh index 26eba90..2b42eb2 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -38,7 +38,7 @@ MODE="symlink" SKILLS_CSV="" SKILLS_LIST=() REPO_DIR="" -REPO_REF="nautilus-v2" +REPO_REF="" UNINSTALL="false" NON_INTERACTIVE="false" @@ -190,21 +190,16 @@ fetch_nautilus_docs() { return fi - local fetch_script="$REPO_DIR/nautilus-docs/scripts/fetch-docs.sh" - if [[ -x "$fetch_script" ]]; then - bash "$fetch_script" - else - echo "Fetching NautilusTrader docs..." - local temp - temp=$(mktemp -d) - trap "rm -rf '$temp'" RETURN - git clone --filter=blob:none --sparse --depth 1 \ - https://github.com/nautechsystems/nautilus_trader.git "$temp" - git -C "$temp" sparse-checkout set docs/ - rm -rf "$temp/docs/api_reference" - mkdir -p "$(dirname "$docs_dir")" - mv "$temp/docs" "$docs_dir" - fi + echo "Fetching NautilusTrader docs..." + local temp + temp=$(mktemp -d) + trap "rm -rf '$temp'" RETURN + git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$temp" + git -C "$temp" sparse-checkout set docs/ + rm -rf "$temp/docs/api_reference" + mkdir -p "$(dirname "$docs_dir")" + mv "$temp/docs" "$docs_dir" } fetch_skill_deps() { From 4fcd0e7fa07c1a8588fa59fb5e90ecefbe889193 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Mon, 23 Mar 2026 22:03:58 +0200 Subject: [PATCH 21/25] fix: add unsubscribe in on_stop and complete Rust Strategy example Prevents dangling handlers in the actor registry after shutdown. Without unsubscribe, closures fire through unsafe get_actor_unchecked on a dead actor. Matches the canonical EmaCross pattern. Co-Authored-By: Claude Opus 4.6 --- nautilus-docs/SKILL.md | 5 ++++- nautilus-docs/references/battle_tested.md | 21 +++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 95617de..97b0222 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -185,7 +185,10 @@ impl DataActor for MyActor { log::info!("px={} qty={}", trade.price.as_f64(), trade.size.as_f64()); Ok(()) } - fn on_stop(&mut self) -> anyhow::Result<()> { Ok(()) } // MUST implement — warns if missing + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_trades(self.instrument_id, None, None); + Ok(()) + } } // Constructor: diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 13b8b04..28a4d12 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -229,12 +229,16 @@ Pre-aggregated top 10 levels — lower overhead than full book subscription for DataActor is data-only. For submitting orders, use `Strategy` from `nautilus-trading`: ```rust +use std::ops::{Deref, DerefMut}; use nautilus_common::actor::{DataActor, DataActorCore}; +use nautilus_model::data::TradeTick; +use nautilus_model::identifiers::InstrumentId; use nautilus_trading::strategy::{Strategy, StrategyConfig, StrategyCore}; #[derive(Debug)] struct MyStrategy { core: StrategyCore, // wraps DataActorCore internally + instrument_id: InstrumentId, } // CRITICAL: Deref target is DataActorCore, NOT StrategyCore. @@ -248,10 +252,19 @@ impl DerefMut for MyStrategy { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.core } } -// DataActor impl — data callbacks + order submission impl DataActor for MyStrategy { - fn on_start(&mut self) -> anyhow::Result<()> { Ok(()) } - fn on_stop(&mut self) -> anyhow::Result<()> { Ok(()) } + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_trades(self.instrument_id, None, None); + Ok(()) + } + fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { + log::info!("trade px={}", trade.price.as_f64()); + Ok(()) + } + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_trades(self.instrument_id, None, None); + Ok(()) + } } impl Strategy for MyStrategy { @@ -260,7 +273,7 @@ impl Strategy for MyStrategy { } ``` -Constructor: `StrategyCore::new(StrategyConfig { strategy_id: Some("MY-001".into()), ..Default::default() })`. +Constructor: `MyStrategy { core: StrategyCore::new(StrategyConfig { strategy_id: Some("MY-001".into()), ..Default::default() }), instrument_id }`. Key event fields: - `PositionOpened`: `entry` (OrderSide), `side` (PositionSide), `quantity`, `last_px`, `avg_px_open`, `currency` From 3982027e599f9e390cea8fb6c398fffff15afe2c Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Wed, 25 Mar 2026 01:49:27 +0200 Subject: [PATCH 22/25] docs: add debugging playbook from live loader session lessons Distills hard-won debugging lessons from a Binance+Bybit L2 book/trades/funding loader session: read response bodies first, per-product API keys, encrypted Ed25519, SBE schema versioning, and the funding rate workaround. Adds guard rail against cloning/modifying NautilusTrader source. Co-Authored-By: Claude Opus 4.6 --- nautilus-docs/SKILL.md | 4 +- nautilus-docs/references/battle_tested.md | 58 +++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 97b0222..a3cb6a8 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -12,6 +12,8 @@ description: > **MUST READ** before any NautilusTrader work: [architecture.md](references/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). +**NEVER clone or modify NautilusTrader source**. Use extension points (DataActor/Strategy), `[patch]` in Cargo.toml for adapter fixes, or the built-in data catalog for persistence. See [battle_tested.md § Debugging Live Issues](references/battle_tested.md) for the full debugging playbook. + ## Doc Navigator ### Concepts @@ -96,7 +98,7 @@ When the navigator doesn't cover your need, grep `references/docs/` for the clas ## Supporting Files -- **[battle_tested.md](references/battle_tested.md)** — Load when: Rust Strategy pattern, credentials, patching deps, on_start() ordering, market making, backtest config, venue gotchas, backtest→live migration, hot path performance, state persistence, fill/latency metrics, log rotation, custom indicators, data pipeline validation. +- **[battle_tested.md](references/battle_tested.md)** — Load when: **debugging live issues** (step 1: read the error body), Rust Strategy pattern, credentials, patching deps, on_start() ordering, market making, backtest config, venue gotchas, backtest→live migration, hot path performance, state persistence, fill/latency metrics, log rotation, custom indicators, data pipeline validation. - **[REBUILD.md](references/REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. ## Rust Standalone Binary diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 28a4d12..f03ce08 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -222,6 +222,64 @@ Pre-aggregated top 10 levels — lower overhead than full book subscription for - No `modify_order` support - Binary outcomes only — `YES`/`NO` tokens +## Debugging Live Issues + +### Step 1: Read the actual error + +Most wasted debugging time comes from treating errors as opaque. The server is almost always telling you exactly what's wrong. + +```bash +# HTTP errors: ALWAYS read the response body, not just the status code +curl -v 'https://fapi.binance.com/fapi/v1/...' -H 'X-MBX-APIKEY: ...' 2>&1 + +# Nautilus logs: grep ERROR first +grep "ERROR" logs/trader.log | head -20 + +# Rust: use {:?} (Debug), not {} (Display) — Debug includes inner error chain +log::error!("Failed: {error:?}"); # shows full chain +log::error!("Failed: {error}"); # may hide root cause +``` + +If a Nautilus error says "400 Bad Request" — **that is not enough information to debug**. Get the response body. One `curl -v` is worth 10 rounds of guessing. + +### Step 2: Venue-specific API key rules + +Binance has **per-product API keys**. A futures Ed25519 key is NOT valid for spot. Check: +- Spot key → spot endpoints only +- Futures/Linear key → futures endpoints only +- You may have separate keys in `.env`: `BINANCE_SPOT_API_KEY`, `BINANCE_LINEAR_API_KEY` + +**Ed25519 encrypted keys**: Binance Ed25519 private keys can be password-encrypted (PKCS#8). NautilusTrader's `SigningCredential` needs the unencrypted key. If your `.env` has a `_DECRYPTED` variant, use that. + +### Step 3: NEVER modify NautilusTrader source + +When something doesn't work (missing feature, adapter bug, data type not supported): + +1. **Use extension points**: wrap missing functionality in a DataActor or Strategy +2. **Use `[patch]` in Cargo.toml**: fork, fix, point to your branch (see "Patching Nautilus dependencies" below) +3. **Use the data catalog**: `nautilus-persistence` writes Parquet natively — don't write custom Parquet loaders + +**DO NOT**: clone the NautilusTrader repo, modify adapter source, and build locally. This creates an unmaintainable fork that diverges immediately and breaks on the next update. + +### Common runtime traps + +| Symptom | Cause | Fix | +|---------|-------|-----| +| "Invalid symbol" on subscribe | Symbol doesn't exist on that venue/product | Check symbol list — not all coins have futures (SHIB, PEPE, LEO) | +| "Invalid X-MBX-APIKEY header" | Wrong API key for the endpoint (spot key on futures) | Use the correct per-product key | +| WebSocket reconnecting repeatedly | Too many subscriptions per connection (Binance limit ~1024 streams) | Reduce symbols or split across connections | +| Signature validation error | Ed25519 key misdetected as HMAC, or encrypted key used | Check `SigningCredential` detection, use unencrypted key | +| SBE decode failure | Exchange upgraded schema version (e.g. v2→v3) | Check exchange API changelog, update Nautilus | +| 0 instruments loaded | Missing `load_ids` in `InstrumentProviderConfig` | Add `load_ids=frozenset({"SYMBOL.VENUE"})` | +| Catalog empty after run | Run too short for feather flush, or wrong catalog path | Run longer (>60s), check `catalog_path` in config | +| `subscribe_funding_rates()` no data | Not all adapters implement the feed (Binance doesn't stream it natively) | Use `BinanceFuturesMarkPriceUpdate` which contains funding_rate | + +### Binance funding rate workaround + +Binance doesn't stream funding rates as a separate feed. They're embedded in `BinanceFuturesMarkPriceUpdate`. To capture: +- Subscribe to mark price updates — funding rate arrives as a field +- Or poll the REST endpoint via a timer-based actor (`queue_for_executor` + async HTTP) + ## Rust Live Trading ### Strategy pattern (for order management) From d148b7dff903640b1fe5d568b2405a4dab7006fe Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Wed, 25 Mar 2026 03:06:29 +0200 Subject: [PATCH 23/25] docs: replace blanket "NEVER modify" with extension decision tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the absolute prohibition with a 4-category decision tree: compose via DataActor/Strategy (90%), [patch] for bugs, fork+modify for upstream PR, clone to read source. Fixes incorrect MarkPriceUpdate example — uses first-class subscribe_funding_rates/on_funding_rate path verified against actual DataActor trait source. Co-Authored-By: Claude Opus 4.6 --- nautilus-docs/SKILL.md | 2 +- nautilus-docs/references/battle_tested.md | 101 +++++++++++++++++++--- 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index a3cb6a8..7cf218b 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -12,7 +12,7 @@ description: > **MUST READ** before any NautilusTrader work: [architecture.md](references/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). -**NEVER clone or modify NautilusTrader source**. Use extension points (DataActor/Strategy), `[patch]` in Cargo.toml for adapter fixes, or the built-in data catalog for persistence. See [battle_tested.md § Debugging Live Issues](references/battle_tested.md) for the full debugging playbook. +**Do not build on a private NautilusTrader fork**. Extend via composition (DataActor/Strategy traits), use `[patch]` for temporary bug fixes (with upstream PR), or fork+modify only when contributing back. Cloning to read source or run tests is fine. See [battle_tested.md § Step 3](references/battle_tested.md) for the full decision tree. ## Doc Navigator diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index f03ce08..61dc95d 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -251,15 +251,92 @@ Binance has **per-product API keys**. A futures Ed25519 key is NOT valid for spo **Ed25519 encrypted keys**: Binance Ed25519 private keys can be password-encrypted (PKCS#8). NautilusTrader's `SigningCredential` needs the unencrypted key. If your `.env` has a `_DECRYPTED` variant, use that. -### Step 3: NEVER modify NautilusTrader source +### Step 3: Extending NautilusTrader without modifying source -When something doesn't work (missing feature, adapter bug, data type not supported): +When NautilusTrader is missing a feature or an adapter does not behave the way you need, follow this decision tree. The architecture smell to avoid is building on a private fork without upstreaming -- not cloning the repo itself. Cloning to read source, run tests, or prepare a PR is perfectly normal. -1. **Use extension points**: wrap missing functionality in a DataActor or Strategy -2. **Use `[patch]` in Cargo.toml**: fork, fix, point to your branch (see "Patching Nautilus dependencies" below) -3. **Use the data catalog**: `nautilus-persistence` writes Parquet natively — don't write custom Parquet loaders +**Decision tree -- "I need something NautilusTrader does not provide":** -**DO NOT**: clone the NautilusTrader repo, modify adapter source, and build locally. This creates an unmaintainable fork that diverges immediately and breaks on the next update. +#### 1. Compose via DataActor or Strategy (first choice, 90% of cases) + +NautilusTrader's extension surface is trait-based. `DataActor` and `Strategy` are traits you implement on your own structs. They give you lifecycle callbacks (`on_start`, `on_stop`), data callbacks (`on_trade`, `on_quote`, `on_mark_price`, `on_book_deltas`, `on_funding_rate`, `on_data`, ...), timer events, and the signal/custom-data pipeline. This is composition, not inheritance -- your struct owns a `DataActorCore` or `StrategyCore` and delegates via `Deref`. + +Use this when: +- An adapter streams data you need to reshape (e.g., use `subscribe_funding_rates` / `on_funding_rate` for funding data — the adapter handles the underlying source format) +- You need to combine multiple data feeds into a derived signal +- You need periodic REST polling (timer + async HTTP in Python via `queue_for_executor`, or timer + `reqwest` in a spawned task in Rust) +- You want to publish custom data types for downstream consumers + +Rust example -- subscribing to funding rates: + +```rust +use nautilus_model::data::FundingRateUpdate; + +impl DataActor for FundingRateCapture { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_funding_rates(self.instrument_id, None, None); + Ok(()) + } + + fn on_funding_rate(&mut self, update: &FundingRateUpdate) -> anyhow::Result<()> { + // FundingRateUpdate has: instrument_id, rate (Decimal), interval, next_funding_ns + log::info!("{} rate={}", update.instrument_id, update.rate); + Ok(()) + } + + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_funding_rates(self.instrument_id, None, None); + Ok(()) + } +} +``` + +Note: whether funding rate data actually arrives depends on adapter support. Binance embeds it in mark price stream at the adapter level — the DataActor callback fires if the adapter parses it out. If the adapter doesn't support it, subscribe to mark prices as a fallback and check what fields arrive. + +This is idiomatic Rust composition: your struct implements the trait, the runtime calls your callbacks. No source modification needed. + +#### 2. Cargo `[patch]` for adapter bugs (temporary fix while upstreaming) + +If an adapter has an actual bug (wrong signature encoding, missing WebSocket message handling, incorrect field parsing), fork on GitHub, fix the bug on a branch, and point your `Cargo.toml` at your fork: + +```toml +[patch.'https://github.com/nautechsystems/nautilus_trader.git'] +nautilus-binance = { git = "https://github.com/YOUR-USER/nautilus_trader.git", branch = "fix/binance-sig-encoding" } +``` + +Only list the crate(s) you modified. Cargo resolves transitive deps automatically. This is not a private fork -- it is a temporary patch with a clear intent to upstream via PR. + +Rules: +- The branch name should describe the fix (`fix/binance-ws-reconnect`, not `my-changes`) +- Open an upstream PR immediately or as soon as the fix is validated +- Remove the `[patch]` once the fix is merged and released +- Never use `[patch]` for feature additions -- that belongs in category 3 + +#### 3. Fork + modify for upstream contribution (the right way to add features) + +When NautilusTrader genuinely lacks functionality that belongs in the core (new adapter message type, new data feed, new order type support), the correct path is: + +1. Clone the repo +2. Implement the feature on a branch +3. Run the existing test suite, add tests for your change +4. Open a PR upstream +5. Use `[patch]` to consume your branch while the PR is in review + +This is how open source works. The smell is NOT cloning or modifying source -- it is building on a local fork without upstreaming. A private fork with adapter modifications that you never PR diverges immediately and breaks on the next Nautilus update. + +#### 4. Cloning to read source or run tests (always fine) + +Cloning the NautilusTrader repo to read adapter source, understand internal behavior, run tests, or check trait signatures is completely normal development practice. The repo is the best documentation for edge cases. + +#### What NOT to do + +Do not clone the repo, modify adapter internals to add missing functionality, build locally, and treat that as your production dependency. This is the architecture smell. Concretely, the failure mode looks like: + +1. "Binance does not stream funding rates" -- correct observation +2. Clone repo, add funding rate streaming to the Binance adapter -- wrong response +3. Build from local path, deploy -- unmaintainable; breaks on next Nautilus version + +The correct response to step 1 is category 1 above: write a `DataActor` that subscribes to `MarkPriceUpdate` (which contains funding rate data) and extracts what you need. No source modification required. ### Common runtime traps @@ -272,13 +349,11 @@ When something doesn't work (missing feature, adapter bug, data type not support | SBE decode failure | Exchange upgraded schema version (e.g. v2→v3) | Check exchange API changelog, update Nautilus | | 0 instruments loaded | Missing `load_ids` in `InstrumentProviderConfig` | Add `load_ids=frozenset({"SYMBOL.VENUE"})` | | Catalog empty after run | Run too short for feather flush, or wrong catalog path | Run longer (>60s), check `catalog_path` in config | -| `subscribe_funding_rates()` no data | Not all adapters implement the feed (Binance doesn't stream it natively) | Use `BinanceFuturesMarkPriceUpdate` which contains funding_rate | +| `subscribe_funding_rates()` no data | Not all adapters implement the feed (Binance doesn't stream it natively) | Use `MarkPriceUpdate` which contains funding_rate (see Step 3 above) | -### Binance funding rate workaround +### Binance funding rate -Binance doesn't stream funding rates as a separate feed. They're embedded in `BinanceFuturesMarkPriceUpdate`. To capture: -- Subscribe to mark price updates — funding rate arrives as a field -- Or poll the REST endpoint via a timer-based actor (`queue_for_executor` + async HTTP) +In Rust, `subscribe_funding_rates` / `on_funding_rate` is the first-class path — the Binance adapter parses funding from its mark price stream internally. In Python, funding arrives via `BinanceFuturesMarkPriceUpdate` (contains mark, index, AND funding_rate). As a fallback, poll the REST endpoint via a timer-based actor (`queue_for_executor` + async HTTP). ## Rust Live Trading @@ -354,14 +429,14 @@ Ed25519 signatures are base64 containing `+`, `/`, `=`. Some adapter HTTP client ### Patching Nautilus dependencies -Fork on GitHub, push fix to a branch, use Cargo `[patch]`: +See "Step 3: Extending NautilusTrader without modifying source" above for the full decision tree. Quick reference for Cargo `[patch]`: ```toml [patch.'https://github.com/nautechsystems/nautilus_trader.git'] nautilus-{venue} = { git = "https://github.com/YOUR-USER/nautilus_trader.git", branch = "fix/my-fix" } ``` -Only list the crate(s) you modified. Cargo resolves transitive deps automatically. Submit upstream PR when the fix is general. +Only list the crate(s) you modified. Cargo resolves transitive deps automatically. Open an upstream PR, then remove the `[patch]` once the fix is merged and released. ### Rust adapter maturity gaps From 89ea3558cd8380c94ba20063fd331bd1cdde9c52 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Thu, 2 Apr 2026 17:16:45 +0300 Subject: [PATCH 24/25] vibe shit. todo:mv+debug+remove Co-Authored-By: Claude Opus 4.6 --- .gitignore | 6 +- nautilus-docs/SKILL.md | 3 +- nautilus-docs/references/REBUILD.md | 62 +++--- nautilus-docs/references/battle_tested.md | 8 +- tests/nautilus-verify/Cargo.toml | 13 ++ tests/nautilus-verify/VERSION | 1 + tests/nautilus-verify/pyproject.toml | 7 + tests/nautilus-verify/python/conftest.py | 3 + .../python/test_anti_hallucination.py | 207 ++++++++++++++++++ .../python/test_backtest_config.py | 43 ++++ .../python/test_constructors.py | 91 ++++++++ .../python/test_custom_data.py | 42 ++++ .../python/test_custom_indicator.py | 50 +++++ .../nautilus-verify/python/test_order_book.py | 32 +++ tests/nautilus-verify/src/lib.rs | 2 + .../tests/anti_hallucination.rs | 152 +++++++++++++ tests/nautilus-verify/tests/crate_map.rs | 120 ++++++++++ .../tests/data_actor_pattern.rs | 63 ++++++ .../nautilus-verify/tests/strategy_pattern.rs | 81 +++++++ tests/nautilus-verify/verify.sh | 47 ++++ 20 files changed, 989 insertions(+), 44 deletions(-) create mode 100644 tests/nautilus-verify/Cargo.toml create mode 100644 tests/nautilus-verify/VERSION create mode 100644 tests/nautilus-verify/pyproject.toml create mode 100644 tests/nautilus-verify/python/conftest.py create mode 100644 tests/nautilus-verify/python/test_anti_hallucination.py create mode 100644 tests/nautilus-verify/python/test_backtest_config.py create mode 100644 tests/nautilus-verify/python/test_constructors.py create mode 100644 tests/nautilus-verify/python/test_custom_data.py create mode 100644 tests/nautilus-verify/python/test_custom_indicator.py create mode 100644 tests/nautilus-verify/python/test_order_book.py create mode 100644 tests/nautilus-verify/src/lib.rs create mode 100644 tests/nautilus-verify/tests/anti_hallucination.rs create mode 100644 tests/nautilus-verify/tests/crate_map.rs create mode 100644 tests/nautilus-verify/tests/data_actor_pattern.rs create mode 100644 tests/nautilus-verify/tests/strategy_pattern.rs create mode 100755 tests/nautilus-verify/verify.sh diff --git a/.gitignore b/.gitignore index 31b173e..42eba18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ .claude/ .tmp/ .env -nautilus-docs/references/docs/ \ No newline at end of file +nautilus-docs/references/docs/ +tests/nautilus-verify/target/ +tests/nautilus-verify/.pytest_cache/ +tests/nautilus-verify/Cargo.lock +__pycache__/ \ No newline at end of file diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index 7cf218b..de3fe59 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -205,6 +205,7 @@ impl MyActor { ``` ## Common Hallucinations + ### Python @@ -232,7 +233,7 @@ impl MyActor { | `on_timer()` as callback | `clock.set_timer(callback=handler)` | | `order.order_side` | `order.side` — events use `event.order_side` | | `request_bars(bar_type)` one arg | Requires `start`: `request_bars(bar_type, start=datetime(...))` | -| `GreeksCalculator(cache, clock, logger)` | Only 2 args: `GreeksCalculator(cache, clock)` | +| `GreeksCalculator(cache, clock, logger)` | Only 2 args: `GreeksCalculator(cache, clock)`. Import from `nautilus_trader.model.greeks`, not `indicators.greeks_calculator` | | `SyntheticInstrument(sym, prec, comps, formula)` | 6 required args — also needs `ts_event`, `ts_init` | | `from nautilus_trader.core.nautilus_pyo3` wrong path | `from nautilus_trader.core.nautilus_pyo3 import black_scholes_greeks` | | `BacktestEngine.add_venue(venue=BacktestVenueConfig(...))` | Takes positional args. `BacktestVenueConfig` is for `BacktestRunConfig` only | diff --git a/nautilus-docs/references/REBUILD.md b/nautilus-docs/references/REBUILD.md index 9d0cd75..2970aec 100644 --- a/nautilus-docs/references/REBUILD.md +++ b/nautilus-docs/references/REBUILD.md @@ -80,46 +80,33 @@ grep -r "pub fn builder" "$CHECKOUT/crates/live/src/" grep -A 20 "pub struct BinanceDataClientConfig" "$CHECKOUT/crates/adapters/binance/src/config.rs" ``` -## Step 4: Rebuild Anti-Hallucination Tables +## Step 4: Run the Verification Suite -This is the highest-value part of the skill. Each row prevents a specific mistake Claude makes repeatedly. +The `tests/nautilus-verify/` suite mechanically tests every hallucination row and pattern: -**How to maintain**: - -1. **Test existing rows** — for each hallucination, verify the "Reality" column is still correct: - ```python - # Python hallucinations — test in a venv with nautilus_trader installed - from nautilus_trader.common.actor import Actor # still correct? - from nautilus_trader.indicators import ExponentialMovingAverage # still correct? - ``` +```bash +# Run everything (Rust compile tests + Python tests) +./tests/nautilus-verify/verify.sh -2. **Discover new hallucinations** — ask Claude to write code for common tasks WITHOUT the skill, then diff against what actually compiles/runs. Common sources: - - New config struct fields that Claude will guess wrong - - Renamed enums or moved types - - Changed method signatures (added/removed args) - - New features that Claude will confuse with old patterns +# Or individually: +cd tests/nautilus-verify && cargo test # Rust: 22 tests +cd tests/nautilus-verify && ../../.venv/bin/python -m pytest python/ -v # Python: 50 tests +``` -3. **Remove stale rows** — if a hallucinated API now exists (e.g., a missing method was added), remove the row. +Failures = stale rows. Fix the test to match the new API, then update the corresponding +hallucination table row in SKILL.md and patterns in battle_tested.md. -4. **Rust hallucinations** — rebuild by attempting to compile a standalone binary: - ```bash - cd my_trading_system && cargo build 2>&1 | grep "error\[E" - ``` - Every compilation error that comes from a wrong import path, missing trait, or wrong struct field is a hallucination row candidate. +## Step 5: Discover New Hallucinations -## Step 5: Verify Working Patterns +Ask Claude to write code for common tasks WITHOUT the skill, then diff against what +actually compiles/runs. Common sources: +- New config struct fields that Claude will guess wrong +- Renamed enums or moved types +- Changed method signatures (added/removed args) +- New features that Claude will confuse with old patterns -The DataActor pattern and LiveNode wiring in SKILL.md must compile. Test: - -```bash -cd my_trading_system && cargo build --release 2>&1 -``` - -If it fails, the Rust section needs updating. Common breakage: -- `DataActor` trait method signatures changed -- `LiveNode::builder()` API changed -- `BinanceDataClientConfig` fields renamed -- New required fields added to configs (no more `..Default::default()`) +Add new test cases to the appropriate file in `tests/nautilus-verify/` and new rows +to the hallucination tables in SKILL.md. --- @@ -228,9 +215,10 @@ context7. If they can't, the skill has gaps. ## Versioning -After rebuild, update the version line in SKILL.md: -``` -**Tested against v1.XXX.0** — all code validated by running tests. -``` +After rebuild: +1. Update the version tag in `tests/nautilus-verify/Cargo.toml` (all git dep `tag = "vX.Y.Z"` lines) +2. Update `tests/nautilus-verify/VERSION` +3. Update the `` comment in SKILL.md hallucination tables +4. Run `./tests/nautilus-verify/verify.sh` — fix any failures, update SKILL.md rows Track what version the anti-hallucination table was last verified against. Stale tables are worse than no table — they give false confidence. diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 61dc95d..702929d 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -524,7 +524,7 @@ Nautilus creates one log file per run. For rotation, use external logrotate or c ## Custom Indicator Development ```python -from nautilus_trader.indicators.base.indicator import Indicator +from nautilus_trader.indicators.base import Indicator class SpreadEMA(Indicator): def __init__(self, period: int): @@ -539,15 +539,13 @@ class SpreadEMA(Indicator): if len(self._values) > self.period: self._values.pop(0) self.value = sum(self._values) / len(self._values) + # _set_initialized inherited from Cython Indicator base — do NOT override self._set_initialized(len(self._values) >= self.period) - def _set_initialized(self, value: bool) -> None: - self.initialized = value - def reset(self) -> None: self._values.clear() self.value = 0.0 - self.initialized = False + self._set_initialized(False) ``` Register: `self.register_indicator_for_quote_ticks(instrument_id, spread_ema)` in `on_start`. diff --git a/tests/nautilus-verify/Cargo.toml b/tests/nautilus-verify/Cargo.toml new file mode 100644 index 0000000..02e687c --- /dev/null +++ b/tests/nautilus-verify/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "nautilus-verify" +version = "0.1.0" +edition = "2021" + +[dependencies] +nautilus-model = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } +nautilus-common = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } +nautilus-trading = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } +nautilus-binance = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } +nautilus-live = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } +anyhow = "1" +log = "0.4" diff --git a/tests/nautilus-verify/VERSION b/tests/nautilus-verify/VERSION new file mode 100644 index 0000000..ee716f9 --- /dev/null +++ b/tests/nautilus-verify/VERSION @@ -0,0 +1 @@ +1.224.0 diff --git a/tests/nautilus-verify/pyproject.toml b/tests/nautilus-verify/pyproject.toml new file mode 100644 index 0000000..fec1077 --- /dev/null +++ b/tests/nautilus-verify/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "nautilus-verify" +version = "0.1.0" +requires-python = ">=3.12" + +[tool.pytest.ini_options] +testpaths = ["python"] diff --git a/tests/nautilus-verify/python/conftest.py b/tests/nautilus-verify/python/conftest.py new file mode 100644 index 0000000..e80ebae --- /dev/null +++ b/tests/nautilus-verify/python/conftest.py @@ -0,0 +1,3 @@ +import nautilus_trader + +NAUTILUS_VERSION = nautilus_trader.__version__ diff --git a/tests/nautilus-verify/python/test_anti_hallucination.py b/tests/nautilus-verify/python/test_anti_hallucination.py new file mode 100644 index 0000000..1262a87 --- /dev/null +++ b/tests/nautilus-verify/python/test_anti_hallucination.py @@ -0,0 +1,207 @@ +"""Tests for every row in the Python anti-hallucination table (SKILL.md). + +Each test verifies the 'Reality' column is still correct for the installed version. +Many Cython classes have opaque signatures (*args, **kwargs), so we test by +constructing objects rather than introspecting signatures. +""" + +import inspect + + +# Row: cache.position_for_instrument(id) → cache.positions_open(instrument_id=id) +def test_cache_positions_open_method(): + from nautilus_trader.cache.cache import Cache + + assert hasattr(Cache, "positions_open") + assert not hasattr(Cache, "position_for_instrument") + + +# Row: engine.trader.cache → engine.cache directly +def test_engine_cache_direct_access(): + from nautilus_trader.backtest.engine import BacktestEngine + + assert hasattr(BacktestEngine, "cache") + + +# Row: GenericDataWrangler → specific wranglers +def test_specific_data_wranglers_exist(): + from nautilus_trader.persistence.wranglers import ( + BarDataWrangler, + OrderBookDeltaDataWrangler, + QuoteTickDataWrangler, + TradeTickDataWrangler, + ) + + +# Row: catalog.data_types() → catalog.list_data_types() +def test_catalog_list_data_types(): + from nautilus_trader.persistence.catalog.parquet import ParquetDataCatalog + + assert hasattr(ParquetDataCatalog, "list_data_types") + + +# Row: BacktestEngineConfig from nautilus_trader.config → from nautilus_trader.backtest +def test_backtest_engine_config_import(): + from nautilus_trader.backtest.engine import BacktestEngineConfig + + +# Row: FillModel(prob_fill_on_stop=...) → only prob_fill_on_limit, prob_slippage, random_seed +def test_fillmodel_constructor_accepts_three_params(): + from nautilus_trader.backtest.models import FillModel + + fm = FillModel(prob_fill_on_limit=0.3, prob_slippage=0.5, random_seed=42) + assert fm is not None + + +# Row: LoggingConfig(log_file_path=) → log_directory= +def test_logging_config_log_directory(): + from nautilus_trader.config import LoggingConfig + + lc = LoggingConfig(log_directory="logs/") + assert lc.log_directory == "logs/" + + +# Row: from nautilus_trader.trading.actor → from nautilus_trader.common.actor import Actor +def test_actor_import_from_common(): + from nautilus_trader.common.actor import Actor + + +# Row: from nautilus_trader.indicators.ema → from nautilus_trader.indicators import EMA +def test_ema_import(): + from nautilus_trader.indicators import ExponentialMovingAverage + + +# Row: BollingerBands(20) → BollingerBands(20, 2.0) — k mandatory +def test_bollinger_bands_k_mandatory(): + from nautilus_trader.indicators import BollingerBands + + bb = BollingerBands(20, 2.0) + assert bb is not None + + +# Row: MACD(12, 26, 9) → 3rd param is MovingAverageType, not signal_period +def test_macd_third_param_is_ma_type(): + from nautilus_trader.indicators import ( + MovingAverageConvergenceDivergence, + MovingAverageType, + ) + + m = MovingAverageConvergenceDivergence(12, 26, MovingAverageType.SIMPLE) + assert m is not None + + +# Row: DYDXDataClientConfig → DydxDataClientConfig (mixed case) +def test_dydx_config_mixed_case(): + from nautilus_trader.adapters.dydx.config import DydxDataClientConfig + + +# Row: RSI value in [0, 1] not [0, 100] +def test_rsi_value_range_note(): + from nautilus_trader.indicators import RelativeStrengthIndex + + rsi = RelativeStrengthIndex(14) + + +# Row: frozen_account=True means checks DISABLED (inverted) +def test_frozen_account_param_exists(): + from nautilus_trader.backtest.engine import BacktestEngine + + sig = inspect.signature(BacktestEngine.add_venue) + assert "frozen_account" in sig.parameters + + +# Row: AccountType.CASH vs MARGIN +def test_account_type_margin_exists(): + from nautilus_trader.model.enums import AccountType + + assert hasattr(AccountType, "MARGIN") + assert hasattr(AccountType, "CASH") + + +# Row: order.order_side → order.side (events use event.order_side) +def test_order_side_field_name(): + from nautilus_trader.model.orders import MarketOrder + + assert hasattr(MarketOrder, "side") + + +# Row: request_bars(bar_type) one arg → requires start +def test_request_bars_signature(): + from nautilus_trader.trading.strategy import Strategy + + sig = inspect.signature(Strategy.request_bars) + assert "start" in sig.parameters + + +# Row: GreeksCalculator import path (moved to nautilus_trader.model.greeks) +def test_greeks_calculator_import(): + from nautilus_trader.model.greeks import GreeksCalculator + + +# Row: from nautilus_trader.core.nautilus_pyo3 → correct path for black_scholes_greeks +def test_pyo3_black_scholes_import(): + from nautilus_trader.core.nautilus_pyo3 import black_scholes_greeks + + +# Row: BinanceAccountType.USDT_FUTURE (no S) → USDT_FUTURES (with S) +def test_binance_usdt_futures_with_s(): + from nautilus_trader.adapters.binance.common.enums import BinanceAccountType + + assert hasattr(BinanceAccountType, "USDT_FUTURES") + assert not hasattr(BinanceAccountType, "USDT_FUTURE") + + +# Row: is_testnet=True (not testnet=True) for dYdX +def test_dydx_is_testnet_param(): + from nautilus_trader.adapters.dydx.config import DydxDataClientConfig + + d = DydxDataClientConfig(is_testnet=True) + assert d.is_testnet is True + + +# Row: is_testnet=True (not testnet=True) for Deribit +def test_deribit_is_testnet_param(): + from nautilus_trader.adapters.deribit.config import DeribitDataClientConfig + + d = DeribitDataClientConfig(is_testnet=True) + assert d.is_testnet is True + + +# Row: book.best_bid_price() returns Price, not float +def test_price_type_exists(): + from nautilus_trader.model.objects import Price + + p = Price.from_str("1.50000") + assert not isinstance(p, float) + + +# Row: subscribe_funding_rates() method exists on Strategy +def test_strategy_has_subscribe_funding_rates(): + from nautilus_trader.trading.strategy import Strategy + + assert hasattr(Strategy, "subscribe_funding_rates") + + +# Row: MarketStatusAction.RESUME does not exist — use TRADING +def test_market_status_action_trading(): + from nautilus_trader.model.enums import MarketStatusAction + + assert hasattr(MarketStatusAction, "TRADING") + assert not hasattr(MarketStatusAction, "RESUME") + + +# Row: SyntheticInstrument needs ts_event, ts_init (Cython — test via hasattr on class) +def test_synthetic_instrument_exists(): + from nautilus_trader.model.instruments import SyntheticInstrument + + assert SyntheticInstrument is not None + + +# Row: from nautilus_trader.common.component import LiveClock (not TestClock from common.clock) +def test_live_clock_import(): + from nautilus_trader.common.component import LiveClock + + +# Row: CacheConfig import path +def test_cache_config_import(): + from nautilus_trader.config import CacheConfig diff --git a/tests/nautilus-verify/python/test_backtest_config.py b/tests/nautilus-verify/python/test_backtest_config.py new file mode 100644 index 0000000..538bd6c --- /dev/null +++ b/tests/nautilus-verify/python/test_backtest_config.py @@ -0,0 +1,43 @@ +"""Tests backtest configuration patterns from battle_tested.md.""" + + +def test_venue_setup_full_pattern(): + """Verify the full venue setup from battle_tested compiles.""" + from nautilus_trader.model.enums import AccountType, OmsType + from nautilus_trader.model.identifiers import Venue + from nautilus_trader.model.objects import Currency, Money + from nautilus_trader.backtest.models import FillModel + + venue = Venue("BINANCE") + oms_type = OmsType.NETTING + account_type = AccountType.MARGIN + usdt = Currency.from_str("USDT") + starting_balances = [Money(10_000, usdt)] + fill_model = FillModel( + prob_fill_on_limit=0.3, + prob_slippage=0.5, + random_seed=42, + ) + + assert venue is not None + assert account_type == AccountType.MARGIN + assert len(starting_balances) == 1 + assert fill_model is not None + + +def test_bar_data_wrangler_ts_init_delta(): + """Verify BarDataWrangler accepts ts_init_delta for timestamp correction.""" + import inspect + from nautilus_trader.persistence.wranglers import BarDataWrangler + + sig = inspect.signature(BarDataWrangler.process) + assert "ts_init_delta" in sig.parameters + + +def test_multi_currency_base_currency_none(): + """Verify base_currency=None is valid for multi-currency accounts.""" + import inspect + from nautilus_trader.backtest.engine import BacktestEngine + + sig = inspect.signature(BacktestEngine.add_venue) + assert "base_currency" in sig.parameters diff --git a/tests/nautilus-verify/python/test_constructors.py b/tests/nautilus-verify/python/test_constructors.py new file mode 100644 index 0000000..1a922e2 --- /dev/null +++ b/tests/nautilus-verify/python/test_constructors.py @@ -0,0 +1,91 @@ +"""Tests for constructor patterns from battle_tested.md and SKILL.md.""" + +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.objects import Price, Quantity + + +# BookOrder: 4 positional args, not keyword-style +def test_book_order_positional_args(): + from nautilus_trader.model.data import BookOrder + + order = BookOrder( + OrderSide.BUY, + Price.from_str("50000.00"), + Quantity.from_str("1.000"), + 0, + ) + assert order.side == OrderSide.BUY + + +# FillModel: only 3 params +def test_fillmodel_constructor(): + from nautilus_trader.backtest.models import FillModel + + fm = FillModel( + prob_fill_on_limit=0.3, + prob_slippage=0.5, + random_seed=42, + ) + assert fm is not None + + +# Venue setup with AccountType.MARGIN +def test_account_type_and_oms_type(): + from nautilus_trader.model.enums import AccountType, OmsType + + assert AccountType.MARGIN is not None + assert OmsType.NETTING is not None + + +# Currency.from_str (not bare constant) +def test_currency_from_str(): + from nautilus_trader.model.objects import Currency + + usdt = Currency.from_str("USDT") + assert usdt is not None + + +# Money constructor +def test_money_constructor(): + from nautilus_trader.model.objects import Currency + from nautilus_trader.model.objects import Money + + usdt = Currency.from_str("USDT") + m = Money(10_000, usdt) + assert m is not None + + +# Price.from_str (not Price(float)) +def test_price_from_str(): + from nautilus_trader.model.objects import Price + + p = Price.from_str("1.50000") + assert float(p) == 1.5 + + +# Venue constructor +def test_venue_constructor(): + from nautilus_trader.model.identifiers import Venue + + v = Venue("BINANCE") + assert str(v) == "BINANCE" + + +# InstrumentId full format "SYMBOL.VENUE" +def test_instrument_id_format(): + from nautilus_trader.model.identifiers import InstrumentId + + iid = InstrumentId.from_str("BTCUSDT-PERP.BINANCE") + assert str(iid) == "BTCUSDT-PERP.BINANCE" + + +# LoggingConfig with log_directory +def test_logging_config_constructor(): + from nautilus_trader.config import LoggingConfig + + lc = LoggingConfig( + log_level="INFO", + log_directory="logs/", + log_colors=False, + ) + assert lc.log_directory == "logs/" diff --git a/tests/nautilus-verify/python/test_custom_data.py b/tests/nautilus-verify/python/test_custom_data.py new file mode 100644 index 0000000..3f14a89 --- /dev/null +++ b/tests/nautilus-verify/python/test_custom_data.py @@ -0,0 +1,42 @@ +"""Tests the ImbalanceData custom data pattern from battle_tested.md.""" + +from nautilus_trader.core.data import Data + + +class ImbalanceData(Data): + def __init__(self, imbalance: float, volume: float, ts_event: int, ts_init: int): + self.imbalance = imbalance + self.volume = volume + self._ts_event = ts_event + self._ts_init = ts_init + + @property + def ts_event(self) -> int: + return self._ts_event + + @property + def ts_init(self) -> int: + return self._ts_init + + +def test_imbalance_data_instantiates(): + data = ImbalanceData( + imbalance=0.75, + volume=1000.0, + ts_event=1_000_000_000, + ts_init=1_000_000_001, + ) + assert data.imbalance == 0.75 + assert data.volume == 1000.0 + assert data.ts_event == 1_000_000_000 + assert data.ts_init == 1_000_000_001 + + +def test_imbalance_data_is_data(): + data = ImbalanceData( + imbalance=0.5, + volume=500.0, + ts_event=0, + ts_init=0, + ) + assert isinstance(data, Data) diff --git a/tests/nautilus-verify/python/test_custom_indicator.py b/tests/nautilus-verify/python/test_custom_indicator.py new file mode 100644 index 0000000..13ccd88 --- /dev/null +++ b/tests/nautilus-verify/python/test_custom_indicator.py @@ -0,0 +1,50 @@ +"""Tests the SpreadEMA custom indicator pattern from battle_tested.md.""" + +from nautilus_trader.indicators.base import Indicator + + +class SpreadEMA(Indicator): + def __init__(self, period: int): + super().__init__([]) + self.period = period + self._values = [] + self.value = 0.0 + + def handle_quote_tick(self, tick) -> None: + spread = float(tick.ask_price - tick.bid_price) + self._values.append(spread) + if len(self._values) > self.period: + self._values.pop(0) + self.value = sum(self._values) / len(self._values) + # _set_initialized is inherited from Cython Indicator base — do NOT override it + self._set_initialized(len(self._values) >= self.period) + + def reset(self) -> None: + self._values.clear() + self.value = 0.0 + self._set_initialized(False) + + +def test_spread_ema_instantiates(): + ema = SpreadEMA(period=10) + assert ema.period == 10 + assert ema.value == 0.0 + assert not ema.initialized + + +def test_spread_ema_warmup_and_reset(): + ema = SpreadEMA(period=3) + ema._values = [1.0, 2.0, 3.0] + ema.value = 2.0 + ema._set_initialized(True) + assert ema.initialized + + ema.reset() + assert ema._values == [] + assert ema.value == 0.0 + assert not ema.initialized + + +def test_spread_ema_is_indicator(): + ema = SpreadEMA(period=5) + assert isinstance(ema, Indicator) diff --git a/tests/nautilus-verify/python/test_order_book.py b/tests/nautilus-verify/python/test_order_book.py new file mode 100644 index 0000000..6376954 --- /dev/null +++ b/tests/nautilus-verify/python/test_order_book.py @@ -0,0 +1,32 @@ +"""Tests order book patterns from battle_tested.md and anti-hallucination table.""" + +from nautilus_trader.model.objects import Price + + +def test_price_is_not_float(): + """book.best_bid_price() returns Price, not float — must cast with float().""" + p = Price.from_str("50000.00") + assert not isinstance(p, float) + assert isinstance(float(p), float) + + +def test_price_arithmetic_requires_cast(): + """Price objects need float() for arithmetic with regular numbers.""" + p = Price.from_str("100.50") + result = float(p) * 2 + assert result == 201.0 + + +def test_book_order_import_from_data(): + """BookOrder import from model.data, NOT model.book.""" + from nautilus_trader.model.data import BookOrder + + +def test_order_book_depth10_exists(): + """OrderBookDepth10 pre-aggregated type exists.""" + from nautilus_trader.model.data import OrderBookDepth10 + + +def test_order_book_delta_exists(): + """OrderBookDelta type exists for L2 data.""" + from nautilus_trader.model.data import OrderBookDelta diff --git a/tests/nautilus-verify/src/lib.rs b/tests/nautilus-verify/src/lib.rs new file mode 100644 index 0000000..c19de38 --- /dev/null +++ b/tests/nautilus-verify/src/lib.rs @@ -0,0 +1,2 @@ +// nautilus-verify: compile and runtime tests for nautilus-docs skill +// This crate exists only so `cargo test` can run integration tests in tests/ diff --git a/tests/nautilus-verify/tests/anti_hallucination.rs b/tests/nautilus-verify/tests/anti_hallucination.rs new file mode 100644 index 0000000..748229c --- /dev/null +++ b/tests/nautilus-verify/tests/anti_hallucination.rs @@ -0,0 +1,152 @@ +// Tests for every row in the Rust anti-hallucination table (SKILL.md). +// Each test verifies the "Reality" column compiles/works correctly. + +use nautilus_binance::common::enums::BinanceProductType; +use nautilus_binance::config::BinanceDataClientConfig; +use nautilus_binance::factories::BinanceExecutionClientFactory; +use nautilus_common::actor::{DataActor, DataActorCore}; +use nautilus_common::enums::Environment; +use nautilus_live::node::LiveNode; +use nautilus_model::data::{FundingRateUpdate, OrderBookDeltas}; +use nautilus_model::events::position::closed::PositionClosed; +use nautilus_model::identifiers::{InstrumentId, TraderId}; +use nautilus_trading::strategy::{StrategyConfig, StrategyCore}; +use std::ops::{Deref, DerefMut}; + +// Row: LiveNode::new(config) → LiveNode::builder(trader_id, Environment::Live)?.build()? +#[test] +fn hallucination_livenode_builder_pattern() { + // Verify the builder method exists with correct signature + // (we can't actually build without a full runtime, but the function must exist) + let _ = LiveNode::builder as fn(TraderId, Environment) -> anyhow::Result<_>; +} + +// Row: BinanceDataClientConfig::builder() → struct literal with ..Default::default() +#[test] +fn hallucination_binance_data_config_struct_literal() { + let config = BinanceDataClientConfig { + product_types: vec![BinanceProductType::UsdM], + ..Default::default() + }; + assert_eq!(config.product_types, vec![BinanceProductType::UsdM]); +} + +// Row: BinanceAccountType::UsdtFutures → BinanceProductType::UsdM +#[test] +fn hallucination_binance_product_type_usdm() { + let _ = BinanceProductType::UsdM; + let _ = BinanceProductType::CoinM; + let _ = BinanceProductType::Spot; + let _ = BinanceProductType::Margin; +} + +// Row: nautilus_core::identifiers::InstrumentId → nautilus_model::identifiers::InstrumentId +#[test] +fn hallucination_instrument_id_in_model() { + let _ = std::any::type_name::(); + // Compiles = nautilus_model::identifiers::InstrumentId is the correct path +} + +// Row: DataActor in nautilus_trading → DataActor in nautilus_common::actor +#[test] +fn hallucination_data_actor_in_common() { + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +// Row: nautilus_trader_model → nautilus_model +#[test] +fn hallucination_crate_name_is_nautilus_model() { + // If this compiles, nautilus_model (not nautilus_trader_model) is correct + let _ = std::any::type_name::(); +} + +// Row: on_book_delta (singular) → on_book_deltas (plural — batch) +#[test] +fn hallucination_on_book_deltas_plural() { + // Verify the trait method signature accepts OrderBookDeltas (batch) + let _ = std::any::type_name::(); + // The DataActor trait has fn on_book_deltas(&mut self, deltas: &OrderBookDeltas) + // If OrderBookDeltas doesn't exist, this test fails to compile. +} + +// Row: event.duration_ns on PositionClosed → event.duration (no _ns suffix) +#[test] +fn hallucination_position_closed_duration_field() { + // Verify the field name is `duration`, not `duration_ns` + // PositionClosed has: pub duration: DurationNanos + let _ = |e: &PositionClosed| e.duration; +} + +// Row: event.realized_pnl is Money → Option +#[test] +fn hallucination_realized_pnl_is_option() { + // PositionClosed has: pub realized_pnl: Option + let _ = |e: &PositionClosed| { + match &e.realized_pnl { + Some(_money) => {} + None => {} + } + }; +} + +// Row: Deref for Strategy → Deref +#[test] +fn hallucination_strategy_core_derefs_to_data_actor_core() { + // StrategyCore implements Deref + fn assert_deref_target(_: &impl Deref) {} + let config = StrategyConfig { + strategy_id: Some("TEST-001".into()), + ..Default::default() + }; + let core = StrategyCore::new(config); + assert_deref_target(&core); +} + +// Row: node.add_actor(strategy) for Strategy → node.add_strategy(strategy) +// Can't test without runtime, but verify the method exists on LiveNode +// (tested transitively by strategy_pattern.rs and livenode builder test) + +// Row: BinanceExecClientFactory → BinanceExecutionClientFactory (full name) +#[test] +fn hallucination_binance_execution_client_factory_full_name() { + let _ = std::any::type_name::(); +} + +// Row: Deref target for user Strategy struct must be DataActorCore +#[test] +fn hallucination_user_strategy_deref_chain() { + #[derive(Debug)] + struct TestStrat { + core: StrategyCore, + } + + impl Deref for TestStrat { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { + &self.core // auto-derefs StrategyCore → DataActorCore + } + } + + impl DerefMut for TestStrat { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.core + } + } + + let config = StrategyConfig { + strategy_id: Some("TEST-002".into()), + ..Default::default() + }; + let strat = TestStrat { + core: StrategyCore::new(config), + }; + // Verify the deref chain works + let _core_ref: &DataActorCore = &strat; +} + +// Row: FundingRateUpdate exists in nautilus_model::data +#[test] +fn hallucination_funding_rate_update_path() { + let _ = std::any::type_name::(); +} diff --git a/tests/nautilus-verify/tests/crate_map.rs b/tests/nautilus-verify/tests/crate_map.rs new file mode 100644 index 0000000..90b00ea --- /dev/null +++ b/tests/nautilus-verify/tests/crate_map.rs @@ -0,0 +1,120 @@ +// Verifies all import paths from the SKILL.md crate map. +// If any `use` statement fails to compile, the crate map is stale. + +// nautilus-model: identifiers +use nautilus_model::identifiers::{ + AccountId, ActorId, ClientId, ClientOrderId, ComponentId, ExecAlgorithmId, InstrumentId, + OrderListId, PositionId, StrategyId, Symbol, TradeId, TraderId, Venue, VenueOrderId, +}; + +// nautilus-model: data types +use nautilus_model::data::{ + Bar, BarSpecification, BarType, BookOrder, FundingRateUpdate, IndexPriceUpdate, + InstrumentClose, MarkPriceUpdate, OrderBookDelta, OrderBookDeltas, OrderBookDepth10, + QuoteTick, TradeTick, +}; + +// nautilus-model: order types +use nautilus_model::orders::{ + LimitOrder, MarketOrder, StopLimitOrder, StopMarketOrder, TrailingStopLimitOrder, + TrailingStopMarketOrder, +}; + +// nautilus-model: events +use nautilus_model::events::position::closed::PositionClosed; + +// nautilus-common: actor +use nautilus_common::actor::{DataActorConfig, DataActorCore}; + +// DataActor is a trait — imported via trait bound check +fn _assert_data_actor_trait_exists() {} + +// nautilus-common: environment +use nautilus_common::enums::Environment; + +// nautilus-trading: strategy +use nautilus_trading::strategy::{StrategyConfig, StrategyCore}; + +// nautilus-binance: config + enums +use nautilus_binance::common::enums::BinanceProductType; +use nautilus_binance::config::{BinanceDataClientConfig, BinanceExecClientConfig}; +use nautilus_binance::factories::{BinanceDataClientFactory, BinanceExecutionClientFactory}; + +// nautilus-live: node +use nautilus_live::node::LiveNode; + +#[test] +fn crate_map_identifiers_compile() { + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +#[test] +fn crate_map_data_types_compile() { + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +#[test] +fn crate_map_order_types_compile() { + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +#[test] +fn crate_map_actor_and_strategy_compile() { + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +#[test] +fn crate_map_binance_compile() { + let _ = BinanceProductType::UsdM; + let _ = BinanceProductType::CoinM; + let _ = BinanceProductType::Spot; + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +#[test] +fn crate_map_live_node_compile() { + let _ = std::any::type_name::(); + let _ = std::any::type_name::(); +} + +#[test] +fn crate_map_events_compile() { + let _ = std::any::type_name::(); +} diff --git a/tests/nautilus-verify/tests/data_actor_pattern.rs b/tests/nautilus-verify/tests/data_actor_pattern.rs new file mode 100644 index 0000000..60b97a5 --- /dev/null +++ b/tests/nautilus-verify/tests/data_actor_pattern.rs @@ -0,0 +1,63 @@ +// Verifies the DataActor pattern from battle_tested.md (FundingRateCapture). +// If this compiles, the pattern is correct for the current nautilus version. + +use std::ops::{Deref, DerefMut}; + +use nautilus_common::actor::{DataActor, DataActorConfig, DataActorCore}; +use nautilus_model::data::FundingRateUpdate; +use nautilus_model::identifiers::InstrumentId; + +#[derive(Debug)] +struct FundingRateCapture { + core: DataActorCore, + instrument_id: InstrumentId, +} + +impl FundingRateCapture { + fn new(instrument_id: InstrumentId) -> Self { + Self { + core: DataActorCore::new(DataActorConfig { + actor_id: Some("FundingCapture-001".into()), + ..Default::default() + }), + instrument_id, + } + } +} + +impl Deref for FundingRateCapture { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { + &self.core + } +} + +impl DerefMut for FundingRateCapture { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.core + } +} + +impl DataActor for FundingRateCapture { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_funding_rates(self.instrument_id, None, None); + Ok(()) + } + + fn on_funding_rate(&mut self, update: &FundingRateUpdate) -> anyhow::Result<()> { + log::info!("{} rate={}", update.instrument_id, update.rate); + Ok(()) + } + + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_funding_rates(self.instrument_id, None, None); + Ok(()) + } +} + +#[test] +fn data_actor_funding_rate_capture_compiles() { + let id = InstrumentId::from("BTCUSDT-PERP.BINANCE"); + let actor = FundingRateCapture::new(id); + assert_eq!(actor.instrument_id.to_string(), "BTCUSDT-PERP.BINANCE"); +} diff --git a/tests/nautilus-verify/tests/strategy_pattern.rs b/tests/nautilus-verify/tests/strategy_pattern.rs new file mode 100644 index 0000000..e13e35f --- /dev/null +++ b/tests/nautilus-verify/tests/strategy_pattern.rs @@ -0,0 +1,81 @@ +// Verifies the Strategy pattern from battle_tested.md (MyStrategy with Deref chain). +// If this compiles, the full Strategy + DataActor trait wiring is correct. + +use std::ops::{Deref, DerefMut}; + +use nautilus_common::actor::{DataActor, DataActorCore}; +use nautilus_model::data::TradeTick; +use nautilus_model::identifiers::InstrumentId; +use nautilus_trading::strategy::{Strategy, StrategyConfig, StrategyCore}; + +#[derive(Debug)] +struct MyStrategy { + core: StrategyCore, + instrument_id: InstrumentId, +} + +impl MyStrategy { + fn new(instrument_id: InstrumentId) -> Self { + Self { + core: StrategyCore::new(StrategyConfig { + strategy_id: Some("MY-001".into()), + ..Default::default() + }), + instrument_id, + } + } +} + +// CRITICAL: Deref target is DataActorCore, NOT StrategyCore. +// StrategyCore implements Deref, so this chains through. +impl Deref for MyStrategy { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { + &self.core // auto-derefs StrategyCore → DataActorCore + } +} + +impl DerefMut for MyStrategy { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.core + } +} + +impl DataActor for MyStrategy { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_trades(self.instrument_id, None, None); + Ok(()) + } + + fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { + log::info!("trade px={}", trade.price.as_f64()); + Ok(()) + } + + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_trades(self.instrument_id, None, None); + Ok(()) + } +} + +impl Strategy for MyStrategy { + fn core(&self) -> &StrategyCore { + &self.core + } + fn core_mut(&mut self) -> &mut StrategyCore { + &mut self.core + } +} + +#[test] +fn strategy_pattern_compiles_and_constructs() { + let id = InstrumentId::from("BTCUSDT-PERP.BINANCE"); + let strat = MyStrategy::new(id); + assert_eq!(strat.instrument_id.to_string(), "BTCUSDT-PERP.BINANCE"); + + // Verify Deref chain: MyStrategy → DataActorCore + let _core_ref: &DataActorCore = &strat; + + // Verify Strategy trait: core() returns StrategyCore reference + let _strat_core: &StrategyCore = strat.core(); +} diff --git a/tests/nautilus-verify/verify.sh b/tests/nautilus-verify/verify.sh new file mode 100755 index 0000000..e848fa4 --- /dev/null +++ b/tests/nautilus-verify/verify.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +VERIFIED="$(cat "$SCRIPT_DIR/VERSION")" +INSTALLED=$("$REPO_ROOT/.venv/bin/python" -c "import nautilus_trader; print(nautilus_trader.__version__)") + +echo "Verified: $VERIFIED | Installed: $INSTALLED" + +if [[ "$VERIFIED" != "$INSTALLED" ]]; then + echo "VERSION CHANGED — full verification required" +fi + +RUST_OK=true +PYTHON_OK=true + +echo "" +echo "=== Rust compile tests ===" +if (cd "$SCRIPT_DIR" && cargo test 2>&1); then + echo "Rust: PASS" +else + echo "Rust: FAIL" + RUST_OK=false +fi + +echo "" +echo "=== Python tests ===" +if (cd "$SCRIPT_DIR" && "$REPO_ROOT/.venv/bin/python" -m pytest python/ -v 2>&1); then + echo "Python: PASS" +else + echo "Python: FAIL" + PYTHON_OK=false +fi + +echo "" +if $RUST_OK && $PYTHON_OK; then + if [[ "$VERIFIED" != "$INSTALLED" ]]; then + echo "$INSTALLED" > "$SCRIPT_DIR/VERSION" + echo "All tests pass. Updated VERSION: $VERIFIED -> $INSTALLED" + else + echo "All tests pass. VERSION unchanged ($VERIFIED)." + fi +else + echo "FAILURES DETECTED — hallucination rows may be stale for v$INSTALLED" + exit 1 +fi From e7c6af44950a0af859fe08ccf062b7bf6af91398 Mon Sep 17 00:00:00 2001 From: Vlad Kochetov Date: Tue, 7 Apr 2026 20:13:10 +0300 Subject: [PATCH 25/25] chore: drop nautilus-verify test suite The suite duplicated work better handled by version-aware manual checks during REBUILD. Learnings from writing the tests are preserved in battle_tested.md (SpreadEMA _set_initialized, dYdX 5% oracle slippage, signal type inspection) and SKILL.md (verified version marker). REBUILD.md steps 4+5 and versioning reverted to manual-testing flow. Co-Authored-By: Claude Opus 4.6 --- .gitignore | 5 +- nautilus-docs/SKILL.md | 4 +- nautilus-docs/references/REBUILD.md | 59 ++--- nautilus-docs/references/battle_tested.md | 63 ++---- tests/nautilus-verify/Cargo.toml | 13 -- tests/nautilus-verify/VERSION | 1 - tests/nautilus-verify/pyproject.toml | 7 - tests/nautilus-verify/python/conftest.py | 3 - .../python/test_anti_hallucination.py | 207 ------------------ .../python/test_backtest_config.py | 43 ---- .../python/test_constructors.py | 91 -------- .../python/test_custom_data.py | 42 ---- .../python/test_custom_indicator.py | 50 ----- .../nautilus-verify/python/test_order_book.py | 32 --- tests/nautilus-verify/src/lib.rs | 2 - .../tests/anti_hallucination.rs | 152 ------------- tests/nautilus-verify/tests/crate_map.rs | 120 ---------- .../tests/data_actor_pattern.rs | 63 ------ .../nautilus-verify/tests/strategy_pattern.rs | 81 ------- tests/nautilus-verify/verify.sh | 47 ---- 20 files changed, 57 insertions(+), 1028 deletions(-) delete mode 100644 tests/nautilus-verify/Cargo.toml delete mode 100644 tests/nautilus-verify/VERSION delete mode 100644 tests/nautilus-verify/pyproject.toml delete mode 100644 tests/nautilus-verify/python/conftest.py delete mode 100644 tests/nautilus-verify/python/test_anti_hallucination.py delete mode 100644 tests/nautilus-verify/python/test_backtest_config.py delete mode 100644 tests/nautilus-verify/python/test_constructors.py delete mode 100644 tests/nautilus-verify/python/test_custom_data.py delete mode 100644 tests/nautilus-verify/python/test_custom_indicator.py delete mode 100644 tests/nautilus-verify/python/test_order_book.py delete mode 100644 tests/nautilus-verify/src/lib.rs delete mode 100644 tests/nautilus-verify/tests/anti_hallucination.rs delete mode 100644 tests/nautilus-verify/tests/crate_map.rs delete mode 100644 tests/nautilus-verify/tests/data_actor_pattern.rs delete mode 100644 tests/nautilus-verify/tests/strategy_pattern.rs delete mode 100755 tests/nautilus-verify/verify.sh diff --git a/.gitignore b/.gitignore index 42eba18..23a0e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,4 @@ .tmp/ .env nautilus-docs/references/docs/ -tests/nautilus-verify/target/ -tests/nautilus-verify/.pytest_cache/ -tests/nautilus-verify/Cargo.lock -__pycache__/ \ No newline at end of file +__pycache__/ diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md index de3fe59..513d31d 100644 --- a/nautilus-docs/SKILL.md +++ b/nautilus-docs/SKILL.md @@ -77,6 +77,8 @@ description: > | Coding standards | [coding_standards.md](references/docs/developer_guide/coding_standards.md) | | Benchmarking | [benchmarking.md](references/docs/developer_guide/benchmarking.md), [criterion_template.rs](references/docs/dev_templates/criterion_template.rs), [iai_template.rs](references/docs/dev_templates/iai_template.rs) | | FFI | [ffi.md](references/docs/developer_guide/ffi.md) | +| Documentation | [docs.md](references/docs/developer_guide/docs.md) | +| Releases | [releases.md](references/docs/developer_guide/releases.md) | ### Tutorials @@ -205,7 +207,7 @@ impl MyActor { ``` ## Common Hallucinations - + ### Python diff --git a/nautilus-docs/references/REBUILD.md b/nautilus-docs/references/REBUILD.md index 2970aec..c16be40 100644 --- a/nautilus-docs/references/REBUILD.md +++ b/nautilus-docs/references/REBUILD.md @@ -80,33 +80,46 @@ grep -r "pub fn builder" "$CHECKOUT/crates/live/src/" grep -A 20 "pub struct BinanceDataClientConfig" "$CHECKOUT/crates/adapters/binance/src/config.rs" ``` -## Step 4: Run the Verification Suite +## Step 4: Rebuild Anti-Hallucination Tables -The `tests/nautilus-verify/` suite mechanically tests every hallucination row and pattern: +This is the highest-value part of the skill. Each row prevents a specific mistake Claude makes repeatedly. -```bash -# Run everything (Rust compile tests + Python tests) -./tests/nautilus-verify/verify.sh +**How to maintain**: -# Or individually: -cd tests/nautilus-verify && cargo test # Rust: 22 tests -cd tests/nautilus-verify && ../../.venv/bin/python -m pytest python/ -v # Python: 50 tests -``` +1. **Test existing rows** — for each hallucination, verify the "Reality" column is still correct: + ```python + # Python hallucinations — test in a venv with nautilus_trader installed + from nautilus_trader.common.actor import Actor # still correct? + from nautilus_trader.indicators import ExponentialMovingAverage # still correct? + ``` + +2. **Discover new hallucinations** — ask Claude to write code for common tasks WITHOUT the skill, then diff against what actually compiles/runs. Common sources: + - New config struct fields that Claude will guess wrong + - Renamed enums or moved types + - Changed method signatures (added/removed args) + - New features that Claude will confuse with old patterns + +3. **Remove stale rows** — if a hallucinated API now exists (e.g., a missing method was added), remove the row. -Failures = stale rows. Fix the test to match the new API, then update the corresponding -hallucination table row in SKILL.md and patterns in battle_tested.md. +4. **Rust hallucinations** — rebuild by attempting to compile a standalone binary: + ```bash + cd my_trading_system && cargo build 2>&1 | grep "error\[E" + ``` + Every compilation error that comes from a wrong import path, missing trait, or wrong struct field is a hallucination row candidate. -## Step 5: Discover New Hallucinations +## Step 5: Verify Working Patterns -Ask Claude to write code for common tasks WITHOUT the skill, then diff against what -actually compiles/runs. Common sources: -- New config struct fields that Claude will guess wrong -- Renamed enums or moved types -- Changed method signatures (added/removed args) -- New features that Claude will confuse with old patterns +The DataActor pattern and LiveNode wiring in SKILL.md must compile. Test: + +```bash +cd my_trading_system && cargo build --release 2>&1 +``` -Add new test cases to the appropriate file in `tests/nautilus-verify/` and new rows -to the hallucination tables in SKILL.md. +If it fails, the Rust section needs updating. Common breakage: +- `DataActor` trait method signatures changed +- `LiveNode::builder()` API changed +- `BinanceDataClientConfig` fields renamed +- New required fields added to configs (no more `..Default::default()`) --- @@ -215,10 +228,6 @@ context7. If they can't, the skill has gaps. ## Versioning -After rebuild: -1. Update the version tag in `tests/nautilus-verify/Cargo.toml` (all git dep `tag = "vX.Y.Z"` lines) -2. Update `tests/nautilus-verify/VERSION` -3. Update the `` comment in SKILL.md hallucination tables -4. Run `./tests/nautilus-verify/verify.sh` — fix any failures, update SKILL.md rows +After rebuild, update the `` comment in SKILL.md hallucination tables. Track what version the anti-hallucination table was last verified against. Stale tables are worse than no table — they give false confidence. diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md index 702929d..2fef201 100644 --- a/nautilus-docs/references/battle_tested.md +++ b/nautilus-docs/references/battle_tested.md @@ -1,48 +1,21 @@ # Battle-Tested Patterns & Tricks -Non-obvious knowledge from live testing NautilusTrader v1.224.0. These patterns are verified against real exchanges and won't be found in official docs. - -## Critical Ordering Rules - -### on_start() must follow this exact sequence - -```python -def on_start(self) -> None: - # 1. Cache instrument FIRST — None if not loaded, crashes later - self.instrument = self.cache.instrument(self.config.instrument_id) - if self.instrument is None: - self.log.error(f"Not found: {self.config.instrument_id}") - return - - # 2. Register indicators BEFORE subscribing - self.register_indicator_for_bars(bar_type, self.ema) - - # 3. Subscribe AFTER instrument + indicators ready - self.subscribe_bars(bar_type) -``` - -Wrong order = silent failures: indicators never receive data, subscriptions produce 0 callbacks. - -### Data loading order for backtest - -```python -# Load multiple instruments with deferred sorting (faster) -engine.add_data(ticks_btc, sort=False) -engine.add_data(ticks_eth, sort=False) -engine.sort_data() # single O(n log n) sort at the end -``` - -### Bar timestamp correction - -If bar data uses opening timestamps (common in CSV exports), set `ts_init_delta` to bar duration: - -```python -bars = BarDataWrangler(bar_type=bar_type, instrument=inst).process( - data=df, ts_init_delta=60_000_000_000 # 1 minute in nanoseconds -) -``` - -Without this: look-ahead bias — strategy sees bar data before the bar closes. +Non-obvious knowledge from live testing NautilusTrader v1.224.0. **Only patterns not covered in official docs.** For anything below marked with a doc link, read the doc first — it has more detail. + +## Covered in Official Docs (read these first) + +| Gotcha | Where | +|--------|-------| +| `on_start()` must be: cache → indicators → subscribe | [strategies.md](docs/concepts/strategies.md) | +| `sort=False` + `sort_data()` for multi-instrument backtest | [backtesting.md](docs/concepts/backtesting.md) | +| `ts_init_delta` for bar open→close timestamps (look-ahead bias) | [backtesting.md](docs/concepts/backtesting.md), [data.md](docs/concepts/data.md) | +| `F_LAST` flag mandatory on last delta in batch | [data.md](docs/concepts/data.md) | +| FillModel: only `prob_fill_on_limit`, `prob_slippage`, `random_seed` | [backtesting.md](docs/concepts/backtesting.md) | +| Binance `-PERP` suffix for futures | [binance.md](docs/integrations/binance.md) | +| dYdX: `is_testnet=True`, no `modify_order`, IOC limit orders | [dydx.md](docs/integrations/dydx.md) | +| Deribit: `is_testnet=True` | [deribit.md](docs/integrations/deribit.md) | +| Polymarket: no `modify_order`, binary outcomes | [polymarket.md](docs/integrations/polymarket.md) | +| Log rotation: `LoggingConfig(log_directory=, log_file_format=)` | [logging.md](docs/concepts/logging.md) | ## Execution Tricks @@ -115,6 +88,8 @@ def on_signal(self, signal) -> None: self.momentum = signal.value ``` +Note: the official docs say "differentiate using `signal.value`" — this is misleading. `type(signal).__name__` works and is more precise when publishing multiple distinct signal types. The `name` param to `publish_signal` is not stored as `.name` on the object, but it IS baked into the generated class name. + ### Custom data for structured payloads When signal values need to be richer than a single number: @@ -208,7 +183,7 @@ Pre-aggregated top 10 levels — lower overhead than full book subscription for ### dYdX -- Market orders implemented as aggressive limit: buy at `oracle_price × 1.01`, sell at `× 0.99` +- Market orders implemented as aggressive limit (IOC): buy at `oracle_price × 1.05`, sell at `× 0.95` (5% buffer, `DEFAULT_MARKET_ORDER_SLIPPAGE = 0.05`). Unfilled slippage not consumed. - Config: `is_testnet=True` (NOT `testnet=True`) - No `modify_order` support diff --git a/tests/nautilus-verify/Cargo.toml b/tests/nautilus-verify/Cargo.toml deleted file mode 100644 index 02e687c..0000000 --- a/tests/nautilus-verify/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "nautilus-verify" -version = "0.1.0" -edition = "2021" - -[dependencies] -nautilus-model = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } -nautilus-common = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } -nautilus-trading = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } -nautilus-binance = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } -nautilus-live = { git = "https://github.com/nautechsystems/nautilus_trader.git", tag = "v1.224.0" } -anyhow = "1" -log = "0.4" diff --git a/tests/nautilus-verify/VERSION b/tests/nautilus-verify/VERSION deleted file mode 100644 index ee716f9..0000000 --- a/tests/nautilus-verify/VERSION +++ /dev/null @@ -1 +0,0 @@ -1.224.0 diff --git a/tests/nautilus-verify/pyproject.toml b/tests/nautilus-verify/pyproject.toml deleted file mode 100644 index fec1077..0000000 --- a/tests/nautilus-verify/pyproject.toml +++ /dev/null @@ -1,7 +0,0 @@ -[project] -name = "nautilus-verify" -version = "0.1.0" -requires-python = ">=3.12" - -[tool.pytest.ini_options] -testpaths = ["python"] diff --git a/tests/nautilus-verify/python/conftest.py b/tests/nautilus-verify/python/conftest.py deleted file mode 100644 index e80ebae..0000000 --- a/tests/nautilus-verify/python/conftest.py +++ /dev/null @@ -1,3 +0,0 @@ -import nautilus_trader - -NAUTILUS_VERSION = nautilus_trader.__version__ diff --git a/tests/nautilus-verify/python/test_anti_hallucination.py b/tests/nautilus-verify/python/test_anti_hallucination.py deleted file mode 100644 index 1262a87..0000000 --- a/tests/nautilus-verify/python/test_anti_hallucination.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Tests for every row in the Python anti-hallucination table (SKILL.md). - -Each test verifies the 'Reality' column is still correct for the installed version. -Many Cython classes have opaque signatures (*args, **kwargs), so we test by -constructing objects rather than introspecting signatures. -""" - -import inspect - - -# Row: cache.position_for_instrument(id) → cache.positions_open(instrument_id=id) -def test_cache_positions_open_method(): - from nautilus_trader.cache.cache import Cache - - assert hasattr(Cache, "positions_open") - assert not hasattr(Cache, "position_for_instrument") - - -# Row: engine.trader.cache → engine.cache directly -def test_engine_cache_direct_access(): - from nautilus_trader.backtest.engine import BacktestEngine - - assert hasattr(BacktestEngine, "cache") - - -# Row: GenericDataWrangler → specific wranglers -def test_specific_data_wranglers_exist(): - from nautilus_trader.persistence.wranglers import ( - BarDataWrangler, - OrderBookDeltaDataWrangler, - QuoteTickDataWrangler, - TradeTickDataWrangler, - ) - - -# Row: catalog.data_types() → catalog.list_data_types() -def test_catalog_list_data_types(): - from nautilus_trader.persistence.catalog.parquet import ParquetDataCatalog - - assert hasattr(ParquetDataCatalog, "list_data_types") - - -# Row: BacktestEngineConfig from nautilus_trader.config → from nautilus_trader.backtest -def test_backtest_engine_config_import(): - from nautilus_trader.backtest.engine import BacktestEngineConfig - - -# Row: FillModel(prob_fill_on_stop=...) → only prob_fill_on_limit, prob_slippage, random_seed -def test_fillmodel_constructor_accepts_three_params(): - from nautilus_trader.backtest.models import FillModel - - fm = FillModel(prob_fill_on_limit=0.3, prob_slippage=0.5, random_seed=42) - assert fm is not None - - -# Row: LoggingConfig(log_file_path=) → log_directory= -def test_logging_config_log_directory(): - from nautilus_trader.config import LoggingConfig - - lc = LoggingConfig(log_directory="logs/") - assert lc.log_directory == "logs/" - - -# Row: from nautilus_trader.trading.actor → from nautilus_trader.common.actor import Actor -def test_actor_import_from_common(): - from nautilus_trader.common.actor import Actor - - -# Row: from nautilus_trader.indicators.ema → from nautilus_trader.indicators import EMA -def test_ema_import(): - from nautilus_trader.indicators import ExponentialMovingAverage - - -# Row: BollingerBands(20) → BollingerBands(20, 2.0) — k mandatory -def test_bollinger_bands_k_mandatory(): - from nautilus_trader.indicators import BollingerBands - - bb = BollingerBands(20, 2.0) - assert bb is not None - - -# Row: MACD(12, 26, 9) → 3rd param is MovingAverageType, not signal_period -def test_macd_third_param_is_ma_type(): - from nautilus_trader.indicators import ( - MovingAverageConvergenceDivergence, - MovingAverageType, - ) - - m = MovingAverageConvergenceDivergence(12, 26, MovingAverageType.SIMPLE) - assert m is not None - - -# Row: DYDXDataClientConfig → DydxDataClientConfig (mixed case) -def test_dydx_config_mixed_case(): - from nautilus_trader.adapters.dydx.config import DydxDataClientConfig - - -# Row: RSI value in [0, 1] not [0, 100] -def test_rsi_value_range_note(): - from nautilus_trader.indicators import RelativeStrengthIndex - - rsi = RelativeStrengthIndex(14) - - -# Row: frozen_account=True means checks DISABLED (inverted) -def test_frozen_account_param_exists(): - from nautilus_trader.backtest.engine import BacktestEngine - - sig = inspect.signature(BacktestEngine.add_venue) - assert "frozen_account" in sig.parameters - - -# Row: AccountType.CASH vs MARGIN -def test_account_type_margin_exists(): - from nautilus_trader.model.enums import AccountType - - assert hasattr(AccountType, "MARGIN") - assert hasattr(AccountType, "CASH") - - -# Row: order.order_side → order.side (events use event.order_side) -def test_order_side_field_name(): - from nautilus_trader.model.orders import MarketOrder - - assert hasattr(MarketOrder, "side") - - -# Row: request_bars(bar_type) one arg → requires start -def test_request_bars_signature(): - from nautilus_trader.trading.strategy import Strategy - - sig = inspect.signature(Strategy.request_bars) - assert "start" in sig.parameters - - -# Row: GreeksCalculator import path (moved to nautilus_trader.model.greeks) -def test_greeks_calculator_import(): - from nautilus_trader.model.greeks import GreeksCalculator - - -# Row: from nautilus_trader.core.nautilus_pyo3 → correct path for black_scholes_greeks -def test_pyo3_black_scholes_import(): - from nautilus_trader.core.nautilus_pyo3 import black_scholes_greeks - - -# Row: BinanceAccountType.USDT_FUTURE (no S) → USDT_FUTURES (with S) -def test_binance_usdt_futures_with_s(): - from nautilus_trader.adapters.binance.common.enums import BinanceAccountType - - assert hasattr(BinanceAccountType, "USDT_FUTURES") - assert not hasattr(BinanceAccountType, "USDT_FUTURE") - - -# Row: is_testnet=True (not testnet=True) for dYdX -def test_dydx_is_testnet_param(): - from nautilus_trader.adapters.dydx.config import DydxDataClientConfig - - d = DydxDataClientConfig(is_testnet=True) - assert d.is_testnet is True - - -# Row: is_testnet=True (not testnet=True) for Deribit -def test_deribit_is_testnet_param(): - from nautilus_trader.adapters.deribit.config import DeribitDataClientConfig - - d = DeribitDataClientConfig(is_testnet=True) - assert d.is_testnet is True - - -# Row: book.best_bid_price() returns Price, not float -def test_price_type_exists(): - from nautilus_trader.model.objects import Price - - p = Price.from_str("1.50000") - assert not isinstance(p, float) - - -# Row: subscribe_funding_rates() method exists on Strategy -def test_strategy_has_subscribe_funding_rates(): - from nautilus_trader.trading.strategy import Strategy - - assert hasattr(Strategy, "subscribe_funding_rates") - - -# Row: MarketStatusAction.RESUME does not exist — use TRADING -def test_market_status_action_trading(): - from nautilus_trader.model.enums import MarketStatusAction - - assert hasattr(MarketStatusAction, "TRADING") - assert not hasattr(MarketStatusAction, "RESUME") - - -# Row: SyntheticInstrument needs ts_event, ts_init (Cython — test via hasattr on class) -def test_synthetic_instrument_exists(): - from nautilus_trader.model.instruments import SyntheticInstrument - - assert SyntheticInstrument is not None - - -# Row: from nautilus_trader.common.component import LiveClock (not TestClock from common.clock) -def test_live_clock_import(): - from nautilus_trader.common.component import LiveClock - - -# Row: CacheConfig import path -def test_cache_config_import(): - from nautilus_trader.config import CacheConfig diff --git a/tests/nautilus-verify/python/test_backtest_config.py b/tests/nautilus-verify/python/test_backtest_config.py deleted file mode 100644 index 538bd6c..0000000 --- a/tests/nautilus-verify/python/test_backtest_config.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Tests backtest configuration patterns from battle_tested.md.""" - - -def test_venue_setup_full_pattern(): - """Verify the full venue setup from battle_tested compiles.""" - from nautilus_trader.model.enums import AccountType, OmsType - from nautilus_trader.model.identifiers import Venue - from nautilus_trader.model.objects import Currency, Money - from nautilus_trader.backtest.models import FillModel - - venue = Venue("BINANCE") - oms_type = OmsType.NETTING - account_type = AccountType.MARGIN - usdt = Currency.from_str("USDT") - starting_balances = [Money(10_000, usdt)] - fill_model = FillModel( - prob_fill_on_limit=0.3, - prob_slippage=0.5, - random_seed=42, - ) - - assert venue is not None - assert account_type == AccountType.MARGIN - assert len(starting_balances) == 1 - assert fill_model is not None - - -def test_bar_data_wrangler_ts_init_delta(): - """Verify BarDataWrangler accepts ts_init_delta for timestamp correction.""" - import inspect - from nautilus_trader.persistence.wranglers import BarDataWrangler - - sig = inspect.signature(BarDataWrangler.process) - assert "ts_init_delta" in sig.parameters - - -def test_multi_currency_base_currency_none(): - """Verify base_currency=None is valid for multi-currency accounts.""" - import inspect - from nautilus_trader.backtest.engine import BacktestEngine - - sig = inspect.signature(BacktestEngine.add_venue) - assert "base_currency" in sig.parameters diff --git a/tests/nautilus-verify/python/test_constructors.py b/tests/nautilus-verify/python/test_constructors.py deleted file mode 100644 index 1a922e2..0000000 --- a/tests/nautilus-verify/python/test_constructors.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Tests for constructor patterns from battle_tested.md and SKILL.md.""" - -from nautilus_trader.model.enums import OrderSide -from nautilus_trader.model.objects import Price, Quantity - - -# BookOrder: 4 positional args, not keyword-style -def test_book_order_positional_args(): - from nautilus_trader.model.data import BookOrder - - order = BookOrder( - OrderSide.BUY, - Price.from_str("50000.00"), - Quantity.from_str("1.000"), - 0, - ) - assert order.side == OrderSide.BUY - - -# FillModel: only 3 params -def test_fillmodel_constructor(): - from nautilus_trader.backtest.models import FillModel - - fm = FillModel( - prob_fill_on_limit=0.3, - prob_slippage=0.5, - random_seed=42, - ) - assert fm is not None - - -# Venue setup with AccountType.MARGIN -def test_account_type_and_oms_type(): - from nautilus_trader.model.enums import AccountType, OmsType - - assert AccountType.MARGIN is not None - assert OmsType.NETTING is not None - - -# Currency.from_str (not bare constant) -def test_currency_from_str(): - from nautilus_trader.model.objects import Currency - - usdt = Currency.from_str("USDT") - assert usdt is not None - - -# Money constructor -def test_money_constructor(): - from nautilus_trader.model.objects import Currency - from nautilus_trader.model.objects import Money - - usdt = Currency.from_str("USDT") - m = Money(10_000, usdt) - assert m is not None - - -# Price.from_str (not Price(float)) -def test_price_from_str(): - from nautilus_trader.model.objects import Price - - p = Price.from_str("1.50000") - assert float(p) == 1.5 - - -# Venue constructor -def test_venue_constructor(): - from nautilus_trader.model.identifiers import Venue - - v = Venue("BINANCE") - assert str(v) == "BINANCE" - - -# InstrumentId full format "SYMBOL.VENUE" -def test_instrument_id_format(): - from nautilus_trader.model.identifiers import InstrumentId - - iid = InstrumentId.from_str("BTCUSDT-PERP.BINANCE") - assert str(iid) == "BTCUSDT-PERP.BINANCE" - - -# LoggingConfig with log_directory -def test_logging_config_constructor(): - from nautilus_trader.config import LoggingConfig - - lc = LoggingConfig( - log_level="INFO", - log_directory="logs/", - log_colors=False, - ) - assert lc.log_directory == "logs/" diff --git a/tests/nautilus-verify/python/test_custom_data.py b/tests/nautilus-verify/python/test_custom_data.py deleted file mode 100644 index 3f14a89..0000000 --- a/tests/nautilus-verify/python/test_custom_data.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Tests the ImbalanceData custom data pattern from battle_tested.md.""" - -from nautilus_trader.core.data import Data - - -class ImbalanceData(Data): - def __init__(self, imbalance: float, volume: float, ts_event: int, ts_init: int): - self.imbalance = imbalance - self.volume = volume - self._ts_event = ts_event - self._ts_init = ts_init - - @property - def ts_event(self) -> int: - return self._ts_event - - @property - def ts_init(self) -> int: - return self._ts_init - - -def test_imbalance_data_instantiates(): - data = ImbalanceData( - imbalance=0.75, - volume=1000.0, - ts_event=1_000_000_000, - ts_init=1_000_000_001, - ) - assert data.imbalance == 0.75 - assert data.volume == 1000.0 - assert data.ts_event == 1_000_000_000 - assert data.ts_init == 1_000_000_001 - - -def test_imbalance_data_is_data(): - data = ImbalanceData( - imbalance=0.5, - volume=500.0, - ts_event=0, - ts_init=0, - ) - assert isinstance(data, Data) diff --git a/tests/nautilus-verify/python/test_custom_indicator.py b/tests/nautilus-verify/python/test_custom_indicator.py deleted file mode 100644 index 13ccd88..0000000 --- a/tests/nautilus-verify/python/test_custom_indicator.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Tests the SpreadEMA custom indicator pattern from battle_tested.md.""" - -from nautilus_trader.indicators.base import Indicator - - -class SpreadEMA(Indicator): - def __init__(self, period: int): - super().__init__([]) - self.period = period - self._values = [] - self.value = 0.0 - - def handle_quote_tick(self, tick) -> None: - spread = float(tick.ask_price - tick.bid_price) - self._values.append(spread) - if len(self._values) > self.period: - self._values.pop(0) - self.value = sum(self._values) / len(self._values) - # _set_initialized is inherited from Cython Indicator base — do NOT override it - self._set_initialized(len(self._values) >= self.period) - - def reset(self) -> None: - self._values.clear() - self.value = 0.0 - self._set_initialized(False) - - -def test_spread_ema_instantiates(): - ema = SpreadEMA(period=10) - assert ema.period == 10 - assert ema.value == 0.0 - assert not ema.initialized - - -def test_spread_ema_warmup_and_reset(): - ema = SpreadEMA(period=3) - ema._values = [1.0, 2.0, 3.0] - ema.value = 2.0 - ema._set_initialized(True) - assert ema.initialized - - ema.reset() - assert ema._values == [] - assert ema.value == 0.0 - assert not ema.initialized - - -def test_spread_ema_is_indicator(): - ema = SpreadEMA(period=5) - assert isinstance(ema, Indicator) diff --git a/tests/nautilus-verify/python/test_order_book.py b/tests/nautilus-verify/python/test_order_book.py deleted file mode 100644 index 6376954..0000000 --- a/tests/nautilus-verify/python/test_order_book.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Tests order book patterns from battle_tested.md and anti-hallucination table.""" - -from nautilus_trader.model.objects import Price - - -def test_price_is_not_float(): - """book.best_bid_price() returns Price, not float — must cast with float().""" - p = Price.from_str("50000.00") - assert not isinstance(p, float) - assert isinstance(float(p), float) - - -def test_price_arithmetic_requires_cast(): - """Price objects need float() for arithmetic with regular numbers.""" - p = Price.from_str("100.50") - result = float(p) * 2 - assert result == 201.0 - - -def test_book_order_import_from_data(): - """BookOrder import from model.data, NOT model.book.""" - from nautilus_trader.model.data import BookOrder - - -def test_order_book_depth10_exists(): - """OrderBookDepth10 pre-aggregated type exists.""" - from nautilus_trader.model.data import OrderBookDepth10 - - -def test_order_book_delta_exists(): - """OrderBookDelta type exists for L2 data.""" - from nautilus_trader.model.data import OrderBookDelta diff --git a/tests/nautilus-verify/src/lib.rs b/tests/nautilus-verify/src/lib.rs deleted file mode 100644 index c19de38..0000000 --- a/tests/nautilus-verify/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -// nautilus-verify: compile and runtime tests for nautilus-docs skill -// This crate exists only so `cargo test` can run integration tests in tests/ diff --git a/tests/nautilus-verify/tests/anti_hallucination.rs b/tests/nautilus-verify/tests/anti_hallucination.rs deleted file mode 100644 index 748229c..0000000 --- a/tests/nautilus-verify/tests/anti_hallucination.rs +++ /dev/null @@ -1,152 +0,0 @@ -// Tests for every row in the Rust anti-hallucination table (SKILL.md). -// Each test verifies the "Reality" column compiles/works correctly. - -use nautilus_binance::common::enums::BinanceProductType; -use nautilus_binance::config::BinanceDataClientConfig; -use nautilus_binance::factories::BinanceExecutionClientFactory; -use nautilus_common::actor::{DataActor, DataActorCore}; -use nautilus_common::enums::Environment; -use nautilus_live::node::LiveNode; -use nautilus_model::data::{FundingRateUpdate, OrderBookDeltas}; -use nautilus_model::events::position::closed::PositionClosed; -use nautilus_model::identifiers::{InstrumentId, TraderId}; -use nautilus_trading::strategy::{StrategyConfig, StrategyCore}; -use std::ops::{Deref, DerefMut}; - -// Row: LiveNode::new(config) → LiveNode::builder(trader_id, Environment::Live)?.build()? -#[test] -fn hallucination_livenode_builder_pattern() { - // Verify the builder method exists with correct signature - // (we can't actually build without a full runtime, but the function must exist) - let _ = LiveNode::builder as fn(TraderId, Environment) -> anyhow::Result<_>; -} - -// Row: BinanceDataClientConfig::builder() → struct literal with ..Default::default() -#[test] -fn hallucination_binance_data_config_struct_literal() { - let config = BinanceDataClientConfig { - product_types: vec![BinanceProductType::UsdM], - ..Default::default() - }; - assert_eq!(config.product_types, vec![BinanceProductType::UsdM]); -} - -// Row: BinanceAccountType::UsdtFutures → BinanceProductType::UsdM -#[test] -fn hallucination_binance_product_type_usdm() { - let _ = BinanceProductType::UsdM; - let _ = BinanceProductType::CoinM; - let _ = BinanceProductType::Spot; - let _ = BinanceProductType::Margin; -} - -// Row: nautilus_core::identifiers::InstrumentId → nautilus_model::identifiers::InstrumentId -#[test] -fn hallucination_instrument_id_in_model() { - let _ = std::any::type_name::(); - // Compiles = nautilus_model::identifiers::InstrumentId is the correct path -} - -// Row: DataActor in nautilus_trading → DataActor in nautilus_common::actor -#[test] -fn hallucination_data_actor_in_common() { - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -// Row: nautilus_trader_model → nautilus_model -#[test] -fn hallucination_crate_name_is_nautilus_model() { - // If this compiles, nautilus_model (not nautilus_trader_model) is correct - let _ = std::any::type_name::(); -} - -// Row: on_book_delta (singular) → on_book_deltas (plural — batch) -#[test] -fn hallucination_on_book_deltas_plural() { - // Verify the trait method signature accepts OrderBookDeltas (batch) - let _ = std::any::type_name::(); - // The DataActor trait has fn on_book_deltas(&mut self, deltas: &OrderBookDeltas) - // If OrderBookDeltas doesn't exist, this test fails to compile. -} - -// Row: event.duration_ns on PositionClosed → event.duration (no _ns suffix) -#[test] -fn hallucination_position_closed_duration_field() { - // Verify the field name is `duration`, not `duration_ns` - // PositionClosed has: pub duration: DurationNanos - let _ = |e: &PositionClosed| e.duration; -} - -// Row: event.realized_pnl is Money → Option -#[test] -fn hallucination_realized_pnl_is_option() { - // PositionClosed has: pub realized_pnl: Option - let _ = |e: &PositionClosed| { - match &e.realized_pnl { - Some(_money) => {} - None => {} - } - }; -} - -// Row: Deref for Strategy → Deref -#[test] -fn hallucination_strategy_core_derefs_to_data_actor_core() { - // StrategyCore implements Deref - fn assert_deref_target(_: &impl Deref) {} - let config = StrategyConfig { - strategy_id: Some("TEST-001".into()), - ..Default::default() - }; - let core = StrategyCore::new(config); - assert_deref_target(&core); -} - -// Row: node.add_actor(strategy) for Strategy → node.add_strategy(strategy) -// Can't test without runtime, but verify the method exists on LiveNode -// (tested transitively by strategy_pattern.rs and livenode builder test) - -// Row: BinanceExecClientFactory → BinanceExecutionClientFactory (full name) -#[test] -fn hallucination_binance_execution_client_factory_full_name() { - let _ = std::any::type_name::(); -} - -// Row: Deref target for user Strategy struct must be DataActorCore -#[test] -fn hallucination_user_strategy_deref_chain() { - #[derive(Debug)] - struct TestStrat { - core: StrategyCore, - } - - impl Deref for TestStrat { - type Target = DataActorCore; - fn deref(&self) -> &Self::Target { - &self.core // auto-derefs StrategyCore → DataActorCore - } - } - - impl DerefMut for TestStrat { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.core - } - } - - let config = StrategyConfig { - strategy_id: Some("TEST-002".into()), - ..Default::default() - }; - let strat = TestStrat { - core: StrategyCore::new(config), - }; - // Verify the deref chain works - let _core_ref: &DataActorCore = &strat; -} - -// Row: FundingRateUpdate exists in nautilus_model::data -#[test] -fn hallucination_funding_rate_update_path() { - let _ = std::any::type_name::(); -} diff --git a/tests/nautilus-verify/tests/crate_map.rs b/tests/nautilus-verify/tests/crate_map.rs deleted file mode 100644 index 90b00ea..0000000 --- a/tests/nautilus-verify/tests/crate_map.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Verifies all import paths from the SKILL.md crate map. -// If any `use` statement fails to compile, the crate map is stale. - -// nautilus-model: identifiers -use nautilus_model::identifiers::{ - AccountId, ActorId, ClientId, ClientOrderId, ComponentId, ExecAlgorithmId, InstrumentId, - OrderListId, PositionId, StrategyId, Symbol, TradeId, TraderId, Venue, VenueOrderId, -}; - -// nautilus-model: data types -use nautilus_model::data::{ - Bar, BarSpecification, BarType, BookOrder, FundingRateUpdate, IndexPriceUpdate, - InstrumentClose, MarkPriceUpdate, OrderBookDelta, OrderBookDeltas, OrderBookDepth10, - QuoteTick, TradeTick, -}; - -// nautilus-model: order types -use nautilus_model::orders::{ - LimitOrder, MarketOrder, StopLimitOrder, StopMarketOrder, TrailingStopLimitOrder, - TrailingStopMarketOrder, -}; - -// nautilus-model: events -use nautilus_model::events::position::closed::PositionClosed; - -// nautilus-common: actor -use nautilus_common::actor::{DataActorConfig, DataActorCore}; - -// DataActor is a trait — imported via trait bound check -fn _assert_data_actor_trait_exists() {} - -// nautilus-common: environment -use nautilus_common::enums::Environment; - -// nautilus-trading: strategy -use nautilus_trading::strategy::{StrategyConfig, StrategyCore}; - -// nautilus-binance: config + enums -use nautilus_binance::common::enums::BinanceProductType; -use nautilus_binance::config::{BinanceDataClientConfig, BinanceExecClientConfig}; -use nautilus_binance::factories::{BinanceDataClientFactory, BinanceExecutionClientFactory}; - -// nautilus-live: node -use nautilus_live::node::LiveNode; - -#[test] -fn crate_map_identifiers_compile() { - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -#[test] -fn crate_map_data_types_compile() { - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -#[test] -fn crate_map_order_types_compile() { - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -#[test] -fn crate_map_actor_and_strategy_compile() { - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -#[test] -fn crate_map_binance_compile() { - let _ = BinanceProductType::UsdM; - let _ = BinanceProductType::CoinM; - let _ = BinanceProductType::Spot; - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -#[test] -fn crate_map_live_node_compile() { - let _ = std::any::type_name::(); - let _ = std::any::type_name::(); -} - -#[test] -fn crate_map_events_compile() { - let _ = std::any::type_name::(); -} diff --git a/tests/nautilus-verify/tests/data_actor_pattern.rs b/tests/nautilus-verify/tests/data_actor_pattern.rs deleted file mode 100644 index 60b97a5..0000000 --- a/tests/nautilus-verify/tests/data_actor_pattern.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Verifies the DataActor pattern from battle_tested.md (FundingRateCapture). -// If this compiles, the pattern is correct for the current nautilus version. - -use std::ops::{Deref, DerefMut}; - -use nautilus_common::actor::{DataActor, DataActorConfig, DataActorCore}; -use nautilus_model::data::FundingRateUpdate; -use nautilus_model::identifiers::InstrumentId; - -#[derive(Debug)] -struct FundingRateCapture { - core: DataActorCore, - instrument_id: InstrumentId, -} - -impl FundingRateCapture { - fn new(instrument_id: InstrumentId) -> Self { - Self { - core: DataActorCore::new(DataActorConfig { - actor_id: Some("FundingCapture-001".into()), - ..Default::default() - }), - instrument_id, - } - } -} - -impl Deref for FundingRateCapture { - type Target = DataActorCore; - fn deref(&self) -> &Self::Target { - &self.core - } -} - -impl DerefMut for FundingRateCapture { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.core - } -} - -impl DataActor for FundingRateCapture { - fn on_start(&mut self) -> anyhow::Result<()> { - self.subscribe_funding_rates(self.instrument_id, None, None); - Ok(()) - } - - fn on_funding_rate(&mut self, update: &FundingRateUpdate) -> anyhow::Result<()> { - log::info!("{} rate={}", update.instrument_id, update.rate); - Ok(()) - } - - fn on_stop(&mut self) -> anyhow::Result<()> { - self.unsubscribe_funding_rates(self.instrument_id, None, None); - Ok(()) - } -} - -#[test] -fn data_actor_funding_rate_capture_compiles() { - let id = InstrumentId::from("BTCUSDT-PERP.BINANCE"); - let actor = FundingRateCapture::new(id); - assert_eq!(actor.instrument_id.to_string(), "BTCUSDT-PERP.BINANCE"); -} diff --git a/tests/nautilus-verify/tests/strategy_pattern.rs b/tests/nautilus-verify/tests/strategy_pattern.rs deleted file mode 100644 index e13e35f..0000000 --- a/tests/nautilus-verify/tests/strategy_pattern.rs +++ /dev/null @@ -1,81 +0,0 @@ -// Verifies the Strategy pattern from battle_tested.md (MyStrategy with Deref chain). -// If this compiles, the full Strategy + DataActor trait wiring is correct. - -use std::ops::{Deref, DerefMut}; - -use nautilus_common::actor::{DataActor, DataActorCore}; -use nautilus_model::data::TradeTick; -use nautilus_model::identifiers::InstrumentId; -use nautilus_trading::strategy::{Strategy, StrategyConfig, StrategyCore}; - -#[derive(Debug)] -struct MyStrategy { - core: StrategyCore, - instrument_id: InstrumentId, -} - -impl MyStrategy { - fn new(instrument_id: InstrumentId) -> Self { - Self { - core: StrategyCore::new(StrategyConfig { - strategy_id: Some("MY-001".into()), - ..Default::default() - }), - instrument_id, - } - } -} - -// CRITICAL: Deref target is DataActorCore, NOT StrategyCore. -// StrategyCore implements Deref, so this chains through. -impl Deref for MyStrategy { - type Target = DataActorCore; - fn deref(&self) -> &Self::Target { - &self.core // auto-derefs StrategyCore → DataActorCore - } -} - -impl DerefMut for MyStrategy { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.core - } -} - -impl DataActor for MyStrategy { - fn on_start(&mut self) -> anyhow::Result<()> { - self.subscribe_trades(self.instrument_id, None, None); - Ok(()) - } - - fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { - log::info!("trade px={}", trade.price.as_f64()); - Ok(()) - } - - fn on_stop(&mut self) -> anyhow::Result<()> { - self.unsubscribe_trades(self.instrument_id, None, None); - Ok(()) - } -} - -impl Strategy for MyStrategy { - fn core(&self) -> &StrategyCore { - &self.core - } - fn core_mut(&mut self) -> &mut StrategyCore { - &mut self.core - } -} - -#[test] -fn strategy_pattern_compiles_and_constructs() { - let id = InstrumentId::from("BTCUSDT-PERP.BINANCE"); - let strat = MyStrategy::new(id); - assert_eq!(strat.instrument_id.to_string(), "BTCUSDT-PERP.BINANCE"); - - // Verify Deref chain: MyStrategy → DataActorCore - let _core_ref: &DataActorCore = &strat; - - // Verify Strategy trait: core() returns StrategyCore reference - let _strat_core: &StrategyCore = strat.core(); -} diff --git a/tests/nautilus-verify/verify.sh b/tests/nautilus-verify/verify.sh deleted file mode 100755 index e848fa4..0000000 --- a/tests/nautilus-verify/verify.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -VERIFIED="$(cat "$SCRIPT_DIR/VERSION")" -INSTALLED=$("$REPO_ROOT/.venv/bin/python" -c "import nautilus_trader; print(nautilus_trader.__version__)") - -echo "Verified: $VERIFIED | Installed: $INSTALLED" - -if [[ "$VERIFIED" != "$INSTALLED" ]]; then - echo "VERSION CHANGED — full verification required" -fi - -RUST_OK=true -PYTHON_OK=true - -echo "" -echo "=== Rust compile tests ===" -if (cd "$SCRIPT_DIR" && cargo test 2>&1); then - echo "Rust: PASS" -else - echo "Rust: FAIL" - RUST_OK=false -fi - -echo "" -echo "=== Python tests ===" -if (cd "$SCRIPT_DIR" && "$REPO_ROOT/.venv/bin/python" -m pytest python/ -v 2>&1); then - echo "Python: PASS" -else - echo "Python: FAIL" - PYTHON_OK=false -fi - -echo "" -if $RUST_OK && $PYTHON_OK; then - if [[ "$VERIFIED" != "$INSTALLED" ]]; then - echo "$INSTALLED" > "$SCRIPT_DIR/VERSION" - echo "All tests pass. Updated VERSION: $VERIFIED -> $INSTALLED" - else - echo "All tests pass. VERSION unchanged ($VERIFIED)." - fi -else - echo "FAILURES DETECTED — hallucination rows may be stale for v$INSTALLED" - exit 1 -fi