diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 33c37d0..7d048f9 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -105,6 +105,12 @@ "description": "Hierarchical microstructure knowledge base for trading venues: exchange mechanics, order books, auctions, Reg NMS, Chinese futures (CTP), feed handlers, execution models", "source": "./venue-expert", "skills": ["./"] + }, + { + "name": "nautilus-docs", + "description": "Official NautilusTrader documentation — concepts, API reference, integration guides, tutorials", + "source": "./nautilus-docs", + "skills": ["./"] } ] } diff --git a/.gitignore b/.gitignore index d803c05..23a0e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .claude/ .tmp/ +.env +nautilus-docs/references/docs/ +__pycache__/ diff --git a/README.md b/README.md index dd5cf1c..47132b2 100644 --- a/README.md +++ b/README.md @@ -4,31 +4,29 @@ Deevs' plugin marketplace for Claude Code with workflow chains, terminal control ## Installation +Use the installer for all platforms (Claude Code, Codex, or both). Interactive by default: + ```bash -/plugin marketplace add git@github.com:DeevsDeevs/agent-system.git +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- ``` -## Codex Setup - -Codex loads repo-scoped skills from `.agents/skills`. This repo includes `.agents/skills` symlinks to each skill folder. -For backward compatibility, `.codex/skills` symlinks are also kept. +The installer clones the repo, fetches any external dependencies (e.g. NautilusTrader docs — only when the nautilus-docs skill is selected), and sets everything up. -Codex multi-agent role config for all personas is included in: -- `.codex/config.toml` -- `.codex/agents/*.toml` - -Usage: -- Invoke a skill with `$skill-name` or `/skills`. -- For chain operations: `$chain-system link `, `$chain-system load `, `$chain-system list`. -- For persona workflows: `$dev-experts `, `$bug-hunters `, `$alpha-squad `, `$mft-research-experts `. - -If Codex was already running, restart it to reload the skills. - -Global (user-scoped) install for any repo: +### Non-interactive ```bash +# Claude Code — all skills +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --non-interactive --platform claude + +# Codex — all skills curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ | bash -s -- --non-interactive --platform codex + +# Both platforms +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --non-interactive --platform both ``` Install selected skills only: @@ -38,20 +36,30 @@ curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/script | bash -s -- --non-interactive --platform codex --skills dev-experts,bug-hunters ``` -Installer (interactive by default): +### Flags + +| Flag | Description | +|------|-------------| +| `--platform claude\|codex\|both` | Target platform | +| `--skills a,b,c` | Install only these skills (default: all, codex only) | +| `--repo-ref ` | Checkout specific ref | +| `--repo ` | Custom repo URL | +| `--clone-dir ` | Where to clone (default: `~/src/agent-system`) | +| `--target ` | Codex skills dir (default: `~/.codex/skills`) | +| `--mode symlink\|copy` | Install mode (default: symlink) | +| `--non-interactive` | Skip prompts | +| `--uninstall` | Remove installed skills | + +### Uninstall ```bash -curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \\ - | bash -s -- +curl -fsSL https://raw.githubusercontent.com/DeevsDeevs/agent-system/main/scripts/install.sh \ + | bash -s -- --uninstall --platform both ``` -Installer flags: `--non-interactive`, `--platform claude|codex|both`, `--repo-ref `, `--uninstall`, `--skills a,b,c`. - -Uninstall: -- Codex: run the installer with `--uninstall`. -- Claude Code: use `/plugin uninstall @deevs-agent-system` inside Claude Code. +### Codex usage -Install a single skill by URL with `$skill-installer`. +Invoke a skill with `$skill-name` or `/skills`. If Codex was already running, restart it to reload skills. ## Plugins diff --git a/nautilus-docs/SKILL.md b/nautilus-docs/SKILL.md new file mode 100644 index 0000000..513d31d --- /dev/null +++ b/nautilus-docs/SKILL.md @@ -0,0 +1,286 @@ +--- +name: nautilus-docs +description: > + NautilusTrader trading platform — strategies, backtesting, live deployment, order management, + market data, Rust native binaries. Use when writing, reviewing, or debugging NautilusTrader + code (Python or Rust), configuring venue adapters, or answering questions about the platform. +--- + +# NautilusTrader v1.224.0 + +**MANDATORY**: Before writing or debugging NautilusTrader code, READ the matching doc from the navigator below. If no entry matches, **grep** `references/docs/` for the class/method/config name. Never guess signatures or constructors. + +**MUST READ** before any NautilusTrader work: [architecture.md](references/docs/concepts/architecture.md) (system diagram, data/execution flow, threading model, component FSM) and [actors.md](references/docs/concepts/actors.md) (lifecycle, callbacks, data handler mapping). + +**Do not build on a private NautilusTrader fork**. Extend via composition (DataActor/Strategy traits), use `[patch]` for temporary bug fixes (with upstream PR), or fork+modify only when contributing back. Cloning to read source or run tests is fine. See [battle_tested.md § Step 3](references/battle_tested.md) for the full decision tree. + +## Doc Navigator + +### Concepts + +| Topic | Doc | +|-------|-----| +| Architecture / overview | [architecture.md](references/docs/concepts/architecture.md), [overview.md](references/docs/concepts/overview.md) | +| Actor development | [actors.md](references/docs/concepts/actors.md) | +| Strategy development | [strategies.md](references/docs/concepts/strategies.md) | +| Backtesting | [backtesting.md](references/docs/concepts/backtesting.md) | +| Live trading | [live.md](references/docs/concepts/live.md) | +| Order types & execution | [orders.md](references/docs/concepts/orders.md), [execution.md](references/docs/concepts/execution.md) | +| Order book | [order_book.md](references/docs/concepts/order_book.md) | +| Data types & custom data | [data.md](references/docs/concepts/data.md), [custom_data.md](references/docs/concepts/custom_data.md) | +| Instruments | [instruments.md](references/docs/concepts/instruments.md) | +| Value types (Price, Quantity, Money) | [value_types.md](references/docs/concepts/value_types.md) | +| Positions & PnL | [positions.md](references/docs/concepts/positions.md) | +| Cache | [cache.md](references/docs/concepts/cache.md) | +| MessageBus | [message_bus.md](references/docs/concepts/message_bus.md) | +| Portfolio | [portfolio.md](references/docs/concepts/portfolio.md) | +| Options & Greeks | [options.md](references/docs/concepts/options.md), [greeks.md](references/docs/concepts/greeks.md) | +| Logging | [logging.md](references/docs/concepts/logging.md) | +| Reports & analysis | [reports.md](references/docs/concepts/reports.md) | +| Visualization | [visualization.md](references/docs/concepts/visualization.md) | +| Adapter development | [concepts/adapters.md](references/docs/concepts/adapters.md), [developer_guide/adapters.md](references/docs/developer_guide/adapters.md) | + +### Venue Integrations + +| Venue | Doc | +|-------|-----| +| Binance | [binance.md](references/docs/integrations/binance.md) | +| Bybit | [bybit.md](references/docs/integrations/bybit.md) | +| OKX | [okx.md](references/docs/integrations/okx.md) | +| dYdX | [dydx.md](references/docs/integrations/dydx.md) | +| Deribit | [deribit.md](references/docs/integrations/deribit.md) | +| Hyperliquid | [hyperliquid.md](references/docs/integrations/hyperliquid.md) | +| Kraken | [kraken.md](references/docs/integrations/kraken.md) | +| Interactive Brokers | [ib.md](references/docs/integrations/ib.md) | +| Betfair | [betfair.md](references/docs/integrations/betfair.md) | +| Polymarket | [polymarket.md](references/docs/integrations/polymarket.md) | +| Databento | [databento.md](references/docs/integrations/databento.md) | +| Tardis | [tardis.md](references/docs/integrations/tardis.md) | +| BitMEX | [bitmex.md](references/docs/integrations/bitmex.md) | +| AX Exchange | [architect_ax.md](references/docs/integrations/architect_ax.md) | +| Blockchain / DeFi | [blockchain.md](references/docs/integrations/blockchain.md) | + +### Dev / Setup + +| Topic | Doc | +|-------|-----| +| Installation | [installation.md](references/docs/getting_started/installation.md) | +| Quickstart | [quickstart.py](references/docs/getting_started/quickstart.py) | +| Backtest examples | [backtest_high_level.py](references/docs/getting_started/backtest_high_level.py), [backtest_low_level.py](references/docs/getting_started/backtest_low_level.py) | +| Environment setup | [environment_setup.md](references/docs/developer_guide/environment_setup.md) | +| Rust development | [rust.md](references/docs/developer_guide/rust.md) | +| Python development | [python.md](references/docs/developer_guide/python.md) | +| Testing | [testing.md](references/docs/developer_guide/testing.md) | +| Data testing spec | [spec_data_testing.md](references/docs/developer_guide/spec_data_testing.md) | +| Execution testing spec | [spec_exec_testing.md](references/docs/developer_guide/spec_exec_testing.md) | +| Test datasets | [test_datasets.md](references/docs/developer_guide/test_datasets.md) | +| Coding standards | [coding_standards.md](references/docs/developer_guide/coding_standards.md) | +| Benchmarking | [benchmarking.md](references/docs/developer_guide/benchmarking.md), [criterion_template.rs](references/docs/dev_templates/criterion_template.rs), [iai_template.rs](references/docs/dev_templates/iai_template.rs) | +| FFI | [ffi.md](references/docs/developer_guide/ffi.md) | +| Documentation | [docs.md](references/docs/developer_guide/docs.md) | +| Releases | [releases.md](references/docs/developer_guide/releases.md) | + +### Tutorials + +| Tutorial | Doc | +|----------|-----| +| FX mean reversion (AX) | [fx_mean_reversion_ax.md](references/docs/tutorials/fx_mean_reversion_ax.md) | +| Gold book imbalance (AX) | [gold_book_imbalance_ax.md](references/docs/tutorials/gold_book_imbalance_ax.md) | +| dYdX grid market maker | [grid_market_maker_dydx.md](references/docs/tutorials/grid_market_maker_dydx.md) | +| BitMEX grid market maker | [grid_market_maker_bitmex.md](references/docs/tutorials/grid_market_maker_bitmex.md) | +| Backtest Binance orderbook | [backtest_orderbook_binance.py](references/docs/tutorials/backtest_orderbook_binance.py) | +| Backtest Bybit orderbook | [backtest_orderbook_bybit.py](references/docs/tutorials/backtest_orderbook_bybit.py) | +| Backtest FX bars | [backtest_fx_bars.py](references/docs/tutorials/backtest_fx_bars.py) | +| Databento data catalog | [data_catalog_databento.py](references/docs/tutorials/data_catalog_databento.py) | +| Loading external data | [loading_external_data.py](references/docs/tutorials/loading_external_data.py) | + +### Search Strategy + +When the navigator doesn't cover your need, grep `references/docs/` for the class, method, or config name. **Concepts docs** have working code examples with correct signatures. **Integration docs** have venue-specific configs, supported features, and gotchas. Read BEFORE writing code — never invent an API call. + +## Supporting Files + +- **[battle_tested.md](references/battle_tested.md)** — Load when: **debugging live issues** (step 1: read the error body), Rust Strategy pattern, credentials, patching deps, on_start() ordering, market making, backtest config, venue gotchas, backtest→live migration, hot path performance, state persistence, fill/latency metrics, log rotation, custom indicators, data pipeline validation. +- **[REBUILD.md](references/REBUILD.md)** — Meta-prompt for regenerating this skill when NautilusTrader API changes. + +## Rust Standalone Binary + +Git dependencies, no workspace clone needed. All crates: `{ git = "https://github.com/nautechsystems/nautilus_trader.git" }`. Naming: `nautilus-{name}` (Cargo) → `nautilus_{name}` (Rust). + +### Crate Map + +| Layer | Crate | Provides | +|-------|-------|----------| +| Foundation | `nautilus-core` | Primitives, FFI | +| Foundation | `nautilus-model` | `InstrumentId`, `TraderId`, `Price`, `Quantity`, orders, positions, instruments | +| Foundation | `nautilus-common` | `DataActor`, `DataActorCore`, `DataActorConfig`, timers, message bus, component lifecycle | +| Trading | `nautilus-trading` | `Strategy` trait (extends DataActor), order/position management | +| Runtime | `nautilus-live` | `LiveNode`, `LiveNodeBuilder` | +| Runtime | `nautilus-backtest` | `BacktestNode`, `BacktestEngine` — needs `features = ["streaming"]` | +| Infra | `nautilus-indicators` | EMA, SMA, RSI, MACD, Bollinger — enable via `nautilus-trading` `features = ["examples"]` | +| Infra | `nautilus-persistence` | Parquet catalog, DataFusion | +| Infra | `nautilus-serialization` | Arrow, MessagePack encoding | +| Infra | `nautilus-network` | HTTP/WebSocket clients | +| Infra | `nautilus-cryptography` | HMAC, Ed25519 signing | +| Infra | `nautilus-infrastructure` | Redis, logging, monitoring | +| Infra | `nautilus-analysis` | Performance stats | +| Engine | `nautilus-data` | Data engine internals | +| Engine | `nautilus-execution` | Order routing internals | +| Engine | `nautilus-portfolio` | Account/position aggregation | +| Engine | `nautilus-risk` | Pre-trade checks | +| System | `nautilus-system` | Kernel orchestration | +| Test | `nautilus-testkit` | Test fixtures, mock instruments | + +### Adapter Crates + +All `nautilus-{venue}`: binance, bybit, okx, kraken, bitmex, deribit, dydx, hyperliquid, databento, tardis, betfair, polymarket, architect-ax, sandbox. See Venue Integrations docs for config details. + +### Which crates for your task + +| Use case | Crates | +|----------|--------| +| Data-only live actor | `nautilus-common`, `nautilus-model`, `nautilus-live`, `nautilus-{venue}` | +| Live strategy (orders) | above + `nautilus-trading` | +| Backtest | `nautilus-common`, `nautilus-model`, `nautilus-backtest` (+ `streaming`), `nautilus-trading` | +| With indicators | add `nautilus-trading` with `features = ["examples"]` | +| With Parquet catalog | add `nautilus-persistence` | +| `high-precision` (128-bit) | Default on most crates — `default-features = false` disables it, re-add explicitly | + +### Rust vs Python + +- **No `load_ids` / `InstrumentProviderConfig` in Rust** — adapters auto-discover all instruments at connect +- **Logging**: Nautilus owns the `log` backend. Use `log::info!()`. Output goes to stdout. No `RUST_LOG` env var, no `env_logger` +- **First build**: minutes (430+ crates). Incremental: ~2s. Use `cargo build` not `--release` for iteration +- **`#[derive(Debug)]` on all actor structs** — Nautilus requires `Debug` trait; missing it = compile error (zero-cost, fine for prod) +- **Callback naming**: Rust drops `_tick` suffix — `on_trade` not `on_trade_tick`, `on_quote` not `on_quote_tick`, `subscribe_trades` not `subscribe_trade_ticks` +- **Trade fields**: `trade.price.as_f64()`, `trade.size.as_f64()`, `trade.aggressor_side` (`AggressorSide::Buyer`/`Seller`) +- **`node.run().await`** blocks the event loop. Callbacks fire during run. `on_stop` fires on shutdown signal +- **Adapter configs** are struct literals with `..Default::default()`. Full names: `{Venue}DataClientConfig`, `{Venue}ExecutionClientConfig`. Factory: `{Venue}DataClientFactory`, `{Venue}ExecutionClientFactory` +- **Ed25519 keys (v0.55.0)**: HTTP signature URL-encoding bug — HMAC keys may also be auto-detected as Ed25519 if valid base64. See battle_tested.md + +### DataActor pattern (Nautilus-specific — don't generalize Deref/DerefMut elsewhere) + +```rust +use std::ops::{Deref, DerefMut}; +use nautilus_common::actor::{DataActor, DataActorConfig, DataActorCore}; +use nautilus_model::data::TradeTick; +use nautilus_model::identifiers::InstrumentId; + +#[derive(Debug)] +struct MyActor { + core: DataActorCore, // REQUIRED — blanket impls derive Actor+Component from this + instrument_id: InstrumentId, +} + +impl Deref for MyActor { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { &self.core } +} +impl DerefMut for MyActor { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.core } +} + +impl DataActor for MyActor { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_trades(self.instrument_id, None, None); // 3 args + Ok(()) + } + fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { + log::info!("px={} qty={}", trade.price.as_f64(), trade.size.as_f64()); + Ok(()) + } + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_trades(self.instrument_id, None, None); + Ok(()) + } +} + +// Constructor: +impl MyActor { + fn new(instrument_id: InstrumentId) -> Self { + Self { + core: DataActorCore::new(DataActorConfig { actor_id: Some("MyActor-001".into()), ..Default::default() }), + instrument_id, + } + } +} +``` + +## Common Hallucinations + + +### Python + +| Hallucination | Reality | +|--------------|---------| +| `cache.position_for_instrument(id)` | `cache.positions_open(instrument_id=id)` returns list | +| `engine.trader.cache` | `engine.cache` directly | +| `book.filtered_view()` | Use `cache.own_order_book()` | +| `book.get_avg_px_qty_for_exposure()` | Use `get_avg_px_for_quantity()` + `get_quantity_for_price()` | +| `level.count` / `book.count` | `book.update_count` | +| `BookOrder(price=, size=, side=)` | 4 positional args: `BookOrder(OrderSide.BUY, Price, Quantity, order_id=0)`. Import from `model.data`, NOT `model.book` | +| `GenericDataWrangler` | TradeTickDataWrangler, QuoteTickDataWrangler, OrderBookDeltaDataWrangler, BarDataWrangler | +| `catalog.data_types()` | `catalog.list_data_types()` | +| `BacktestEngineConfig` from `nautilus_trader.config` | from `nautilus_trader.backtest.engine` or `nautilus_trader.backtest.config` | +| `FillModel(prob_fill_on_stop=...)` | Only: prob_fill_on_limit, prob_slippage, random_seed | +| `LoggingConfig(log_file_path=)` | `log_directory=` | +| `cache.orders_filled()` | `cache.orders_closed()` | +| `pos.signed_qty / Decimal(...)` | TypeError: returns float — `Decimal(str(pos.signed_qty))` | +| `from nautilus_trader.trading.actor` | `from nautilus_trader.common.actor import Actor` | +| `from nautilus_trader.indicators.ema` | `from nautilus_trader.indicators import ExponentialMovingAverage` | +| `BollingerBands(20)` | `BollingerBands(20, 2.0)` — k mandatory | +| `MACD(12, 26, 9)` | 3rd param is `MovingAverageType`, not signal_period | +| `publish_signal(value=dict(...))` | KeyError — values must be int, float, or str | +| Encrypted Ed25519 private key | Must be unencrypted PKCS#8 | +| `on_timer()` as callback | `clock.set_timer(callback=handler)` | +| `order.order_side` | `order.side` — events use `event.order_side` | +| `request_bars(bar_type)` one arg | Requires `start`: `request_bars(bar_type, start=datetime(...))` | +| `GreeksCalculator(cache, clock, logger)` | Only 2 args: `GreeksCalculator(cache, clock)`. Import from `nautilus_trader.model.greeks`, not `indicators.greeks_calculator` | +| `SyntheticInstrument(sym, prec, comps, formula)` | 6 required args — also needs `ts_event`, `ts_init` | +| `from nautilus_trader.core.nautilus_pyo3` wrong path | `from nautilus_trader.core.nautilus_pyo3 import black_scholes_greeks` | +| `BacktestEngine.add_venue(venue=BacktestVenueConfig(...))` | Takes positional args. `BacktestVenueConfig` is for `BacktestRunConfig` only | +| `DYDXDataClientConfig` (uppercase) | `DydxDataClientConfig` (mixed case) | +| `DydxOraclePrice` custom data type | Does not exist in v1.224.0 | +| `from nautilus_trader.common.clock import TestClock` | Use `from nautilus_trader.common.component import LiveClock` | +| `from nautilus_trader.common.config import CacheConfig` | `from nautilus_trader.config import CacheConfig` or `nautilus_trader.cache.config` | +| `Equity(..., max_price=, min_price=)` | Constructor rejects these kwargs — properties exist but return None | +| `FuturesContract(..., size_precision=, size_increment=)` | Hardcoded to 0/1 in Cython | +| `RSI` value in [0, 100] | Value in [0, 1] — divide by 100 if comparing to standard | +| Indicator warmup returns NaN | Returns partial values (silently wrong, not NaN) — guard with `indicators_initialized()` | +| `BookType.L3_MBO` for crypto | L3 not available on crypto exchanges — L2 at best. L3 is for traditional exchanges only | +| `subscribe_funding_rates()` everywhere | Method exists on Strategy but not all adapters support the feed | +| `subscribe_instrument_status()` on Binance | Binance does NOT implement this — not all adapters support it | +| `MarketStatusAction.RESUME` | Does not exist — use `TRADING` to detect resumption | +| `BinanceAccountType.USDT_FUTURE` (no S) | Must be `USDT_FUTURES` (with S) | +| `load_ids=frozenset({"BTCUSDT-PERP"})` bare symbol | Must be full InstrumentId: `frozenset({"BTCUSDT-PERP.BINANCE"})` — crashes at connect with `missing '.' separator` | +| `modify_order` auto-fallback | Adapter errors if venue doesn't support — no auto cancel+replace fallback | +| `InstrumentStatus` stops order flow | Does NOT automatically stop orders — strategy must react manually | +| Omitting `load_ids` from `InstrumentProviderConfig` | REQUIRED — without it, 0 instruments load, all subscriptions silently produce 0 data | +| `frozen_account=True` means checks active | INVERTED: `True` = checks DISABLED, `False` = checks ACTIVE | +| `testnet=True` for Deribit/dYdX | `is_testnet=True` | +| `AccountType.CASH` for perps backtest | Use `AccountType.MARGIN` — CASH + perps = 0 fills silently | +| `book.best_bid_price()` returns float | Returns `Price` object — cast with `float()` for arithmetic | + +### Rust + +| Hallucination | Reality | +|--------------|---------| +| `LiveNode::new(config)` or `LiveNodeConfig::builder()` | `LiveNode::builder(trader_id, Environment::Live)?.build()?` | +| `BinanceDataClientConfig::builder()` | Struct literal: `BinanceDataClientConfig { product_types: vec![...], ..Default::default() }` | +| `BinanceAccountType::UsdtFutures` (Rust) | `BinanceProductType::UsdM` from `nautilus_binance::common::enums` | +| `nautilus_core::identifiers::InstrumentId` | `nautilus_model::identifiers::InstrumentId` | +| `DataActor` in `nautilus_trading` | `DataActor` in `nautilus_common::actor` | +| `env_logger` alongside Nautilus | Crashes — Nautilus owns `log` backend. Use `log::info!()` | +| Nautilus crates from crates.io | Git source only | +| `nautilus_trader_model` | `nautilus_model` | +| `on_book_delta` (singular) | `on_book_deltas` (plural — batch) | +| Skip `on_stop` in DataActor | Warns "on_stop handler was called when not overridden" — always implement | +| `cancel_all_orders` in `on_stop` | Trading channel closed before `on_stop` — use earlier callbacks | +| `node.stop()` exits cleanly | WebSocket lingers ~10s. Use `tokio::time::timeout` on shutdown | +| `event.duration_ns` on PositionClosed | `event.duration` — no `_ns` suffix | +| `event.realized_pnl` is `Money` | `Option` — use `{:?}` format, not `{}` | +| `node.add_actor(strategy)` for Strategy | `node.add_strategy(strategy)?;` — Strategy ≠ DataActor | +| `Deref` for Strategy | `Deref` — StrategyCore auto-derefs through | +| `BinanceExecClientFactory` | `BinanceExecutionClientFactory` — full name, same for config | +| Patching a nautilus crate | Fork on GitHub + `[patch]` in Cargo.toml. See battle_tested.md | +| Ed25519 keys in Rust HTTP adapters (v0.55.0) | URL-encoding bug — base64 `+/=` not encoded. Needs fork | diff --git a/nautilus-docs/agents/openai.yaml b/nautilus-docs/agents/openai.yaml new file mode 100644 index 0000000..cae5d5e --- /dev/null +++ b/nautilus-docs/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "NautilusDocs" + short_description: "NautilusTrader library expert knowledge" + default_prompt: "Use $nautilus-docs to design/implement or fix trading systems with NautilusTrader." diff --git a/nautilus-docs/references/REBUILD.md b/nautilus-docs/references/REBUILD.md new file mode 100644 index 0000000..c16be40 --- /dev/null +++ b/nautilus-docs/references/REBUILD.md @@ -0,0 +1,233 @@ +# Rebuilding the nautilus-docs Skill + +Prompt and checklist for regenerating SKILL.md when NautilusTrader's API changes. + +## When to Rebuild + +- NautilusTrader version bump (check `nautilus_trader.__version__` or workspace `Cargo.toml`) +- New adapter added or removed +- Rust crate renamed, split, or merged +- Breaking API changes (trait signatures, config structs, builder patterns) +- New docs added to the official `docs/` folder + +## Step 1: Update the Docs + +Docs are fetched on install into `nautilus-docs/references/docs/` (gitignored). To refresh: + +```bash +rm -rf nautilus-docs/references/docs +# Re-run install.sh or manually: +temp=$(mktemp -d) +git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$temp" +git -C "$temp" sparse-checkout set docs/ +mv "$temp/docs" nautilus-docs/references/docs +rm -rf "$temp" +``` + +Check for: +- New files (new concepts, new adapters, new tutorials) +- Removed files (deprecated features) +- Changed files (API changes, renamed types) + +## Step 2: Rebuild the Doc Navigator + +For every new `.md` or `.py` file in `docs/`, add a row to the appropriate navigator table in SKILL.md (Concepts, Venue Integrations, Dev/Setup, or Tutorials). + +For removed files, delete the row. For renamed files, update the path. + +## Step 3: Rebuild the Rust Crate Map + +The crate map is the most fragile part — it breaks on every workspace restructure. + +**Source of truth**: The workspace `Cargo.toml` at the root of the nautilus_trader repo. + +```bash +# After pulling the latest nautilus_trader git dep: +CHECKOUT=$(find ~/.cargo/git/checkouts/nautilus_trader-* -maxdepth 1 -type d | sort | tail -1) + +# List all crate package names +find "$CHECKOUT/crates" -name "Cargo.toml" -maxdepth 3 -exec grep -H '^name' {} \; + +# List all adapter crates +ls "$CHECKOUT/crates/adapters/" + +# Check MSRV +grep 'rust-version' "$CHECKOUT/Cargo.toml" +``` + +For each crate, verify: +- Package name matches SKILL.md crate map +- Key re-exports haven't moved (especially `DataActor`, `Strategy`, `InstrumentId`) +- Feature flags still work as documented + +**Critical re-exports to verify**: + +```bash +# DataActor location +grep -r "pub use.*DataActor" "$CHECKOUT/crates/common/src/" + +# Strategy location +grep -r "pub use.*Strategy" "$CHECKOUT/crates/trading/src/" + +# InstrumentId location +grep -r "pub struct InstrumentId" "$CHECKOUT/crates/model/src/" + +# LiveNode builder +grep -r "pub fn builder" "$CHECKOUT/crates/live/src/" + +# BinanceDataClientConfig fields +grep -A 20 "pub struct BinanceDataClientConfig" "$CHECKOUT/crates/adapters/binance/src/config.rs" +``` + +## Step 4: Rebuild Anti-Hallucination Tables + +This is the highest-value part of the skill. Each row prevents a specific mistake Claude makes repeatedly. + +**How to maintain**: + +1. **Test existing rows** — for each hallucination, verify the "Reality" column is still correct: + ```python + # Python hallucinations — test in a venv with nautilus_trader installed + from nautilus_trader.common.actor import Actor # still correct? + from nautilus_trader.indicators import ExponentialMovingAverage # still correct? + ``` + +2. **Discover new hallucinations** — ask Claude to write code for common tasks WITHOUT the skill, then diff against what actually compiles/runs. Common sources: + - New config struct fields that Claude will guess wrong + - Renamed enums or moved types + - Changed method signatures (added/removed args) + - New features that Claude will confuse with old patterns + +3. **Remove stale rows** — if a hallucinated API now exists (e.g., a missing method was added), remove the row. + +4. **Rust hallucinations** — rebuild by attempting to compile a standalone binary: + ```bash + cd my_trading_system && cargo build 2>&1 | grep "error\[E" + ``` + Every compilation error that comes from a wrong import path, missing trait, or wrong struct field is a hallucination row candidate. + +## Step 5: Verify Working Patterns + +The DataActor pattern and LiveNode wiring in SKILL.md must compile. Test: + +```bash +cd my_trading_system && cargo build --release 2>&1 +``` + +If it fails, the Rust section needs updating. Common breakage: +- `DataActor` trait method signatures changed +- `LiveNode::builder()` API changed +- `BinanceDataClientConfig` fields renamed +- New required fields added to configs (no more `..Default::default()`) + +--- + +## Fundamental Questions the Skill Must Always Answer + +These are the enduring questions any trading infrastructure user will ask. The skill must provide correct answers for ALL of these regardless of API version. If a section doesn't cover one, it's a gap. + +### Data Ingestion +- How do I subscribe to trades, quotes, order book, bars? +- What's the difference between INTERNAL and EXTERNAL bars? +- How do I handle custom data types? +- What happens when I subscribe to data that doesn't exist? (silent failure) +- How does the DataEngine buffer and dispatch data? + +### Order Lifecycle +- How do I submit, modify, cancel orders? +- What order types are available? (Market, Limit, StopMarket, StopLimit, TrailingStop) +- How do bracket orders work? +- What happens when modify_order isn't supported by the venue? +- How do I track order state changes? (callbacks: on_order_accepted, on_order_filled, etc.) + +### Position Management +- How do I query open positions? +- NETTING vs HEDGING — when to use which? +- How do I compute unrealized PnL? +- How does signed_qty work? (float, not Decimal) + +### Instrument Discovery +- How do I load instruments? (load_ids is REQUIRED) +- What's the InstrumentId format? ("SYMBOL.VENUE") +- What instrument types exist? +- How do venue-specific instrument IDs map? + +### Actor / Strategy Architecture +- When to use Actor vs Strategy? +- What's the component lifecycle FSM? +- How do I publish/subscribe signals between actors? +- How do timers and alerts work? +- What's the correct on_start() ordering? (cache instrument → register indicators → subscribe) + +### Backtesting +- How do I set up a BacktestEngine? +- How do I load historical data? (wranglers, catalog) +- How does the FillModel work? +- What's the difference between BacktestEngine (low-level) and BacktestRunConfig (high-level)? + +### Live Deployment +- How do I configure a TradingNode (Python) / LiveNode (Rust)? +- What are the adapter config patterns for each venue? +- How do I handle shutdown gracefully? +- What are the expected warnings/errors at startup and shutdown? + +### Venue Connectivity +- Which adapters exist and what markets do they support? +- What data subscriptions does each adapter support? +- What execution features does each adapter support? (modify_order, etc.) +- How do API keys and authentication work per venue? + +### Rust-Specific +- Which crates do I need for my use case? +- How do I implement DataActor / Strategy traits? +- How do I build a standalone binary outside the workspace? +- What are the correct import paths? (nautilus_model vs nautilus_core) +- How does LiveNode builder work? +- What feature flags do I need? + +### Performance & Operations +- Why must callbacks return fast? (single-threaded event loop) +- How do I avoid blocking the event loop? +- How do I use the clock for scheduling? +- What logging/monitoring is available? +- How do I handle degraded/faulted states? + +--- + +## Meta-Prompt for Full Rebuild + +Use this prompt to have Claude regenerate the entire SKILL.md from scratch: + +``` +You are rebuilding the nautilus-docs skill for NautilusTrader. + +The official docs are in docs/ (read the full directory tree). +The current SKILL.md is your starting point — keep the structure but update all content. + +Steps: +1. Read every file in docs/concepts/, docs/integrations/, docs/developer_guide/, docs/tutorials/, docs/getting_started/ +2. Build the Doc Navigator tables from the actual files present +3. Check the Rust workspace Cargo.toml for current crate names and versions +4. Verify all anti-hallucination table entries by checking actual source code: + - For Python: test imports in the installed nautilus_trader package + - For Rust: check actual struct/trait definitions in the crate source +5. Update the Rust crate map from workspace members +6. Update the DataActor pattern and LiveNode wiring from actual working examples +7. Test that the Rust example compiles: cd my_trading_system && cargo build +8. Verify word count is under 2,000 + +For anti-hallucination entries, try to write code for each of the "Fundamental Questions" +listed in REBUILD.md WITHOUT looking at the skill, then compare what you generated +against what actually works. Every mistake you made is a hallucination row. + +The skill must be self-sufficient — a user should be able to write correct +NautilusTrader code (Python or Rust) using ONLY this skill, without deepwiki or +context7. If they can't, the skill has gaps. +``` + +## Versioning + +After rebuild, update the `` comment in SKILL.md hallucination tables. + +Track what version the anti-hallucination table was last verified against. Stale tables are worse than no table — they give false confidence. diff --git a/nautilus-docs/references/battle_tested.md b/nautilus-docs/references/battle_tested.md new file mode 100644 index 0000000..2fef201 --- /dev/null +++ b/nautilus-docs/references/battle_tested.md @@ -0,0 +1,558 @@ +# Battle-Tested Patterns & Tricks + +Non-obvious knowledge from live testing NautilusTrader v1.224.0. **Only patterns not covered in official docs.** For anything below marked with a doc link, read the doc first — it has more detail. + +## Covered in Official Docs (read these first) + +| Gotcha | Where | +|--------|-------| +| `on_start()` must be: cache → indicators → subscribe | [strategies.md](docs/concepts/strategies.md) | +| `sort=False` + `sort_data()` for multi-instrument backtest | [backtesting.md](docs/concepts/backtesting.md) | +| `ts_init_delta` for bar open→close timestamps (look-ahead bias) | [backtesting.md](docs/concepts/backtesting.md), [data.md](docs/concepts/data.md) | +| `F_LAST` flag mandatory on last delta in batch | [data.md](docs/concepts/data.md) | +| FillModel: only `prob_fill_on_limit`, `prob_slippage`, `random_seed` | [backtesting.md](docs/concepts/backtesting.md) | +| Binance `-PERP` suffix for futures | [binance.md](docs/integrations/binance.md) | +| dYdX: `is_testnet=True`, no `modify_order`, IOC limit orders | [dydx.md](docs/integrations/dydx.md) | +| Deribit: `is_testnet=True` | [deribit.md](docs/integrations/deribit.md) | +| Polymarket: no `modify_order`, binary outcomes | [polymarket.md](docs/integrations/polymarket.md) | +| Log rotation: `LoggingConfig(log_directory=, log_file_format=)` | [logging.md](docs/concepts/logging.md) | + +## Execution Tricks + +### modify_order > cancel+replace + +- Single message, lower latency, less detectable +- But NOT supported everywhere: dYdX, Binance Spot, Polymarket don't support it +- Check venue support before using — adapter errors with no auto-fallback + +### Bracket order pattern (manual) + +```python +entry = self.order_factory.market(instrument_id, OrderSide.BUY, quantity) +self.submit_order(entry) + +# In on_order_filled: +sl = self.order_factory.stop_market(instrument_id, OrderSide.SELL, quantity, trigger_price=sl_price) +tp = self.order_factory.limit(instrument_id, OrderSide.SELL, quantity, price=tp_price) +self.submit_order(sl) +self.submit_order(tp) +``` + +For Rust backtest: `support_contingent_orders = Some(true)` required in `add_venue`. + +### Market maker: on_order_filled resets order IDs + +```python +def on_order_filled(self, event) -> None: + if event.client_order_id == self._bid_id: + self._bid_id = None + elif event.client_order_id == self._ask_id: + self._ask_id = None +``` + +Without this: stale `_bid_id`/`_ask_id` → `cache.order(stale_id)` returns filled order → `is_open` is False → submits duplicate instead of modifying. + +### Reconciliation delay + +`reconciliation_startup_delay_secs >= 10` — lower values cause false positive "missing order" resolutions on startup. + +## Actor Patterns + +### Timer-based REST polling + +```python +def on_start(self) -> None: + self.clock.set_timer("poll_oi", interval=timedelta(seconds=5), callback=self._on_poll) + +def _on_poll(self, event) -> None: + self.queue_for_executor(self._fetch_open_interest) + +async def _fetch_open_interest(self) -> None: + # async HTTP call here — non-blocking + pass +``` + +### Signal pipeline (Actor → Strategy) + +**Actor publishes:** +```python +self.publish_signal(name="momentum", value=score, ts_event=tick.ts_event) +``` + +Name `"momentum"` auto-generates class `SignalMomentum` (capitalized). Values must be int/float/str — dict causes KeyError. + +**Strategy receives:** +```python +def on_signal(self, signal) -> None: + if type(signal).__name__ == "SignalMomentum": + self.momentum = signal.value +``` + +Note: the official docs say "differentiate using `signal.value`" — this is misleading. `type(signal).__name__` works and is more precise when publishing multiple distinct signal types. The `name` param to `publish_signal` is not stored as `.name` on the object, but it IS baked into the generated class name. + +### Custom data for structured payloads + +When signal values need to be richer than a single number: + +```python +class ImbalanceData(Data): + def __init__(self, imbalance: float, volume: float, ts_event: int, ts_init: int): + self.imbalance = imbalance + self.volume = volume + self._ts_event = ts_event + self._ts_init = ts_init + + @property + def ts_event(self) -> int: return self._ts_event + @property + def ts_init(self) -> int: return self._ts_init +``` + +Actor: `self.publish_data(data_type=DataType(ImbalanceData), data=data)` +Strategy: subscribe in `on_start`, receive in `on_data` + +## Backtest Configuration + +### FillModel tuning + +```python +FillModel( + prob_fill_on_limit=0.3, # 30% fill probability on limit orders + prob_slippage=0.5, # 50% chance of 1-tick slippage + random_seed=42, # reproducibility +) +``` + +No `prob_fill_on_stop` parameter exists. + +### frozen_account naming is INVERTED + +- `frozen_account=False` → margin checks ARE active (not frozen) +- `frozen_account=True` → margin checks DISABLED (frozen) + +For realistic backtesting, use `frozen_account=False` (the confusing one). + +### Multi-currency accounts + +`base_currency=None` for multi-currency (standard for crypto). `Currency.from_str("USDT")` not bare constant. + +### Venue setup (low-level API) + +```python +engine.add_venue( + venue=Venue("BINANCE"), + oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, # MUST be MARGIN for perps/futures + starting_balances=[Money(10_000, Currency.from_str("USDT"))], + fill_model=fill_model, +) +``` + +`AccountType.CASH` + perps = 0 fills silently. + +## Order Book + +### F_LAST flag is mandatory + +Every delta batch MUST end with `RecordFlag.F_LAST` or DataEngine buffers indefinitely — subscribers never receive data. + +- Single delta: always set F_LAST +- Batch: set F_LAST only on last delta + +### book.best_bid_price() returns Price, not float + +Cast with `float(book.best_bid_price())` for arithmetic. Same for `best_ask_price()`, `spread()`, `midpoint()`. + +### L2_MBP is the ceiling for crypto + +L3_MBO not available on any crypto exchange. L2 is sufficient for all HFT strategies. L3 is only for traditional exchanges (equities). + +### OrderBookDepth10 for lightweight signals + +Pre-aggregated top 10 levels — lower overhead than full book subscription for signal generation. + +## Venue-Specific Gotchas + +### Binance + +- `-PERP` suffix mandatory: `BTCUSDT-PERP.BINANCE` for futures, `BTCUSDT.BINANCE` for spot +- `BinanceAccountType.USDT_FUTURES` (with S) — `USDT_FUTURE` doesn't exist +- `BinanceFuturesMarkPriceUpdate` contains mark, index, AND funding_rate — don't subscribe separately +- Spot doesn't support `modify_order` +- High-supply tokens (SHIB, PEPE) need `high-precision` feature flag + +### dYdX + +- Market orders implemented as aggressive limit (IOC): buy at `oracle_price × 1.05`, sell at `× 0.95` (5% buffer, `DEFAULT_MARKET_ORDER_SLIPPAGE = 0.05`). Unfilled slippage not consumed. +- Config: `is_testnet=True` (NOT `testnet=True`) +- No `modify_order` support + +### Deribit + +- Config: `is_testnet=True` (NOT `testnet=True`) +- `CryptoOption.underlying` is `Currency`, `OptionContract.underlying` is `str` — different types + +### Polymarket + +- No `modify_order` support +- Binary outcomes only — `YES`/`NO` tokens + +## Debugging Live Issues + +### Step 1: Read the actual error + +Most wasted debugging time comes from treating errors as opaque. The server is almost always telling you exactly what's wrong. + +```bash +# HTTP errors: ALWAYS read the response body, not just the status code +curl -v 'https://fapi.binance.com/fapi/v1/...' -H 'X-MBX-APIKEY: ...' 2>&1 + +# Nautilus logs: grep ERROR first +grep "ERROR" logs/trader.log | head -20 + +# Rust: use {:?} (Debug), not {} (Display) — Debug includes inner error chain +log::error!("Failed: {error:?}"); # shows full chain +log::error!("Failed: {error}"); # may hide root cause +``` + +If a Nautilus error says "400 Bad Request" — **that is not enough information to debug**. Get the response body. One `curl -v` is worth 10 rounds of guessing. + +### Step 2: Venue-specific API key rules + +Binance has **per-product API keys**. A futures Ed25519 key is NOT valid for spot. Check: +- Spot key → spot endpoints only +- Futures/Linear key → futures endpoints only +- You may have separate keys in `.env`: `BINANCE_SPOT_API_KEY`, `BINANCE_LINEAR_API_KEY` + +**Ed25519 encrypted keys**: Binance Ed25519 private keys can be password-encrypted (PKCS#8). NautilusTrader's `SigningCredential` needs the unencrypted key. If your `.env` has a `_DECRYPTED` variant, use that. + +### Step 3: Extending NautilusTrader without modifying source + +When NautilusTrader is missing a feature or an adapter does not behave the way you need, follow this decision tree. The architecture smell to avoid is building on a private fork without upstreaming -- not cloning the repo itself. Cloning to read source, run tests, or prepare a PR is perfectly normal. + +**Decision tree -- "I need something NautilusTrader does not provide":** + +#### 1. Compose via DataActor or Strategy (first choice, 90% of cases) + +NautilusTrader's extension surface is trait-based. `DataActor` and `Strategy` are traits you implement on your own structs. They give you lifecycle callbacks (`on_start`, `on_stop`), data callbacks (`on_trade`, `on_quote`, `on_mark_price`, `on_book_deltas`, `on_funding_rate`, `on_data`, ...), timer events, and the signal/custom-data pipeline. This is composition, not inheritance -- your struct owns a `DataActorCore` or `StrategyCore` and delegates via `Deref`. + +Use this when: +- An adapter streams data you need to reshape (e.g., use `subscribe_funding_rates` / `on_funding_rate` for funding data — the adapter handles the underlying source format) +- You need to combine multiple data feeds into a derived signal +- You need periodic REST polling (timer + async HTTP in Python via `queue_for_executor`, or timer + `reqwest` in a spawned task in Rust) +- You want to publish custom data types for downstream consumers + +Rust example -- subscribing to funding rates: + +```rust +use nautilus_model::data::FundingRateUpdate; + +impl DataActor for FundingRateCapture { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_funding_rates(self.instrument_id, None, None); + Ok(()) + } + + fn on_funding_rate(&mut self, update: &FundingRateUpdate) -> anyhow::Result<()> { + // FundingRateUpdate has: instrument_id, rate (Decimal), interval, next_funding_ns + log::info!("{} rate={}", update.instrument_id, update.rate); + Ok(()) + } + + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_funding_rates(self.instrument_id, None, None); + Ok(()) + } +} +``` + +Note: whether funding rate data actually arrives depends on adapter support. Binance embeds it in mark price stream at the adapter level — the DataActor callback fires if the adapter parses it out. If the adapter doesn't support it, subscribe to mark prices as a fallback and check what fields arrive. + +This is idiomatic Rust composition: your struct implements the trait, the runtime calls your callbacks. No source modification needed. + +#### 2. Cargo `[patch]` for adapter bugs (temporary fix while upstreaming) + +If an adapter has an actual bug (wrong signature encoding, missing WebSocket message handling, incorrect field parsing), fork on GitHub, fix the bug on a branch, and point your `Cargo.toml` at your fork: + +```toml +[patch.'https://github.com/nautechsystems/nautilus_trader.git'] +nautilus-binance = { git = "https://github.com/YOUR-USER/nautilus_trader.git", branch = "fix/binance-sig-encoding" } +``` + +Only list the crate(s) you modified. Cargo resolves transitive deps automatically. This is not a private fork -- it is a temporary patch with a clear intent to upstream via PR. + +Rules: +- The branch name should describe the fix (`fix/binance-ws-reconnect`, not `my-changes`) +- Open an upstream PR immediately or as soon as the fix is validated +- Remove the `[patch]` once the fix is merged and released +- Never use `[patch]` for feature additions -- that belongs in category 3 + +#### 3. Fork + modify for upstream contribution (the right way to add features) + +When NautilusTrader genuinely lacks functionality that belongs in the core (new adapter message type, new data feed, new order type support), the correct path is: + +1. Clone the repo +2. Implement the feature on a branch +3. Run the existing test suite, add tests for your change +4. Open a PR upstream +5. Use `[patch]` to consume your branch while the PR is in review + +This is how open source works. The smell is NOT cloning or modifying source -- it is building on a local fork without upstreaming. A private fork with adapter modifications that you never PR diverges immediately and breaks on the next Nautilus update. + +#### 4. Cloning to read source or run tests (always fine) + +Cloning the NautilusTrader repo to read adapter source, understand internal behavior, run tests, or check trait signatures is completely normal development practice. The repo is the best documentation for edge cases. + +#### What NOT to do + +Do not clone the repo, modify adapter internals to add missing functionality, build locally, and treat that as your production dependency. This is the architecture smell. Concretely, the failure mode looks like: + +1. "Binance does not stream funding rates" -- correct observation +2. Clone repo, add funding rate streaming to the Binance adapter -- wrong response +3. Build from local path, deploy -- unmaintainable; breaks on next Nautilus version + +The correct response to step 1 is category 1 above: write a `DataActor` that subscribes to `MarkPriceUpdate` (which contains funding rate data) and extracts what you need. No source modification required. + +### Common runtime traps + +| Symptom | Cause | Fix | +|---------|-------|-----| +| "Invalid symbol" on subscribe | Symbol doesn't exist on that venue/product | Check symbol list — not all coins have futures (SHIB, PEPE, LEO) | +| "Invalid X-MBX-APIKEY header" | Wrong API key for the endpoint (spot key on futures) | Use the correct per-product key | +| WebSocket reconnecting repeatedly | Too many subscriptions per connection (Binance limit ~1024 streams) | Reduce symbols or split across connections | +| Signature validation error | Ed25519 key misdetected as HMAC, or encrypted key used | Check `SigningCredential` detection, use unencrypted key | +| SBE decode failure | Exchange upgraded schema version (e.g. v2→v3) | Check exchange API changelog, update Nautilus | +| 0 instruments loaded | Missing `load_ids` in `InstrumentProviderConfig` | Add `load_ids=frozenset({"SYMBOL.VENUE"})` | +| Catalog empty after run | Run too short for feather flush, or wrong catalog path | Run longer (>60s), check `catalog_path` in config | +| `subscribe_funding_rates()` no data | Not all adapters implement the feed (Binance doesn't stream it natively) | Use `MarkPriceUpdate` which contains funding_rate (see Step 3 above) | + +### Binance funding rate + +In Rust, `subscribe_funding_rates` / `on_funding_rate` is the first-class path — the Binance adapter parses funding from its mark price stream internally. In Python, funding arrives via `BinanceFuturesMarkPriceUpdate` (contains mark, index, AND funding_rate). As a fallback, poll the REST endpoint via a timer-based actor (`queue_for_executor` + async HTTP). + +## Rust Live Trading + +### Strategy pattern (for order management) + +DataActor is data-only. For submitting orders, use `Strategy` from `nautilus-trading`: + +```rust +use std::ops::{Deref, DerefMut}; +use nautilus_common::actor::{DataActor, DataActorCore}; +use nautilus_model::data::TradeTick; +use nautilus_model::identifiers::InstrumentId; +use nautilus_trading::strategy::{Strategy, StrategyConfig, StrategyCore}; + +#[derive(Debug)] +struct MyStrategy { + core: StrategyCore, // wraps DataActorCore internally + instrument_id: InstrumentId, +} + +// CRITICAL: Deref target is DataActorCore, NOT StrategyCore. +// Strategy extends DataActor which requires Deref. +// StrategyCore itself implements Deref, so this chains through. +impl Deref for MyStrategy { + type Target = DataActorCore; + fn deref(&self) -> &Self::Target { &self.core } // auto-derefs StrategyCore → DataActorCore +} +impl DerefMut for MyStrategy { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.core } +} + +impl DataActor for MyStrategy { + fn on_start(&mut self) -> anyhow::Result<()> { + self.subscribe_trades(self.instrument_id, None, None); + Ok(()) + } + fn on_trade(&mut self, trade: &TradeTick) -> anyhow::Result<()> { + log::info!("trade px={}", trade.price.as_f64()); + Ok(()) + } + fn on_stop(&mut self) -> anyhow::Result<()> { + self.unsubscribe_trades(self.instrument_id, None, None); + Ok(()) + } +} + +impl Strategy for MyStrategy { + fn core(&self) -> &StrategyCore { &self.core } + fn core_mut(&mut self) -> &mut StrategyCore { &mut self.core } +} +``` + +Constructor: `MyStrategy { core: StrategyCore::new(StrategyConfig { strategy_id: Some("MY-001".into()), ..Default::default() }), instrument_id }`. + +Key event fields: +- `PositionOpened`: `entry` (OrderSide), `side` (PositionSide), `quantity`, `last_px`, `avg_px_open`, `currency` +- `PositionClosed`: `realized_pnl` (Option — use `{:?}` format), `duration` (not `duration_ns`), `avg_px_open`, `avg_px_close` +- `OrderFilled`: `last_qty`, `last_px`, `commission`, `liquidity_side`, `trade_id` + +Add to node: `node.add_strategy(my_strategy)?;` (not `add_actor`). + +### Credential handling + +Adapters using `SigningCredential` auto-detect key type: +1. Tries to strip PEM headers + base64 decode → Ed25519 if valid +2. Falls back to HMAC + +**Gotcha**: HMAC secrets that happen to be valid base64 (≥32 bytes) get misdetected as Ed25519. Symptoms: signature validation errors on execution client connect. This affects any adapter using `SigningCredential`. + +### Ed25519 HTTP signature encoding (v0.55.0) + +Ed25519 signatures are base64 containing `+`, `/`, `=`. Some adapter HTTP clients insert signatures into query strings without URL-encoding. HMAC signatures are hex (URL-safe) so they work fine. If you see signature errors only with Ed25519 keys, check whether the adapter's HTTP client URL-encodes the signature. Most have a `percent_encode()` helper already. + +### Patching Nautilus dependencies + +See "Step 3: Extending NautilusTrader without modifying source" above for the full decision tree. Quick reference for Cargo `[patch]`: + +```toml +[patch.'https://github.com/nautechsystems/nautilus_trader.git'] +nautilus-{venue} = { git = "https://github.com/YOUR-USER/nautilus_trader.git", branch = "fix/my-fix" } +``` + +Only list the crate(s) you modified. Cargo resolves transitive deps automatically. Open an upstream PR, then remove the `[patch]` once the fix is merged and released. + +### Rust adapter maturity gaps + +The Rust adapters are newer than Python. Expect: +- Some WebSocket event types unhandled (fall to `Unknown`, log warnings) — usually harmless if the critical event path works +- Some config options existing in Python but missing in Rust (e.g. `use_trade_lite`) +- HTTP signing edge cases with non-HMAC key types +- Check the venue's adapter crate README and compare with Python adapter for feature parity + +## Backtest → Live Checklist + +1. `InstrumentProviderConfig(load_ids=frozenset({"BTCUSDT-PERP.BINANCE"}))` — backtest uses `engine.add_instrument()`, live requires `load_ids` +2. Replace `TestInstrumentProvider` instruments with real venue instruments +3. Add `exec_clients` config (backtest doesn't need it) +4. Add `reconciliation=True`, `reconciliation_lookback_mins=1440` to `LiveExecEngineConfig` +5. Add `on_order_rejected` / `on_order_modify_rejected` handlers — backtest rarely rejects +6. Handle `modify_order` unsupported — fall back to cancel+new +7. Set realistic `LoggingConfig(log_level="INFO", log_directory="logs/")` +8. Expect `fills > orders` (partials), `frozen_account` semantics differ +9. Test on venue testnet first (`testnet=True` or `is_testnet=True` depending on venue) +10. WebSocket shutdown lingers ~10s — use `tokio::time::timeout` (Rust) or KeyboardInterrupt (Python) + +## State Persistence + +### on_save / on_load for strategy restart + +```python +def on_save(self) -> dict[str, bytes]: + return {"position_state": msgspec.json.encode(self._state)} + +def on_load(self, state: dict[str, bytes]) -> None: + if "position_state" in state: + self._state = msgspec.json.decode(state["position_state"]) +``` + +State is saved on graceful shutdown and restored on restart. Keys are strategy-scoped. + +### Redis for shared state + +```python +ExecEngineConfig( + cache_database=CacheDatabaseConfig(type="redis", host="localhost", port=6379), +) +``` + +Enables order/position state recovery across restarts. Without Redis, in-memory cache is lost on crash. + +## Metrics + +### Fill quality tracking + +```python +def on_order_filled(self, event) -> None: + expected = self._last_signal_price + actual = float(event.last_px) + slippage_bps = (actual - expected) / expected * 10_000 + self.log.info(f"Slippage: {slippage_bps:.1f} bps") +``` + +Track per-venue, per-side. Asymmetric slippage (buys worse than sells) indicates toxic flow detection. + +### Latency measurement + +```python +def on_order_filled(self, event) -> None: + round_trip_ns = event.ts_init - self._order_submitted_ns + self.log.info(f"Round-trip: {round_trip_ns / 1_000_000:.1f}ms") +``` + +`ts_init` on the fill event is when the exchange response arrived locally. Compare against submission timestamp for round-trip latency. + +## Log Rotation + +```python +LoggingConfig( + log_level="INFO", + log_directory="logs/", + log_file_format="{trader_id}_{instance_id}", + log_colors=False, +) +``` + +Nautilus creates one log file per run. For rotation, use external logrotate or cron. File names include trader_id for multi-strategy separation. + +## Custom Indicator Development + +```python +from nautilus_trader.indicators.base import Indicator + +class SpreadEMA(Indicator): + def __init__(self, period: int): + super().__init__([]) # no input params for registration + self.period = period + self._values = [] + self.value = 0.0 + + def handle_quote_tick(self, tick: QuoteTick) -> None: + spread = float(tick.ask_price - tick.bid_price) + self._values.append(spread) + if len(self._values) > self.period: + self._values.pop(0) + self.value = sum(self._values) / len(self._values) + # _set_initialized inherited from Cython Indicator base — do NOT override + self._set_initialized(len(self._values) >= self.period) + + def reset(self) -> None: + self._values.clear() + self.value = 0.0 + self._set_initialized(False) +``` + +Register: `self.register_indicator_for_quote_ticks(instrument_id, spread_ema)` in `on_start`. + +## Data Pipeline + +### Gap detection in tick data + +```python +def check_gaps(ticks, max_gap_ms=60_000): + for i in range(1, len(ticks)): + gap = (ticks[i].ts_event - ticks[i-1].ts_event) / 1_000_000 + if gap > max_gap_ms: + print(f"Gap: {gap:.0f}ms at index {i}") +``` + +Run before backtesting. Gaps > 1 minute during trading hours = suspect data. Gaps during maintenance windows (Binance: Tue 06:00-06:30 UTC) are normal. + +### Timestamp sanity + +- `ts_event` should increase monotonically +- `ts_init >= ts_event` always (init is local receipt time) +- `ts_init - ts_event > 60s` = suspect clock skew or stale data + +## Performance Checklist + +1. **Rust hot path**: `Price::new(f64, u8)` — never `Price::from(format!("{:.prec$}", val, prec=p).as_str())` (heap alloc per tick) +2. Cache `self.instrument` in `on_start()` — avoid repeated `cache.instrument()` lookups in hot path +3. Keep `on_order_book_deltas` tight — fires at tick frequency +4. Never `time.sleep()` in callbacks — single-threaded event loop blocks everything +5. Use `modify_order` over cancel+replace where supported — single message, less latency +6. `sort=False` + `engine.sort_data()` for multi-instrument backtest loading +7. Order books are Rust-native — all operations execute as native code, no Python overhead +8. `indicators_initialized()` guard prevents acting on partial warmup values +9. `Vec::with_capacity(n)` for known-size collections in hot path — avoid reallocation diff --git a/scripts/install.sh b/scripts/install.sh index a2c79a1..2b42eb2 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3,7 +3,7 @@ set -euo pipefail usage() { cat <<'USAGE' -Install Codex skills and/or print Claude Code plugin install commands. +Install skills for Codex and/or Claude Code. Usage: install.sh [--platform codex|claude|both] @@ -33,7 +33,7 @@ USAGE PLATFORM="codex" REPO_URL="https://github.com/DeevsDeevs/agent-system.git" CLONE_DIR="$HOME/src/agent-system" -TARGET="${AGENTS_HOME:-$HOME/.agents}/skills" +TARGET="" MODE="symlink" SKILLS_CSV="" SKILLS_LIST=() @@ -128,25 +128,26 @@ run_interactive() { UNINSTALL="false" fi + prompt SKILLS_CSV "Skills (comma-separated, blank = all)" "$SKILLS_CSV" if [[ "$PLATFORM" == "codex" || "$PLATFORM" == "both" ]]; then - prompt TARGET "Codex skills target dir" "$TARGET" - prompt SKILLS_CSV "Skills (comma-separated, blank = all)" "$SKILLS_CSV" - local advanced="n" - ask_yn advanced "Show advanced options? (y/N)" "N" - if [[ "$advanced" == "y" || "$advanced" == "Y" ]]; then - prompt MODE "Mode (symlink|copy)" "$MODE" - - local repo_dir_input="" - prompt repo_dir_input "Local repo dir (blank to clone)" "$REPO_DIR" - if [[ -n "$repo_dir_input" ]]; then - REPO_DIR="$repo_dir_input" - else - prompt REPO_URL "Repo URL" "$REPO_URL" - prompt CLONE_DIR "Clone dir" "$CLONE_DIR" - fi - - prompt REPO_REF "Repo ref (tag/branch/commit, optional)" "$REPO_REF" + prompt TARGET "Codex skills target dir" "${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" + fi + + local advanced="n" + ask_yn advanced "Show advanced options? (y/N)" "N" + if [[ "$advanced" == "y" || "$advanced" == "Y" ]]; then + prompt MODE "Mode (symlink|copy)" "$MODE" + + local repo_dir_input="" + prompt repo_dir_input "Local repo dir (blank to clone)" "$REPO_DIR" + if [[ -n "$repo_dir_input" ]]; then + REPO_DIR="$repo_dir_input" + else + prompt REPO_URL "Repo URL" "$REPO_URL" + prompt CLONE_DIR "Clone dir" "$CLONE_DIR" fi + + prompt REPO_REF "Repo ref (tag/branch/commit, optional)" "$REPO_REF" fi } @@ -162,6 +163,7 @@ ensure_repo() { if [[ -z "$REPO_DIR" ]]; then if [[ -d "$CLONE_DIR/.git" ]]; then REPO_DIR="$CLONE_DIR" + git -C "$REPO_DIR" fetch --all --prune else git clone "$REPO_URL" "$CLONE_DIR" REPO_DIR="$CLONE_DIR" @@ -169,9 +171,43 @@ ensure_repo() { fi if [[ -n "$REPO_REF" ]]; then - git -C "$REPO_DIR" fetch --tags --prune git -C "$REPO_DIR" checkout "$REPO_REF" + git -C "$REPO_DIR" pull --ff-only 2>/dev/null || true + fi +} + +cleanup_repo() { + if [[ -d "$CLONE_DIR/.git" ]]; then + rm -rf "$CLONE_DIR" + echo "Removed cloned repo: $CLONE_DIR" + fi +} + +fetch_nautilus_docs() { + local docs_dir="$REPO_DIR/nautilus-docs/references/docs" + + if [[ -d "$docs_dir" ]]; then + return fi + + echo "Fetching NautilusTrader docs..." + local temp + temp=$(mktemp -d) + trap "rm -rf '$temp'" RETURN + git clone --filter=blob:none --sparse --depth 1 \ + https://github.com/nautechsystems/nautilus_trader.git "$temp" + git -C "$temp" sparse-checkout set docs/ + rm -rf "$temp/docs/api_reference" + mkdir -p "$(dirname "$docs_dir")" + mv "$temp/docs" "$docs_dir" +} + +fetch_skill_deps() { + for skill in "${SKILLS_LIST[@]}"; do + if [[ "$(basename "$skill")" == "nautilus-docs" ]]; then + fetch_nautilus_docs + fi + done } collect_skills() { @@ -196,10 +232,12 @@ collect_skills() { install_codex() { ensure_repo + collect_skills + fetch_skill_deps + TARGET="${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" mkdir -p "$TARGET" - collect_skills for skill in "${SKILLS_LIST[@]}"; do local src dst src="$REPO_DIR/$skill" @@ -219,76 +257,89 @@ install_codex() { printf '\nDone. Restart Codex to reload skills.\n' } -uninstall_codex() { - if [[ -z "$SKILLS_CSV" ]]; then - ensure_repo - elif [[ -n "$REPO_REF" ]]; then - echo "Warning: --repo-ref is ignored when --skills is provided for uninstall." >&2 +uninstall_from() { + local target_dir="$1" + + if [[ ! -d "$target_dir" ]]; then + echo "Nothing to uninstall — $target_dir does not exist." + return fi - collect_skills - for skill in "${SKILLS_LIST[@]}"; do - local dst link_target - dst="$TARGET/$skill" + local skills_to_remove=() + if [[ -n "$SKILLS_CSV" ]]; then + IFS=',' read -r -a skills_to_remove <<< "$SKILLS_CSV" + else + for entry in "$target_dir"/*/; do + [[ -f "${entry}SKILL.md" ]] && skills_to_remove+=("$(basename "$entry")") + done + fi + + for skill_name in "${skills_to_remove[@]}"; do + local dst="$target_dir/$skill_name" if [[ -L "$dst" ]]; then - if [[ -n "$REPO_DIR" ]]; then - link_target="$(readlink "$dst")" - if [[ "$link_target" == "$REPO_DIR"* ]]; then - rm -f "$dst" - echo "Removed $dst" - else - echo "Skipping $dst (symlink not pointing to repo)" >&2 - fi - else - rm -f "$dst" - echo "Removed $dst" - fi - elif [[ -d "$dst" ]]; then - if [[ -f "$dst/SKILL.md" ]]; then - rm -rf "$dst" - echo "Removed $dst" - else - echo "Skipping $dst (no SKILL.md)" >&2 - fi + rm -f "$dst" + echo "Removed $dst" + elif [[ -d "$dst" && -f "$dst/SKILL.md" ]]; then + rm -rf "$dst" + echo "Removed $dst" else echo "Not found: $dst" fi done +} +uninstall_codex() { + TARGET="${TARGET:-${CODEX_HOME:-$HOME/.codex}/skills}" + uninstall_from "$TARGET" + cleanup_repo printf '\nDone. Restart Codex to reload skills.\n' } -print_claude_instructions() { - cat <<'EOC' -Run the following commands inside Claude Code: - -/plugin marketplace add git@github.com:DeevsDeevs/agent-system.git -/plugin install chain-system@deevs-agent-system -/plugin install dev-experts@deevs-agent-system -/plugin install bug-hunters@deevs-agent-system -/plugin install alpha-squad@deevs-agent-system -/plugin install mft-research-experts@deevs-agent-system -/plugin install cost-status@deevs-agent-system -/plugin install arxiv-search@deevs-agent-system - -Note: These are Claude Code commands and will not run in a shell. -EOC -} +install_claude() { + ensure_repo + collect_skills + fetch_skill_deps + + local clone_abs marketplace + clone_abs="$(cd "$REPO_DIR" && pwd)" + marketplace="$(basename "$clone_abs")" -print_claude_uninstall() { - cat <<'EOC' -Run the following commands inside Claude Code: + clean_plugin_cache "$marketplace" + + printf 'Run inside Claude Code:\n\n' + printf ' /plugin marketplace add %s\n\n' "$clone_abs" + for skill in "${SKILLS_LIST[@]}"; do + printf ' /plugin install %s@%s\n' "$(basename "$skill")" "$marketplace" + done + printf '\nDone. Restart Claude Code to reload skills.\n' +} -/plugin uninstall chain-system@deevs-agent-system -/plugin uninstall dev-experts@deevs-agent-system -/plugin uninstall bug-hunters@deevs-agent-system -/plugin uninstall alpha-squad@deevs-agent-system -/plugin uninstall mft-research-experts@deevs-agent-system -/plugin uninstall cost-status@deevs-agent-system -/plugin uninstall arxiv-search@deevs-agent-system +uninstall_claude() { + local marketplace + marketplace="$(basename "$CLONE_DIR")" + clean_plugin_cache "$marketplace" + cleanup_repo + printf '\nDone. Remove any installed plugins inside Claude Code with:\n' + printf ' /plugin marketplace remove %s\n' "$marketplace" +} -Note: These are Claude Code commands and will not run in a shell. -EOC +clean_plugin_cache() { + local marketplace="$1" + local plugins_dir="$HOME/.claude/plugins" + if [[ -d "$plugins_dir/cache/$marketplace" || -d "$plugins_dir/marketplaces/$marketplace" ]]; then + rm -rf "$plugins_dir/cache/$marketplace" "$plugins_dir/marketplaces/$marketplace" + if command -v jq &>/dev/null; then + [[ -f "$plugins_dir/installed_plugins.json" ]] && \ + jq ".plugins |= with_entries(select(.key | test(\"@${marketplace}\$\") | not))" \ + "$plugins_dir/installed_plugins.json" > "$plugins_dir/installed_plugins.json.tmp" && \ + mv "$plugins_dir/installed_plugins.json.tmp" "$plugins_dir/installed_plugins.json" + [[ -f "$plugins_dir/known_marketplaces.json" ]] && \ + jq "del(.\"$marketplace\")" \ + "$plugins_dir/known_marketplaces.json" > "$plugins_dir/known_marketplaces.json.tmp" && \ + mv "$plugins_dir/known_marketplaces.json.tmp" "$plugins_dir/known_marketplaces.json" + fi + echo "Cleaned plugin cache for $marketplace" + fi } case "$PLATFORM" in @@ -301,20 +352,20 @@ case "$PLATFORM" in ;; claude) if [[ "$UNINSTALL" == "true" ]]; then - print_claude_uninstall + uninstall_claude else - print_claude_instructions + install_claude fi ;; both) if [[ "$UNINSTALL" == "true" ]]; then uninstall_codex printf '\n' - print_claude_uninstall + uninstall_claude else install_codex printf '\n' - print_claude_instructions + install_claude fi ;; *)