From ee187a7afe512a3ad3363da9585b938c371e2aba Mon Sep 17 00:00:00 2001 From: Christos KK Loverdos Date: Thu, 21 May 2026 19:54:27 +0000 Subject: [PATCH 1/3] memory-memwal: add Nexus Tools for MemWal persistent memory (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the `memory-memwal` tool bundle: seven operations (`remember`, `remember_bulk`, `recall`, `ask`, `analyze`, `forget`, `stats`) against a MemWal relayer for SEAL-encrypted memory on Walrus over Sui. Pinned to relayer tag `@mysten-incubation/memwal@0.0.4`; `health_check` enforces the version against `GET /health` at runtime. Wire protocol: Ed25519-signed requests with canonical message `{ts}.{METHOD}.{path}.{sha256hex(body)}.{nonce}.{account_id}` and a fresh UUID v4 nonce per request. Boot-time URL validation (`https://` only; `MEMWAL_ALLOW_INSECURE=1` for dev). Delegate key parsed once into an `Arc`; hex intermediates zeroized; `KeyValidation` `Debug` redacts the `Ok` variant. Upstream errors map to terse client reasons; full body logged at `target = "memwal::upstream"`. Process-wide `reqwest::Client` via `OnceLock` with `gzip`/`brotli`. `429` surfaced as `RateLimited { retry_after_secs }`. Polling uses exponential backoff (100 ms → 4 s within a 60 s budget) with 0–25% jitter. `MAX_BULK_ITEMS = 20`; 1 MiB cap on text inputs. A `just memwal …` module wraps server lifecycle, per-tool curl invocations, and an end-to-end `demo` recipe. --- Cargo.lock | 1045 +++++++++++++--- Cargo.toml | 45 +- justfile | 3 + tools/memory-memwal/.env.example | 23 + tools/memory-memwal/.just | 329 +++++ tools/memory-memwal/Cargo.toml | 36 + tools/memory-memwal/README.md | 373 ++++++ tools/memory-memwal/src/analyze.rs | 202 +++ tools/memory-memwal/src/ask.rs | 246 ++++ tools/memory-memwal/src/auth.rs | 240 ++++ tools/memory-memwal/src/client.rs | 1455 ++++++++++++++++++++++ tools/memory-memwal/src/error.rs | 42 + tools/memory-memwal/src/forget.rs | 191 +++ tools/memory-memwal/src/main.rs | 51 + tools/memory-memwal/src/recall.rs | 249 ++++ tools/memory-memwal/src/remember.rs | 310 +++++ tools/memory-memwal/src/remember_bulk.rs | 327 +++++ tools/memory-memwal/src/stats.rs | 223 ++++ 18 files changed, 5199 insertions(+), 191 deletions(-) create mode 100644 tools/memory-memwal/.env.example create mode 100644 tools/memory-memwal/.just create mode 100644 tools/memory-memwal/Cargo.toml create mode 100644 tools/memory-memwal/README.md create mode 100644 tools/memory-memwal/src/analyze.rs create mode 100644 tools/memory-memwal/src/ask.rs create mode 100644 tools/memory-memwal/src/auth.rs create mode 100644 tools/memory-memwal/src/client.rs create mode 100644 tools/memory-memwal/src/error.rs create mode 100644 tools/memory-memwal/src/forget.rs create mode 100644 tools/memory-memwal/src/main.rs create mode 100644 tools/memory-memwal/src/recall.rs create mode 100644 tools/memory-memwal/src/remember.rs create mode 100644 tools/memory-memwal/src/remember_bulk.rs create mode 100644 tools/memory-memwal/src/stats.rs diff --git a/Cargo.lock b/Cargo.lock index e04ed9f..de1b13f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -25,6 +31,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -36,9 +63,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -57,9 +84,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -86,9 +113,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "assert-json-diff" @@ -100,6 +127,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-openai" version = "0.27.2" @@ -113,12 +152,12 @@ dependencies = [ "eventsource-stream", "futures", "rand 0.8.5", - "reqwest", + "reqwest 0.12.25", "reqwest-eventsource", "secrecy", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", @@ -148,6 +187,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backoff" version = "0.4.0" @@ -247,7 +308,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -259,6 +320,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bnum" version = "0.13.0" @@ -271,6 +341,27 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.5.1" @@ -345,9 +436,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -357,6 +448,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -372,12 +472,46 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.9.4" @@ -413,6 +547,24 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -435,6 +587,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -442,9 +603,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -509,7 +670,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -560,12 +721,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -577,6 +749,18 @@ dependencies = [ "syn", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -590,7 +774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ "der", - "digest", + "digest 0.10.7", "elliptic-curve", "rfc6979", "signature", @@ -616,7 +800,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -635,7 +819,7 @@ checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -679,9 +863,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "0.1.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -689,9 +873,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -735,11 +919,11 @@ dependencies = [ "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -754,6 +938,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -788,6 +983,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fluent-uri" version = "0.3.2" @@ -799,6 +1004,17 @@ dependencies = [ "serde", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fmt-cmp" version = "0.1.2" @@ -811,6 +1027,18 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -845,6 +1073,12 @@ dependencies = [ "num", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -982,11 +1216,24 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "glob" version = "0.3.3" @@ -1042,11 +1289,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "headers" @@ -1090,7 +1351,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1109,16 +1370,16 @@ name = "http" version = "1.0.0" dependencies = [ "backon", - "base64 0.21.7", - "jsonschema", + "base64 0.22.1", + "jsonschema 0.46.5", "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "url", ] @@ -1179,6 +1440,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -1291,7 +1561,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -1404,6 +1674,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -1438,7 +1714,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -1509,9 +1787,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "log", @@ -1522,15 +1800,64 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -1543,10 +1870,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1561,18 +1890,47 @@ dependencies = [ "base64 0.22.1", "bytecount", "email_address", - "fancy-regex", + "fancy-regex 0.14.0", "fraction", "idna", "itoa", "num-cmp", "once_cell", "percent-encoding", - "referencing", + "referencing 0.28.3", "regex-syntax", - "reqwest", + "reqwest 0.12.25", + "serde", + "serde_json", + "uuid-simd", +] + +[[package]] +name = "jsonschema" +version = "0.46.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5fe5206f06e589caf25e79fc05ccdf91fca745685fe9fe1a13bbdfb479a631" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex 0.18.0", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing 0.46.5", + "regex", + "regex-syntax", + "reqwest 0.13.3", + "rustls", "serde", "serde_json", + "unicode-general-category", "uuid-simd", ] @@ -1585,7 +1943,7 @@ dependencies = [ "cfg-if", "ecdsa", "elliptic-curve", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -1637,17 +1995,23 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.178" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1662,13 +2026,13 @@ dependencies = [ "anyhow", "async-openai", "chrono", - "jsonschema", + "jsonschema 0.46.5", "log", "mockito", "nexus-sdk", "nexus-toolkit", "portpicker", - "reqwest", + "reqwest 0.12.25", "rstest", "schemars", "serde", @@ -1712,10 +2076,47 @@ dependencies = [ ] [[package]] -name = "memchr" -version = "2.7.6" +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" + +[[package]] +name = "memory-memwal" +version = "1.0.0" +dependencies = [ + "anyhow", + "dotenvy", + "ed25519-dalek", + "env_logger", + "hex", + "log", + "mockito", + "nexus-sdk", + "nexus-toolkit", + "reqwest 0.12.25", + "schemars", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "url", + "uuid", + "zeroize", +] + +[[package]] +name = "micromap" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" [[package]] name = "mime" @@ -1735,10 +2136,11 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.13.0" +version = "2.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0adbe6e92a6ce0fd6c4aac593fdfd3e3950b0f61b1a63aa9731eb6fd85776fa3" +checksum = "2929e494b2280e1e18959bb2e121da03347ae896896fdfaceaab43c88a02803f" dependencies = [ + "memo-map", "serde", ] @@ -1748,11 +2150,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "log", @@ -1762,9 +2174,9 @@ dependencies = [ [[package]] name = "mockito" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0603425789b4a70fcc4ac4f5a46a566c116ee3e2a6b768dc623f7719c611de" +checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" dependencies = [ "assert-json-diff", "bytes", @@ -1805,17 +2217,17 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.2.1", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework", "security-framework-sys", "tempfile", ] @@ -1834,7 +2246,7 @@ dependencies = [ "futures", "futures-util", "hex", - "jsonschema", + "jsonschema 0.28.3", "lazy-regex", "lazy_static", "petgraph", @@ -1842,15 +2254,15 @@ dependencies = [ "prost-types", "rand 0.8.5", "regex", - "reqwest", + "reqwest 0.12.25", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sui-crypto", "sui-rpc", "sui-sdk-types", "sui-transaction-builder", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "toml", @@ -1872,7 +2284,7 @@ dependencies = [ "log", "nexus-sdk", "notify", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", @@ -2018,7 +2430,7 @@ checksum = "498447ee9c67221e1a558d5d49f63594c6ccaa2cdd437abec55c92e8c61887bc" dependencies = [ "base64 0.22.1", "cfg-if", - "digest", + "digest 0.10.7", "fmt-cmp", "hmac", "oauth-credentials", @@ -2055,15 +2467,14 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags 2.10.0", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -2085,11 +2496,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -2112,7 +2529,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -2259,6 +2676,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -2321,7 +2748,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -2389,8 +2816,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.1", - "thiserror 2.0.17", + "socket2 0.6.3", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -2411,7 +2838,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -2426,7 +2853,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -2446,6 +2873,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -2541,17 +2974,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0dcb5ab28989ad7c91eb1b9531a37a1a137cc69a0499aee4117cae4a107c464" dependencies = [ "ahash", - "fluent-uri", + "fluent-uri 0.3.2", "once_cell", "percent-encoding", "serde_json", ] +[[package]] +name = "referencing" +version = "0.46.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4e17ef386c5383591d07623d3de49cbc601156e7582973e6db98d66a57de2" +dependencies = [ + "ahash", + "fluent-uri 0.4.1", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -2561,9 +3011,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -2632,6 +3082,45 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.12", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "reqwest-eventsource" version = "0.6.0" @@ -2644,7 +3133,7 @@ dependencies = [ "mime", "nom", "pin-project-lite", - "reqwest", + "reqwest 0.12.25", "thiserror 1.0.69", ] @@ -2684,21 +3173,20 @@ dependencies = [ [[package]] name = "rstest" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", "rstest_macros", - "rustc_version", ] [[package]] name = "rstest_macros" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", @@ -2729,9 +3217,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags 2.10.0", "errno", @@ -2746,6 +3234,7 @@ version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -2761,10 +3250,10 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ - "openssl-probe", + "openssl-probe 0.1.6", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -2777,12 +3266,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2829,9 +3346,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -2842,9 +3359,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" dependencies = [ "proc-macro2", "quote", @@ -2894,19 +3411,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -2979,15 +3483,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -3038,11 +3542,12 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" dependencies = [ - "futures", + "futures-executor", + "futures-util", "log", "once_cell", "parking_lot", @@ -3052,9 +3557,9 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2", "quote", @@ -3068,8 +3573,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -3079,8 +3584,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3104,10 +3620,32 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" @@ -3136,11 +3674,11 @@ dependencies = [ "nexus-sdk", "nexus-toolkit", "oauth1-request", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -3156,12 +3694,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3194,15 +3732,15 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", @@ -3289,7 +3827,7 @@ dependencies = [ "serde_json", "serde_with", "sui-sdk-types", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -3352,12 +3890,12 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3387,11 +3925,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -3407,9 +3945,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -3462,9 +4000,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3472,16 +4010,16 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3680,13 +4218,18 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ + "async-compression", "bitflags 2.10.0", "bytes", + "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", + "http-body-util", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -3763,9 +4306,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicase" @@ -3773,12 +4316,24 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -3787,9 +4342,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -3817,10 +4372,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.19.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ + "getrandom 0.4.2", "js-sys", "wasm-bindgen", ] @@ -3869,15 +4425,15 @@ name = "walrus" version = "1.0.0" dependencies = [ "anyhow", - "jsonschema", + "jsonschema 0.46.5", "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -3931,14 +4487,23 @@ version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -3949,22 +4514,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3972,9 +4534,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -3985,13 +4547,35 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -4005,11 +4589,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -4025,6 +4621,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.4" @@ -4293,6 +4898,94 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.2" @@ -4416,6 +5109,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 7f6c092..576d6f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,25 +14,34 @@ keywords = ["nexus", "talus", "tools", "walrus"] categories = [] [workspace.dependencies] -anyhow = "1.0.97" -base64 = "0.21" -chrono = { version = "0.4", features = ["serde"] } -jsonschema = "0.28.3" -log = "^0.4.26" -minijinja = "2.8.0" -mockito = "1.7.0" +anyhow = "1.0.102" +base64 = "0.22.1" +bytes = "1.11.1" +dotenvy = "0.15.7" +ed25519-dalek = "2.2.0" +env_logger = "0.11.10" +hex = "0.4.3" +sha2 = "0.11.0" +chrono = { version = "0.4.44", features = ["serde"] } +jsonschema = "0.46.5" +log = "0.4.29" +minijinja = "2.20.0" +mockito = "1.7.2" portpicker = "0.1.1" -reqwest = { version = "0.12.25", features = ["json", "stream"] } -rstest = "0.25.0" -schemars = "1.0.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -serial_test = "3.2.0" -strum = "0.27" -strum_macros = "0.27" -thiserror = "2.0.12" -tokio = { version = "1.49.0", features = ["full"] } -warp = "0.3.7" +reqwest = { version = "0.12.25", features = ["json", "stream", "gzip", "brotli"] } +rstest = "0.26.1" +schemars = "1.2.1" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.149" +serial_test = "3.4.0" +strum = "0.28.0" +strum_macros = "0.28.0" +thiserror = "2.0.18" +tokio = { version = "1.52.3", features = ["full"] } +url = "2.5.8" +uuid = { version = "1.23.1", features = ["v4"] } +warp = "0.4.3" +zeroize = { version = "1", features = ["zeroize_derive"] } # === Nexus deps === [workspace.dependencies.nexus-sdk] diff --git a/justfile b/justfile index af0071c..957f5d7 100644 --- a/justfile +++ b/justfile @@ -8,6 +8,9 @@ # Commands concerning native Nexus Tools mod tools 'tools/.just' +# Example invocations for the memory-memwal Tools bundle +mod memwal 'tools/memory-memwal/.just' + # Pre-commit hooks mod pre-commit '.pre-commit/.just' diff --git a/tools/memory-memwal/.env.example b/tools/memory-memwal/.env.example new file mode 100644 index 0000000..8fcc2fa --- /dev/null +++ b/tools/memory-memwal/.env.example @@ -0,0 +1,23 @@ +# memory-memwal — environment template. Copy to `.env` (gitignored) or +# export in the deployment environment. + +# Required. Hex-encoded 32-byte Ed25519 delegate private key (64 hex chars). +# Unset → boot continues, every signed call fails. Malformed → FATAL exit. +MEMWAL_DELEGATE_PRIVATE_KEY= + +# Recommended. MemWal account object ID (`0x…`, 64 hex chars). Sent as +# `x-account-id` and signed into the canonical message. Unset → relayer +# falls back to an on-chain registry scan by public key (slower, flakier). +MEMWAL_ACCOUNT_ID= + +# Optional. Default `https://relayer.staging.memwal.ai` (testnet). +# Use `https://relayer.memwal.ai` for mainnet. Validated at startup; +# only `https://` accepted unless MEMWAL_ALLOW_INSECURE=1. +MEMWAL_SERVER_URL= + +# Dev/test only. Accept `http://` URLs (e.g. mockito). Leave unset in prod. +# MEMWAL_ALLOW_INSECURE=1 + +# Optional. Bind address for the bundled HTTP server. Default `127.0.0.1:8080`. +# Use `0.0.0.0:8080` to accept external connections. +BIND_ADDR= diff --git a/tools/memory-memwal/.just b/tools/memory-memwal/.just new file mode 100644 index 0000000..c29aaa1 --- /dev/null +++ b/tools/memory-memwal/.just @@ -0,0 +1,329 @@ +# Example invocations for the memory-memwal bundle. Recipes target the +# bundle's local HTTP server and pretty-print responses via `jq`. Requires +# `curl`, `jq`, `cargo` on PATH and `MEMWAL_DELEGATE_PRIVATE_KEY` exported +# in the binary's environment. Default relayer is testnet; set +# `MEMWAL_SERVER_URL=https://relayer.memwal.ai` for mainnet. + +# Host:port the recipes target. Override on the CLI: +# just memwal --set host 0.0.0.0:9000 health +host := "127.0.0.1:8080" + +[private] +_default: + @just --list memwal + +# --- Process lifecycle --- +# The server runs detached; PID at target/memwal-server.pid, logs at +# target/memwal-server.log. server-start blocks until /tools answers. + +_pid_subpath := "target/memwal-server.pid" +_log_subpath := "target/memwal-server.log" + +# Print an array of tokens as a copy-paste-ready shell command. Quote-only- +# when-needed: single-quote any token that contains characters outside +# `[A-Za-z0-9_/.@:=,+-]`. Single source of truth for command tracing. +_print_cmd_fn := ''' +_print_cmd() { + local a + for a in "$@"; do + if [[ -z "$a" || "$a" =~ [^A-Za-z0-9_/.@:=,+-] ]]; then + printf "'%s' " "${a//\'/\'\\\'\'}" + else + printf '%s ' "$a" + fi + done + echo +} +''' + +# `cargo locate-project --workspace` is the authoritative anchor. Defined as +# a private recipe (not a backticked variable) so the cargo invocation only +# runs for recipes that need the path, not for every `just --list`. +[private] +_workspace-root: + @dirname "$(cargo locate-project --workspace --message-format plain)" + +# Override bind: just memwal server-start 0.0.0.0:9000 +# (also set host: just memwal --set host 0.0.0.0:9000 server-start) +# Loads tools/memory-memwal/.env before spawning; existing shell exports win. +# Build the release binary, spawn it detached, wait until /tools answers. +server-start bind=host: + #!/usr/bin/env bash + set -euo pipefail + workspace_root="$(just memwal::_workspace-root)" + pid_file="$workspace_root/{{_pid_subpath}}" + log_file="$workspace_root/{{_log_subpath}}" + env_file="$workspace_root/tools/memory-memwal/.env" + cd "$workspace_root" + + # Load .env defaults without clobbering vars already exported in the + # calling shell: snapshot current exports, source .env, then replay the + # snapshot so original values override any reassignments. Variables that + # only exist in .env survive the replay (they aren't in the snapshot). + if [[ -f "$env_file" ]]; then + echo "Loading env defaults from $env_file" + env_snapshot=$(declare -px | grep '^declare -x ' || true) + set -a + # shellcheck disable=SC1090 + source "$env_file" + set +a + eval "$env_snapshot" + fi + + if [[ -f "$pid_file" ]]; then + existing=$(cat "$pid_file") + if kill -0 "$existing" 2>/dev/null; then + echo "memwal server already running (PID $existing)"; exit 1 + fi + echo "Stale PID file; removing." + rm -f "$pid_file" + fi + echo "Building memory-memwal (release)..." + cargo +stable build --package memory-memwal --release --quiet + mkdir -p "$(dirname "$pid_file")" + BIND_ADDR={{bind}} nohup ./target/release/memory-memwal \ + > "$log_file" 2>&1 & + pid=$! + echo "$pid" > "$pid_file" + # The bind probe address may differ from {{bind}} (e.g. 0.0.0.0 is not + # routable from a client); always probe via {{host}}. + # If the binary exits before answering (e.g. FATAL on malformed + # MEMWAL_DELEGATE_PRIVATE_KEY), break early and dump the log. + for i in $(seq 1 30); do + if ! kill -0 "$pid" 2>/dev/null; then + echo "FAIL: server PID $pid exited during startup. Last log lines:" >&2 + tail -n 30 "$log_file" >&2 || true + rm -f "$pid_file" + exit 1 + fi + if curl -sSf "http://{{host}}/tools" > /dev/null 2>&1; then + echo "Ready at http://{{host}} (bind={{bind}}, PID $pid, log $log_file)" + exit 0 + fi + sleep 1 + done + echo "FAIL: server did not answer on http://{{host}}/tools within 30s." >&2 + echo "Last log lines:" >&2 + tail -n 30 "$log_file" >&2 || true + exit 1 + +# Stop the detached server: SIGTERM, wait 5 s, escalate to SIGKILL. +server-stop: + #!/usr/bin/env bash + set -euo pipefail + pid_file="$(just memwal::_workspace-root)/{{_pid_subpath}}" + if [[ ! -f "$pid_file" ]]; then + echo "No PID file at $pid_file; nothing to stop."; exit 0 + fi + pid=$(cat "$pid_file") + if ! kill -0 "$pid" 2>/dev/null; then + echo "PID $pid is not alive; clearing stale PID file." + rm -f "$pid_file"; exit 0 + fi + kill -TERM "$pid" + for i in $(seq 1 5); do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$pid_file" + echo "Stopped PID $pid." + exit 0 + fi + sleep 1 + done + echo "PID $pid did not exit after SIGTERM; sending SIGKILL." >&2 + kill -KILL "$pid" || true + rm -f "$pid_file" + +# PID + process liveness + /tools reachability. Exit 0 if fully up. +server-status: + #!/usr/bin/env bash + set -euo pipefail + pid_file="$(just memwal::_workspace-root)/{{_pid_subpath}}" + if [[ ! -f "$pid_file" ]]; then + echo "stopped (no PID file)"; exit 1 + fi + pid=$(cat "$pid_file") + if ! kill -0 "$pid" 2>/dev/null; then + echo "stopped (PID $pid is not alive; PID file is stale)"; exit 1 + fi + if curl -sSf "http://{{host}}/tools" > /dev/null 2>&1; then + echo "running (PID $pid, http://{{host}})" + else + echo "process alive (PID $pid) but http://{{host}}/tools not answering" + exit 1 + fi + +# Tail the detached server's log (`tail -n 50 -F`). +server-logs: + tail -n 50 -F "$(just memwal::_workspace-root)/{{_log_subpath}}" + +# --- Read-only inspection (no auth, no relayer round-trip) --- + +# GET /tools — list registered tool FQNs and JSON schemas. No auth. +tools: + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + cmd=(curl --fail-with-body -sS "http://{{host}}/tools") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# --- Liveness (auth required; hits the relayer) --- + +# GET /health — checks delegate key + relayer reachability + version match. +health: + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + cmd=(curl --fail-with-body -sS "http://{{host}}/health") + _print_cmd "${cmd[@]}" + "${cmd[@]}" + +# --- Tool invocations --- + +# Store a memory; blocks until written. Returns blob_id. +remember text namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg text {{quote(text)}} --arg ns {{quote(namespace)}} \ + '{text: $text, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/remember/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Semantic search over stored memories; returns ranked matches. +recall query limit="5" namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc \ + --arg q {{quote(query)}} \ + --argjson limit {{quote(limit)}} \ + --arg ns {{quote(namespace)}} \ + '{query: $q, limit: $limit, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/recall/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Memory-augmented Q&A; returns LLM answer + source memories. +ask question limit="3" namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc \ + --arg q {{quote(question)}} \ + --argjson limit {{quote(limit)}} \ + --arg ns {{quote(namespace)}} \ + '{question: $q, limit: $limit, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/ask/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Extract facts and enqueue one memory-write per fact. Fire-and-forget. +analyze text namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg text {{quote(text)}} --arg ns {{quote(namespace)}} \ + '{text: $text, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/analyze/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Delete every memory in a namespace; returns the count removed (owner-scoped). +forget namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg ns {{quote(namespace)}} '{namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/forget/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Memory count + storage bytes for a namespace (quota guard; 1 GB per account). +stats namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg ns {{quote(namespace)}} '{namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/stats/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# items_json: JSON array of `{"text": "...", "namespace": "..."}` (namespace optional per item). +# Store up to 20 memories in one batched call. +remember-bulk items_json: + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --argjson items {{quote(items_json)}} '{items: $items}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/remember_bulk/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# --- End-to-end smoke test --- + +# remember_bulk is omitted (weight 10 would push the demo over the 30/min cap). +# Total signed-request weight ~18 — within the cap, but back-to-back re-runs +# may 429 because the sliding window hasn't cleared yet. +# End-to-end demo: auto-manages the server, runs one of each tool. +demo: + #!/usr/bin/env bash + set -euo pipefail + + # Auto-manage the server: start it iff it isn't already running, and + # tear it down on exit only in that case so a developer-started server + # is left alone. + if just memwal::server-status > /dev/null 2>&1; then + echo "Using already-running server." + else + echo "Server not running — starting it now." + just memwal::server-start + # `set -e` ensures any failure below still hits this trap. + trap 'echo; echo "Demo done — stopping the server we started."; just memwal::server-stop' EXIT + fi + echo + + # Each wrapper recipe prints the actual curl command it's about to run + # (single source of truth via a bash array), so we just call them + # directly — no need for a wrapper echo helper. + ns="demo-$(date +%s)" + echo "== namespace: $ns ==" + echo + just memwal::tools + echo + just memwal::health + echo + echo + just memwal::remember "The Eiffel Tower is in Paris, France." "$ns" + echo + just memwal::recall "famous landmarks in France" 3 "$ns" + echo + just memwal::ask "Where is the Eiffel Tower?" 3 "$ns" + echo + just memwal::analyze "Carol leads design. Dave handles ops." "$ns" + echo + just memwal::stats "$ns" + echo + just memwal::forget "$ns" diff --git a/tools/memory-memwal/Cargo.toml b/tools/memory-memwal/Cargo.toml new file mode 100644 index 0000000..f7ce1dc --- /dev/null +++ b/tools/memory-memwal/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "memory-memwal" +description = "Nexus Tools for MemWal persistent memory" + +edition.workspace = true +version.workspace = true +repository.workspace = true +homepage.workspace = true +license.workspace = true +readme.workspace = true +authors.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +anyhow.workspace = true +dotenvy.workspace = true +ed25519-dalek.workspace = true +env_logger.workspace = true +hex.workspace = true +log.workspace = true +nexus-sdk.workspace = true +nexus-toolkit.workspace = true +reqwest.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true +tokio.workspace = true +url.workspace = true +uuid.workspace = true +zeroize.workspace = true + +[dev-dependencies] +mockito.workspace = true diff --git a/tools/memory-memwal/README.md b/tools/memory-memwal/README.md new file mode 100644 index 0000000..d612827 --- /dev/null +++ b/tools/memory-memwal/README.md @@ -0,0 +1,373 @@ +# Memory Tools (MemWal) + +A Nexus Tool bundle exposing seven memory operations backed by the +[MemWal](https://memwal.ai) relayer: + +- `remember` / `remember_bulk` — store one or up to 20 memories +- `recall` — semantic search over stored memories +- `ask` — memory-augmented Q&A (LLM with retrieved context) +- `analyze` — extract facts from text and store each as a memory +- `forget` — delete every memory in a namespace +- `stats` — report memory count and storage bytes for a namespace + +## Pinned MemWal release + +Wire format pinned to +[`@mysten-incubation/memwal@0.0.4`](https://github.com/MystenLabs/MemWal/tree/%40mysten-incubation%2Fmemwal%400.0.4/services/server) +(server `Cargo.toml` v`0.1.0`). `health_check()` enforces the version at +runtime against `GET /health`. On a new relayer tag with a Cargo bump or +any change to `auth.rs` / `types.rs` / `routes.rs` / `rate_limit.rs`, +update `MEMWAL_API_VERSION` in `src/client.rs` and re-audit those four files. + +## Build & Run + +```sh +# Build (release) +cargo build --package memory-memwal --release + +# Run locally — binds to 127.0.0.1:8080 by default +cargo run --package memory-memwal + +# Override the bind address +BIND_ADDR=0.0.0.0:9000 cargo run --package memory-memwal +``` + +## Environment Variables + +`.env` (cwd-only) is loaded at startup via [`dotenvy`](https://crates.io/crates/dotenvy); +existing exports win. + +| Variable | Required | Default | Description | +| --- | --- | --- | --- | +| `MEMWAL_DELEGATE_PRIVATE_KEY` | **yes** | — | Hex-encoded 32-byte Ed25519 (Elliptic Curve Digital Signature Algorithm) delegate private key | +| `MEMWAL_ACCOUNT_ID` | recommended | — | MemWal account object ID (`0x…`). Sent as `x-account-id` and signed into the canonical message. Unset → relayer falls back to a slower on-chain registry scan | +| `MEMWAL_SERVER_URL` | no | `https://relayer.staging.memwal.ai` (testnet) | Relayer base URL. Validated as `https://` (no path/query/fragment/userinfo). Mainnet: `https://relayer.memwal.ai` | +| `MEMWAL_ALLOW_INSECURE` | no | unset | Set to `1` to accept `http://` relayer URLs (dev/mockito only) | + +## Server Endpoints + +```sh +# List registered tool paths +GET /tools + +# Liveness check (validates key + reaches the relayer) +GET /health + +# Invoke a tool (POST, JSON body) +POST //invoke +``` + +## Tools + +All seven tools return `Err { reason: String }` on failure (`reason` is a +short, terse description; full upstream bodies are not inlined). Per-tool +sections below cover only the `Ok` variant unless a failure mode is +tool-specific. + +--- + +# `xyz.taluslabs.memory.memwal.remember@1` + +Store a single piece of text as a persistent memory. The call blocks until the +memory is durably written to Walrus and returns the resulting blob ID — the +next vertex in a Nexus DAG (Directed Acyclic Graph) will not activate until +the write is confirmed. + +## Input + +**`text`: `String`** *(required)* + +The text to store as a memory. + +**`namespace`: `String`** *(optional)* + +Namespace used to scope this memory. Defaults to `"default"` on the server +when omitted. + +## Output Variants & Ports + +**`ok`** + +The memory was durably stored. + +- **`ok.blob_id`: `String`** — Walrus blob identifier for the stored memory. + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/remember/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "Paris is the capital of France"}' +# {"ok":{"blob_id":""}} + +# With an explicit namespace +curl -s -X POST http://127.0.0.1:8080/memory/remember/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "Alice works at ACME", "namespace": "people"}' +# {"ok":{"blob_id":""}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.recall@1` + +Semantic search over stored memories. Returns the closest matches ranked by +cosine distance — lower distance means more relevant. + +## Input + +**`query`: `String`** *(required)* + +Natural-language query to search for relevant memories. + +**`limit`: `u32`** *(optional)* + +Maximum number of results to return. Server default applies when omitted. + +**`namespace`: `String`** *(optional)* + +Namespace to search within. Searches the `"default"` namespace when omitted. + +## Output Variants & Ports + +**`ok`** + +Search completed. The result list may be empty if nothing matched. + +- **`ok.results`: `Array`** — Memories ranked by relevance, each containing: + - **`text`: `String`** — The stored text of the memory. + - **`blob_id`: `String`** — Walrus blob identifier. + - **`distance`: `f64`** — Cosine distance from the query vector (lower = more relevant). +- **`ok.namespace`: `String`** — The namespace that was searched (one query = one namespace). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/recall/invoke \ + -H 'Content-Type: application/json' \ + -d '{"query": "capital cities in Europe", "limit": 3}' +# {"ok":{"results":[{"text":"Paris is the capital of France","blob_id":"...","distance":0.12,"namespace":"default"}]}} + +# Empty result when nothing matches +curl -s -X POST http://127.0.0.1:8080/memory/recall/invoke \ + -H 'Content-Type: application/json' \ + -d '{"query": "something completely unknown"}' +# {"ok":{"results":[]}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.ask@1` + +Memory-augmented question answering. The relayer retrieves the most relevant +stored memories for the question, injects them as context into an LLM (Large +Language Model) prompt, and returns the generated answer together with the +source memories that informed it. + +## Input + +**`question`: `String`** *(required)* + +The question to answer using stored memories as context. + +**`namespace`: `String`** *(optional)* + +Namespace to retrieve memories from. Uses the `"default"` namespace when +omitted. + +**`limit`: `u32`** *(optional)* + +Maximum number of source memories to inject as context. Server default applies +when omitted. + +## Output Variants & Ports + +**`ok`** + +Question answered successfully. + +- **`ok.answer`: `String`** — LLM-generated answer. +- **`ok.sources`: `Array`** — Memories used as context, each containing: + - **`blob_id`: `String`** — Walrus blob identifier. + - **`text`: `String`** — The stored text of the memory. + - **`distance`: `f64`** — Cosine distance from the question vector. + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/ask/invoke \ + -H 'Content-Type: application/json' \ + -d '{"question": "What is the capital of France?"}' +# {"ok":{"answer":"Paris","sources":[{"blob_id":"...","text":"Paris is the capital of France","distance":0.05}]}} + +# No relevant memories → answer with empty sources +curl -s -X POST http://127.0.0.1:8080/memory/ask/invoke \ + -H 'Content-Type: application/json' \ + -d '{"question": "What is the speed of light?"}' +# {"ok":{"answer":"I don't know","sources":[]}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.analyze@1` + +Extract discrete facts from a text document and store each fact as an +individual memory. The relayer runs an LLM fact-extraction pass and enqueues +one memory-write job per fact found. + +This tool returns immediately after the jobs are enqueued — it does **not** +block on individual Walrus writes. Use `job_count` for downstream monitoring; +use `remember` if you need a confirmed blob ID. + +## Input + +**`text`: `String`** *(required)* + +The text from which to extract and store facts. + +**`namespace`: `String`** *(optional)* + +Namespace to store the extracted facts in. Uses the `"default"` namespace when +omitted. + +## Output Variants & Ports + +**`ok`** + +Facts were submitted for storage. + +- **`ok.job_count`: `u32`** — Number of individual memory-write jobs enqueued. + Zero means no facts were extracted from the text. + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/analyze/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "Alice lives in Paris. Bob works at ACME. Paris is in France."}' +# {"ok":{"job_count":3}} + +# No facts extracted → job_count is 0 +curl -s -X POST http://127.0.0.1:8080/memory/analyze/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "..."}' +# {"ok":{"job_count":0}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.remember_bulk@1` + +Store up to 20 memories in a single batched call. The relayer rate-limits +`/api/remember` at weight 5 per call but `/api/remember/bulk` at weight 10 for +up to 20 items — a 10× efficiency gain when batching is feasible. The call +blocks until every item is durably written; a single failed item fails the +whole batch. + +## Input + +**`items`: `Array`** *(required, 1–20 entries)* + +Each entry has: + +- **`text`: `String`** *(required)* — the text to store. +- **`namespace`: `String`** *(optional)* — namespace for this item. A single + bulk call can write to multiple namespaces. + +## Output Variants & Ports + +**`ok`** + +Every item was durably stored. + +- **`ok.blob_ids`: `Array`** — Walrus blob identifiers, in the same + order as the input `items`. + +**`err`** — additionally: the whole batch fails if any individual item fails (no partial success). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/remember_bulk/invoke \ + -H 'Content-Type: application/json' \ + -d '{ + "items": [ + {"text": "Paris is the capital of France"}, + {"text": "Tokyo is the capital of Japan"}, + {"text": "Madrid is the capital of Spain"} + ] + }' +# {"ok":{"blob_ids":["","",""]}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.forget@1` + +Delete every memory in a namespace. Owner-scoped — only memories belonging to +the authenticated account are removed. Lifecycle complement to `remember`: +useful for clearing scratch namespaces at the end of a DAG run, or for +recovering quota before the 1 GB per-account cap is hit. + +## Input + +**`namespace`: `String`** *(optional)* + +The namespace to clear. Defaults to `"default"` on the server when omitted. + +## Output Variants & Ports + +**`ok`** + +Deletion completed. + +- **`ok.deleted`: `u64`** — number of memories removed. Zero is a valid + success (the namespace was empty or did not exist). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/forget/invoke \ + -H 'Content-Type: application/json' \ + -d '{"namespace": "scratch-pad"}' +# {"ok":{"deleted":12}} + +# Clearing an empty namespace is fine +curl -s -X POST http://127.0.0.1:8080/memory/forget/invoke \ + -H 'Content-Type: application/json' \ + -d '{"namespace": "never-used"}' +# {"ok":{"deleted":0}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.stats@1` + +Report the number of memories and total encrypted byte size stored in a +namespace for the authenticated account. Useful as a quota guard before +heavy writes — the relayer enforces 1 GB per-account storage and starts +returning HTTP 402 once that's exceeded. + +## Input + +**`namespace`: `String`** *(optional)* + +Defaults to `"default"` on the server when omitted. + +## Output Variants & Ports + +**`ok`** + +- **`ok.memory_count`: `i64`** — number of memories in the namespace. +- **`ok.storage_bytes`: `i64`** — total encrypted byte size on Walrus. +- **`ok.namespace`: `String`** — the resolved namespace (mirrors what the + server interpreted; `"default"` when the input was omitted). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/stats/invoke \ + -H 'Content-Type: application/json' \ + -d '{"namespace": "people"}' +# {"ok":{"memory_count":17,"storage_bytes":5242880,"namespace":"people"}} +``` diff --git a/tools/memory-memwal/src/analyze.rs b/tools/memory-memwal/src/analyze.rs new file mode 100644 index 0000000..f412ff9 --- /dev/null +++ b/tools/memory-memwal/src/analyze.rs @@ -0,0 +1,202 @@ +//! # `xyz.taluslabs.memory.memwal.analyze@1` +//! +//! Extract facts from a document via an LLM pass and enqueue one memory-write +//! job per fact. Returns the count of jobs submitted (fire-and-forget — no +//! per-job polling). + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// The text from which to extract and store facts. + text: String, + /// Namespace to store the extracted facts in. Uses the default namespace + /// when omitted. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Facts were submitted for storage. `job_count` is the number of + /// individual memory-write jobs enqueued on the server. + Ok { + job_count: u32, + }, + Err { + reason: String, + }, +} + +pub(crate) struct AnalyzeAndRemember { + client: MemWalClient, +} + +impl NexusTool for AnalyzeAndRemember { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.analyze@1") + } + + fn path() -> &'static str { + "/memory/analyze" + } + + fn description() -> &'static str { + "Extract facts from a text document and store each fact as a separate memory." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self + .client + .analyze(&input.text, input.namespace.as_deref()) + .await + { + Ok(count) => Output::Ok { + job_count: count as u32, + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> AnalyzeAndRemember { + AnalyzeAndRemember { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns `Ok { job_count }` matching the number of job_ids returned + /// by the server. + /// Failure mode caught: job_ids list not counted, job_count is always 0 or wrong. + #[tokio::test] + async fn invoke_returns_job_count() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/analyze") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_ids": ["j1", "j2", "j3"]}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "Alice lives in Paris. Bob works at ACME. Paris is in France.".into(), + namespace: None, + }) + .await; + + assert!( + matches!(output, Output::Ok { job_count: 3 }), + "expected Ok{{job_count: 3}}" + ); + } + + /// `invoke` returns `Ok { job_count: 0 }` when the server returns no jobs + /// (e.g., no facts extracted from the text). + /// Failure mode caught: empty job_ids list treated as an error. + #[tokio::test] + async fn invoke_returns_zero_for_empty_job_ids() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/analyze") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_ids": []}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "...".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Ok { job_count: 0 }), + "empty job_ids must produce Ok{{job_count: 0}}" + ); + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP error from analyze endpoint swallowed. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/analyze") + .with_status(500) + .with_body("error") + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "any text".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } +} diff --git a/tools/memory-memwal/src/ask.rs b/tools/memory-memwal/src/ask.rs new file mode 100644 index 0000000..a58eb7e --- /dev/null +++ b/tools/memory-memwal/src/ask.rs @@ -0,0 +1,246 @@ +//! # `xyz.taluslabs.memory.memwal.ask@1` +//! +//! Nexus Tool for memory-augmented Q&A. The relayer retrieves the most +//! relevant stored memories for the question, injects them into an LLM prompt, +//! and returns both the generated answer and the source memories that informed +//! it. + +use { + crate::client::{AskSource, MemWalClient}, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// The question to answer using stored memories as context. + question: String, + /// Namespace to retrieve memories from. Uses the default namespace when omitted. + namespace: Option, + /// Maximum number of source memories to inject as context. + limit: Option, +} + +/// A memory that contributed to the answer. +#[derive(Serialize, JsonSchema)] +pub(crate) struct AnswerSource { + /// Walrus blob identifier for the encrypted memory. + blob_id: String, + /// The stored text of the memory. + text: String, + /// Cosine distance from the question vector — lower means more relevant. + distance: f64, +} + +impl From for AnswerSource { + fn from(s: AskSource) -> Self { + Self { + blob_id: s.blob_id, + text: s.text, + distance: s.distance, + } + } +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// LLM-generated answer and the memories used as context. + Ok { + answer: String, + sources: Vec, + }, + Err { + reason: String, + }, +} + +pub(crate) struct AskMemory { + client: MemWalClient, +} + +impl NexusTool for AskMemory { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.ask@1") + } + + fn path() -> &'static str { + "/memory/ask" + } + + fn description() -> &'static str { + "Answer a question by retrieving relevant memories and generating a response with an LLM." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self + .client + .ask(&input.question, input.namespace.as_deref(), input.limit) + .await + { + Ok(resp) => Output::Ok { + answer: resp.answer, + sources: resp.memories.into_iter().map(AnswerSource::from).collect(), + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> AskMemory { + AskMemory { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns the LLM answer and its sources on a successful call. + /// Failure mode caught: successful ask response incorrectly mapped to `Err`. + #[tokio::test] + async fn invoke_returns_answer_and_sources() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/ask") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "answer": "Paris", + "memories_used": 1, + "memories": [ + { + "blob_id": "blob-1", + "text": "Paris is the capital of France", + "distance": 0.05 + } + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + question: "What is the capital of France?".into(), + namespace: None, + limit: None, + }) + .await; + match output { + Output::Ok { answer, sources } => { + assert_eq!(answer, "Paris"); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].text, "Paris is the capital of France"); + assert_eq!(sources[0].blob_id, "blob-1"); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok` with an empty sources list when the server returns none. + /// Failure mode caught: empty sources array causes a parse error or Err variant. + #[tokio::test] + async fn invoke_handles_empty_sources() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/ask") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"answer": "I don't know", "memories_used": 0, "memories": []}).to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + question: "unknown topic".into(), + namespace: None, + limit: None, + }) + .await; + assert!( + matches!(output, Output::Ok { sources, .. } if sources.is_empty()), + "empty sources must produce Ok with empty vec" + ); + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP 500 swallowed, returning a garbage answer. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/ask") + .with_status(503) + .with_body("service unavailable") + .create_async() + .await; + + let output = tool + .invoke(Input { + question: "anything".into(), + namespace: None, + limit: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 503 must produce Err variant" + ); + } +} diff --git a/tools/memory-memwal/src/auth.rs b/tools/memory-memwal/src/auth.rs new file mode 100644 index 0000000..190396e --- /dev/null +++ b/tools/memory-memwal/src/auth.rs @@ -0,0 +1,240 @@ +//! Ed25519 request signing for the MemWal relayer. +//! +//! ```text +//! x-public-key : hex(public_key) +//! x-timestamp : unix_seconds +//! x-nonce : UUID v4 (fresh per request; server checks against a 600 s +//! Redis replay cache, missing header → HTTP 426) +//! x-signature : hex(Ed25519_sign(key, +//! "{ts}.{METHOD}.{path}.{sha256hex(body)}.{nonce}.{account_id}")) +//! ``` +//! +//! `{account_id}` is `""` when `x-account-id` is not sent. Server rejects +//! timestamps outside ±5 min — host clock must be reasonably accurate. + +use { + crate::error::AuthError, + ed25519_dalek::{Signer, SigningKey}, + sha2::{Digest, Sha256}, + std::time::{SystemTime, UNIX_EPOCH}, + zeroize::{Zeroize, Zeroizing}, +}; + +/// Headers produced by [`sign_request`]. +pub(crate) struct AuthHeaders { + pub(crate) public_key: String, + pub(crate) signature: String, + pub(crate) timestamp: String, + pub(crate) nonce: String, +} + +/// Parse a hex 32-byte Ed25519 secret into a `SigningKey` plus its public +/// key in hex. Called once at startup so signing skips per-request hex +/// decode, SHA-512 derivation, and Curve25519 scalar mult. +pub(crate) fn parse_signing_key(private_key_hex: &str) -> Result<(SigningKey, String), AuthError> { + if private_key_hex.is_empty() { + return Err(AuthError::MissingKey); + } + // Zeroize the heap buffer + stack copy; SigningKey is ZeroizeOnDrop but + // these intermediates aren't — a core dump in between would recover the key. + let raw: Zeroizing> = Zeroizing::new(hex::decode(private_key_hex)?); + if raw.len() != 32 { + return Err(AuthError::InvalidKeyLength(raw.len())); + } + let mut key_bytes: [u8; 32] = [0u8; 32]; + key_bytes.copy_from_slice(&raw); + let signing_key = SigningKey::from_bytes(&key_bytes); + key_bytes.zeroize(); + let public_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + Ok((signing_key, public_key_hex)) +} + +/// Canonical signed message per the relayer's `services/server/src/auth.rs` +/// at tag `@mysten-incubation/memwal@0.0.4`. Pure so the format can be +/// locked by a unit test independent of the crypto. +fn canonical_message( + ts: &str, + method: &str, + path: &str, + body_hash: &str, + nonce: &str, + account_id: &str, +) -> String { + format!("{ts}.{method}.{path}.{body_hash}.{nonce}.{account_id}") +} + +/// Build the auth headers for one request. Mints a fresh UUID v4 nonce per +/// call. `method` uppercase, `path` includes the leading `/`, `body` is the +/// exact bytes that go on the wire (empty slice for GET), `account_id` must +/// match what's sent as `x-account-id` (or `""` if the header is omitted). +pub(crate) fn sign_request( + signing_key: &SigningKey, + public_key_hex: &str, + method: &str, + path: &str, + body: &[u8], + account_id: &str, +) -> Result { + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| AuthError::Clock(e.to_string()))? + .as_secs() + .to_string(); + + let nonce = uuid::Uuid::new_v4().to_string(); + let body_hash = hex::encode(Sha256::digest(body)); + + let message = canonical_message(&ts, method, path, &body_hash, &nonce, account_id); + let signature = signing_key.sign(message.as_bytes()); + + Ok(AuthHeaders { + public_key: public_key_hex.to_string(), + signature: hex::encode(signature.to_bytes()), + timestamp: ts, + nonce, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_key() -> (SigningKey, String) { + // Deterministic test key: 32 bytes of 0x42. + parse_signing_key(&hex::encode([0x42u8; 32])).expect("test key must parse") + } + + /// `canonical_message` matches the relayer's expected 6-segment dot-separated + /// wire format byte-for-byte. + /// Failure mode caught: any drift in the canonical-message format (separator + /// swap, segment reorder, missing field) would pass every existing shape + /// test and only fail with HTTP 401 against the live relayer. + #[test] + fn canonical_message_format_locked() { + let m = canonical_message( + "1700000000", + "POST", + "/api/remember", + "abc123", + "11111111-2222-3333-4444-555555555555", + "0xacct", + ); + assert_eq!( + m, + "1700000000.POST./api/remember.abc123.11111111-2222-3333-4444-555555555555.0xacct" + ); + } + + /// `parse_signing_key` rejects an empty hex string with `MissingKey`. + /// Failure mode caught: missing key is silently ignored, producing + /// unauthenticated requests. + #[test] + fn parse_signing_key_rejects_empty() { + assert!(matches!(parse_signing_key(""), Err(AuthError::MissingKey))); + } + + /// `parse_signing_key` rejects non-hex with `InvalidHex`. + /// Failure mode caught: bad key bypasses validation and produces garbage + /// headers. + #[test] + fn parse_signing_key_rejects_non_hex() { + assert!(matches!( + parse_signing_key("not-hex!!"), + Err(AuthError::InvalidHex(_)) + )); + } + + /// `parse_signing_key` rejects an under-length key with `InvalidKeyLength`. + /// Failure mode caught: a 16-byte key would silently produce a 32-byte + /// signing key by zero-extension if the byte-length check were removed. + #[test] + fn parse_signing_key_rejects_short_key() { + let short = hex::encode([0u8; 16]); + assert!(matches!( + parse_signing_key(&short), + Err(AuthError::InvalidKeyLength(16)) + )); + } + + /// `parse_signing_key` returns a public key whose hex form is 64 chars. + /// Failure mode caught: public-key derivation drops bytes or returns an + /// empty string, so all subsequent x-public-key headers are malformed. + #[test] + fn parse_signing_key_derives_public_key_hex() { + let (_sk, pk_hex) = test_key(); + assert_eq!(pk_hex.len(), 64); + } + + /// `sign_request` produces non-empty headers with the right lengths for a + /// valid key and request. + /// Failure mode caught: auth module silently returns empty headers. + #[test] + fn sign_request_returns_headers() { + let (sk, pk_hex) = test_key(); + let h = sign_request( + &sk, + &pk_hex, + "POST", + "/api/remember", + b"{\"text\":\"hello\"}", + "", + ) + .expect("should sign successfully"); + + assert_eq!(h.public_key, pk_hex); + assert!(!h.signature.is_empty()); + assert!(!h.timestamp.is_empty()); + assert!(!h.nonce.is_empty()); + assert_eq!(h.public_key.len(), 64); + assert_eq!(h.signature.len(), 128); + } + + /// `sign_request` mints a fresh UUID-formatted nonce on every call. + /// Failure mode caught: a static or empty nonce would trigger replay + /// rejection (or HTTP 426) on the second invocation against the relayer. + #[test] + fn sign_request_nonce_is_unique_uuid_per_call() { + let (sk, pk_hex) = test_key(); + let h1 = sign_request(&sk, &pk_hex, "POST", "/api/remember", b"{}", "").unwrap(); + let h2 = sign_request(&sk, &pk_hex, "POST", "/api/remember", b"{}", "").unwrap(); + assert_ne!(h1.nonce, h2.nonce); + assert!(uuid::Uuid::parse_str(&h1.nonce).is_ok()); + assert!(uuid::Uuid::parse_str(&h2.nonce).is_ok()); + } + + /// Repeated signing with the same key returns the same public key. + /// Failure mode caught: per-call public-key derivation produces a different + /// public key, which would break attribution server-side. + #[test] + fn sign_request_same_key_same_pubkey() { + let (sk, pk_hex) = test_key(); + let h1 = sign_request( + &sk, + &pk_hex, + "POST", + "/api/recall", + b"{\"query\":\"foo\"}", + "", + ) + .unwrap(); + let h2 = sign_request( + &sk, + &pk_hex, + "POST", + "/api/recall", + b"{\"query\":\"foo\"}", + "", + ) + .unwrap(); + assert_eq!(h1.public_key, h2.public_key); + } + + /// `sign_request` with an empty body does not panic. + /// Failure mode caught: empty-body path panics or errors for GET requests. + #[test] + fn sign_request_handles_empty_body() { + let (sk, pk_hex) = test_key(); + let result = sign_request(&sk, &pk_hex, "GET", "/api/remember/some-job-id", b"", ""); + assert!(result.is_ok()); + } +} diff --git a/tools/memory-memwal/src/client.rs b/tools/memory-memwal/src/client.rs new file mode 100644 index 0000000..7053920 --- /dev/null +++ b/tools/memory-memwal/src/client.rs @@ -0,0 +1,1455 @@ +//! MemWal HTTP client. +//! +//! Credentials (`MEMWAL_DELEGATE_PRIVATE_KEY`, optional `MEMWAL_ACCOUNT_ID`) +//! and the relayer URL (`MEMWAL_SERVER_URL`) come from env at startup — never +//! from tool inputs, which flow through the Nexus DAG as on-chain data. + +use { + crate::{ + auth::{parse_signing_key, sign_request}, + error::{AuthError, MemWalError}, + }, + ed25519_dalek::SigningKey, + serde::{Deserialize, Serialize}, + std::{ + collections::HashMap, + sync::{Arc, Once, OnceLock}, + time::Duration, + }, + tokio::time::{sleep, Instant}, + url::Url, + zeroize::Zeroizing, +}; + +/// One-shot guard so the "missing delegate key" warning fires once even +/// though `from_env` runs once per registered tool. Keep the closure +/// trivially fast — `Once::call_once` blocks every other caller until it +/// returns. +static WARN_MISSING_KEY: Once = Once::new(); + +/// Load `/.env` if present. Cwd-only (no parent walk) so a planted +/// `.env` above the binary's cwd cannot influence the process. Existing +/// exports always win. Must be called before the tokio runtime is built — +/// `set_var` is unsound from a multi-threaded process. +pub(crate) fn load_dotenv_if_present() { + let candidate = match std::env::current_dir() { + Ok(d) => d.join(".env"), + Err(e) => { + log::warn!("could not read cwd while looking for .env: {e}"); + return; + } + }; + if !candidate.is_file() { + return; + } + match dotenvy::from_path(&candidate) { + Ok(()) => log::info!("loaded env vars from {}", candidate.display()), + Err(e) => log::warn!("failed to load {}: {e}", candidate.display()), + } +} + +/// Classification of the delegate key env var. Hand-written `Debug` redacts +/// the secret in `Ok` — never `derive` Debug on this type. +enum KeyValidation { + /// Valid: 32-byte Ed25519 scalar, hex-encoded. Wrapped in `Zeroizing` so + /// the heap buffer is wiped on drop. + Ok(Zeroizing), + /// Unset or empty. Boot continues; signed calls fail with MissingKey. + Missing, + /// Set but malformed. Carries an operator-facing reason. + Invalid(String), +} + +impl std::fmt::Debug for KeyValidation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Ok(_) => write!(f, "Ok()"), + Self::Missing => write!(f, "Missing"), + Self::Invalid(reason) => f.debug_tuple("Invalid").field(reason).finish(), + } + } +} + +impl PartialEq for KeyValidation { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Ok(a), Self::Ok(b)) => a.as_str() == b.as_str(), + (Self::Missing, Self::Missing) => true, + (Self::Invalid(a), Self::Invalid(b)) => a == b, + _ => false, + } + } +} +impl Eq for KeyValidation {} + +/// Pure classifier — no I/O, no exits, deterministic. Side effects live in +/// [`read_validated_private_key`]. +fn classify_key(value: Option<&str>) -> KeyValidation { + let raw = match value { + Some(s) if !s.is_empty() => s, + _ => return KeyValidation::Missing, + }; + let bytes = match hex::decode(raw) { + Ok(b) => b, + Err(e) => { + return KeyValidation::Invalid(format!( + "is set but is not valid hex ({e}). Expected 64 hex chars \ + encoding a 32-byte Ed25519 scalar." + )); + } + }; + if bytes.len() != 32 { + return KeyValidation::Invalid(format!( + "decoded to {} bytes, expected 32 (Ed25519 scalar).", + bytes.len() + )); + } + KeyValidation::Ok(Zeroizing::new(raw.to_string())) +} + +/// Eager startup-time key validation. Without this, a malformed key would +/// only surface on the first `/invoke` (well after `server-start` reports +/// "Ready") because `bootstrap!` constructs `NexusTool` instances lazily. +/// Called from `main`, which owns the only `process::exit` site. +pub(crate) fn validate_credentials_at_startup() -> Result<(), String> { + read_validated_private_key().map(|_| ()) +} + +/// `Ok(Some(hex))` valid; `Ok(None)` unset (warn-and-continue); `Err(reason)` +/// malformed. `main` exits on `Err`; `from_env` downgrades it to a log so a +/// mid-process key rotation doesn't kill in-flight requests. +fn read_validated_private_key() -> Result>, String> { + // `env::var` returns a fresh `String` — wrap it immediately so that + // intermediate copy is also zeroed on drop. + let raw = std::env::var(ENV_PRIVATE_KEY).ok().map(Zeroizing::new); + match classify_key(raw.as_deref().map(|z| z.as_str())) { + KeyValidation::Ok(k) => Ok(Some(k)), + KeyValidation::Missing => { + WARN_MISSING_KEY.call_once(|| { + log::warn!( + "{ENV_PRIVATE_KEY} is not set — every signed call and \ + per-tool health check will fail with MissingKey until \ + this env var is exported in the process environment." + ); + }); + Ok(None) + } + KeyValidation::Invalid(reason) => Err(reason), + } +} + +/// MemWal relayer version this crate is pinned to. +/// +/// Source of truth: tag [`@mysten-incubation/memwal@0.0.4`][tag], +/// `services/server/Cargo.toml`. Every wire-format invariant in this crate +/// (canonical signed message, header names, endpoint paths, request/ +/// response shapes) was derived from +/// `services/server/src/{auth,types,routes,rate_limit}.rs` at that tag. +/// +/// **Maintenance contract:** on every new relayer tag with a Cargo version +/// bump *or* any change to those four files, re-audit and update this +/// constant. [`MemWalClient::health_check`] enforces the version match at +/// runtime against `GET /health`'s `version` field. +/// +/// [tag]: https://github.com/MystenLabs/MemWal/tree/%40mysten-incubation%2Fmemwal%400.0.4/services/server +pub(crate) const MEMWAL_API_VERSION: &str = "0.1.0"; + +#[allow(dead_code)] +pub(crate) const RELAYER_URL_MAINNET: &str = "https://relayer.memwal.ai"; + +/// Walrus Foundation's pre-production deployment is wired to Sui testnet — +/// the `staging` hostname is not a typo. +pub(crate) const RELAYER_URL_TESTNET: &str = "https://relayer.staging.memwal.ai"; + +/// Default points at testnet so the beta API can be exercised without +/// real SUI; mainnet requires opting in via `MEMWAL_SERVER_URL`. +pub(crate) const DEFAULT_SERVER_URL: &str = RELAYER_URL_TESTNET; + +pub(crate) const ENV_SERVER_URL: &str = "MEMWAL_SERVER_URL"; +pub(crate) const ENV_PRIVATE_KEY: &str = "MEMWAL_DELEGATE_PRIVATE_KEY"; +pub(crate) const ENV_ACCOUNT_ID: &str = "MEMWAL_ACCOUNT_ID"; + +/// First poll fires after a short delay so fast jobs resolve fast. +const POLL_INITIAL: Duration = Duration::from_millis(100); +/// Cap on inter-poll delay — bounds the rate-limit cost of a slow job. +const POLL_MAX: Duration = Duration::from_secs(4); +/// Wall-clock budget per `poll_job` / `poll_bulk_jobs`. Sized for Walrus +/// erasure-coded tail latency (30–60 s under load). +const POLL_BUDGET: Duration = Duration::from_secs(60); + +fn next_backoff(current: Duration) -> Duration { + current.saturating_mul(2).min(POLL_MAX) +} + +/// Add 0..25% positive jitter to `delay` so concurrent polls of the same +/// job (e.g. two clients waiting on a shared job_id) don't lock-step on the +/// exact 100/200/400 ms ticks. Entropy comes from the system clock's +/// sub-second nanoseconds — adequate for spreading-out, not for security. +fn jittered(delay: Duration) -> Duration { + let jitter_ms = (delay.as_millis() as u64 / 4).max(1); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0); + delay + Duration::from_millis(nanos % jitter_ms) +} + +fn check_text_len(field: &str, value: &str) -> Result<(), MemWalError> { + if value.len() > MAX_TEXT_BYTES { + return Err(MemWalError::Config(format!( + "{field} exceeds maximum allowed size of {MAX_TEXT_BYTES} bytes (got {})", + value.len() + ))); + } + Ok(()) +} + +/// Mirrors the relayer's `MAX_BULK_ITEMS` so a 21-item batch fails at the +/// tool boundary with a clear reason instead of an opaque HTTP 400. +pub(crate) const MAX_BULK_ITEMS: usize = 20; + +/// Defensive cap on text-carrying tool inputs. Oversized inputs fail before +/// burning a signed-request slot. +pub(crate) const MAX_TEXT_BYTES: usize = 1 << 20; // 1 MiB + +/// Map non-2xx into `MemWalError`: 429 → `RateLimited` (parses +/// `Retry-After`); everything else → a terse `Server(_)` that does NOT +/// inline the upstream body, since `/invoke` callers are unauthenticated. +/// +/// The full body (capped 256 chars) is logged under `target=memwal::upstream`. +/// It may quote stored memory text or other relayer internals — operators +/// who treat logs as lower-trust than the binary should filter via +/// `RUST_LOG=memwal::upstream=off`. +async fn map_error_response(resp: reqwest::Response) -> MemWalError { + let status = resp.status(); + if status.as_u16() == 429 { + let retry_after_secs = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.trim().parse::().ok()); + let body = resp.text().await.unwrap_or_default(); + log::warn!( + target: "memwal::upstream", + "HTTP 429 (retry-after={retry_after_secs:?}): {}", + body.chars().take(256).collect::() + ); + return MemWalError::RateLimited { retry_after_secs }; + } + let body = resp.text().await.unwrap_or_default(); + log::warn!( + target: "memwal::upstream", + "HTTP {}: {}", + status, + body.chars().take(256).collect::() + ); + let terse = match status.as_u16() { + 401 | 403 => "upstream auth failure".to_string(), + 408 | 504 => "upstream timeout".to_string(), + s if (400..500).contains(&s) => format!("upstream rejected the request (HTTP {s})"), + s => format!("upstream unavailable (HTTP {s})"), + }; + MemWalError::Server(terse) +} + +// --------------------------------------------------------------------------- +// API response shapes +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +pub(crate) struct RememberResponse { + pub(crate) job_id: String, +} + +/// Job state from the relayer. `Unknown` (via `#[serde(other)]`) captures +/// future statuses — polling loops surface it as an error rather than +/// looping on something they don't recognize. +#[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum JobStatus { + Pending, + Running, + Uploaded, + Done, + Failed, + #[serde(other)] + Unknown, +} + +impl JobStatus { + fn is_in_progress(self) -> bool { + matches!( + self, + JobStatus::Pending | JobStatus::Running | JobStatus::Uploaded + ) + } +} + +#[derive(Deserialize)] +pub(crate) struct JobStatusResponse { + pub(crate) status: JobStatus, + pub(crate) blob_id: Option, +} + +#[derive(Deserialize)] +pub(crate) struct AnalyzeResponse { + pub(crate) job_ids: Vec, +} + +/// One result item from `POST /api/recall` (`RecallResult` in server types). +/// The server does not include `namespace` in per-result items. +#[derive(Deserialize, Clone)] +pub(crate) struct MemoryResult { + pub(crate) text: String, + pub(crate) blob_id: String, + pub(crate) distance: f64, +} + +#[derive(Deserialize)] +pub(crate) struct RecallResponse { + pub(crate) results: Vec, +} + +/// One source memory from `POST /api/ask` (`RecallResult` in server types). +#[derive(Deserialize, Clone)] +pub(crate) struct AskSource { + pub(crate) blob_id: String, + pub(crate) text: String, + pub(crate) distance: f64, +} + +/// Response from `POST /api/ask`. The server field is `memories`, not `sources`. +#[derive(Deserialize)] +pub(crate) struct AskResponse { + pub(crate) answer: String, + pub(crate) memories: Vec, +} + +/// Response from `POST /api/forget` — count of memories removed. +#[derive(Deserialize)] +pub(crate) struct ForgetResponse { + pub(crate) deleted: u64, +} + +/// Response from `POST /api/stats` — per-namespace usage summary. +#[derive(Deserialize)] +pub(crate) struct StatsResponse { + pub(crate) memory_count: i64, + pub(crate) storage_bytes: i64, + pub(crate) namespace: String, +} + +/// Response from `POST /api/remember/bulk` — 202 Accepted with one job_id per item. +#[derive(Deserialize)] +pub(crate) struct RememberBulkResponse { + pub(crate) job_ids: Vec, +} + +/// One result entry from `POST /api/remember/bulk/status`. +#[derive(Deserialize, Clone)] +pub(crate) struct BulkStatusItem { + pub(crate) job_id: String, + pub(crate) status: JobStatus, + pub(crate) blob_id: Option, +} + +/// Response from `POST /api/remember/bulk/status` — one entry per requested job_id. +#[derive(Deserialize)] +pub(crate) struct BulkStatusResponse { + pub(crate) results: Vec, +} + +/// `GET /health` body. `version` is required (enforces the maintenance-pin +/// contract on [`MEMWAL_API_VERSION`]); other fields are allowed and ignored. +#[derive(Deserialize)] +struct HealthResponse { + version: Option, +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +/// Process-wide HTTP client shared across every `MemWalClient` so the +/// connection pool, TLS session cache, and HTTP/2 multiplexing survive +/// across `invoke` calls. `reqwest::Client` clone is a cheap `Arc` bump. +static SHARED_HTTP: OnceLock = OnceLock::new(); + +fn shared_http() -> reqwest::Client { + SHARED_HTTP + .get_or_init(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .pool_idle_timeout(Duration::from_secs(90)) + .pool_max_idle_per_host(8) + .tcp_keepalive(Duration::from_secs(30)) + .build() + .expect("reqwest client builder must succeed with these options") + }) + .clone() +} + +/// `MEMWAL_ALLOW_INSECURE=1` opts in to non-HTTPS relayer URLs (dev/test only). +fn allow_insecure() -> bool { + std::env::var("MEMWAL_ALLOW_INSECURE") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) +} + +/// Validate a relayer base URL. Rejects: non-https (unless `allow_insecure`), +/// path beyond root, query/fragment, userinfo. `allow_insecure` is a +/// parameter (not read from env) so the policy is pure-testable. +fn parse_relayer_url(raw: &str, allow_insecure: bool) -> Result { + // Don't echo the raw URL in parse errors: an embedded + // `user:secret@host` plus a typo would leak the secret into stderr, + // log files, and (because each tool's `new()` panics on Config) into + // the panic message itself. + let url = + Url::parse(raw).map_err(|e| MemWalError::Config(format!("invalid relayer URL: {e}")))?; + let scheme = url.scheme(); + let scheme_ok = scheme == "https" || (allow_insecure && scheme == "http"); + if !scheme_ok { + return Err(MemWalError::Config(format!( + "relayer URL must use https (got scheme `{scheme}`)" + ))); + } + let path = url.path(); + if !path.is_empty() && path != "/" { + return Err(MemWalError::Config(format!( + "relayer URL must not carry a path (got `{path}`)" + ))); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(MemWalError::Config( + "relayer URL must not carry a query or fragment".into(), + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(MemWalError::Config( + "relayer URL must not embed credentials (userinfo)".into(), + )); + } + Ok(url) +} + +/// Per-tool HTTP client. All non-trivial state is behind `Arc`/`reqwest::Client` +/// so cloning is cheap — every `NexusTool` holds one and reuses it across +/// every `invoke`. The signing key is parsed at construction so the per-call +/// path never re-decodes hex or rebuilds the public key. +#[derive(Clone)] +pub(crate) struct MemWalClient { + http: reqwest::Client, + api_base: Url, + /// `None` when the env key was missing or malformed; signed calls then + /// return `AuthError::MissingKey` instead of taking down the process. + signing: Option>, + /// Empty when unconfigured (relayer falls back to a registry scan). + /// Signed into the canonical message; sent as `x-account-id` only when + /// non-empty (mirrors the JS SDK). + account_id: Arc, +} + +struct SigningMaterial { + signing_key: SigningKey, + public_key_hex: String, +} + +impl MemWalClient { + /// Production constructor: validates `MEMWAL_SERVER_URL` (`Err(Config)` + /// on parse failure) and parses the delegate key. A missing/malformed + /// key logs but doesn't error here — `main`'s startup validation is the + /// fail-fast site. + pub(crate) fn from_env() -> Result { + let api_base_raw = std::env::var(ENV_SERVER_URL) + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| DEFAULT_SERVER_URL.to_string()); + let api_base = parse_relayer_url(&api_base_raw, allow_insecure())?; + + let signing = match read_validated_private_key() { + Ok(Some(hex)) => match parse_signing_key(&hex) { + Ok((signing_key, public_key_hex)) => Some(Arc::new(SigningMaterial { + signing_key, + public_key_hex, + })), + Err(e) => { + log::error!("{ENV_PRIVATE_KEY} could not be parsed: {e}"); + None + } + }, + Ok(None) => None, + Err(reason) => { + log::error!("{ENV_PRIVATE_KEY} {reason}"); + None + } + }; + + let account_id: Arc = std::env::var(ENV_ACCOUNT_ID).unwrap_or_default().into(); + + Ok(Self { + http: shared_http(), + api_base, + signing, + account_id, + }) + } + + /// Test-only constructor. Accepts an unparsed URL (HTTP loopback for + /// mockito) and a hex key; panics on parse failure since tests should + /// not be exercising the error path. + #[cfg(test)] + pub(crate) fn with_test_config( + api_base: &str, + private_key_hex: &str, + account_id: &str, + ) -> Self { + let url = Url::parse(api_base).expect("test URL must parse"); + let (signing_key, public_key_hex) = + parse_signing_key(private_key_hex).expect("test key must parse"); + Self { + http: reqwest::Client::new(), + api_base: url, + signing: Some(Arc::new(SigningMaterial { + signing_key, + public_key_hex, + })), + account_id: account_id.into(), + } + } + + /// Sign and dispatch a request, returning the `reqwest::Response` so + /// callers can `.json()` it. + fn signed_headers( + &self, + method: &str, + path: &str, + body: &[u8], + ) -> Result { + let signing = self + .signing + .as_ref() + .ok_or(MemWalError::Auth(AuthError::MissingKey))?; + Ok(sign_request( + &signing.signing_key, + &signing.public_key_hex, + method, + path, + body, + &self.account_id, + )?) + } + + fn join_path(&self, path: &str) -> Result { + self.api_base + .join(path) + .map_err(|e| MemWalError::Config(format!("invalid path `{path}`: {e}"))) + } + + // ----------------------------------------------------------------------- + // Low-level request helpers + // ----------------------------------------------------------------------- + + async fn post( + &self, + path: &str, + body: &B, + ) -> Result { + let body_bytes = serde_json::to_vec(body).expect("serializable body"); + let headers = self.signed_headers("POST", path, &body_bytes)?; + let url = self.join_path(path)?; + + let mut req = self + .http + .post(url) + .header("x-public-key", &headers.public_key) + .header("x-signature", &headers.signature) + .header("x-timestamp", &headers.timestamp) + .header("x-nonce", &headers.nonce) + .header("content-type", "application/json"); + if !self.account_id.is_empty() { + req = req.header("x-account-id", self.account_id.as_ref()); + } + let resp = req.body(body_bytes).send().await?; + + if !resp.status().is_success() { + return Err(map_error_response(resp).await); + } + + Ok(resp) + } + + async fn get(&self, path: &str) -> Result { + let headers = self.signed_headers("GET", path, b"")?; + let url = self.join_path(path)?; + + let mut req = self + .http + .get(url) + .header("x-public-key", &headers.public_key) + .header("x-signature", &headers.signature) + .header("x-timestamp", &headers.timestamp) + .header("x-nonce", &headers.nonce); + if !self.account_id.is_empty() { + req = req.header("x-account-id", self.account_id.as_ref()); + } + let resp = req.send().await?; + + if !resp.status().is_success() { + return Err(map_error_response(resp).await); + } + + Ok(resp) + } + + // ----------------------------------------------------------------------- + // Domain-level API calls + // ----------------------------------------------------------------------- + + /// `POST /api/remember` — enqueue a single memory. + pub(crate) async fn remember( + &self, + text: &str, + namespace: Option<&str>, + ) -> Result { + check_text_len("text", text)?; + #[derive(Serialize)] + struct Req<'a> { + text: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: RememberResponse = self + .post("/api/remember", &Req { text, namespace }) + .await? + .json() + .await?; + Ok(resp.job_id) + } + + /// Poll `GET /api/remember/:job_id` to terminal state. Exponential + /// backoff (POLL_INITIAL → POLL_MAX) within POLL_BUDGET; an unrecognized + /// status from the server is a hard error, not an in-progress signal. + /// + /// **Cancel-safety:** the loop is cancel-safe locally, but the + /// server-side write is **not cancellable** — dropping the future after + /// the initial 202 leaks one Walrus blob. + pub(crate) async fn poll_job(&self, job_id: &str) -> Result { + let path = format!("/api/remember/{job_id}"); + let deadline = Instant::now() + POLL_BUDGET; + let mut delay = POLL_INITIAL; + + loop { + sleep(jittered(delay)).await; + let status: JobStatusResponse = self.get(&path).await?.json().await?; + match status.status { + JobStatus::Done => { + return status.blob_id.ok_or_else(|| { + MemWalError::Server(format!( + "job {job_id} reached terminal status `done` but \ + blob_id is missing from the response" + )) + }); + } + JobStatus::Failed => return Err(MemWalError::JobFailed(job_id.to_string())), + s if s.is_in_progress() => {} + JobStatus::Unknown => { + return Err(MemWalError::Server(format!( + "job {job_id} returned an unrecognized status" + ))); + } + _ => unreachable!("is_in_progress covers Pending/Running/Uploaded"), + } + if Instant::now() >= deadline { + return Err(MemWalError::Timeout(job_id.to_string())); + } + delay = next_backoff(delay); + } + } + + /// `POST /api/recall` — semantic search over stored memories. + pub(crate) async fn recall( + &self, + query: &str, + limit: Option, + namespace: Option<&str>, + ) -> Result, MemWalError> { + check_text_len("query", query)?; + #[derive(Serialize)] + struct Req<'a> { + query: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: RecallResponse = self + .post( + "/api/recall", + &Req { + query, + limit, + namespace, + }, + ) + .await? + .json() + .await?; + Ok(resp.results) + } + + /// `POST /api/ask` — memory-augmented Q&A. + pub(crate) async fn ask( + &self, + question: &str, + namespace: Option<&str>, + limit: Option, + ) -> Result { + check_text_len("question", question)?; + #[derive(Serialize)] + struct Req<'a> { + question: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + limit: Option, + } + // The server field is `memories`, not `sources` — AskResponse reflects this. + let resp: AskResponse = self + .post( + "/api/ask", + &Req { + question, + namespace, + limit, + }, + ) + .await? + .json() + .await?; + Ok(resp) + } + + /// `POST /api/analyze` — extract facts from text and enqueue each as a memory. + /// Returns the number of jobs submitted. + pub(crate) async fn analyze( + &self, + text: &str, + namespace: Option<&str>, + ) -> Result { + check_text_len("text", text)?; + #[derive(Serialize)] + struct Req<'a> { + text: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: AnalyzeResponse = self + .post("/api/analyze", &Req { text, namespace }) + .await? + .json() + .await?; + Ok(resp.job_ids.len()) + } + + /// `POST /api/forget` — delete every memory in the given namespace. + /// Owner-scoped: only memories whose owner matches the authenticated + /// account can be deleted, regardless of the namespace string sent. + pub(crate) async fn forget(&self, namespace: Option<&str>) -> Result { + #[derive(Serialize)] + struct Req<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: ForgetResponse = self + .post("/api/forget", &Req { namespace }) + .await? + .json() + .await?; + Ok(resp.deleted) + } + + /// `POST /api/stats` — memory count + stored byte total for a namespace. + pub(crate) async fn stats( + &self, + namespace: Option<&str>, + ) -> Result { + #[derive(Serialize)] + struct Req<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + self.post("/api/stats", &Req { namespace }) + .await? + .json() + .await + .map_err(MemWalError::from) + } + + /// Submit up to `MAX_BULK_ITEMS` texts in one 202-Accepted call. Callers + /// must enforce the `1..=MAX_BULK_ITEMS` cap before this — oversized + /// batches get an opaque HTTP 400 from the relayer. Pair with + /// [`poll_bulk_jobs`] for the batched status endpoint. + pub(crate) async fn remember_bulk( + &self, + items: &[(&str, Option<&str>)], + ) -> Result, MemWalError> { + for (text, _) in items { + check_text_len("text", text)?; + } + #[derive(Serialize)] + struct Item<'a> { + text: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + #[derive(Serialize)] + struct Req<'a> { + items: Vec>, + } + let req = Req { + items: items + .iter() + .map(|(text, ns)| Item { + text, + namespace: *ns, + }) + .collect(), + }; + let resp: RememberBulkResponse = + self.post("/api/remember/bulk", &req).await?.json().await?; + Ok(resp.job_ids) + } + + /// Poll batched jobs to terminal state, returning blob_ids in input + /// order. First `failed` surfaces as `Err(JobFailed)`. Each poll queries + /// only the still-pending subset (cached terminal jobs in a HashMap), + /// so rate-limit cost shrinks with completion. Same cancel-safety + /// caveat as [`MemWalClient::poll_job`], scaled to MAX_BULK_ITEMS. + pub(crate) async fn poll_bulk_jobs( + &self, + job_ids: &[String], + ) -> Result, MemWalError> { + if job_ids.is_empty() { + return Ok(Vec::new()); + } + let deadline = Instant::now() + POLL_BUDGET; + let mut delay = POLL_INITIAL; + let mut terminal: HashMap = HashMap::with_capacity(job_ids.len()); + + loop { + let pending: Vec = job_ids + .iter() + .filter(|id| !terminal.contains_key(*id)) + .cloned() + .collect(); + if pending.is_empty() { + break; + } + + sleep(jittered(delay)).await; + let statuses = self.poll_bulk_status_once(&pending).await?; + let by_id: HashMap<&str, &BulkStatusItem> = + statuses.iter().map(|s| (s.job_id.as_str(), s)).collect(); + + for id in &pending { + let Some(item) = by_id.get(id.as_str()) else { + return Err(MemWalError::Server(format!( + "bulk status response missing job {id}" + ))); + }; + match item.status { + JobStatus::Done | JobStatus::Failed => { + terminal.insert(id.clone(), (*item).clone()); + } + s if s.is_in_progress() => {} + JobStatus::Unknown => { + return Err(MemWalError::Server(format!( + "job {id} returned an unrecognized status" + ))); + } + _ => unreachable!("is_in_progress covers Pending/Running/Uploaded"), + } + } + + if Instant::now() >= deadline { + return Err(MemWalError::Timeout(format!( + "{} bulk jobs did not finish in time", + job_ids.len() - terminal.len() + ))); + } + delay = next_backoff(delay); + } + + // All jobs terminal — assemble in input order, surface first failure. + job_ids + .iter() + .map(|id| { + let item = terminal + .get(id) + .expect("loop only exits when terminal covers job_ids"); + match item.status { + JobStatus::Done => item.blob_id.clone().ok_or_else(|| { + MemWalError::Server(format!( + "job {id} reached terminal status `done` but \ + blob_id is missing" + )) + }), + JobStatus::Failed => Err(MemWalError::JobFailed(id.clone())), + _ => unreachable!("terminal cache only stores Done/Failed"), + } + }) + .collect() + } + + /// Single shot of `POST /api/remember/bulk/status`. Returns whatever the + /// server says without polling. Exposed for callers that want to inspect + /// per-job state directly (e.g. tolerate partial failures). + pub(crate) async fn poll_bulk_status_once( + &self, + job_ids: &[String], + ) -> Result, MemWalError> { + #[derive(Serialize)] + struct Req<'a> { + job_ids: &'a [String], + } + let resp: BulkStatusResponse = self + .post("/api/remember/bulk/status", &Req { job_ids }) + .await? + .json() + .await?; + Ok(resp.results) + } + + /// `GET /health` — reachability + API-version match. Public (no auth). + /// Required: JSON body with `version` matching [`MEMWAL_API_VERSION`]. + /// Missing field / wrong type / non-JSON / mismatch all return `Err`; + /// "best-effort" version check would void the maintenance-pin contract. + pub(crate) async fn health_check(&self) -> Result<(), MemWalError> { + let url = self.join_path("/health")?; + let resp = self.http.get(url).send().await?; + let status = resp.status(); + if !status.is_success() { + return Err(MemWalError::Server(format!( + "relayer health returned HTTP {}", + status.as_u16() + ))); + } + let body: HealthResponse = resp.json().await.map_err(|e| { + MemWalError::Server(format!("relayer /health did not return JSON: {e}")) + })?; + let server_ver = body.version.ok_or_else(|| { + MemWalError::Server( + "relayer /health did not return a `version` field — pre-pinned-tag relayer?".into(), + ) + })?; + if server_ver != MEMWAL_API_VERSION { + return Err(MemWalError::Server(format!( + "relayer version mismatch: tools expect {MEMWAL_API_VERSION}, \ + server reports {server_ver} — update the tools or pin the relayer" + ))); + } + Ok(()) + } + + /// Returns `Err(MissingKey)` when the delegate key was missing or + /// malformed at construction time. Validity follows from the type: + /// `signing` only carries `Some` once `parse_signing_key` succeeded, + /// so there is no per-call hex decode or length check to repeat. + pub(crate) fn validate_key(&self) -> Result<(), AuthError> { + if self.signing.is_none() { + return Err(AuthError::MissingKey); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `classify_key(None)` reports `Missing`. + /// Failure mode caught: an unset env var is silently treated as a valid + /// empty key, masking the misconfiguration behind apparent boot success. + #[test] + fn classify_key_missing_when_unset() { + assert_eq!(classify_key(None), KeyValidation::Missing); + } + + /// `classify_key(Some(""))` reports `Missing`, matching shells that export + /// the variable with an empty value. + /// Failure mode caught: an empty-string env var is treated as a valid key + /// and produces empty hex headers on every signed request. + #[test] + fn classify_key_missing_when_empty() { + assert_eq!(classify_key(Some("")), KeyValidation::Missing); + } + + /// A canonical 32-byte hex key is classified as `Ok` and the original + /// hex string is preserved verbatim (the signer uses the hex form, not + /// re-encoded bytes). + /// Failure mode caught: a valid key is misclassified as Invalid, which + /// would abort the binary on the happy path. + #[test] + fn classify_key_ok_on_valid_32_byte_hex() { + let key = hex::encode([0x42u8; 32]); + match classify_key(Some(&key)) { + KeyValidation::Ok(returned) => assert_eq!(returned.as_str(), key.as_str()), + other => panic!("expected Ok(\"{key}\"), got {other:?}"), + } + } + + /// Non-hex input is classified as `Invalid` with a reason mentioning hex. + /// Failure mode caught: malformed input passes validation, exits would be + /// missing, and the binary boots with garbage credentials that produce + /// confusing relayer errors at request time. + #[test] + fn classify_key_invalid_on_non_hex() { + match classify_key(Some("not-hex!!")) { + KeyValidation::Invalid(reason) => assert!( + reason.contains("hex"), + "reason must mention hex format; got: {reason}" + ), + other => panic!("expected Invalid, got {other:?}"), + } + } + + /// Hex that decodes to fewer than 32 bytes is classified as `Invalid` + /// with a byte-count reason. + /// Failure mode caught: a truncated key is accepted, leading to silent + /// signature verification failures the operator cannot diagnose. + #[test] + fn classify_key_invalid_on_too_few_bytes() { + let short = hex::encode([0u8; 16]); // 16 bytes + match classify_key(Some(&short)) { + KeyValidation::Invalid(reason) => { + assert!( + reason.contains("16") && reason.contains("32"), + "reason must mention actual (16) and expected (32) byte counts; got: {reason}" + ); + } + other => panic!("expected Invalid, got {other:?}"), + } + } + + /// Hex that decodes to more than 32 bytes is also classified as `Invalid`. + /// Failure mode caught: an over-long key (e.g. a Sui-formatted key + /// pasted with extra flag bytes) is silently truncated or accepted. + #[test] + fn classify_key_invalid_on_too_many_bytes() { + let long = hex::encode([0u8; 64]); // 64 bytes + match classify_key(Some(&long)) { + KeyValidation::Invalid(reason) => { + assert!( + reason.contains("64") && reason.contains("32"), + "reason must mention actual (64) and expected (32) byte counts; got: {reason}" + ); + } + other => panic!("expected Invalid, got {other:?}"), + } + } + + /// All-`0xff` 32-byte input is `Ok` — Ed25519 has no scalar-range + /// rejection in dalek's `from_bytes`, which SHA-512-derives the scalar. + /// Failure mode caught: an over-strict client-side filter rejects a key + /// the relayer would actually accept, blocking legitimate deployments. + #[test] + fn classify_key_ok_on_all_ones_32_bytes() { + let key = hex::encode([0xffu8; 32]); + assert_eq!( + classify_key(Some(&key)), + KeyValidation::Ok(Zeroizing::new(key)) + ); + } + + /// `Debug` impl on `KeyValidation::Ok` redacts the hex secret. + /// Failure mode caught: a future refactor switches the manual `Debug` impl + /// back to a derive, or someone accidentally prints `{:?}` on a + /// KeyValidation — either way, the delegate key would land in a log line + /// or panic message. This test asserts the redaction is structural. + #[test] + fn key_validation_debug_redacts_secret() { + let key_hex = hex::encode([0x42u8; 32]); + let v = KeyValidation::Ok(Zeroizing::new(key_hex.clone())); + let debug_output = format!("{v:?}"); + assert!( + !debug_output.contains(&key_hex), + "Debug output must not contain the hex secret. Got: {debug_output}" + ); + assert_eq!(debug_output, "Ok()"); + } + + /// `Debug` impl on the other two variants still emits informative text + /// (no redaction needed — they don't carry secrets). + /// Failure mode caught: an over-zealous redaction sweep silently strips + /// the `Invalid` reason, hiding the actual misconfiguration from the + /// startup log line. + #[test] + fn key_validation_debug_preserves_non_secret_variants() { + assert_eq!(format!("{:?}", KeyValidation::Missing), "Missing"); + let reason = "is set but is not valid hex (...)".to_string(); + let dbg = format!("{:?}", KeyValidation::Invalid(reason.clone())); + assert!(dbg.contains(&reason), "Invalid must show its reason: {dbg}"); + } + + /// `PartialEq` distinguishes every cross-variant pair. + /// Failure mode caught: a hand-written `eq` that accidentally returns + /// `true` on cross-variant comparisons would let miscategorized inputs + /// slip through equality checks. + #[test] + fn key_validation_partial_eq_cross_variants_differ() { + let v_ok = KeyValidation::Ok(Zeroizing::new("x".into())); + let v_missing = KeyValidation::Missing; + let v_invalid = KeyValidation::Invalid("r".into()); + assert_ne!(v_ok, v_missing); + assert_ne!(v_ok, v_invalid); + assert_ne!(v_missing, v_invalid); + } + + /// `jittered(delay)` returns a value in `[delay, delay + ceil(delay/4))`. + /// Failure mode caught: a regression that overshoots the 25% jitter cap + /// would silently extend the polling deadline, or a missing-jitter + /// regression (returning exactly `delay`) would re-introduce lock-step + /// polling between concurrent clients on the same job. + #[test] + fn jittered_bounded() { + for d_ms in [100u64, 250, 500, 1000, 4000] { + let d = Duration::from_millis(d_ms); + // The function isn't deterministic, but a small batch will + // exercise enough nanosecond values to expose an out-of-range + // result. + for _ in 0..50 { + let got = jittered(d); + assert!( + got >= d, + "jittered({d:?}) returned {got:?}, less than the base delay" + ); + let max_jitter = Duration::from_millis((d_ms / 4).max(1)); + assert!( + got < d + max_jitter, + "jittered({d:?}) returned {got:?}, exceeded base + 25%" + ); + } + } + } + + /// `check_text_len` accepts an input exactly at the cap, rejects one + /// byte over, and the error message names the cap and the actual size. + /// Failure mode caught: an off-by-one on the `>` vs `>=` comparison, + /// or an error message that doesn't help the operator diagnose what + /// they sent. + #[test] + fn check_text_len_boundary() { + let at_limit = "x".repeat(MAX_TEXT_BYTES); + assert!(check_text_len("text", &at_limit).is_ok()); + + let over_limit = "x".repeat(MAX_TEXT_BYTES + 1); + match check_text_len("text", &over_limit) { + Err(MemWalError::Config(reason)) => { + assert!( + reason.contains(&MAX_TEXT_BYTES.to_string()), + "reason must mention the cap; got: {reason}" + ); + assert!( + reason.contains(&(MAX_TEXT_BYTES + 1).to_string()), + "reason must mention the actual size; got: {reason}" + ); + } + other => panic!("expected Config(...), got {other:?}"), + } + } + + // ----------------------------------------------------------------------- + // parse_relayer_url policy tests + // + // The flag is passed explicitly (rather than read from env) so each test + // is hermetic — no global state, no test ordering, no MEMWAL_ALLOW_INSECURE + // bleed-through across the suite. + // ----------------------------------------------------------------------- + + /// Accepts a canonical https URL with no trailing slash. + /// Failure mode caught: an over-strict validator that rejects the most + /// common form would block the default deployment. + #[test] + fn parse_relayer_url_accepts_canonical_https() { + assert!(parse_relayer_url("https://relayer.memwal.ai", false).is_ok()); + } + + /// Accepts a trailing slash — url::Url normalises the path to "/". + /// Failure mode caught: a regression where the path check rejects "/" + /// would break operators who copy-paste from a browser bar. + #[test] + fn parse_relayer_url_accepts_trailing_slash() { + assert!(parse_relayer_url("https://relayer.memwal.ai/", false).is_ok()); + } + + /// Accepts an explicit port. + /// Failure mode caught: a host:port URL is misparsed or rejected. + #[test] + fn parse_relayer_url_accepts_port() { + assert!(parse_relayer_url("https://relayer.memwal.ai:8443", false).is_ok()); + } + + /// Rejects http:// when allow_insecure is false. + /// Failure mode caught: a regression that defaults to permissive + /// scheme handling would let an operator (or a tool input that hadn't + /// been removed) downgrade signing material to cleartext. + #[test] + fn parse_relayer_url_rejects_http_when_insecure_disallowed() { + let err = + parse_relayer_url("http://relayer.memwal.ai", false).expect_err("must reject http"); + assert!(err.to_string().contains("https")); + } + + /// Accepts http:// when allow_insecure is true (for local dev / mockito). + /// Failure mode caught: the escape hatch is broken; mockito tests can't + /// construct a real `MemWalClient::from_env` path. + #[test] + fn parse_relayer_url_accepts_http_when_insecure_allowed() { + assert!(parse_relayer_url("http://127.0.0.1:1234", true).is_ok()); + } + + /// Rejects exotic schemes regardless of allow_insecure. + /// Failure mode caught: a `ws://` or `file://` URL bypasses scheme + /// checks and reaches reqwest, which would either error obscurely or + /// open an unintended transport. + #[test] + fn parse_relayer_url_rejects_non_http_schemes() { + for s in ["ws://x", "file:///etc/passwd", "data:text/plain,foo"] { + assert!( + parse_relayer_url(s, true).is_err(), + "expected rejection of `{s}` even with allow_insecure" + ); + } + } + + /// Rejects URLs that carry a path beyond the root. + /// Failure mode caught: `Url::join("/api/remember")` against a base + /// like `https://host/v1` would yield `https://host/api/remember` + /// (Url::join treats absolute paths as replacements), silently + /// rewriting what the operator configured. + #[test] + fn parse_relayer_url_rejects_path() { + let err = parse_relayer_url("https://relayer.memwal.ai/v1", false) + .expect_err("path must be rejected"); + assert!(err.to_string().contains("path")); + } + + /// Rejects URLs that carry a query string. + /// Failure mode caught: `?leak=/api/recall` smuggled in the base URL + /// would either be silently dropped by Url::join or merged into the + /// signed-path semantics, depending on the relayer's parsing. + #[test] + fn parse_relayer_url_rejects_query() { + let err = parse_relayer_url("https://relayer.memwal.ai/?leak=1", false) + .expect_err("query must be rejected"); + assert!(err.to_string().contains("query") || err.to_string().contains("fragment")); + } + + /// Rejects URLs that carry a fragment. + /// Failure mode caught: same shape as the query case; fragments are + /// usually stripped client-side but should not be silently accepted. + #[test] + fn parse_relayer_url_rejects_fragment() { + let err = parse_relayer_url("https://relayer.memwal.ai/#frag", false) + .expect_err("fragment must be rejected"); + assert!(err.to_string().contains("query") || err.to_string().contains("fragment")); + } + + /// Rejects URLs that embed a username. + /// Failure mode caught: an operator who pastes `https://user@host` + /// (e.g. from a curl command) would otherwise have the username + /// tagged onto every outbound request as a Basic-Auth identifier + /// — pointless against this relayer and an unintended exposure. + #[test] + fn parse_relayer_url_rejects_userinfo_username_only() { + let err = parse_relayer_url("https://user@relayer.memwal.ai", false) + .expect_err("userinfo must be rejected"); + assert!(err.to_string().contains("credentials")); + } + + /// Rejects URLs that embed a username and password. + /// Failure mode caught: the more dangerous variant of the userinfo + /// case — a real secret tagged onto every outbound request, AND a + /// secret that would have leaked through parse-error echoes before + /// the no-echo policy was put in place. + #[test] + fn parse_relayer_url_rejects_userinfo_with_password() { + let err = parse_relayer_url("https://user:secret@relayer.memwal.ai", false) + .expect_err("userinfo must be rejected"); + assert!(err.to_string().contains("credentials")); + } + + /// Parse-failure messages do NOT echo the raw input. + /// Failure mode caught: a future refactor reintroduces `{raw}` into + /// the error string and a typoed `https://user:secret@host/bad path` + /// (unparsable due to space) leaks the secret into logs and panic + /// messages. + #[test] + fn parse_relayer_url_does_not_echo_raw_on_parse_failure() { + let err = parse_relayer_url("https://user:secret@host/bad path", false) + .expect_err("malformed URL must Err"); + let msg = err.to_string(); + assert!(!msg.contains("secret"), "secret leaked into error: {msg}"); + assert!(!msg.contains("user"), "userinfo leaked into error: {msg}"); + } + + fn make_client(server_url: &str) -> MemWalClient { + MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), "") + } + + /// `health_check` returns `Ok(())` when `/health` returns the pinned version. + /// Failure mode caught: a version comparison that silently accepts any + /// response would surface here as a passing test against a deliberately + /// matching version mock; we then can flip the mock to verify failure + /// in the sibling tests. + #[tokio::test] + async fn health_check_ok_on_matching_version() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + serde_json::json!({"status": "ok", "version": MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let client = make_client(&server.url()); + assert!(client.health_check().await.is_ok()); + } + + /// `health_check` returns `Err` when `/health` reports a different + /// version. The MEMWAL_API_VERSION pin is the maintenance contract; + /// silently accepting a mismatch defeats the audit trail. + #[tokio::test] + async fn health_check_err_on_mismatched_version() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::json!({"status": "ok", "version": "9.9.9"}).to_string()) + .create_async() + .await; + let client = make_client(&server.url()); + let err = client + .health_check() + .await + .expect_err("version mismatch must Err"); + assert!( + err.to_string().contains("9.9.9"), + "error must mention the server version; got: {err}" + ); + } + + /// `health_check` returns `Err` when `/health` omits `version`. + /// Failure mode caught: a degraded relayer that drops the version field + /// would silently look healthy under the previous best-effort check. + #[tokio::test] + async fn health_check_err_when_version_missing() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::json!({"status": "ok"}).to_string()) + .create_async() + .await; + let client = make_client(&server.url()); + let err = client + .health_check() + .await + .expect_err("missing version must Err"); + assert!( + err.to_string().contains("version"), + "error must mention version; got: {err}" + ); + } + + /// A 429 from the relayer surfaces as `MemWalError::RateLimited` with + /// the parsed `Retry-After` seconds, not the generic `Server` variant. + /// Failure mode caught: rate-limit responses look identical to other + /// upstream failures, so a DAG retry policy cannot distinguish "back + /// off" from "try a different relayer". + #[tokio::test] + async fn post_translates_429_to_rate_limited() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/forget") + .with_status(429) + .with_header("retry-after", "42") + .with_body("rate limited") + .create_async() + .await; + let client = make_client(&server.url()); + let err = client.forget(None).await.expect_err("429 must Err"); + match err { + MemWalError::RateLimited { retry_after_secs } => { + assert_eq!(retry_after_secs, Some(42)); + } + other => panic!("expected RateLimited, got {other}"), + } + } + + /// A 429 with no `Retry-After` header surfaces with `None` rather than + /// failing to parse. + /// Failure mode caught: a missing header is treated as a malformed + /// response and the call becomes Server(...) instead of RateLimited. + #[tokio::test] + async fn post_translates_429_without_retry_after() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/forget") + .with_status(429) + .with_body("rate limited") + .create_async() + .await; + let client = make_client(&server.url()); + let err = client.forget(None).await.expect_err("429 must Err"); + assert!(matches!( + err, + MemWalError::RateLimited { + retry_after_secs: None + } + )); + } + + /// A 500 from the relayer surfaces with a terse client-facing reason — + /// the upstream body is logged but NOT inlined into the error string. + /// Failure mode caught: a relayer that leaks account internals or + /// moderation messages in its 500 body would forward that text verbatim + /// to the unauthenticated /invoke caller. + #[tokio::test] + async fn post_terse_message_on_5xx() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/forget") + .with_status(500) + .with_body("secret internal payload that should not leak") + .create_async() + .await; + let client = make_client(&server.url()); + let err = client.forget(None).await.expect_err("500 must Err"); + let msg = err.to_string(); + assert!(!msg.contains("secret internal payload")); + assert!(msg.contains("upstream")); + } + + /// `health_check` returns `Err` when the body is not JSON at all. + /// Failure mode caught: a misconfigured reverse proxy serving an HTML + /// error page would parse-fail silently and the call would short-circuit + /// to Ok under a permissive `if let Ok(...)` pattern. + #[tokio::test] + async fn health_check_err_on_non_json_body() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_body("nginx error page") + .create_async() + .await; + let client = make_client(&server.url()); + assert!(client.health_check().await.is_err()); + } +} diff --git a/tools/memory-memwal/src/error.rs b/tools/memory-memwal/src/error.rs new file mode 100644 index 0000000..99f71a6 --- /dev/null +++ b/tools/memory-memwal/src/error.rs @@ -0,0 +1,42 @@ +//! Shared error types for the MemWal tool crate. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub(crate) enum MemWalError { + #[error("Auth error: {0}")] + Auth(#[from] AuthError), + + #[error("HTTP request failed: {0}")] + Request(#[from] reqwest::Error), + + #[error("Server returned an error: {0}")] + Server(String), + + #[error("Job {0} failed on the server")] + JobFailed(String), + + #[error("Timed out waiting for job {0} to complete")] + Timeout(String), + + #[error("Configuration error: {0}")] + Config(String), + + #[error("rate-limited by relayer (retry after {retry_after_secs:?}s)")] + RateLimited { retry_after_secs: Option }, +} + +#[derive(Debug, Error)] +pub(crate) enum AuthError { + #[error("MEMWAL_DELEGATE_PRIVATE_KEY is not set or empty")] + MissingKey, + + #[error("Private key is not valid hex: {0}")] + InvalidHex(#[from] hex::FromHexError), + + #[error("Private key must be 32 bytes, got {0}")] + InvalidKeyLength(usize), + + #[error("System clock error: {0}")] + Clock(String), +} diff --git a/tools/memory-memwal/src/forget.rs b/tools/memory-memwal/src/forget.rs new file mode 100644 index 0000000..c32d946 --- /dev/null +++ b/tools/memory-memwal/src/forget.rs @@ -0,0 +1,191 @@ +//! # `xyz.taluslabs.memory.memwal.forget@1` +//! +//! Nexus Tool that deletes every memory in a MemWal namespace. The call is +//! owner-scoped — the relayer's auth middleware identifies the account from +//! the signed request; only memories belonging to that account are removed, +//! regardless of which namespace string is sent. + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// Namespace to clear. When omitted the server interprets it as + /// `"default"`. The relayer scopes the delete to the authenticated + /// account — you cannot delete other accounts' memories. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Namespace was cleared. `deleted` is the number of memories removed + /// (zero is a valid success — the namespace existed but was empty, or + /// did not exist at all). + Ok { + deleted: u64, + }, + Err { + reason: String, + }, +} + +pub(crate) struct ForgetMemories { + client: MemWalClient, +} + +impl NexusTool for ForgetMemories { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.forget@1") + } + + fn path() -> &'static str { + "/memory/forget" + } + + fn description() -> &'static str { + "Delete every memory in a MemWal namespace owned by the authenticated account." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self.client.forget(input.namespace.as_deref()).await { + Ok(deleted) => Output::Ok { deleted }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> ForgetMemories { + ForgetMemories { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` surfaces the server's `deleted` count on a successful call. + /// Failure mode caught: server's deleted count is silently dropped or replaced with a constant. + #[tokio::test] + async fn invoke_returns_deleted_count_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/forget") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "deleted": 42, + "namespace": "scratch", + "owner": "0xowner" + }) + .to_string(), + ) + .create_async() + .await; + + match tool + .invoke(Input { + namespace: Some("scratch".into()), + }) + .await + { + Output::Ok { deleted } => assert_eq!(deleted, 42), + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok { deleted: 0 }` for an empty namespace — a valid success. + /// Failure mode caught: empty namespace mapped to Err, masking idempotent retries. + #[tokio::test] + async fn invoke_returns_zero_for_empty_namespace() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/forget") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"deleted": 0, "namespace": "default", "owner": "0xowner"}).to_string(), + ) + .create_async() + .await; + + match tool.invoke(Input { namespace: None }).await { + Output::Ok { deleted } => assert_eq!(deleted, 0), + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP 500 swallowed, returning a misleading zero count. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/forget") + .with_status(500) + .with_body("internal error") + .create_async() + .await; + + let output = tool.invoke(Input { namespace: None }).await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } +} diff --git a/tools/memory-memwal/src/main.rs b/tools/memory-memwal/src/main.rs new file mode 100644 index 0000000..3b06fe2 --- /dev/null +++ b/tools/memory-memwal/src/main.rs @@ -0,0 +1,51 @@ +//! Nexus Tools for MemWal persistent memory. See `README.md` for the tool +//! list, env vars, and per-tool I/O contracts. +//! +//! Wire format pinned to `@mysten-incubation/memwal@0.0.4` (server Cargo +//! version `0.1.0`). [`client::MEMWAL_API_VERSION`] documents the +//! re-audit contract on tag bumps. + +use nexus_toolkit::bootstrap; + +mod analyze; +mod ask; +mod auth; +mod client; +mod error; +mod forget; +mod recall; +mod remember; +mod remember_bulk; +mod stats; + +fn main() { + // Install env_logger before anything else so dotenv/credential paths + // emit through `log::*`; `bootstrap!`'s own `try_init()` becomes a no-op. + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .format_timestamp_millis() + .init(); + + // dotenv + credential validation run single-threaded — `set_var` is + // unsound from a multi-threaded process. `main` is the only exit site. + client::load_dotenv_if_present(); + if let Err(reason) = client::validate_credentials_at_startup() { + log::error!("{} {reason}", client::ENV_PRIVATE_KEY); + std::process::exit(1); + } + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime") + .block_on(async { + bootstrap!([ + remember::RememberMemory, + remember_bulk::RememberBulkMemories, + recall::RecallMemories, + ask::AskMemory, + analyze::AnalyzeAndRemember, + forget::ForgetMemories, + stats::StatsForAccount, + ]) + }); +} diff --git a/tools/memory-memwal/src/recall.rs b/tools/memory-memwal/src/recall.rs new file mode 100644 index 0000000..81d6a0c --- /dev/null +++ b/tools/memory-memwal/src/recall.rs @@ -0,0 +1,249 @@ +//! # `xyz.taluslabs.memory.memwal.recall@1` +//! +//! Nexus Tool that performs a semantic search over stored memories using a +//! natural-language query and returns the closest matches ranked by vector +//! distance (cosine similarity, lower = closer). + +use { + crate::client::{MemWalClient, MemoryResult}, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// Natural-language query to search for relevant memories. + query: String, + /// Maximum number of results to return. + limit: Option, + /// Namespace to search within. Searches the default namespace when omitted. + namespace: Option, +} + +/// A single memory returned by a recall query. +#[derive(Serialize, JsonSchema)] +pub(crate) struct RecalledMemory { + /// The stored text of the memory. + pub(crate) text: String, + /// Walrus blob identifier for the encrypted memory. + pub(crate) blob_id: String, + /// Cosine distance from the query vector — lower means more relevant. + pub(crate) distance: f64, +} + +impl From for RecalledMemory { + fn from(r: MemoryResult) -> Self { + Self { + text: r.text, + blob_id: r.blob_id, + distance: r.distance, + } + } +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Memories ranked by relevance. May be empty if nothing matched. + /// + /// The `namespace` field is the namespace that was searched, hoisted + /// to the variant since every result in a single recall belongs to + /// the same namespace (the relayer does not currently support + /// cross-namespace recall). + Ok { + results: Vec, + namespace: String, + }, + Err { + reason: String, + }, +} + +pub(crate) struct RecallMemories { + client: MemWalClient, +} + +impl NexusTool for RecallMemories { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.recall@1") + } + + fn path() -> &'static str { + "/memory/recall" + } + + fn description() -> &'static str { + "Search stored memories by natural-language query and return the closest matches." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + let ns = input.namespace.as_deref().unwrap_or("default").to_string(); + + match self + .client + .recall(&input.query, input.limit, input.namespace.as_deref()) + .await + { + Ok(results) => Output::Ok { + results: results.into_iter().map(RecalledMemory::from).collect(), + namespace: ns, + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> RecallMemories { + RecallMemories { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns ranked results when the server responds with memories. + /// Failure mode caught: successful recall response incorrectly mapped to `Err`. + #[tokio::test] + async fn invoke_returns_results_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/recall") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "results": [ + { + "text": "Paris is the capital of France", + "blob_id": "blob-1", + "distance": 0.12 + } + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + query: "capital of France".into(), + limit: None, + namespace: None, + }) + .await; + match output { + Output::Ok { results, namespace } => { + assert_eq!(results.len(), 1); + assert_eq!(results[0].text, "Paris is the capital of France"); + assert_eq!(results[0].blob_id, "blob-1"); + assert!((results[0].distance - 0.12).abs() < f64::EPSILON); + assert_eq!(namespace, "default"); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok` with an empty list when no memories match. + /// Failure mode caught: empty result set treated as an error. + #[tokio::test] + async fn invoke_returns_empty_results_on_no_match() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/recall") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"results": []}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + query: "something obscure".into(), + limit: None, + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Ok { results, .. } if results.is_empty()), + "empty results must produce Ok with empty vec" + ); + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP 500 from the server swallowed, returns empty results. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/recall") + .with_status(500) + .with_body("internal error") + .create_async() + .await; + + let output = tool + .invoke(Input { + query: "anything".into(), + limit: None, + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } +} diff --git a/tools/memory-memwal/src/remember.rs b/tools/memory-memwal/src/remember.rs new file mode 100644 index 0000000..38cdb9e --- /dev/null +++ b/tools/memory-memwal/src/remember.rs @@ -0,0 +1,310 @@ +//! # `xyz.taluslabs.memory.memwal.remember@1` +//! +//! Nexus Tool that stores a single piece of text as a persistent memory in +//! MemWal. The call blocks until the memory is durably stored on Walrus and +//! returns the resulting blob ID, making it safe to chain in a Nexus DAG — +//! the next vertex will not activate until the write is confirmed. + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// The text to store as a memory. + text: String, + /// Namespace used to scope this memory. Defaults to `"default"` on the + /// server when omitted. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Memory was durably stored. `blob_id` is its Walrus blob identifier. + Ok { + blob_id: String, + }, + Err { + reason: String, + }, +} + +pub(crate) struct RememberMemory { + client: MemWalClient, +} + +impl NexusTool for RememberMemory { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.remember@1") + } + + fn path() -> &'static str { + "/memory/remember" + } + + fn description() -> &'static str { + "Store a text memory in MemWal and return its blob ID once durably written." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + let job_id = match self + .client + .remember(&input.text, input.namespace.as_deref()) + .await + { + Ok(id) => id, + Err(e) => { + return Output::Err { + reason: e.to_string(), + } + } + }; + + match self.client.poll_job(&job_id).await { + Ok(blob_id) => Output::Ok { blob_id }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + mockito::{Matcher, Server}, + serde_json::json, + }; + + fn make_tool(server_url: &str) -> RememberMemory { + RememberMemory { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's `NexusTool::health` wrapper drops the + /// validate_key + health_check composition (e.g. a refactor that forgets + /// to await the inner future) — only an end-to-end test through the trait + /// catches that. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns `Ok { blob_id }` when the job completes successfully. + /// Failure mode caught: successful server response mapped to `Err` variant. + #[tokio::test] + async fn invoke_returns_blob_id_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-abc", "status": "pending"}).to_string()) + .create_async() + .await; + + let _m2 = server + .mock("GET", "/api/remember/job-abc") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"job_id": "job-abc", "status": "done", "blob_id": "blob-xyz"}).to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "hello world".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Ok { blob_id } if blob_id == "blob-xyz"), + "expected Ok{{blob_id: blob-xyz}}" + ); + } + + /// `invoke` returns `Err` when the server signals job failure. + /// Failure mode caught: failed job silently returns empty blob_id instead of error. + #[tokio::test] + async fn invoke_returns_err_on_job_failure() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-fail", "status": "pending"}).to_string()) + .create_async() + .await; + + let _m2 = server + .mock("GET", "/api/remember/job-fail") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-fail", "status": "failed"}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "will fail".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "failed job must produce Err variant" + ); + } + + /// `invoke` returns `Err` when the POST to `/api/remember` itself fails (4xx/5xx). + /// Failure mode caught: HTTP error from the initial POST is swallowed. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/remember") + .with_status(500) + .with_body("internal error") + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "text that will not be stored".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } + + /// `invoke` returns `Err` when the server replies `"done"` without a + /// `blob_id` field — the missing identifier must NOT be silently coerced + /// to an empty string. + /// Failure mode caught: a regression of `unwrap_or_default()` in + /// `poll_job` would propagate `blob_id: ""` to downstream DAG vertices + /// as if the write had succeeded. + #[tokio::test] + async fn invoke_returns_err_when_done_missing_blob_id() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-noid", "status": "pending"}).to_string()) + .create_async() + .await; + + // status=done but no blob_id field. + let _m2 = server + .mock("GET", "/api/remember/job-noid") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-noid", "status": "done"}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "text".into(), + namespace: None, + }) + .await; + match output { + Output::Err { reason } => { + assert!( + reason.contains("blob_id"), + "error must mention missing blob_id; got: {reason}" + ); + } + Output::Ok { blob_id } => { + panic!("expected Err, got Ok(blob_id={blob_id:?}); silent empty-blob bug") + } + } + } + + /// `invoke` sends the `text` field in the POST body. + /// Failure mode caught: body serialisation drops the `text` field. + #[tokio::test] + async fn invoke_sends_correct_body() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .match_body(Matcher::PartialJson(json!({"text": "exact text"}))) + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "j1", "status": "pending"}).to_string()) + .create_async() + .await; + + let _m2 = server + .mock("GET", "/api/remember/j1") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "j1", "status": "done", "blob_id": "b1"}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "exact text".into(), + namespace: None, + }) + .await; + assert!(matches!(output, Output::Ok { .. })); + } +} diff --git a/tools/memory-memwal/src/remember_bulk.rs b/tools/memory-memwal/src/remember_bulk.rs new file mode 100644 index 0000000..806024d --- /dev/null +++ b/tools/memory-memwal/src/remember_bulk.rs @@ -0,0 +1,327 @@ +//! # `xyz.taluslabs.memory.memwal.remember_bulk@1` +//! +//! Store up to 20 memories in one batched call. Returns blob_ids in input +//! order. ~10× more rate-limit-efficient than N separate `remember` calls +//! (weight 10 for up to 20 items vs 5 each). + +use { + crate::client::{MemWalClient, MAX_BULK_ITEMS}, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +/// One item in the bulk request. `namespace` is per-item: a single bulk call +/// can write to multiple namespaces in one go. +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct BulkItem { + /// The text to store. + text: String, + /// Namespace for this specific item. Defaults to `"default"` on the + /// server when omitted. + namespace: Option, +} + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// 1–20 items to store. The relayer rejects empty arrays and any batch + /// over MAX_BULK_ITEMS = 20. + items: Vec, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Every item was durably stored. `blob_ids` aligns positionally with the + /// input `items` array (i.e. `blob_ids[i]` is the blob for `items[i]`). + Ok { + blob_ids: Vec, + }, + Err { + reason: String, + }, +} + +pub(crate) struct RememberBulkMemories { + client: MemWalClient, +} + +impl NexusTool for RememberBulkMemories { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.remember_bulk@1") + } + + fn path() -> &'static str { + "/memory/remember_bulk" + } + + fn description() -> &'static str { + "Store up to 20 memories in one batched call; returns one blob_id per item." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + // Validate batch size before signing anything — empty arrays waste a + // signed request slot; oversized batches surface as opaque HTTP 400 + // from the relayer without this guard. + if input.items.is_empty() { + return Output::Err { + reason: "bulk batch must contain at least 1 item".into(), + }; + } + if input.items.len() > MAX_BULK_ITEMS { + return Output::Err { + reason: format!( + "bulk batch capped at {MAX_BULK_ITEMS} items, got {}", + input.items.len() + ), + }; + } + + // Map our owned `Vec` into the borrowed `(&str, Option<&str>)` + // tuple shape the client expects. This avoids `Item<'_>` lifetime + // gymnastics by keeping the inputs alive in `items_refs`. + let items_refs: Vec<(&str, Option<&str>)> = input + .items + .iter() + .map(|i| (i.text.as_str(), i.namespace.as_deref())) + .collect(); + + let job_ids = match self.client.remember_bulk(&items_refs).await { + Ok(ids) => ids, + Err(e) => { + return Output::Err { + reason: e.to_string(), + }; + } + }; + + match self.client.poll_bulk_jobs(&job_ids).await { + Ok(blob_ids) => Output::Ok { blob_ids }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> RememberBulkMemories { + RememberBulkMemories { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + fn bulk_input(texts: &[&str]) -> Input { + Input { + items: texts + .iter() + .map(|t| BulkItem { + text: t.to_string(), + namespace: None, + }) + .collect(), + } + } + + /// `invoke` returns `Ok { blob_ids }` aligned with the input order when + /// every job reaches "done". + /// Failure mode caught: blob_ids returned in random order, breaking + /// callers that index-correlate items with results. + #[tokio::test] + async fn invoke_returns_blob_ids_in_input_order() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _bulk = server + .mock("POST", "/api/remember/bulk") + .with_status(202) + .with_header("content-type", "application/json") + .with_body( + json!({ + "job_ids": ["j1", "j2", "j3"], + "total": 3, + "status": "running" + }) + .to_string(), + ) + .create_async() + .await; + + // Server returns results in reversed order intentionally — the tool + // must reorder by job_id to align with the input order. + let _status = server + .mock("POST", "/api/remember/bulk/status") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "results": [ + {"job_id": "j3", "status": "done", "blob_id": "blob3"}, + {"job_id": "j2", "status": "done", "blob_id": "blob2"}, + {"job_id": "j1", "status": "done", "blob_id": "blob1"}, + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool.invoke(bulk_input(&["one", "two", "three"])).await; + match output { + Output::Ok { blob_ids } => { + assert_eq!(blob_ids, vec!["blob1", "blob2", "blob3"]); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Err` when any individual job ends `failed`. + /// Failure mode caught: a partial failure is silently presented as + /// success with truncated blob_ids, hiding data loss. + #[tokio::test] + async fn invoke_returns_err_on_any_job_failure() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _bulk = server + .mock("POST", "/api/remember/bulk") + .with_status(202) + .with_header("content-type", "application/json") + .with_body( + json!({"job_ids": ["jA", "jB"], "total": 2, "status": "running"}).to_string(), + ) + .create_async() + .await; + + let _status = server + .mock("POST", "/api/remember/bulk/status") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "results": [ + {"job_id": "jA", "status": "done", "blob_id": "blobA"}, + {"job_id": "jB", "status": "failed", "error": "encrypt timeout"}, + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool.invoke(bulk_input(&["a", "b"])).await; + assert!( + matches!(output, Output::Err { .. }), + "any failed job must produce Err" + ); + } + + /// `invoke` returns `Err` for an empty `items` array without hitting the network. + /// Failure mode caught: an empty batch silently round-trips, burning a + /// signed request slot and a rate-limit point for an opaque 400. + #[tokio::test] + async fn invoke_returns_err_on_empty_items() { + let server = Server::new_async().await; + let tool = make_tool(&server.url()); + let output = tool.invoke(bulk_input(&[])).await; + match output { + Output::Err { reason } => { + assert!( + reason.contains("1 item"), + "reason must mention the lower bound; got: {reason}" + ); + } + Output::Ok { .. } => panic!("empty batch must produce Err"), + } + } + + /// `invoke` returns `Err` for a batch of MAX_BULK_ITEMS+1 items without + /// contacting the relayer. + /// Failure mode caught: oversized batch reaches the relayer and gets an + /// opaque HTTP 400 instead of a tool-side cap explanation. + #[tokio::test] + async fn invoke_returns_err_on_oversized_batch() { + let server = Server::new_async().await; + let tool = make_tool(&server.url()); + let texts: Vec = (0..MAX_BULK_ITEMS + 1) + .map(|i| format!("item-{i}")) + .collect(); + let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); + let output = tool.invoke(bulk_input(&refs)).await; + match output { + Output::Err { reason } => { + assert!( + reason.contains(&MAX_BULK_ITEMS.to_string()), + "reason must mention the cap; got: {reason}" + ); + } + Output::Ok { .. } => panic!("oversized batch must produce Err"), + } + } + + /// `invoke` returns `Err` when the initial bulk POST fails (4xx/5xx). + /// Failure mode caught: the initial-submit error is swallowed. + #[tokio::test] + async fn invoke_returns_err_on_submit_failure() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _bulk = server + .mock("POST", "/api/remember/bulk") + .with_status(400) + .with_body("too many items") + .create_async() + .await; + + let output = tool.invoke(bulk_input(&["x"])).await; + assert!( + matches!(output, Output::Err { .. }), + "submit-time 400 must produce Err" + ); + } +} diff --git a/tools/memory-memwal/src/stats.rs b/tools/memory-memwal/src/stats.rs new file mode 100644 index 0000000..74120bb --- /dev/null +++ b/tools/memory-memwal/src/stats.rs @@ -0,0 +1,223 @@ +//! # `xyz.taluslabs.memory.memwal.stats@1` +//! +//! Nexus Tool that reports per-namespace usage for the authenticated MemWal +//! account: how many memories are stored and how many bytes they take. + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// Namespace to report on. Defaults to `"default"` on the server when + /// omitted. Only namespaces owned by the authenticated account return + /// meaningful data. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Per-namespace usage figures. + Ok { + /// Number of memories stored in this namespace. + memory_count: i64, + /// Total encrypted byte size of those memories on Walrus. + storage_bytes: i64, + /// The resolved namespace (mirrors what the server interpreted). + namespace: String, + }, + Err { + reason: String, + }, +} + +pub(crate) struct StatsForAccount { + client: MemWalClient, +} + +impl NexusTool for StatsForAccount { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!("xyz.taluslabs.memory.memwal.stats@1") + } + + fn path() -> &'static str { + "/memory/stats" + } + + fn description() -> &'static str { + "Report memory count and total stored bytes for a MemWal namespace." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self.client.stats(input.namespace.as_deref()).await { + Ok(s) => Output::Ok { + memory_count: s.memory_count, + storage_bytes: s.storage_bytes, + namespace: s.namespace, + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> StatsForAccount { + StatsForAccount { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` echoes the server's count + bytes verbatim on a healthy response. + /// Failure mode caught: numeric fields silently zeroed or truncated on the boundary. + #[tokio::test] + async fn invoke_returns_counts_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/stats") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "memory_count": 17, + "storage_bytes": 5_242_880, + "namespace": "people", + "owner": "0xowner" + }) + .to_string(), + ) + .create_async() + .await; + + match tool + .invoke(Input { + namespace: Some("people".into()), + }) + .await + { + Output::Ok { + memory_count, + storage_bytes, + namespace, + } => { + assert_eq!(memory_count, 17); + assert_eq!(storage_bytes, 5_242_880); + assert_eq!(namespace, "people"); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok` with zeros for a namespace that exists but is empty. + /// Failure mode caught: zero-memory namespaces misclassified as Err. + #[tokio::test] + async fn invoke_handles_zero_counts() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/stats") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "memory_count": 0, + "storage_bytes": 0, + "namespace": "empty-ns", + "owner": "0xowner" + }) + .to_string(), + ) + .create_async() + .await; + + match tool + .invoke(Input { + namespace: Some("empty-ns".into()), + }) + .await + { + Output::Ok { + memory_count, + storage_bytes, + .. + } => { + assert_eq!(memory_count, 0); + assert_eq!(storage_bytes, 0); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Err` on server error. + /// Failure mode caught: HTTP 500 swallowed, returning misleading zero stats. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/stats") + .with_status(503) + .with_body("service unavailable") + .create_async() + .await; + + let output = tool.invoke(Input { namespace: None }).await; + assert!( + matches!(output, Output::Err { .. }), + "server 503 must produce Err variant" + ); + } +} From 15d31eacd3e511fbdc906d297d0dc09c4f3a64c3 Mon Sep 17 00:00:00 2001 From: tuky191 <25343132+tuky191@users.noreply.github.com> Date: Mon, 25 May 2026 11:45:26 +0200 Subject: [PATCH 2/3] Offchain tools CI pipeline + Nexus v1.0.0 cutover (#33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: offchain tools pipeline design * docs: offchain tools pipeline implementation plan Bite-sized task plan implementing the design in docs/superpowers/specs/2026-05-19-offchain-tools-pipeline-design.md. Eight phases: repo reorg, per-tool conventions (tools.json + build.rs across 7 crates), shared Dockerfile, five composite GH actions, five callable workflows (discover/deploy/prepare/register/readiness), top-level ci.yml orchestrator, README + bootstrap notes, final verification. Co-Authored-By: Claude Opus 4.7 (1M context) * offchain: move workspace, tools, and helpers into offchain/ Prepares the repo for a future onchain/ sibling. Files relocated via git mv to preserve history. Workspace builds verified with cargo check. * onchain: add placeholder dir for future Move tools Co-Authored-By: Claude Sonnet 4.6 * justfile: point modules into offchain/ Co-Authored-By: Claude Sonnet 4.6 * just: fix import + .nightly-version paths after offchain/ move Co-Authored-By: Claude Sonnet 4.6 * ci: repoint coverage, sync-docs, pre-commit deps to offchain/ paths Co-Authored-By: Claude Sonnet 4.6 * pre-commit: cd into offchain/ before cargo invocations Recipes run with PWD = .pre-commit/ after the offchain/ move, so cargo could no longer find Cargo.toml. Each cargo-* recipe now cds into ../offchain first. Verified cargo-check and cargo-clippy succeed. Co-Authored-By: Claude Opus 4.7 (1M context) * tools/math: add tools.json, build.rs, [[bin]] declaration Establishes the per-tool convention used by the offchain CI pipeline. build.rs validates the bin/tool name match at compile time and injects TOOL_FQN_VERSION from env (default "1" for local builds). * tools/exchanges-coinbase: add tools.json, build.rs, [[bin]] declaration * tools/http: add tools.json, build.rs, [[bin]] declaration * tools/llm-openai-chat-completion: add tools.json, build.rs, [[bin]] declaration * tools/social-twitter: add tools.json, build.rs, [[bin]] declaration * tools/storage-walrus: add tools.json, build.rs, [[bin]] declaration * tools/templating-jinja: add tools.json, build.rs, [[bin]] declaration * offchain: update Cargo.lock for toml build-dep build.rs in each tool crate pulls in toml = "0.8". The lockfile update was missed across the seven per-tool commits, breaking cargo check --workspace --locked on a fresh clone. Co-Authored-By: Claude Opus 4.7 (1M context) * offchain: add shared multi-stage Dockerfile Build args PACKAGE, BINARY, TOOL_FQN_VERSION. Three stages: rust slim builder with buildkit cache mounts, debian lib-collector via ldd, and a distroless cc-debian12 final image. Smoke-tested locally with math. * ci/actions: add install-sui composite (suiup + cache) * ci/actions: add install-nexus-cli composite * ci/actions: add gcp-auth-protocol composite * ci/actions: add gcp-auth-infra composite * ci/actions: add retrigger-pr composite (signed empty commit) * ci: add offchain-tools.discover callable workflow Globs offchain/tools/*/tools.json, computes per-tool content versions (sha256(git rev-parse HEAD:/) first 8 hex chars as u32), and emits two matrices: all tools and tools changed since base-ref. A change to a shared file (Cargo.toml/Lock, Dockerfile, rust-toolchain) treats every tool as changed. Co-Authored-By: Claude Sonnet 4.6 * ci: add offchain-tools.deploy callable workflow Per-tool matrix build of offchain/Dockerfile. Pushes to GHCR (ghcr.io//:sha-<7>) and GCR (gcr.io//nexus-tools/:sha-<7>) when dry-run=false; only builds (no push) when dry-run=true. * ci: add offchain-tools.prepare callable workflow Per-tool matrix: pulls the pushed image, runs --meta, renders the Cloud Run config, generates signed-HTTP keys (idempotent on FQN set), uploads tool config to GCS. Aggregator job builds the manifest and the offchain-tools-prepare GitHub artifact consumed by register. Co-Authored-By: Claude Sonnet 4.6 * ci: add offchain-tools.register callable workflow Single-job (serialized Sui state) registration. Skips already-registered FQNs; for new ones, runs the PTB, extracts OwnerCap, saves registration JSON to GCS, registers signing key, reconciles the toolkit-config secret with the on-chain tool_kid (sole-writer rule). Co-Authored-By: Claude Sonnet 4.6 * ci: add offchain-tools.readiness PR merge gate Per-tool matrix that checks GCS for the Cloud Run config + at least one FQN registration JSON. Fails red if either is missing, with a message pointing the author at the publish workflow_dispatch. * ci: add top-level orchestrator ci.yml Wires discover/deploy/prepare/register/readiness per the trigger matrix: - PR (any base): discover, dry-run deploy on changed matrix. - PR with base testnet/mainnet: also readiness gate. - Push to main: discover, full deploy push (no chain ops). - Push to testnet/mainnet: discover, deploy, prepare, register on full matrix. - workflow_dispatch with PR#: resolves PR head, runs full chain, then retrigger-pr flips readiness green. - ci-gate aggregates job results for branch protection. Co-Authored-By: Claude Sonnet 4.6 * docs: README — document offchain/ layout and deploy flow * docs: add offchain bootstrap notes for first deploy * ci/register: drop workflow-level env block vars.* lookups in a workflow-level env block are resolved before the job's environment: is bound, so NEXUS_TAG and SUI_CHANNEL would be empty. Reference vars.* directly in the steps that need them. * ci: fix PR target-ref env-resolution On pull_request events, github.ref_name is the merge-ref (e.g. "123/merge"), not the base branch. Without explicit handling, PRs to mainnet/testnet would resolve to the wrong environment in callable workflows. Prefer github.base_ref on pull_request. * ci: align matrix selection with spec + tighten dispatch gating Two changes: - Push to main now uses matrix-changed (per the design.md trigger table). matrix-all on main was a deviation from spec. - workflow_dispatch without a pr-number no longer fires prepare/register. Previously, a bare dispatch from any branch (including main) would try to register against the resolved environment. * ci/discover: quote literal path in case pattern case ${DIR}/*) was unquoted; quoting the literal prefix avoids issues with shell metacharacters in DIR (low real risk; DIR comes from dirname over a hardcoded glob, but the quoted form is cleaner). * ci/deploy: consolidate image tags into a single output Dry-run runs no longer emit an empty trailing tag line. The "Compute image tags" step writes a heredoc-encoded multi-line list, and the build-push step consumes a single output. * pre-commit: use 'set working-directory' for cargo recipes Cleaner than prefixing every cargo recipe with `cd ../offchain &&`. Recipes that need repo root (md-lint, typos, shebang checks) already cd themselves; check-container-runtime doesn't touch the filesystem. Works on just >= 1.33 (the file-level setting predates the per-recipe [working-directory] attribute from 1.38). Co-Authored-By: Claude Opus 4.7 (1M context) * ci/deploy: build via --manifest-path to fix http crate-name ambiguity cargo build -p http was ambiguous because the workspace member `http` shares its name with transitive crates.io deps (http@0.2.12, http@1.4.0). Switch the Dockerfile to use --manifest-path scoped to the tool's Cargo.toml, which uniquely identifies the workspace member. Pass TOOL_DIR as a build-arg from deploy.yml so the matrix's `dir` field flows through. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/coverage: run cargo llvm-cov from offchain/ workspace dir Cargo.toml lives under offchain/ now; running cargo llvm-cov from repo root failed to locate it. Set working-directory: offchain on the Generate Coverage step; emit lcov.info one level up so codecov-action still picks it up at ./lcov.info. Co-Authored-By: Claude Opus 4.7 (1M context) * helpers/npm-install-g: resolve txt path via script dir The previous `done < helpers/npm-install-g.txt` was relative to PWD = repo root, which broke after the helpers/ move under offchain/. Use BASH_SOURCE-derived SCRIPT_DIR so the txt is found regardless of where the script is invoked from. Co-Authored-By: Claude Opus 4.7 (1M context) * docs: gitignore docs/superpowers/, untrack planning artifacts The spec/plan/notes under docs/superpowers/ are internal brainstorming output and don't need to be in git. They follow brainstorming/writing-plans conventions (mixed ordered-list styles, fence variations) that conflict with the project's MD029/MD031/MD032 markdownlint rules, causing spurious md-lint failures. Move them to local-only and drop the README back-reference. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: enable deploy-from-branch iteration paths Adds two ways to fire the full chain pipeline (deploy + prepare + register) against testnet/mainnet without merging to main: 1. Push trigger on iterate/testnet/** and iterate/mainnet/** git push origin HEAD:iterate/testnet/bootstrap-1 fires the chain against testnet using that branch's content. Branch prefix selects the target env. 2. workflow_dispatch input target-env (testnet|mainnet) gh workflow run ci.yml --ref -f target-env=testnet runs the chain against testnet using the dispatched ref. Requires ci.yml on the default branch for the dispatch button to be discoverable; once landed, this is the ergonomic iteration loop. Existing paths unchanged: PR (dry-run), push to main (build only), push to testnet/mainnet (full chain), dispatch with pr-number (full chain against PR head + retrigger-pr). The env-resolution chain in deploy/prepare/register's target-ref now honors (in order): PR base from resolve-target, inputs.target-env, iterate// branch prefix, PR base on pull_request, github.ref_name. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/register: rename SUI_DEPLOYER_MNEMONIC to SUI_DEPLOYER_PK The secret holds a `suiprivkey...` bech32 private key (sui keytool export format), not a mnemonic. Rename to match reality. Same secret will drive future onchain Move deploys via `sui client publish`, so the bech32 form is the right canonical store. Workflow flow unchanged: sui keytool import converts the bech32 PK to the base64 keystore entry, then we read it back and pass to nexus conf set --sui.pk. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/prepare: bake NEXUS_TAG into each tool's Cloud Run config blob Each per-(tool, version) blob written to gs:////offchain/tools/-v.json now carries nexus_contracts_tag captured from vars.NEXUS_TAG at prepare time. Terraform reads this per-entry so the allowed-leaders mount path stays pinned to the nexus version the tool was deployed against. Upgrades become content-addressed: bump NEXUS_TAG, push, and only new deployments pick up the new tag. Existing services keep their original mount until explicitly torn down (delete the GCS blob and apply). Co-Authored-By: Claude Opus 4.7 (1M context) * ci: trigger tf-nexus-tools terraform apply after successful register After deploy + prepare + register complete, dispatch tf-nexus-tools' terraform.yml workflow against the `production` ref so Cloud Run services / ALB / DNS records pick up the new tool's GCS state blob. Uses the org-level GH_ACCESS_TOKEN PAT (same pattern ava-game uses for all cross-repo gh operations) because the default GITHUB_TOKEN is single-repo-scoped and cannot dispatch workflows in other repos. Fires on every chain-op path: iterate/* push, push to testnet/mainnet, and workflow_dispatch with pr-number or target-env. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/actions: drop token_format from gcp-auth-infra (use Direct WIF) token_format: 'access_token' requires google-github-actions/auth to impersonate a real service account, which we don't have configured for this WIF setup. Removing token_format makes the action use Direct WIF — the federated token is still exposed via the auth_token output, which docker/login-action consumes for `oauth2accesstoken` GCR auth. Mirrors ava-game's pattern in protocol.deploy.yml. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/deploy: split build (no auth) from push (auth) to avoid token expiry Previously a single build-push-action did both compile + push, which holds the GCP access token across the slow Rust compile. With cold buildkit caches, that compile can exceed the 1h token lifetime and the push fails at the end. Mirror ava-game's protocol.deploy.yml pattern: 1. Build (push=false) — slow Rust compile, no auth needed. 2. Login to GHCR + GCP auth — fresh tokens minted here. 3. Push (push=true) — reuses cached layers via cache-from=gha, completes within the token's 1h window. Same gha cache scope is used for cache-from/cache-to on both steps so the push step pulls the just-built layers instead of recompiling. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: rename WIF provider from nexus-tools-protocol to nexus-tools ava-game uses two separate WIF providers per env (ava-game and ava-game-protocol). For nexus-tools we only configured a single nexus-tools provider in tf-talus-infra/params/production/github.json, and granted GCS bucket / Secret Manager IAM on the same principal in tf-nexus-tools (iam.tf + bucket.tf). The -protocol suffix was inherited from blindly mirroring ava-game's prepare/register/readiness scaffolds. Drop it; both gcp-auth-protocol and gcp-auth-infra now hit the same nexus-tools provider. Permissions verified: artifactregistry.writer/reader (tf-talus-infra) storage.objectAdmin on bucket (tf-nexus-tools/bucket.tf) secretmanager.admin prefix=nexus-tools-* (tf-nexus-tools/iam.tf) Co-Authored-By: Claude Opus 4.7 (1M context) * ci: add missing checkout to prepare aggregate + readiness jobs Both jobs reference local composite actions (./.github/actions/ gcp-auth-protocol) but never check out the repo, so the action's action.yml is not present on disk when GHA tries to resolve it. Per-tool prepare matrix already has checkout. Only the aggregate fan-in job and the readiness gate job were missing it. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/register: add tracing + explicit meta-file check Previous run exited 1 right after "Using coin: ..." with no visible error. set -e killed the script silently somewhere in the first for-loop iteration. Add: - set -x for full command tracing - ls of $(dirname MANIFEST) and META_DIR before the loop - cat of the manifest - explicit `[ -f $META ]` guard with directory listing on miss - per-tool echo at loop top + values dump Strictly diagnostic; once we see the failing command we can revert or keep the parts that pay for themselves. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/register: tolerate empty string in to_bytes() The exchanges-coinbase tool's `get-spot-price` FQN returns an empty .description from --meta. The to_bytes() pipe `... | grep '0x' | ...` finds no match on empty input → grep exits 1 → pipefail propagates → the surrounding `DESC_VEC=$(to_bytes "$DESC")` assignment fires set -e and kills the register script silently right after "Using coin: ...". Short-circuit to_bytes() to return empty (and exit 0) when input is empty. The resulting Move literal `vector[]` is a valid empty vector, so the PTB registers the tool with an empty description string. (Empty descriptions are still a quality issue worth fixing upstream in the tool's --meta output, but that's separate.) Co-Authored-By: Claude Opus 4.7 (1M context) * tools: thread TOOL_FQN_VERSION into every fqn!() call site Every FQN literal across the 7 tools was hardcoded as @1, ignoring the TOOL_FQN_VERSION env that build.rs + Dockerfile + CI matrix plumb through end-to-end. As a result, register-on-chain saw @1 forever and no content-addressed versioning ever reached the chain. Mirror ava-game's pattern: each `fqn!("ns.tool@1")` becomes `fqn!(concat!("ns.tool@", env!("TOOL_FQN_VERSION")))`. The fqn! macro from nexus-sdk accepts any &'static str expression, so the concat!+env! expression works inline; no per-tool TOOL_FQN const wrapper required. Verified locally: TOOL_FQN_VERSION=999 cargo build -p math --release ./target/release/math --meta | jq -r '.[].fqn' → xyz.taluslabs.math.i64.{add,mul,cmp,sum}@999 48 FQN sites rewritten across all 7 tools. No stray @1 left. Co-Authored-By: Claude Opus 4.7 (1M context) * ci/register: drive registration via `nexus tool register offchain --from-meta` Replaces the hand-rolled PTB loop (which drained a single COIN_ID across all FQNs and failed with InsufficientGas after ~33 calls) with one CLI invocation per FQN. The CLI acquires/releases a gas coin per call, so the loop is no longer gas-bound. Idempotent: already-registered FQNs return `{"already_registered": true}` and the loop preserves the existing GCS record. GCS layout adds `owner_cap_over_gas` alongside `owner_cap_over_tool` per FQN. Existing fields (`signing_key_hash`, `tool_kid`) survive via read-modify-write merge. Drops the PTB-only `Load Nexus objects` step and the `to_bytes`, `COIN_ID`, and `set -x` plumbing that went with it. Co-Authored-By: Claude Opus 4.7 * ci/register: treat MoveAbort in register_tool_@14 as already-registered The v1.0.0 CLI flags already-registered by string-matching the outer entry name `register_off_chain_tool_`, but EFqnAlreadyExists fires from the inner helper `register_tool_` (tool_registry.move:594), so the CLI surfaces it as a generic error. Detect that exact MoveAbort signature (function_name=register_tool_, instruction=14) and treat it as already-registered so re-runs over prior on-chain state succeed. Co-Authored-By: Claude Opus 4.7 * ci/register: pin gas + collateral coins per call to avoid drain The v1.0.0 CLI builds its gas pool with a single coin (cli/src/sui.rs:294 `with_gas(vec![one_coin])`) picked by-order=0 at startup; collateral picks by-order=1 from a fresh fetch (tool_register_offchain.rs:88). Repeated CLI invocations kept hitting the same coin and draining it below the 100M budget, yielding InsufficientGas after ~11 successful PTBs. Re-query the wallet per FQN, pin the largest coin as --sui-gas-coin and the smallest distinct coin as --collateral-coin. Fresh object refs each call, no implicit collision between the gas and collateral picks. Co-Authored-By: Claude Opus 4.7 * ci/register: instrument loop with wallet/CLI debug dumps Add per-FQN diagnostic groups so we can see what coins were picked, what the CLI saw on stdin/stderr, and what stdout came back when registration fails. Splits CLI stderr/stdout into separate buffers so the JSON output stays parseable. Co-Authored-By: Claude Opus 4.7 * ci/register: bump --sui-gas-budget to 500M to cover tool storage cost The default 100M budget covers computation + ~67M storage for a small schema (verified against the v0.8.2 spot-price register tx), but larger schemas (twitter response shapes, llm chat-completion, etc.) hit InsufficientGas at execution time with `command: None`. Wallet's largest coin is 569M MIST, leaving 500M as the safe budget ceiling. Co-Authored-By: Claude Opus 4.7 * ci/register: consolidate to 4 coins + 1 SUI budget + pre-flight balance check Heavy schemas (twitter get-tweet, llm chat-completion) need ~1 SUI of gas for storage cost — confirmed from the prior static deploy (tf-talus-nexus-v2/.../social-twitter.json:gasBudgetAmount=1e9). The deployer wallet's 22 small coins (largest 0.6 SUI) can't individually cover that budget. Three changes: - Add a Consolidate Wallet step that merges the two smallest coins iteratively until 4 remain, levelling balances to ~2.7 SUI each. - Bump --sui-gas-budget on each register call to 1 SUI. - Pre-flight balance check: if the largest available coin is below the budget, fail loudly with "wallet drained" instead of surfacing an opaque RPC error from Sui validation. Co-Authored-By: Claude Opus 4.7 * ci/register: pick any coin ≥ budget, rotate when prior drained Previously: pick the largest coin, fail with "wallet drained" if it's below budget. But the prior-largest can drop below budget after a call while other coins still have plenty — failing on it instead of rotating wastes the wallet headroom. Now: filter to coins with balance ≥ budget, take the largest of those. Only fail if no coin in the wallet meets budget. Co-Authored-By: Claude Opus 4.7 * ci: trigger-tf-apply waits for dispatched run; drop retrigger-pr Two cleanups now that register is green and trigger-tf-apply successfully reaches tf-nexus-tools: - trigger-tf-apply now polls for the run it just dispatched (since `gh workflow run` returns no ID), then `gh run watch --exit-status` to inherit the conclusion. Register+apply become one atomic gate; ci-gate fails if terraform fails. - Drop retrigger-pr (job + composite action). It was a hack for the workflow_dispatch+PR case to nudge readiness back to green; no longer needed once the apply is awaited inline. Co-Authored-By: Claude Opus 4.7 * ci/register: preserve per-FQN URL path + force trailing slash Two related fixes to make registered URLs leader-compatible: 1. Each FQN has its own URL path (e.g. /i64/add, /get-tweet) baked into the meta JSON by the toolkit binary. Previously we overrode --url with a base hostname for the entire tool, losing the path and causing the leader to call the wrong endpoint. Now we extract the path from the meta URL and combine with the per-network internal hostname. 2. The leader appends `invoke` via RFC 3986 relative-URL join: /i64/add/ + invoke = /i64/add/invoke ✓ /i64/add + invoke = /invoke ✗ (basename replace) Force a trailing slash so the join produces the right endpoint. This matches the legacy v0.8 static-deploy URL convention. Smoke-tested with the math DAG end-to-end on testnet — add/cmp/mul all returned 200 from /i64//invoke. Co-Authored-By: Claude Opus 4.7 * ci: replace gh run watch with the-actions-org/workflow-dispatch `gh run watch` fetches check annotations, which requires the `checks:read` permission. Fine-grained PATs cannot carry that scope (GitHub limitation), so our trigger-tf-apply job was failing with 403 even though the dispatched terraform run itself succeeded. Switch to `the-actions-org/workflow-dispatch@v4.0.0` — the same action ava-game uses for the same dispatch+wait pattern. It polls `actions:read` endpoints which fine-grained PATs CAN carry. Co-Authored-By: Claude Opus 4.7 * ci/register: skip already-registered FQNs via on-chain snapshot Take a single `nexus tool list --json` snapshot at the top of the register step and use it to skip the CLI call entirely for FQNs already on chain. Each register call costs ~1-2s even when it MoveAborts on EFqnAlreadyExists, so this turns a re-run from minutes into seconds. The existing post-call MoveAbort handler stays as a defensive net against snapshot races (e.g. a parallel run registers the same FQN between the snapshot and our call). We still verify the GCS reg file exists for skipped FQNs so the signing-key + toolkit-config steps don't run blind. Co-Authored-By: Claude Opus 4.7 * tools: cargo +nightly fmt across all 48 fqn!(concat!()) sites The earlier perl rewrite of fqn!("...@1") -> fqn!(concat!("...@", env!("TOOL_FQN_VERSION"))) left long single-line invocations. Nightly rustfmt wraps them across three lines, which the pre-commit 03-cargo-nightly-fmt hook enforces. Co-Authored-By: Claude Opus 4.7 * docs: README — current state of the pipeline + tool-author guide Replaces the original PR-opening README with one that reflects how the pipeline actually works after the v1.0.0 cutover: - Tool-author conventions (tools.json, build.rs, [[bin]], fqn!() threading, per-FQN URL paths). - Branch model: main (no chain ops), iterate/testnet/**, iterate/mainnet/**, workflow_dispatch with target-env / pr-number. - What "full pipeline" does end-to-end (discover → deploy → prepare → register → trigger-tf-apply). - Operational notes on deployer wallet, NEXUS_TAG pinning, stale config cleanup, and the parallel-ALB cost tradeoff. Co-Authored-By: Claude Opus 4.7 * docs: README — explicit lifecycle (PR → main → promote to env) Add a Lifecycle section walking from "open a PR" through "merge to main" to "promote to a deploy env" (long-lived branch merge, iterate branch, or manual workflow_dispatch). Expanded the trigger table to also list the long-lived testnet/mainnet branches that are wired into ci.yml. Co-Authored-By: Claude Opus 4.7 * ci/codecov: make patch coverage informational Patch check was blocking PRs whose changes are mechanical and don't warrant new tests — e.g. the 48-site fqn!(concat!()) rewrite where every tool's `fn fqn()` body changed but is still a one-line const return. Project-level check stays strict and still gates real coverage regressions. Co-Authored-By: Claude Opus 4.7 * pre-commit: check-tool-conventions enforces fqn!() threading Catches the one tool-author contract rule the build system can't: fqn!("xyz...@") compiles fine but ships the wrong content-version on chain (or collides with legacy @1 registrations). The hook scans offchain/tools/*/src/ for any fqn!("...@") literal and fails the commit with a pointer to the correct pattern. Also asserts each tool dir has tools.json + build.rs. Co-Authored-By: Claude Opus 4.7 * ci/discover: source BLOCKED_TOOLS from repo variable Skip-list now read from the repo-level `BLOCKED_TOOLS` variable (space-separated tool_name values). Security blocks can be flipped via repo settings → Variables without a code change. Empty/unset variable means no tools blocked. Initial value (set via gh variable set): BLOCKED_TOOLS=http Blocks the http tool pending its SSRF guard. Both the on-chain unregister and this CI block keep it from re-shipping. Co-Authored-By: Claude Opus 4.7 * ci: tag-driven deploy (testnet-*/mainnet-*); drop long-lived deploy branches Switch the canonical promotion path from "merge to long-lived testnet/mainnet branch" to "push a testnet-*/mainnet-* tag". Tags are immutable, so each deployment is attributable to an exact ref — no merge-strategy gotchas (squash collapsing shared ancestry and breaking future promotes), no long-lived branches to keep clean, no ephemeral promote/* branches needed. ci.yml changes: - `on.push.tags: [testnet-*, mainnet-*]` triggers the full chain. - Target-env resolution adds tag-prefix matching above the existing iterate//** rules. - Prepare gate accepts tag pushes; drops the now-unused testnet / mainnet long-lived branch matches. - ref_type guards distinguish tag pushes from same-named branches. README: branch & tag model section + lifecycle promotion step rewritten around the new flow. Co-Authored-By: Claude Opus 4.7 * memory-memwal: cargo +nightly fmt across 7 fqn!(concat!()) sites Same wrap-across-three-lines pattern nightly fmt applied to the other 48 sites after the original perl rewrite. Co-Authored-By: Claude Opus 4.7 * add concurrency group --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/actions/coverage/action.yml | 3 +- .github/actions/gcp-auth-infra/action.yml | 37 ++ .github/actions/gcp-auth-protocol/action.yml | 33 ++ .github/actions/install-nexus-cli/action.yml | 29 ++ .github/actions/install-sui/action.yml | 45 ++ .../pre-commit/dependencies/action.yml | 2 +- .github/workflows/ci.yml | 252 +++++++++++ .github/workflows/coverage-baseline.yml | 4 +- .github/workflows/coverage.yaml | 4 +- .github/workflows/offchain-tools.deploy.yml | 112 +++++ .github/workflows/offchain-tools.discover.yml | 151 +++++++ .github/workflows/offchain-tools.prepare.yml | 262 +++++++++++ .../workflows/offchain-tools.readiness.yml | 66 +++ .github/workflows/offchain-tools.register.yml | 409 ++++++++++++++++++ .github/workflows/sync_docs.yml | 14 +- .gitignore | 3 + .pre-commit/.just | 50 ++- .pre-commit/14-check-tool-conventions | 3 + README.md | 196 ++++++++- codecov.yml | 11 +- justfile | 12 +- Cargo.lock => offchain/Cargo.lock | 9 + Cargo.toml => offchain/Cargo.toml | 0 offchain/Dockerfile | 77 ++++ deny.toml => offchain/deny.toml | 0 {helpers => offchain/helpers}/helpers.just | 2 +- .../helpers}/npm-install-g.sh | 9 +- .../helpers}/npm-install-g.txt | 0 .../rust-toolchain.toml | 0 rustfmt.toml => offchain/rustfmt.toml | 0 {tools => offchain/tools}/.just | 0 .../tools}/exchanges-coinbase/Cargo.toml | 8 + .../tools}/exchanges-coinbase/README.md | 0 offchain/tools/exchanges-coinbase/build.rs | 50 +++ .../exchanges-coinbase/src/coinbase_client.rs | 0 .../tools}/exchanges-coinbase/src/error.rs | 0 .../tools}/exchanges-coinbase/src/main.rs | 0 .../src/tools/get_product_stats.rs | 5 +- .../src/tools/get_product_ticker.rs | 5 +- .../src/tools/get_spot_price.rs | 5 +- .../exchanges-coinbase/src/tools/mod.rs | 0 .../exchanges-coinbase/src/tools/models.rs | 0 offchain/tools/exchanges-coinbase/tools.json | 7 + {tools => offchain/tools}/http/Cargo.toml | 8 + {tools => offchain/tools}/http/README.md | 0 offchain/tools/http/build.rs | 50 +++ {tools => offchain/tools}/http/src/errors.rs | 0 {tools => offchain/tools}/http/src/http.rs | 5 +- .../tools}/http/src/http_client.rs | 0 {tools => offchain/tools}/http/src/main.rs | 0 {tools => offchain/tools}/http/src/models.rs | 0 {tools => offchain/tools}/http/src/utils.rs | 0 offchain/tools/http/tools.json | 7 + .../llm-openai-chat-completion/Cargo.toml | 8 + .../llm-openai-chat-completion/README.md | 0 .../tools/llm-openai-chat-completion/build.rs | 50 +++ .../llm-openai-chat-completion/src/main.rs | 5 +- .../llm-openai-chat-completion/src/status.rs | 0 .../llm-openai-chat-completion/tools.json | 7 + {tools => offchain/tools}/math/Cargo.toml | 8 + {tools => offchain/tools}/math/README.md | 0 offchain/tools/math/build.rs | 50 +++ {tools => offchain/tools}/math/src/i64/add.rs | 5 +- {tools => offchain/tools}/math/src/i64/cmp.rs | 5 +- {tools => offchain/tools}/math/src/i64/mod.rs | 0 {tools => offchain/tools}/math/src/i64/mul.rs | 5 +- {tools => offchain/tools}/math/src/i64/sum.rs | 5 +- {tools => offchain/tools}/math/src/main.rs | 0 offchain/tools/math/tools.json | 7 + .../tools}/memory-memwal/.env.example | 0 {tools => offchain/tools}/memory-memwal/.just | 0 .../tools}/memory-memwal/Cargo.toml | 8 + .../tools}/memory-memwal/README.md | 0 offchain/tools/memory-memwal/build.rs | 50 +++ .../tools}/memory-memwal/src/analyze.rs | 5 +- .../tools}/memory-memwal/src/ask.rs | 5 +- .../tools}/memory-memwal/src/auth.rs | 0 .../tools}/memory-memwal/src/client.rs | 0 .../tools}/memory-memwal/src/error.rs | 0 .../tools}/memory-memwal/src/forget.rs | 5 +- .../tools}/memory-memwal/src/main.rs | 22 +- .../tools}/memory-memwal/src/recall.rs | 5 +- .../tools}/memory-memwal/src/remember.rs | 5 +- .../tools}/memory-memwal/src/remember_bulk.rs | 5 +- .../tools}/memory-memwal/src/stats.rs | 5 +- offchain/tools/memory-memwal/tools.json | 7 + .../tools}/social-twitter/Cargo.toml | 8 + .../tools}/social-twitter/README.md | 0 offchain/tools/social-twitter/build.rs | 50 +++ .../tools}/social-twitter/src/auth.rs | 0 .../create_group_conversation.rs | 5 +- .../get_conversation_messages.rs | 5 +- .../get_conversation_messages_by_id.rs | 5 +- .../social-twitter/src/direct_message/mod.rs | 0 .../src/direct_message/models.rs | 0 .../src/direct_message/send_direct_message.rs | 5 +- .../send_message_to_group_conversation.rs | 5 +- .../tools}/social-twitter/src/error.rs | 0 .../social-twitter/src/list/add_member.rs | 5 +- .../social-twitter/src/list/create_list.rs | 5 +- .../social-twitter/src/list/delete_list.rs | 5 +- .../social-twitter/src/list/get_list.rs | 5 +- .../src/list/get_list_members.rs | 5 +- .../src/list/get_list_tweets.rs | 5 +- .../social-twitter/src/list/get_user_lists.rs | 5 +- .../tools}/social-twitter/src/list/mod.rs | 0 .../tools}/social-twitter/src/list/models.rs | 0 .../social-twitter/src/list/remove_member.rs | 5 +- .../social-twitter/src/list/update_list.rs | 5 +- .../tools}/social-twitter/src/main.rs | 0 .../tools}/social-twitter/src/media/mod.rs | 0 .../tools}/social-twitter/src/media/models.rs | 0 .../social-twitter/src/media/upload_media.rs | 5 +- .../social-twitter/src/tweet/delete_tweet.rs | 5 +- .../src/tweet/get_mentioned_tweets.rs | 5 +- .../src/tweet/get_recent_search_tweets.rs | 5 +- .../src/tweet/get_recent_tweet_count.rs | 5 +- .../social-twitter/src/tweet/get_tweet.rs | 5 +- .../social-twitter/src/tweet/get_tweets.rs | 5 +- .../src/tweet/get_user_tweets.rs | 5 +- .../social-twitter/src/tweet/like_tweet.rs | 5 +- .../tools}/social-twitter/src/tweet/mod.rs | 0 .../tools}/social-twitter/src/tweet/models.rs | 0 .../social-twitter/src/tweet/post_tweet.rs | 5 +- .../social-twitter/src/tweet/retweet_tweet.rs | 5 +- .../src/tweet/undo_retweet_tweet.rs | 5 +- .../social-twitter/src/tweet/unlike_tweet.rs | 5 +- .../social-twitter/src/twitter_client.rs | 0 .../social-twitter/src/user/follow_user.rs | 5 +- .../social-twitter/src/user/get_user_by_id.rs | 5 +- .../src/user/get_user_by_username.rs | 5 +- .../src/user/get_users_by_id.rs | 5 +- .../src/user/get_users_by_username.rs | 5 +- .../tools}/social-twitter/src/user/mod.rs | 0 .../tools}/social-twitter/src/user/models.rs | 0 .../social-twitter/src/user/unfollow_user.rs | 5 +- offchain/tools/social-twitter/tools.json | 7 + .../tools}/storage-walrus/Cargo.toml | 8 + .../tools}/storage-walrus/README.md | 0 offchain/tools/storage-walrus/build.rs | 50 +++ .../tools}/storage-walrus/src/client.rs | 0 .../tools}/storage-walrus/src/main.rs | 0 .../tools}/storage-walrus/src/read_file.rs | 5 +- .../tools}/storage-walrus/src/read_json.rs | 5 +- .../tools}/storage-walrus/src/upload_file.rs | 5 +- .../tools}/storage-walrus/src/upload_json.rs | 5 +- .../tools}/storage-walrus/src/utils.rs | 0 .../tools}/storage-walrus/src/verify_blob.rs | 5 +- offchain/tools/storage-walrus/tools.json | 7 + .../tools}/templating-jinja/Cargo.toml | 10 +- .../tools}/templating-jinja/README.md | 0 offchain/tools/templating-jinja/build.rs | 50 +++ .../tools}/templating-jinja/src/main.rs | 0 .../tools}/templating-jinja/src/template.rs | 5 +- offchain/tools/templating-jinja/tools.json | 7 + onchain/README.md | 6 + 156 files changed, 2528 insertions(+), 92 deletions(-) create mode 100644 .github/actions/gcp-auth-infra/action.yml create mode 100644 .github/actions/gcp-auth-protocol/action.yml create mode 100644 .github/actions/install-nexus-cli/action.yml create mode 100644 .github/actions/install-sui/action.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/offchain-tools.deploy.yml create mode 100644 .github/workflows/offchain-tools.discover.yml create mode 100644 .github/workflows/offchain-tools.prepare.yml create mode 100644 .github/workflows/offchain-tools.readiness.yml create mode 100644 .github/workflows/offchain-tools.register.yml create mode 100755 .pre-commit/14-check-tool-conventions rename Cargo.lock => offchain/Cargo.lock (99%) rename Cargo.toml => offchain/Cargo.toml (100%) create mode 100644 offchain/Dockerfile rename deny.toml => offchain/deny.toml (100%) rename {helpers => offchain/helpers}/helpers.just (98%) rename {helpers => offchain/helpers}/npm-install-g.sh (55%) rename {helpers => offchain/helpers}/npm-install-g.txt (100%) rename rust-toolchain.toml => offchain/rust-toolchain.toml (100%) rename rustfmt.toml => offchain/rustfmt.toml (100%) rename {tools => offchain/tools}/.just (100%) rename {tools => offchain/tools}/exchanges-coinbase/Cargo.toml (84%) rename {tools => offchain/tools}/exchanges-coinbase/README.md (100%) create mode 100644 offchain/tools/exchanges-coinbase/build.rs rename {tools => offchain/tools}/exchanges-coinbase/src/coinbase_client.rs (100%) rename {tools => offchain/tools}/exchanges-coinbase/src/error.rs (100%) rename {tools => offchain/tools}/exchanges-coinbase/src/main.rs (100%) rename {tools => offchain/tools}/exchanges-coinbase/src/tools/get_product_stats.rs (99%) rename {tools => offchain/tools}/exchanges-coinbase/src/tools/get_product_ticker.rs (99%) rename {tools => offchain/tools}/exchanges-coinbase/src/tools/get_spot_price.rs (99%) rename {tools => offchain/tools}/exchanges-coinbase/src/tools/mod.rs (100%) rename {tools => offchain/tools}/exchanges-coinbase/src/tools/models.rs (100%) create mode 100644 offchain/tools/exchanges-coinbase/tools.json rename {tools => offchain/tools}/http/Cargo.toml (86%) rename {tools => offchain/tools}/http/README.md (100%) create mode 100644 offchain/tools/http/build.rs rename {tools => offchain/tools}/http/src/errors.rs (100%) rename {tools => offchain/tools}/http/src/http.rs (99%) rename {tools => offchain/tools}/http/src/http_client.rs (100%) rename {tools => offchain/tools}/http/src/main.rs (100%) rename {tools => offchain/tools}/http/src/models.rs (100%) rename {tools => offchain/tools}/http/src/utils.rs (100%) create mode 100644 offchain/tools/http/tools.json rename {tools => offchain/tools}/llm-openai-chat-completion/Cargo.toml (86%) rename {tools => offchain/tools}/llm-openai-chat-completion/README.md (100%) create mode 100644 offchain/tools/llm-openai-chat-completion/build.rs rename {tools => offchain/tools}/llm-openai-chat-completion/src/main.rs (99%) rename {tools => offchain/tools}/llm-openai-chat-completion/src/status.rs (100%) create mode 100644 offchain/tools/llm-openai-chat-completion/tools.json rename {tools => offchain/tools}/math/Cargo.toml (81%) rename {tools => offchain/tools}/math/README.md (100%) create mode 100644 offchain/tools/math/build.rs rename {tools => offchain/tools}/math/src/i64/add.rs (94%) rename {tools => offchain/tools}/math/src/i64/cmp.rs (94%) rename {tools => offchain/tools}/math/src/i64/mod.rs (100%) rename {tools => offchain/tools}/math/src/i64/mul.rs (94%) rename {tools => offchain/tools}/math/src/i64/sum.rs (97%) rename {tools => offchain/tools}/math/src/main.rs (100%) create mode 100644 offchain/tools/math/tools.json rename {tools => offchain/tools}/memory-memwal/.env.example (100%) rename {tools => offchain/tools}/memory-memwal/.just (100%) rename {tools => offchain/tools}/memory-memwal/Cargo.toml (88%) rename {tools => offchain/tools}/memory-memwal/README.md (100%) create mode 100644 offchain/tools/memory-memwal/build.rs rename {tools => offchain/tools}/memory-memwal/src/analyze.rs (98%) rename {tools => offchain/tools}/memory-memwal/src/ask.rs (98%) rename {tools => offchain/tools}/memory-memwal/src/auth.rs (100%) rename {tools => offchain/tools}/memory-memwal/src/client.rs (100%) rename {tools => offchain/tools}/memory-memwal/src/error.rs (100%) rename {tools => offchain/tools}/memory-memwal/src/forget.rs (98%) rename {tools => offchain/tools}/memory-memwal/src/main.rs (59%) rename {tools => offchain/tools}/memory-memwal/src/recall.rs (98%) rename {tools => offchain/tools}/memory-memwal/src/remember.rs (98%) rename {tools => offchain/tools}/memory-memwal/src/remember_bulk.rs (98%) rename {tools => offchain/tools}/memory-memwal/src/stats.rs (98%) create mode 100644 offchain/tools/memory-memwal/tools.json rename {tools => offchain/tools}/social-twitter/Cargo.toml (84%) rename {tools => offchain/tools}/social-twitter/README.md (100%) create mode 100644 offchain/tools/social-twitter/build.rs rename {tools => offchain/tools}/social-twitter/src/auth.rs (100%) rename {tools => offchain/tools}/social-twitter/src/direct_message/create_group_conversation.rs (99%) rename {tools => offchain/tools}/social-twitter/src/direct_message/get_conversation_messages.rs (98%) rename {tools => offchain/tools}/social-twitter/src/direct_message/get_conversation_messages_by_id.rs (99%) rename {tools => offchain/tools}/social-twitter/src/direct_message/mod.rs (100%) rename {tools => offchain/tools}/social-twitter/src/direct_message/models.rs (100%) rename {tools => offchain/tools}/social-twitter/src/direct_message/send_direct_message.rs (98%) rename {tools => offchain/tools}/social-twitter/src/direct_message/send_message_to_group_conversation.rs (99%) rename {tools => offchain/tools}/social-twitter/src/error.rs (100%) rename {tools => offchain/tools}/social-twitter/src/list/add_member.rs (98%) rename {tools => offchain/tools}/social-twitter/src/list/create_list.rs (99%) rename {tools => offchain/tools}/social-twitter/src/list/delete_list.rs (98%) rename {tools => offchain/tools}/social-twitter/src/list/get_list.rs (99%) rename {tools => offchain/tools}/social-twitter/src/list/get_list_members.rs (99%) rename {tools => offchain/tools}/social-twitter/src/list/get_list_tweets.rs (99%) rename {tools => offchain/tools}/social-twitter/src/list/get_user_lists.rs (99%) rename {tools => offchain/tools}/social-twitter/src/list/mod.rs (100%) rename {tools => offchain/tools}/social-twitter/src/list/models.rs (100%) rename {tools => offchain/tools}/social-twitter/src/list/remove_member.rs (98%) rename {tools => offchain/tools}/social-twitter/src/list/update_list.rs (99%) rename {tools => offchain/tools}/social-twitter/src/main.rs (100%) rename {tools => offchain/tools}/social-twitter/src/media/mod.rs (100%) rename {tools => offchain/tools}/social-twitter/src/media/models.rs (100%) rename {tools => offchain/tools}/social-twitter/src/media/upload_media.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/delete_tweet.rs (98%) rename {tools => offchain/tools}/social-twitter/src/tweet/get_mentioned_tweets.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/get_recent_search_tweets.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/get_recent_tweet_count.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/get_tweet.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/get_tweets.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/get_user_tweets.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/like_tweet.rs (98%) rename {tools => offchain/tools}/social-twitter/src/tweet/mod.rs (100%) rename {tools => offchain/tools}/social-twitter/src/tweet/models.rs (100%) rename {tools => offchain/tools}/social-twitter/src/tweet/post_tweet.rs (99%) rename {tools => offchain/tools}/social-twitter/src/tweet/retweet_tweet.rs (98%) rename {tools => offchain/tools}/social-twitter/src/tweet/undo_retweet_tweet.rs (98%) rename {tools => offchain/tools}/social-twitter/src/tweet/unlike_tweet.rs (98%) rename {tools => offchain/tools}/social-twitter/src/twitter_client.rs (100%) rename {tools => offchain/tools}/social-twitter/src/user/follow_user.rs (98%) rename {tools => offchain/tools}/social-twitter/src/user/get_user_by_id.rs (99%) rename {tools => offchain/tools}/social-twitter/src/user/get_user_by_username.rs (99%) rename {tools => offchain/tools}/social-twitter/src/user/get_users_by_id.rs (99%) rename {tools => offchain/tools}/social-twitter/src/user/get_users_by_username.rs (99%) rename {tools => offchain/tools}/social-twitter/src/user/mod.rs (100%) rename {tools => offchain/tools}/social-twitter/src/user/models.rs (100%) rename {tools => offchain/tools}/social-twitter/src/user/unfollow_user.rs (98%) create mode 100644 offchain/tools/social-twitter/tools.json rename {tools => offchain/tools}/storage-walrus/Cargo.toml (86%) rename {tools => offchain/tools}/storage-walrus/README.md (100%) create mode 100644 offchain/tools/storage-walrus/build.rs rename {tools => offchain/tools}/storage-walrus/src/client.rs (100%) rename {tools => offchain/tools}/storage-walrus/src/main.rs (100%) rename {tools => offchain/tools}/storage-walrus/src/read_file.rs (98%) rename {tools => offchain/tools}/storage-walrus/src/read_json.rs (99%) rename {tools => offchain/tools}/storage-walrus/src/upload_file.rs (99%) rename {tools => offchain/tools}/storage-walrus/src/upload_json.rs (99%) rename {tools => offchain/tools}/storage-walrus/src/utils.rs (100%) rename {tools => offchain/tools}/storage-walrus/src/verify_blob.rs (98%) create mode 100644 offchain/tools/storage-walrus/tools.json rename {tools => offchain/tools}/templating-jinja/Cargo.toml (76%) rename {tools => offchain/tools}/templating-jinja/README.md (100%) create mode 100644 offchain/tools/templating-jinja/build.rs rename {tools => offchain/tools}/templating-jinja/src/main.rs (100%) rename {tools => offchain/tools}/templating-jinja/src/template.rs (99%) create mode 100644 offchain/tools/templating-jinja/tools.json create mode 100644 onchain/README.md diff --git a/.github/actions/coverage/action.yml b/.github/actions/coverage/action.yml index efadc06..a7894ca 100644 --- a/.github/actions/coverage/action.yml +++ b/.github/actions/coverage/action.yml @@ -59,8 +59,9 @@ runs: - name: Generate Coverage shell: bash + working-directory: offchain run: | - cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + cargo llvm-cov --all-features --workspace --lcov --output-path ../lcov.info env: CARGO_TERM_COLOR: always CACHE_ON_FAILURE: true diff --git a/.github/actions/gcp-auth-infra/action.yml b/.github/actions/gcp-auth-infra/action.yml new file mode 100644 index 0000000..9f47415 --- /dev/null +++ b/.github/actions/gcp-auth-infra/action.yml @@ -0,0 +1,37 @@ +name: Authenticate to GCP (infra project) +description: Authenticate via workload identity to the GCP project that owns GCR + shared images +inputs: + project-id: + description: 'GCP infra project ID' + required: true + project-number: + description: 'GCP infra project number' + required: true + provider-name: + description: 'Workload identity pool provider name (e.g. nexus-tools)' + required: true +outputs: + auth_token: + description: 'OAuth2 access token for GCR login' + value: ${{ steps.auth.outputs.auth_token }} +runs: + using: composite + steps: + - name: Authenticate + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ inputs.project-id }} + workload_identity_provider: "projects/${{ inputs.project-number }}/locations/global/workloadIdentityPools/${{ inputs.project-id }}/providers/${{ inputs.provider-name }}" + # No service_account / token_format — use Direct WIF. + # google-github-actions/auth exposes the federated token via + # `auth_token` output, which docker/login-action consumes below. + # `token_format: 'access_token'` would require a service_account + # to impersonate, which we don't have for this WIF setup. + + - name: Login to GCR + uses: docker/login-action@v3 + with: + registry: gcr.io + username: oauth2accesstoken + password: ${{ steps.auth.outputs.auth_token }} diff --git a/.github/actions/gcp-auth-protocol/action.yml b/.github/actions/gcp-auth-protocol/action.yml new file mode 100644 index 0000000..c161081 --- /dev/null +++ b/.github/actions/gcp-auth-protocol/action.yml @@ -0,0 +1,33 @@ +name: Authenticate to GCP (protocol project) +description: Authenticate via workload identity to the GCP project that owns GCS + Secret Manager +inputs: + project-id: + description: 'GCP project ID' + required: true + project-number: + description: 'GCP project number' + required: true + provider-name: + description: 'Workload identity pool provider name (e.g. nexus-tools)' + required: true +outputs: + credentials_file_path: + description: 'Path to the generated credentials file' + value: ${{ steps.auth.outputs.credentials_file_path }} +runs: + using: composite + steps: + - name: Authenticate + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ inputs.project-id }} + workload_identity_provider: "projects/${{ inputs.project-number }}/locations/global/workloadIdentityPools/${{ inputs.project-id }}/providers/${{ inputs.provider-name }}" + + - name: Setup Python (gcloud needs it) + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup Cloud SDK + uses: google-github-actions/setup-gcloud@v2 diff --git a/.github/actions/install-nexus-cli/action.yml b/.github/actions/install-nexus-cli/action.yml new file mode 100644 index 0000000..bfe2227 --- /dev/null +++ b/.github/actions/install-nexus-cli/action.yml @@ -0,0 +1,29 @@ +name: Install Nexus CLI +description: Extract the nexus CLI from the nexus-sdk shell image +inputs: + nexus-tag: + description: 'Tag of gcr.io/.../nexus-sdk/shell to extract from' + required: true + infra-registry: + description: 'Infra GCR registry prefix (e.g. gcr.io/production-tf-talus-infra)' + required: true +runs: + using: composite + steps: + - name: Extract nexus binary + shell: bash + env: + NEXUS_TAG: ${{ inputs.nexus-tag }} + INFRA_REGISTRY: ${{ inputs.infra-registry }} + run: | + set -euo pipefail + mkdir -p "$HOME/.local/bin" + CONTAINER_ID=$(docker create "${INFRA_REGISTRY}/nexus-sdk/shell:${NEXUS_TAG}") + docker cp "${CONTAINER_ID}:/usr/local/bin/nexus" "$HOME/.local/bin/nexus" + docker rm "${CONTAINER_ID}" + chmod +x "$HOME/.local/bin/nexus" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Verify + shell: bash + run: nexus --version diff --git a/.github/actions/install-sui/action.yml b/.github/actions/install-sui/action.yml new file mode 100644 index 0000000..08cc1ca --- /dev/null +++ b/.github/actions/install-sui/action.yml @@ -0,0 +1,45 @@ +name: Install Sui CLI +description: Install the Sui CLI from suiup, with cache support +inputs: + sui-channel: + description: 'Sui release channel (e.g. testnet, mainnet)' + required: true + cache-version: + description: 'Bump to invalidate the cache' + required: false + default: 'v1' +runs: + using: composite + steps: + - name: Cache Sui binary + id: cache + uses: actions/cache@v4 + with: + path: ~/.local/bin/sui + key: sui-${{ runner.os }}-${{ inputs.sui-channel }}-${{ inputs.cache-version }} + + - name: Install suiup and Sui CLI + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + SUI_CHANNEL: ${{ inputs.sui-channel }} + run: | + set -euo pipefail + install_dir="$HOME/.local/bin" + temp_dir="$(mktemp -d)" + mkdir -p "$install_dir" + curl --retry 5 --retry-all-errors -sSfL \ + -o "$temp_dir/suiup.tar.gz" \ + https://github.com/MystenLabs/suiup/releases/latest/download/suiup-Linux-musl-x86_64.tar.gz + tar -xzf "$temp_dir/suiup.tar.gz" -C "$temp_dir" + install -m 0755 "$temp_dir/suiup" "$install_dir/suiup" + "$install_dir/suiup" install "sui@$SUI_CHANNEL" -y + + - name: Add Sui to PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Verify + shell: bash + run: sui --version diff --git a/.github/actions/pre-commit/dependencies/action.yml b/.github/actions/pre-commit/dependencies/action.yml index 469ce99..d862cc1 100644 --- a/.github/actions/pre-commit/dependencies/action.yml +++ b/.github/actions/pre-commit/dependencies/action.yml @@ -20,4 +20,4 @@ runs: - name: Install NPM stuff from helpers/npm-install-g.txt shell: bash - run: ./helpers/npm-install-g.sh + run: ./offchain/helpers/npm-install-g.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..53c7529 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,252 @@ +name: CI + +on: + workflow_dispatch: + inputs: + pr-number: + description: 'PR number to publish for (chain ops fire against the PR HEAD)' + required: false + type: string + target-env: + description: 'Deploy the dispatched ref directly against this env. For iteration on the deploy pipeline without merging to main. Leave empty if using pr-number.' + required: false + type: choice + default: '' + options: + - '' + - testnet + - mainnet + pull_request: + types: [opened, synchronize, reopened] + push: + branches: + - main + - 'iterate/testnet/**' + - 'iterate/mainnet/**' + # Tag-driven deploy. The tag prefix selects the env: + # testnet-* → full pipeline against testnet + # mainnet-* → full pipeline against mainnet + # Tags are immutable per the convention, so each release-style + # promotion lands on a specific commit and the deploy run is + # attributable to that exact ref. No merge-strategy gotchas, no + # ephemeral promote/* branches, no long-lived deploy branches to + # keep clean. + tags: + - 'testnet-*' + - 'mainnet-*' + +permissions: + contents: write + packages: write + id-token: write + pull-requests: write + checks: write + +# Serialize chain-touching runs per target network so two pushes can't race on +# wallet consolidation, register PTBs, or terraform apply locks. Different +# networks (testnet vs mainnet) run in parallel; PRs (dry-run only) are keyed +# per-PR; push-to-main (image builds only, no chain ops) is keyed per-ref. +# `cancel-in-progress: false` queues subsequent runs instead of cancelling — +# we never want to abandon a half-finished register/apply sequence. +concurrency: + group: >- + ${{ + (startsWith(github.ref_name, 'mainnet-') || startsWith(github.ref_name, 'iterate/mainnet/')) && 'chain-mainnet' || + (startsWith(github.ref_name, 'testnet-') || startsWith(github.ref_name, 'iterate/testnet/')) && 'chain-testnet' || + (github.event_name == 'workflow_dispatch' && inputs.target-env != '') && format('chain-{0}', inputs.target-env) || + github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || + format('ref-{0}', github.ref) + }} + cancel-in-progress: false + +jobs: + # ── Resolve target ref for workflow_dispatch with PR# ──────── + resolve-target: + if: github.event_name == 'workflow_dispatch' && inputs.pr-number != '' + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.r.outputs.ref }} + head-sha: ${{ steps.r.outputs.head-sha }} + base-ref: ${{ steps.r.outputs.base-ref }} + steps: + - name: Resolve + id: r + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + BASE=$(gh api "repos/${{ github.repository }}/pulls/${{ inputs.pr-number }}" --jq '.base.ref') + HEAD_REF=$(gh api "repos/${{ github.repository }}/pulls/${{ inputs.pr-number }}" --jq '.head.ref') + HEAD_SHA=$(gh api "repos/${{ github.repository }}/pulls/${{ inputs.pr-number }}" --jq '.head.sha') + echo "base-ref=$BASE" >> "$GITHUB_OUTPUT" + echo "ref=$HEAD_REF" >> "$GITHUB_OUTPUT" + echo "head-sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + # ── Discover tools (always) ────────────────────────────────── + discover: + uses: ./.github/workflows/offchain-tools.discover.yml + with: + base-ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }} + + # target-ref passed to callables. The callables' `environment:` expression + # treats this string as the env hint: it resolves to 'mainnet' when this + # evaluates to 'mainnet', else 'testnet'. Precedence: + # 1. PR# dispatch → PR base branch (testnet/mainnet) + # 2. dispatch with target-env input → that env literal + # 3. push to iterate//... → that env literal + # 4. PR event → PR base + # 5. push to a deploy branch → branch name (main/testnet/mainnet) + + # ── Build / push ───────────────────────────────────────────── + # PR (any base): dry-run on changed matrix. + # Push main: push on changed matrix (no chain ops). + # Push iterate//** or tag testnet-*/mainnet-*: push on all matrix. + # workflow_dispatch with PR# or target-env: push on all matrix. + deploy: + needs: [discover, resolve-target] + if: always() && !cancelled() && needs.discover.result == 'success' + uses: ./.github/workflows/offchain-tools.deploy.yml + with: + target-ref: >- + ${{ + needs.resolve-target.outputs.base-ref || + inputs.target-env || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'mainnet-') && 'mainnet') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'testnet-') && 'testnet') || + (startsWith(github.ref_name, 'iterate/mainnet/') && 'mainnet') || + (startsWith(github.ref_name, 'iterate/testnet/') && 'testnet') || + (github.event_name == 'pull_request' && github.base_ref) || + github.ref_name + }} + matrix-json: >- + ${{ + (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'main')) + && needs.discover.outputs.matrix-changed + || needs.discover.outputs.matrix-all + }} + dry-run: ${{ github.event_name == 'pull_request' }} + secrets: inherit + + # ── Prepare ────────────────────────────────────────────────── + # Fires on: + # - push to iterate//** branches + # - push of testnet-* / mainnet-* tags + # - workflow_dispatch with pr-number OR target-env + prepare: + needs: [discover, deploy, resolve-target] + if: >- + always() && !cancelled() && needs.deploy.result == 'success' && + ((github.event_name == 'workflow_dispatch' && (inputs.pr-number != '' || inputs.target-env != '')) || + (github.event_name == 'push' && github.ref_type == 'branch' && ( + startsWith(github.ref_name, 'iterate/testnet/') || + startsWith(github.ref_name, 'iterate/mainnet/') + )) || + (github.event_name == 'push' && github.ref_type == 'tag' && ( + startsWith(github.ref_name, 'testnet-') || + startsWith(github.ref_name, 'mainnet-') + ))) + uses: ./.github/workflows/offchain-tools.prepare.yml + with: + target-ref: >- + ${{ + needs.resolve-target.outputs.base-ref || + inputs.target-env || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'mainnet-') && 'mainnet') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'testnet-') && 'testnet') || + (startsWith(github.ref_name, 'iterate/mainnet/') && 'mainnet') || + (startsWith(github.ref_name, 'iterate/testnet/') && 'testnet') || + (github.event_name == 'pull_request' && github.base_ref) || + github.ref_name + }} + matrix-json: ${{ needs.discover.outputs.matrix-all }} + secrets: inherit + + # ── Register (after prepare) ──────────────────────────────── + register: + needs: [prepare, resolve-target] + if: always() && !cancelled() && needs.prepare.result == 'success' + uses: ./.github/workflows/offchain-tools.register.yml + with: + target-ref: >- + ${{ + needs.resolve-target.outputs.base-ref || + inputs.target-env || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'mainnet-') && 'mainnet') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'testnet-') && 'testnet') || + (startsWith(github.ref_name, 'iterate/mainnet/') && 'mainnet') || + (startsWith(github.ref_name, 'iterate/testnet/') && 'testnet') || + (github.event_name == 'pull_request' && github.base_ref) || + github.ref_name + }} + secrets: inherit + + # ── Readiness (PRs targeting testnet/mainnet only) ────────── + readiness: + needs: discover + if: >- + github.event_name == 'pull_request' && + (github.base_ref == 'testnet' || github.base_ref == 'mainnet') && + fromJson(needs.discover.outputs.matrix-changed).include[0] != null + uses: ./.github/workflows/offchain-tools.readiness.yml + with: + matrix-json: ${{ needs.discover.outputs.matrix-changed }} + secrets: inherit + + # ── Trigger tf-nexus-tools apply (reconcile Cloud Run from GCS state) ── + # After register has written the new tool's config blob + on-chain entry, + # kick the terraform stack so Cloud Run services / ALB / DNS records get + # materialized. We wait for the dispatched run to complete and inherit + # its conclusion, so register-then-apply is one atomic gate. Uses + # GH_ACCESS_TOKEN PAT because GITHUB_TOKEN can't dispatch in other repos. + trigger-tf-apply: + needs: [register] + if: always() && !cancelled() && needs.register.result == 'success' + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + steps: + - name: Require GH_ACCESS_TOKEN + run: | + if [ -z "${GH_TOKEN:-}" ]; then + echo "::error::GH_ACCESS_TOKEN secret is not configured on this repo (or not scoped to it)." + exit 1 + fi + + # Polls actions:read endpoints only — works with fine-grained PATs. + # `gh run watch` would also fetch check annotations (checks:read), + # which fine-grained PATs cannot carry. ava-game uses the same + # action for the same reason. + - name: Dispatch tf-nexus-tools terraform apply and wait + uses: the-actions-org/workflow-dispatch@v4.0.0 + with: + repo: Talus-Network/tf-nexus-tools + ref: production + workflow: Terraform + token: ${{ secrets.GH_ACCESS_TOKEN }} + inputs: '{}' + + # ── ci-gate: single required status check for branch protection ── + ci-gate: + if: always() + needs: [discover, deploy, prepare, register, readiness, trigger-tf-apply] + runs-on: ubuntu-latest + steps: + - name: Check job results + run: | + set -euo pipefail + declare -A r=( + [discover]="${{ needs.discover.result }}" + [deploy]="${{ needs.deploy.result }}" + [prepare]="${{ needs.prepare.result }}" + [register]="${{ needs.register.result }}" + [readiness]="${{ needs.readiness.result }}" + [trigger-tf-apply]="${{ needs.trigger-tf-apply.result }}" + ) + for j in "${!r[@]}"; do + v="${r[$j]}" + case "$v" in + success|skipped) echo "✓ $j: $v" ;; + *) echo "✗ $j: $v"; FAIL=1 ;; + esac + done + [ -z "${FAIL:-}" ] || exit 1 diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 02162ca..9f37020 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -11,7 +11,9 @@ jobs: uses: ./.github/workflows/detect_changes.yml with: files: | - tools/** + offchain/tools/** + offchain/Cargo.toml + offchain/Cargo.lock update-baseline: runs-on: ubuntu-latest diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index ab7427b..b6a2307 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -9,7 +9,9 @@ jobs: uses: ./.github/workflows/detect_changes.yml with: files: | - tools/** + offchain/tools/** + offchain/Cargo.toml + offchain/Cargo.lock coverage: runs-on: ubuntu-latest needs: detect-changes diff --git a/.github/workflows/offchain-tools.deploy.yml b/.github/workflows/offchain-tools.deploy.yml new file mode 100644 index 0000000..c3aae37 --- /dev/null +++ b/.github/workflows/offchain-tools.deploy.yml @@ -0,0 +1,112 @@ +name: Offchain Tools Deploy + +on: + workflow_call: + inputs: + target-ref: + required: false + type: string + matrix-json: + required: true + type: string + dry-run: + required: false + type: boolean + default: false + +permissions: + contents: read + packages: write + id-token: write + +jobs: + build: + name: "Build ${{ matrix.tool }} v${{ matrix.version }}" + runs-on: ubuntu-latest + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + strategy: + matrix: ${{ fromJson(inputs.matrix-json) }} + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute short SHA + id: sha + run: echo "short=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Compute image tags + id: tags + run: | + set -euo pipefail + SHA=${{ steps.sha.outputs.short }} + GHCR="ghcr.io/${{ github.repository }}/${{ matrix.tool }}:sha-${SHA}" + { + echo "list<> "$GITHUB_OUTPUT" + + # Build first, no auth required. The Rust compile is the slow part — + # doing auth before this risks the GCP access token (1h lifetime) + # expiring before the push window. Same pattern as ava-game's + # protocol.deploy.yml. + - name: Build (no push) + uses: docker/build-push-action@v6 + with: + context: . + file: ./offchain/Dockerfile + platforms: linux/amd64 + push: false + build-args: | + BINARY=${{ matrix.command }} + TOOL_DIR=${{ matrix.dir }} + TOOL_FQN_VERSION=${{ matrix.version }} + cache-from: type=gha,scope=offchain-${{ matrix.tool }} + cache-to: type=gha,mode=max,scope=offchain-${{ matrix.tool }} + + - name: Login to GHCR + if: ${{ !inputs.dry-run }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Authenticate to GCP (infra) + if: ${{ !inputs.dry-run }} + id: auth-infra + uses: ./.github/actions/gcp-auth-infra + with: + project-id: ${{ vars.GCP_INFRA_PROJECT_ID }} + project-number: ${{ vars.GCP_INFRA_PROJECT_NUMBER }} + provider-name: nexus-tools + + # Push pulls the layers we just built from the gha cache and emits + # them under the resolved tags. Token freshly minted in the step + # above, so this completes well within the 1h lifetime. + - name: Push + if: ${{ !inputs.dry-run }} + uses: docker/build-push-action@v6 + with: + context: . + file: ./offchain/Dockerfile + platforms: linux/amd64 + push: true + build-args: | + BINARY=${{ matrix.command }} + TOOL_DIR=${{ matrix.dir }} + TOOL_FQN_VERSION=${{ matrix.version }} + tags: ${{ steps.tags.outputs.list }} + cache-from: type=gha,scope=offchain-${{ matrix.tool }} + cache-to: type=gha,mode=max,scope=offchain-${{ matrix.tool }} diff --git a/.github/workflows/offchain-tools.discover.yml b/.github/workflows/offchain-tools.discover.yml new file mode 100644 index 0000000..fa6bb10 --- /dev/null +++ b/.github/workflows/offchain-tools.discover.yml @@ -0,0 +1,151 @@ +name: Offchain Tools Discover + +on: + workflow_call: + inputs: + base-ref: + description: 'Base ref to compute changed-files diff against' + required: false + type: string + default: '' + outputs: + matrix-all: + description: 'JSON matrix of every discovered tool' + value: ${{ jobs.discover.outputs.matrix-all }} + matrix-changed: + description: 'JSON matrix of tools whose subtree changed since base-ref' + value: ${{ jobs.discover.outputs.matrix-changed }} + content-hash: + description: 'Aggregate hash of all tools.json + their subtree versions' + value: ${{ jobs.discover.outputs.content-hash }} + +jobs: + discover: + name: Discover offchain tools + runs-on: ubuntu-latest + outputs: + matrix-all: ${{ steps.build.outputs.matrix-all }} + matrix-changed: ${{ steps.build.outputs.matrix-changed }} + content-hash: ${{ steps.build.outputs.content-hash }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed files + id: changes + if: inputs.base-ref != '' + uses: tj-actions/changed-files@v46 + with: + base_sha: ${{ inputs.base-ref }} + files: | + offchain/tools/** + offchain/Cargo.toml + offchain/Cargo.lock + offchain/Dockerfile + offchain/rust-toolchain.toml + + - name: Build matrices + id: build + shell: bash + env: + CHANGED_FILES: ${{ steps.changes.outputs.all_changed_files }} + # Space-separated list of tool_name values to skip. Sourced + # from the repo-level `BLOCKED_TOOLS` variable so security + # blocks can be flipped via repo settings without a PR. + # Unset = no tools blocked. + BLOCKED_TOOLS: ${{ vars.BLOCKED_TOOLS }} + run: | + set -euo pipefail + ALL='[]' + CHANGED='[]' + HASH_INPUT="" + + # Shared files force every tool into the "changed" set. + SHARED_TOUCHED=false + if [ -n "${CHANGED_FILES:-}" ]; then + for f in $CHANGED_FILES; do + case "$f" in + offchain/Cargo.toml|offchain/Cargo.lock|offchain/Dockerfile|offchain/rust-toolchain.toml) + SHARED_TOUCHED=true + ;; + esac + done + fi + echo "Shared files touched: $SHARED_TOUCHED" + + # Tools blocked from deployment are sourced from the repo + # variable `BLOCKED_TOOLS` (space-separated tool_name list). + # Discover excludes them from both matrices, which prevents + # the entire downstream pipeline (deploy → prepare → register + # → tf-apply) from touching them. Flip the variable in repo + # settings → Variables to unblock (no PR needed). + # + # Current blocks (2026-05-21): + # http — no SSRF guard; would let a DAG reach the GCP + # metadata server and exfiltrate an SA OAuth token. + # Unregistered on chain; re-enable after the guard + # ships in offchain/tools/http/. + read -ra BLOCKED_TOOLS_ARR <<< "${BLOCKED_TOOLS:-}" + echo "Blocked tools: ${BLOCKED_TOOLS_ARR[*]:-}" + + for TOOLS_JSON in offchain/tools/*/tools.json; do + [ -f "$TOOLS_JSON" ] || continue + DIR=$(dirname "$TOOLS_JSON") + TOOL_NAME=$(jq -r '.tool_name' "$TOOLS_JSON") + COMMAND=$(jq -r '.command' "$TOOLS_JSON") + + SKIP=false + for blocked in "${BLOCKED_TOOLS_ARR[@]}"; do + if [ "$TOOL_NAME" = "$blocked" ]; then + echo "::warning::Skipping blocked tool: $TOOL_NAME (in BLOCKED_TOOLS repo variable)" + SKIP=true + break + fi + done + [ "$SKIP" = "true" ] && continue + + TREE_HASH=$(git rev-parse "HEAD:${DIR}/") + CONTENT_HASH=$(printf '%s' "$TREE_HASH" | sha256sum | cut -d' ' -f1) + VERSION=$(printf '%d' "0x${CONTENT_HASH:0:8}") + + ENTRY=$(jq -nc \ + --arg tool "$TOOL_NAME" \ + --arg dir "$DIR" \ + --arg cmd "$COMMAND" \ + --arg ver "$VERSION" \ + '{tool: $tool, dir: $dir, command: $cmd, version: $ver}') + + ALL=$(echo "$ALL" | jq -c ". + [${ENTRY}]") + + # Is this tool changed? + TOOL_CHANGED=false + if [ "$SHARED_TOUCHED" = "true" ] || [ -z "${CHANGED_FILES:-}" ]; then + TOOL_CHANGED=true + else + for f in $CHANGED_FILES; do + case "$f" in + "${DIR}/"*) TOOL_CHANGED=true ;; + esac + done + fi + if [ "$TOOL_CHANGED" = "true" ]; then + CHANGED=$(echo "$CHANGED" | jq -c ". + [${ENTRY}]") + fi + + HASH_INPUT="${HASH_INPUT}${TOOL_NAME}:${VERSION}\n" + echo " - ${TOOL_NAME} v${VERSION} (changed=${TOOL_CHANGED})" + done + + CONTENT_HASH=$(printf '%s' "$HASH_INPUT" | sha256sum | cut -c1-16) + echo "matrix-all={\"include\":${ALL}}" >> "$GITHUB_OUTPUT" + echo "matrix-changed={\"include\":${CHANGED}}" >> "$GITHUB_OUTPUT" + echo "content-hash=${CONTENT_HASH}" >> "$GITHUB_OUTPUT" + + echo "::group::matrix-all" + echo "$ALL" | jq . + echo "::endgroup::" + echo "::group::matrix-changed" + echo "$CHANGED" | jq . + echo "::endgroup::" diff --git a/.github/workflows/offchain-tools.prepare.yml b/.github/workflows/offchain-tools.prepare.yml new file mode 100644 index 0000000..ddd3af5 --- /dev/null +++ b/.github/workflows/offchain-tools.prepare.yml @@ -0,0 +1,262 @@ +name: Offchain Tools Prepare + +on: + workflow_call: + inputs: + target-ref: + required: false + type: string + matrix-json: + required: true + type: string + +permissions: + contents: read + packages: read + id-token: write + +jobs: + prepare: + name: "Prepare ${{ matrix.tool }} v${{ matrix.version }}" + runs-on: ubuntu-latest + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + strategy: + matrix: ${{ fromJson(inputs.matrix-json) }} + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute short SHA + versioned name + id: names + run: | + SHA=${GITHUB_SHA::7} + echo "sha=${SHA}" >> "$GITHUB_OUTPUT" + echo "versioned=${{ matrix.tool }}-v${{ matrix.version }}" >> "$GITHUB_OUTPUT" + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull image + run: | + IMAGE="ghcr.io/${{ github.repository }}/${{ matrix.tool }}:sha-${{ steps.names.outputs.sha }}" + IMAGE="${IMAGE,,}" + docker pull "$IMAGE" + echo "IMAGE=$IMAGE" >> "$GITHUB_ENV" + + - name: Extract --meta + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/prepare/meta" + docker run --rm --entrypoint /usr/local/bin/${{ matrix.command }} \ + "$IMAGE" --meta \ + > "${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json" + echo "::group::Meta for ${{ matrix.tool }}" + jq '.[].fqn' "${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json" + echo "::endgroup::" + + - name: Render Cloud Run tool config + id: render + run: | + set -euo pipefail + DIR="${RUNNER_TEMP}/prepare/gcs/tools" + mkdir -p "$DIR" + VERSIONED="${{ steps.names.outputs.versioned }}" + # matrix.dir is the directory path (e.g. offchain/tools/storage-walrus). + TOOLS_JSON="${{ matrix.dir }}/tools.json" + EXTRA_ENV=$(jq -c '.environment // {}' "$TOOLS_JSON") + RESOURCES=$(jq -c '.resources // {}' "$TOOLS_JSON") + # nexus_contracts_tag is captured at prepare time and persisted into + # the per-tool config blob, so the (tool, version) pair is pinned to + # whichever nexus version was current when it was first prepared. + # Future NEXUS_TAG bumps only affect NEW deployments; existing + # services keep their original allowed-leaders mount path. + jq -n \ + --arg name "$VERSIONED" \ + --arg network "${{ vars.SUI_NETWORK }}" \ + --arg image "nexus-tools/${{ matrix.tool }}" \ + --arg tag "sha-${{ steps.names.outputs.sha }}" \ + --arg cmd "${{ matrix.command }}" \ + --arg nexus_tag "${{ vars.NEXUS_TAG }}" \ + --argjson extra_env "$EXTRA_ENV" \ + --argjson resources "$RESOURCES" \ + '{ + name: $name, + network: $network, + image: $image, + tag: $tag, + nexus_contracts_tag: $nexus_tag, + replicas: 1, + environment: ({ + RUST_LOG: "info", + BIND_ADDR: "0.0.0.0:8080", + NEXUS_TOOLKIT_CONFIG_PATH: "/app/secrets/toolkit-config.json" + } + $extra_env), + command: $cmd, + ports: [{containerPort: 8080}], + signed_http: {enabled: true}, + resources: $resources + }' > "$DIR/${VERSIONED}.json" + echo "config=$DIR/${VERSIONED}.json" >> "$GITHUB_OUTPUT" + + - name: Authenticate to GCP (protocol) + id: auth-gcs + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Authenticate to GCP (infra) + id: auth-infra + uses: ./.github/actions/gcp-auth-infra + with: + project-id: ${{ vars.GCP_INFRA_PROJECT_ID }} + project-number: ${{ vars.GCP_INFRA_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Generate signed HTTP keys (idempotent) + run: | + set -euo pipefail + VERSIONED="${{ steps.names.outputs.versioned }}" + PROJECT="${{ vars.GCP_PROJECT_ID }}" + META="${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json" + + FQNS=$(jq -r '.[].fqn' "$META" | tr '\n' ' ' | xargs) + echo "FQNs for $VERSIONED: $FQNS" + + FORCE_FLAG="" + EXISTING=$(gcloud secrets versions access latest \ + --secret="nexus-tools-${VERSIONED}-signed-http-keys" \ + --project="$PROJECT" 2>/dev/null || echo "") + if [ -n "$EXISTING" ]; then + EXISTING_FQNS=$(echo "$EXISTING" | jq -r '[.keys_registry[].fqn] | sort | join(" ")') + WANTED_FQNS=$(echo "$FQNS" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs) + if [ "$EXISTING_FQNS" != "$WANTED_FQNS" ]; then + echo "FQNs changed: '$EXISTING_FQNS' -> '$WANTED_FQNS' — forcing new keys" + FORCE_FLAG="--force" + else + echo "FQNs unchanged — skipping keygen" + exit 0 + fi + fi + + docker run --rm \ + -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/creds.json \ + -v "${{ steps.auth-gcs.outputs.credentials_file_path }}:/tmp/creds.json:ro" \ + gcr.io/${{ vars.GCP_INFRA_PROJECT_ID }}/nexus-next/generate-signed-http-keys:latest \ + python /app/bin/generate_signed_http_keys.py "nexus-tools" "$VERSIONED" \ + --fqns $FQNS \ + --project "$PROJECT" $FORCE_FLAG + + - name: Upload tool config to GCS + run: | + set -euo pipefail + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + VERSIONED="${{ steps.names.outputs.versioned }}" + LOCAL="${{ steps.render.outputs.config }}" + REMOTE="gs://${BUCKET}/${NETWORK}/offchain/tools/${VERSIONED}.json" + + # Preserve existing image tag pinning (only NEW versions adopt build SHA). + EXISTING_TAG=$(gsutil cat "$REMOTE" 2>/dev/null | jq -r '.tag // empty' || true) + if [ -n "$EXISTING_TAG" ]; then + echo "Preserving existing image tag: $EXISTING_TAG" + jq --arg t "$EXISTING_TAG" '.tag = $t' "$LOCAL" > "${LOCAL}.tmp" + mv "${LOCAL}.tmp" "$LOCAL" + fi + + LOCAL_HASH=$(sha256sum "$LOCAL" | cut -c1-16) + REMOTE_HASH=$(gsutil cat "$REMOTE" 2>/dev/null | sha256sum | cut -c1-16 || echo "") + if [ "$LOCAL_HASH" = "$REMOTE_HASH" ]; then + echo "Tool config unchanged — skipping upload" + else + gsutil cp "$LOCAL" "$REMOTE" + echo "Uploaded $REMOTE" + fi + + - name: Upload per-tool artifact + uses: actions/upload-artifact@v4 + with: + name: prepare-${{ matrix.tool }} + retention-days: 1 + path: | + ${{ runner.temp }}/prepare/meta/${{ matrix.tool }}.json + ${{ runner.temp }}/prepare/gcs/tools/${{ steps.names.outputs.versioned }}.json + + aggregate: + name: Aggregate prepare artifacts + runs-on: ubuntu-latest + needs: prepare + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + steps: + # Required so the local composite action gcp-auth-protocol resolves. + - name: Checkout + uses: actions/checkout@v4 + + - name: Download all prepare artifacts + uses: actions/download-artifact@v4 + with: + pattern: prepare-* + merge-multiple: true + path: ${{ runner.temp }}/prepare + + - name: Build manifest + id: manifest + run: | + set -euo pipefail + PREPARE="${{ runner.temp }}/prepare" + MANIFEST="${PREPARE}/manifest.json" + echo '{}' > "$MANIFEST" + MATRIX='${{ inputs.matrix-json }}' + echo "$MATRIX" | jq -c '.include[]' | while read -r entry; do + TOOL=$(echo "$entry" | jq -r '.tool') + VERSION=$(echo "$entry" | jq -r '.version') + COMMAND=$(echo "$entry" | jq -r '.command') + jq --arg t "$TOOL" --arg v "$VERSION" --arg c "$COMMAND" \ + '.[$t] = {command: $c, version: $v}' \ + "$MANIFEST" > "${MANIFEST}.tmp" && mv "${MANIFEST}.tmp" "$MANIFEST" + done + jq . "$MANIFEST" + HASH=$(sha256sum "$MANIFEST" | cut -c1-16) + echo "hash=$HASH" >> "$GITHUB_OUTPUT" + + - name: Authenticate to GCP (protocol) + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Upload manifest to GCS (content-addressed) + run: | + set -euo pipefail + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + HASH="${{ steps.manifest.outputs.hash }}" + REMOTE="gs://${BUCKET}/${NETWORK}/offchain/manifest/${HASH}.json" + if gsutil -q stat "$REMOTE" 2>/dev/null; then + echo "Manifest ${HASH}.json already exists — skipping" + else + gsutil cp "${{ runner.temp }}/prepare/manifest.json" "$REMOTE" + echo "Uploaded $REMOTE" + fi + + - name: Upload bundled prepare artifact + uses: actions/upload-artifact@v4 + with: + name: offchain-tools-prepare + retention-days: 1 + path: ${{ runner.temp }}/prepare/ diff --git a/.github/workflows/offchain-tools.readiness.yml b/.github/workflows/offchain-tools.readiness.yml new file mode 100644 index 0000000..c805f24 --- /dev/null +++ b/.github/workflows/offchain-tools.readiness.yml @@ -0,0 +1,66 @@ +name: Offchain Tools Readiness + +on: + workflow_call: + inputs: + matrix-json: + required: true + type: string + +permissions: + contents: read + id-token: write + +jobs: + check: + name: "Readiness ${{ matrix.tool }} v${{ matrix.version }}" + runs-on: ubuntu-latest + environment: >- + ${{ + (github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + strategy: + matrix: ${{ fromJson(inputs.matrix-json) }} + fail-fast: false + steps: + # Required so the local composite action gcp-auth-protocol resolves. + - name: Checkout + uses: actions/checkout@v4 + + - name: Authenticate to GCP (protocol) + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Check GCS artifacts + run: | + set -euo pipefail + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + TOOL="${{ matrix.tool }}" + VERSION="${{ matrix.version }}" + VERSIONED="${TOOL}-v${VERSION}" + + MISSING=() + + CFG="gs://${BUCKET}/${NETWORK}/offchain/tools/${VERSIONED}.json" + gsutil -q stat "$CFG" 2>/dev/null || MISSING+=("Cloud Run config: $CFG") + + REG_DIR="gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/" + REG_COUNT=$(gsutil ls "$REG_DIR" 2>/dev/null | wc -l || echo 0) + if [ "$REG_COUNT" = "0" ]; then + MISSING+=("FQN registrations under $REG_DIR") + fi + + if [ ${#MISSING[@]} -gt 0 ]; then + echo "::error::Missing artifacts for ${VERSIONED}:" + for m in "${MISSING[@]}"; do echo " - $m"; done + echo "" + echo "Trigger the Offchain Tools publish workflow with this PR number to seed them." + exit 1 + fi + + echo "✓ All artifacts present for ${VERSIONED}" diff --git a/.github/workflows/offchain-tools.register.yml b/.github/workflows/offchain-tools.register.yml new file mode 100644 index 0000000..f59e8cd --- /dev/null +++ b/.github/workflows/offchain-tools.register.yml @@ -0,0 +1,409 @@ +name: Offchain Tools Register + +on: + workflow_call: + inputs: + target-ref: + required: false + type: string + +permissions: + contents: read + id-token: write + +jobs: + register: + name: Register offchain tools + runs-on: ubuntu-latest + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Sui CLI + uses: ./.github/actions/install-sui + with: + sui-channel: ${{ vars.SUI_CHANNEL }} + cache-version: ${{ vars.SUI_CACHE_VERSION }} + + - name: Download prepare artifacts + uses: actions/download-artifact@v4 + with: + name: offchain-tools-prepare + path: ${{ runner.temp }}/prepare + + - name: Show artifacts + id: artifacts + run: | + PREPARE="${{ runner.temp }}/prepare" + echo "manifest=${PREPARE}/manifest.json" >> "$GITHUB_OUTPUT" + echo "meta_dir=${PREPARE}/meta" >> "$GITHUB_OUTPUT" + jq . "${PREPARE}/manifest.json" + + - name: Authenticate to GCP (protocol) + id: auth-gcs + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Authenticate to GCP (infra) + uses: ./.github/actions/gcp-auth-infra + with: + project-id: ${{ vars.GCP_INFRA_PROJECT_ID }} + project-number: ${{ vars.GCP_INFRA_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Install Nexus CLI + uses: ./.github/actions/install-nexus-cli + with: + nexus-tag: ${{ vars.NEXUS_TAG }} + infra-registry: gcr.io/${{ vars.GCP_INFRA_PROJECT_ID }} + + - name: Configure wallet and Nexus + run: | + set -euo pipefail + yes "" | sui client new-env --alias tool-register --rpc "${{ vars.NEXT_PUBLIC_SUI_RPC_URL }}" || true + sui client switch --env tool-register + + # SUI_DEPLOYER_PK is a `suiprivkey...` bech32 string (sui keytool + # export format). Same secret will drive future onchain Move + # deploys (`sui client publish`). + IMPORT=$(sui keytool import "${{ secrets.SUI_DEPLOYER_PK }}" ed25519) + ADDR=$(echo "$IMPORT" | grep -oE '0x[0-9a-fA-F]{64}') + sui client switch --address "$ADDR" + echo "DEPLOYER_ADDR=$ADDR" >> "$GITHUB_ENV" + + KEY_INDEX=$(sui keytool list --json | jq -r --arg addr "$ADDR" 'to_entries[] | select(.value.suiAddress == $addr) | .key') + SUI_PK=$(jq -r --argjson idx "$KEY_INDEX" '.[$idx]' "$HOME/.sui/sui_config/sui.keystore") + + NETWORK="${{ vars.SUI_NETWORK }}" + wget -q -O "$RUNNER_TEMP/objects.toml" \ + "https://storage.googleapis.com/production-talus-sui-objects/${{ vars.NEXUS_TAG }}/objects.${NETWORK}.toml?t=$(date +%s)" + + mkdir -p "$HOME/.nexus" + cp "$RUNNER_TEMP/objects.toml" "$HOME/.nexus/objects.${NETWORK}.toml" + nexus conf set \ + --nexus.objects "$HOME/.nexus/objects.${NETWORK}.toml" \ + --sui.rpc-url "${{ vars.NEXT_PUBLIC_SUI_RPC_URL }}" \ + --sui.pk "$SUI_PK" + + - name: Consolidate wallet gas coins + shell: bash + run: | + set -euo pipefail + # Merge most coins into one big primary so the register loop + # has enough headroom for budgets up to ~1 SUI per call + # (static deploy needed 1e9 MIST for heavy twitter schemas). + # Leaves a tiny leftover so we still have a distinct + # collateral coin per call. + echo "::group::wallet before consolidation" + sui client gas --json | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + echo "::endgroup::" + + # Target 4 coins so we have headroom for several register + # calls in parallel and across drain. Merge the two smallest + # each iteration; this levels small fragments up rather than + # leaving 1 huge + 3 small (which can't cover the 1 SUI + # budget for heavy schemas). + MAX_MERGES=40 + i=0 + while [ "$(sui client gas --json | jq 'length')" -gt 4 ] && [ "$i" -lt "$MAX_MERGES" ]; do + INFO=$(sui client gas --json) + TO_MERGE=$(echo "$INFO" | jq -r 'sort_by(.mistBalance|tonumber) | .[0].gasCoinId') + PRIMARY=$(echo "$INFO" | jq -r 'sort_by(.mistBalance|tonumber) | .[1].gasCoinId') + echo "merge[$i]: $TO_MERGE -> $PRIMARY" + sui client merge-coin \ + --primary-coin "$PRIMARY" \ + --coin-to-merge "$TO_MERGE" \ + --gas-budget 5000000 >/dev/null + i=$((i+1)) + done + + echo "::group::wallet after consolidation" + sui client gas --json | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + echo "::endgroup::" + + - name: Register offchain tools + shell: bash + run: | + set -euo pipefail + + MANIFEST="${{ steps.artifacts.outputs.manifest }}" + META_DIR="${{ steps.artifacts.outputs.meta_dir }}" + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + + # Snapshot already-registered FQNs once up front. Each + # `nexus tool register` call costs ~1-2s even for an + # already-registered FQN (it submits a tx, the Move call + # aborts with EFqnAlreadyExists, we detect it and preserve). + # Skipping the CLI call entirely for known-registered FQNs + # shortens a typical re-run from minutes to seconds. + REGISTERED_FQNS="${RUNNER_TEMP}/registered_fqns.txt" + nexus tool list --json 2>/dev/null | jq -r '.[].fqn' > "$REGISTERED_FQNS" || true + echo "::group::Already-registered FQNs on chain ($(wc -l < "$REGISTERED_FQNS"))" + cat "$REGISTERED_FQNS" + echo "::endgroup::" + + for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do + META="${META_DIR}/${TOOL}.json" + if [ ! -f "$META" ]; then + echo "::error::Meta file not found: $META" + ls -la "$META_DIR" >&2 || true + exit 1 + fi + VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST") + TOOL_HOST="${NETWORK}-${TOOL}-v${VERSION}.tools.internal" + echo "=== TOOL=$TOOL VERSION=$VERSION HOST=$TOOL_HOST ===" + + # Each tool binary's --meta emits an array of ToolMeta (one per + # FQN). The CLI's --from-meta takes a single ToolMeta on stdin, + # so iterate and pipe each entry. Each FQN has its own URL + # *path* (e.g. /i64/add/) baked into the meta — we keep that + # path and only swap the host from the in-container BIND_ADDR + # to the external internal-DNS hostname. + jq -c '.[]' "$META" | while read -r TOOL_META; do + FQN=$(echo "$TOOL_META" | jq -r '.fqn') + REG_PATH="gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json" + + # Skip the CLI call entirely if the FQN is already + # registered on chain (snapshot taken above). We still + # verify the GCS file is present so downstream + # signing-key / toolkit-config steps don't run blind. + if grep -qxF "$FQN" "$REGISTERED_FQNS"; then + if ! gsutil -q stat "$REG_PATH" 2>/dev/null; then + echo "::error::$FQN registered on-chain but no caps in GCS ($REG_PATH)" + exit 1 + fi + echo " ✓ $FQN already registered — skipping CLI call" + continue + fi + + META_URL=$(echo "$TOOL_META" | jq -r '.url') + # Strip scheme + host:port, keep the path (defaulting to /). + URL_PATH=$(echo "$META_URL" | sed -E 's|^[a-z]+://[^/]*||') + [ -z "$URL_PATH" ] && URL_PATH="/" + # Leader appends `invoke` via RFC 3986 relative-URL join: + # /i64/add/ + invoke = /i64/add/invoke ✓ + # /i64/add + invoke = /invoke ✗ + # Force a trailing slash so the join produces the right + # endpoint, matching the legacy v0.8 static-deploy URLs. + [ "${URL_PATH: -1}" != "/" ] && URL_PATH="${URL_PATH}/" + TOOL_URL="http://${TOOL_HOST}${URL_PATH}" + + # The CLI's gas pool is a single coin picked at startup + # (sui.rs:294 `with_gas(vec![one_coin])`); reusing the same + # coin across many CLI invocations drains it below the + # 100M budget and yields InsufficientGas. Re-query the + # wallet per call and pin the largest coin as gas and + # smallest (distinct) coin as collateral — fresh object + # refs every call, no chance of overlap. + GAS_INFO=$(sui client gas --json) + echo "::group::wallet coins before $FQN" + echo "$GAS_INFO" | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + echo "$GAS_INFO" | jq 'length as $n | (map(.mistBalance|tonumber) | add) as $sum | {coin_count: $n, total_mist: $sum, total_sui: ($sum/1000000000)}' + echo "::endgroup::" + GAS_BUDGET=1000000000 + # Pick any coin with balance ≥ budget — preferring the + # largest, but rotating to a different one if the previous + # call drained the prior-largest below budget. Sui rejects + # at validation (not execution) when the gas coin can't + # cover the budget reservation, so this guard saves us + # from opaque RPC errors and from staying stuck on a + # drained coin while others in the wallet still have + # plenty. + GAS_COIN=$(echo "$GAS_INFO" | jq -r --argjson b "$GAS_BUDGET" \ + 'map(select((.mistBalance|tonumber) >= $b)) + | sort_by(.mistBalance|tonumber) | last.gasCoinId // empty') + if [ -z "$GAS_COIN" ]; then + echo "::error::No coin in wallet has balance ≥ $GAS_BUDGET MIST." + echo "$GAS_INFO" | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + exit 1 + fi + GAS_BAL=$(echo "$GAS_INFO" | jq -r --arg g "$GAS_COIN" '.[] | select(.gasCoinId == $g) | .mistBalance') + # Collateral can be any other coin (split is 1 MIST), so + # take the smallest one not chosen as gas. + COLL_COIN=$(echo "$GAS_INFO" | jq -r --arg g "$GAS_COIN" \ + 'map(select(.gasCoinId != $g)) | sort_by(.mistBalance|tonumber) | first.gasCoinId // empty') + if [ -z "$COLL_COIN" ]; then + echo "::error::Wallet has only one coin; need ≥2 (gas + collateral)" + exit 1 + fi + COLL_BAL=$(echo "$GAS_INFO" | jq -r --arg c "$COLL_COIN" '.[] | select(.gasCoinId == $c) | .mistBalance') + echo " selected gas=$GAS_COIN ($GAS_BAL MIST) collateral=$COLL_COIN ($COLL_BAL MIST) budget=$GAS_BUDGET" + + echo "::group::CLI invocation for $FQN" + echo " url: $TOOL_URL" + echo " tool_meta: $(echo "$TOOL_META" | jq -c '{fqn, url, timeout, description: (.description|.[0:40])}')" + echo "::endgroup::" + + STDERR_LOG="${RUNNER_TEMP}/nexus-register.stderr" + set +e + # Heavy schemas (twitter get-tweet, llm chat-completion) + # need ~1 SUI of gas to cover storage; the proven value + # from the prior static deploy + # (tf-talus-nexus-v2 social-twitter.json:gasBudgetAmount). + # Pre-consolidated wallet coins (above) cover this. + OUT=$(echo "$TOOL_META" \ + | nexus tool register offchain \ + --from-meta - \ + --url "$TOOL_URL" \ + --sui-gas-coin "$GAS_COIN" \ + --collateral-coin "$COLL_COIN" \ + --sui-gas-budget "$GAS_BUDGET" \ + --invocation-cost 0 \ + --no-save \ + --json 2>"$STDERR_LOG") + RC=$? + set -e + echo "::group::CLI stderr for $FQN (rc=$RC)" + cat "$STDERR_LOG" || true + echo "::endgroup::" + echo "::group::CLI stdout for $FQN" + echo "$OUT" + echo "::endgroup::" + if [ $RC -ne 0 ]; then + echo "::error::nexus CLI exited $RC for $FQN" + exit $RC + fi + + ENTRY=$(echo "$OUT" | jq -c '.[0]') + ALREADY=$(echo "$ENTRY" | jq -r '.already_registered // false') + ERR=$(echo "$ENTRY" | jq -r '.error // empty') + + # v1.0.0 CLI flags already-registered by string-matching the + # outer entry name, but the MoveAbort surfaces from the inner + # helper `register_tool_` at instruction 14 (the + # EFqnAlreadyExists assertion in tool_registry.move:594), so + # the CLI misses it. Treat that exact signature as + # already-registered. TODO: fix upstream in nexus-sdk. + if [ "$ALREADY" != "true" ] && [ -n "$ERR" ] \ + && echo "$ERR" | grep -q 'function_name: Some(Identifier("register_tool_"))' \ + && echo "$ERR" | grep -q 'instruction: 14'; then + ALREADY="true" + fi + + if [ "$ALREADY" = "true" ]; then + # Caps must already exist in GCS from a prior run; refusing + # to proceed when they're missing keeps downstream steps + # (signing keys, toolkit-config) from running blind. + if ! gsutil -q stat "$REG_PATH" 2>/dev/null; then + echo "::error::$FQN registered on-chain but no caps in GCS ($REG_PATH)" + exit 1 + fi + echo " ✓ $FQN already registered — caps preserved" + continue + fi + + TOOL_CAP=$(echo "$ENTRY" | jq -r '.owner_cap_over_tool_id // empty') + GAS_CAP=$(echo "$ENTRY" | jq -r '.owner_cap_over_gas_id // empty') + if [ -z "$TOOL_CAP" ] || [ -z "$GAS_CAP" ]; then + echo "::error::register output missing caps for $FQN: $ENTRY" + exit 1 + fi + + # Merge with any existing record so downstream-added fields + # (signing_key_hash, tool_kid) survive re-registration. + EXISTING=$(gsutil cat "$REG_PATH" 2>/dev/null || echo '{}') + echo "$EXISTING" | jq \ + --arg fqn "$FQN" --arg t "$TOOL_CAP" --arg g "$GAS_CAP" \ + '. + {fqn: $fqn, owner_cap_over_tool: $t, owner_cap_over_gas: $g}' \ + | gsutil cp - "$REG_PATH" + echo " + $FQN registered (tool_cap=$TOOL_CAP gas_cap=$GAS_CAP)" + done + done + + - name: Register signing keys + shell: bash + run: | + set -euo pipefail + MANIFEST="${{ steps.artifacts.outputs.manifest }}" + META_DIR="${{ steps.artifacts.outputs.meta_dir }}" + PROJECT="${{ vars.GCP_PROJECT_ID }}" + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + + for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do + VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST") + VERSIONED="${TOOL}-v${VERSION}" + + KEYS=$(gcloud secrets versions access latest \ + --secret="nexus-tools-${VERSIONED}-signed-http-keys" \ + --project="$PROJECT") + KEYS_HASH=$(echo "$KEYS" | jq -c '[.keys_registry[] | {fqn, public_key_hex}]' | sha256sum | cut -c1-16) + + for FQN in $(jq -r '.[].fqn' "${META_DIR}/${TOOL}.json"); do + REG=$(gsutil cat "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json") + OWNER_CAP=$(echo "$REG" | jq -r '.owner_cap_over_tool') + REGISTERED_HASH=$(echo "$REG" | jq -r '.signing_key_hash // ""') + if [ "$REGISTERED_HASH" = "$KEYS_HASH" ]; then + echo " ✓ $FQN signing key unchanged — skipping" + continue + fi + + PRIVATE_KEY=$(echo "$KEYS" | jq -re --arg f "$FQN" '.keys_registry[] | select(.fqn == $f) | .private_key_hex') + REG_OUT=$(nexus tool auth register-key \ + --json \ + --tool-fqn "$FQN" \ + --owner-cap "$OWNER_CAP" \ + --signing-key "$PRIVATE_KEY" \ + --description "ci-managed") + KID=$(echo "$REG_OUT" | jq -r '.tool_kid') + + echo "$REG" | jq --arg h "$KEYS_HASH" --argjson kid "$KID" \ + '. + {signing_key_hash: $h, tool_kid: $kid}' \ + | gsutil cp - "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json" + echo " + $FQN registered with tool_kid=$KID" + done + done + + - name: Reconcile toolkit-config secret + shell: bash + run: | + set -euo pipefail + MANIFEST="${{ steps.artifacts.outputs.manifest }}" + META_DIR="${{ steps.artifacts.outputs.meta_dir }}" + PROJECT="${{ vars.GCP_PROJECT_ID }}" + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + + for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do + VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST") + VERSIONED="${TOOL}-v${VERSION}" + SECRET="nexus-tools-${VERSIONED}-signed-http-toolkit-config" + + gcloud secrets describe "$SECRET" --project="$PROJECT" >/dev/null 2>&1 \ + || gcloud secrets create "$SECRET" --project="$PROJECT" --replication-policy=automatic + + SKELETON=$(gcloud secrets versions access latest \ + --secret="nexus-tools-${VERSIONED}-signed-http-keys" \ + --project="$PROJECT" \ + | jq -c '.toolkit_config') + + DESIRED="$SKELETON" + for FQN in $(jq -r '.[].fqn' "${META_DIR}/${TOOL}.json"); do + REG=$(gsutil cat "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json") + KID=$(echo "$REG" | jq -r '.tool_kid // "MISSING"') + if [ "$KID" = "MISSING" ] || [ "$KID" = "null" ]; then + echo "::error::No tool_kid for $FQN — refusing to write toolkit-config" + exit 1 + fi + DESIRED=$(echo "$DESIRED" | jq -c --arg fqn "$FQN" --argjson kid "$KID" \ + '.signed_http.tools[$fqn].tool_kid = $kid') + done + + CURRENT=$(gcloud secrets versions access latest \ + --secret="$SECRET" --project="$PROJECT" 2>/dev/null | jq -c . 2>/dev/null || echo "") + if [ "$DESIRED" = "$CURRENT" ]; then + echo " ✓ toolkit-config for $VERSIONED unchanged" + else + echo -n "$DESIRED" | gcloud secrets versions add "$SECRET" --project="$PROJECT" --data-file=- + echo " + Wrote new toolkit-config for $VERSIONED" + fi + done diff --git a/.github/workflows/sync_docs.yml b/.github/workflows/sync_docs.yml index cc4f4ef..2c95b4d 100644 --- a/.github/workflows/sync_docs.yml +++ b/.github/workflows/sync_docs.yml @@ -16,7 +16,7 @@ jobs: with: files: | docs/** - tools/*/README.md + offchain/tools/*/README.md sync-docs: runs-on: ubuntu-latest @@ -47,13 +47,11 @@ jobs: cp -r docs/* gitbook-docs/${{ env.REPO_NAME }}/ fi - # Sync README.md files from tools/*/ - for readme in tools/*/README.md; do - if [ -f "$readme" ]; then - tool_name=$(basename $(dirname "$readme")) - mkdir -p "gitbook-docs/tools/$tool_name" - cp "$readme" "gitbook-docs/tools/$tool_name/" - fi + # Sync README.md files from offchain/tools/*/ + for readme in offchain/tools/*/README.md; do + tool_name=$(basename "$(dirname "$readme")") + mkdir -p "gitbook-docs/tools/$tool_name" + cp "$readme" "gitbook-docs/tools/$tool_name/" done # Check if any changes were made diff --git a/.gitignore b/.gitignore index f857cae..f6e8d30 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ sdk/tests/move/onchain_tool_test/Move.lock .DS_Store **/Move.lock + +# Local-only planning artifacts (brainstorming specs, plans, notes) +docs/superpowers/ diff --git a/.pre-commit/.just b/.pre-commit/.just index ef4485d..7ce8a23 100644 --- a/.pre-commit/.just +++ b/.pre-commit/.just @@ -1,4 +1,8 @@ -import '../helpers/helpers.just' +import '../offchain/helpers/helpers.just' + +# Cargo lives under offchain/; default all recipes there. Recipes that +# need a different PWD (md-lint, typos, shebang checks) `cd` explicitly. +set working-directory := '../offchain' [private] _default: @@ -227,6 +231,50 @@ executable-scripts: exit $error +# Verify every offchain/tools// follows the tool author contract: +# 1. has tools.json +# 2. has build.rs (which validates [[bin]] == crate name at compile time) +# 3. no fqn!("...@") hardcoded — must use +# fqn!(concat!("...@", env!("TOOL_FQN_VERSION"))) +# A hardcoded @N would compile fine but ship the wrong content-version +# to chain (or collide with the legacy @1 registrations), so this is the +# main thing the build system can't catch on its own. +check-tool-conventions: + #!/usr/bin/env bash + + set -euo pipefail + + cd $(git rev-parse --show-toplevel) + + error=0 + + for d in offchain/tools/*/; do + tool=$(basename "$d") + if [[ ! -f "${d}tools.json" ]]; then + echo "❌ ${d}: missing tools.json" >&2 + error=1 + fi + if [[ ! -f "${d}build.rs" ]]; then + echo "❌ ${d}: missing build.rs (needed to thread TOOL_FQN_VERSION)" >&2 + error=1 + fi + done + + # Any fqn!("...@") with a literal numeric version is a bug: + # the version must come from env!("TOOL_FQN_VERSION") via concat!(). + bad=$(grep -rEn 'fqn!\("[^"]+@[0-9]+"' offchain/tools/*/src/ 2>/dev/null || true) + if [[ -n "$bad" ]]; then + echo "❌ Hardcoded fqn! version found — use concat!(\"...@\", env!(\"TOOL_FQN_VERSION\")):" >&2 + echo "$bad" >&2 + error=1 + fi + + if [[ $error -eq 0 ]]; then + echo "✅ Tool conventions OK" >&2 + fi + exit $error + + nightly-version: #!/usr/bin/env bash diff --git a/.pre-commit/14-check-tool-conventions b/.pre-commit/14-check-tool-conventions new file mode 100755 index 0000000..5f23846 --- /dev/null +++ b/.pre-commit/14-check-tool-conventions @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +exec just pre-commit::check-tool-conventions diff --git a/README.md b/README.md index 67c390c..9fbdf10 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,203 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://github.com/Talus-Network/nexus-tools/blob/main/LICENSE) [![Actions](https://img.shields.io/badge/GitHub_Actions-Active-brightgreen)](https://github.com/Talus-Network/nexus-tools/actions) -This is a collection of [tools] that the Talus core team maintains. +A collection of [tools] that the Talus core team maintains. ## Development -We use [just][just-repo], a straightforward command runner similar to `make`. +We use [just][just-repo] as a command runner. List tasks with `just --list`. -To explore the available tasks, run `just --list`. +## Layout + +~~~text +nexus-tools/ +├── offchain/ # Rust workspace + tools (cd here before cargo) +│ ├── Cargo.toml +│ ├── tools// +│ │ ├── Cargo.toml +│ │ ├── tools.json # per-tool deploy config +│ │ ├── build.rs # validates bin/tool name, threads TOOL_FQN_VERSION +│ │ └── src/... +│ └── Dockerfile # shared, parameterized by PACKAGE/BINARY/TOOL_FQN_VERSION +├── onchain/ # reserved for future Move tools +└── .github/ + ├── actions/ # composite actions (install-sui, install-nexus-cli, gcp-auth-*) + └── workflows/ # CI: discover → deploy → prepare → register → trigger-tf-apply +~~~ + +All cargo commands run from `offchain/`: + +~~~bash +cd offchain +cargo build --workspace +cargo test --workspace +~~~ + +## Adding a new tool + +A "tool" is a single Rust crate under `offchain/tools//` with a few +conventions the CI pipeline relies on: + +1. **`tools.json`** — declares the deploy shape. The crate's name is the + key under `offchain/tools/`. Minimal example: + + ~~~json + { + "environment": { "RUST_LOG": "info" }, + "resources": { "cpu": "1", "memory": "512Mi" }, + "signed_http": { "enabled": true } + } + ~~~ + +1. **`build.rs`** — must compile-time validate that the crate name + matches the binary and emit `TOOL_FQN_VERSION` from the Docker + build-arg (defaults to `"1"` for local builds). Copy from an + existing tool like `offchain/tools/math/build.rs`. + +1. **`[[bin]]` in `Cargo.toml`** — binary name must equal the crate + name (the build.rs assertion enforces this). + +1. **`fqn!()` source threading** — every FQN literal in your tool must + use `concat!()` + `env!("TOOL_FQN_VERSION")` so the content version + flows from the build args into the registered FQN: + + ~~~rust + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.math.i64.add@", + env!("TOOL_FQN_VERSION") + )) + } + ~~~ + +1. **Optional per-route URL path** — the toolkit binary's `--meta` + output reports a per-FQN URL like `http://localhost/i64/add`. The + register step preserves that path (forcing a trailing slash) when + composing the on-chain URL, so the leader can reach the right + endpoint. + +Once those four pieces are in place, the CI pipeline discovers the +tool automatically by globbing `offchain/tools/*/tools.json`. + +## Branch & tag model + +Two roles for branches, one for tags: + +- **`main`** — dev integration. Tip of `main` is the source of truth + for what code exists. Pushes to `main` build + push images to GCR but + never touch the deployer wallet and never register FQNs on chain. +- **`iterate/testnet/` / `iterate/mainnet/`** — short- + lived branches for iterating on the deploy pipeline itself. Push + fires the full chain. Delete the branch once the rollout settles. +- **`testnet-*` / `mainnet-*` tags** — the canonical deploy path. Tag + a commit on `main` (for testnet) or any prior testnet-deployed + commit (for mainnet), push the tag, and CI runs the full chain + against the corresponding env. Tags are immutable, so each + deployment is attributable to an exact ref — no merge-strategy + gotchas, no long-lived deploy branches to keep clean. + +| Trigger | Matrix | Build | Push images | Chain ops | TF apply | +| --- | --- | --- | --- | --- | --- | +| PR (any base) | changed | ✓ | dry-run | — | — | +| Push to `main` | changed | ✓ | ✓ | — | — | +| Push to `iterate/testnet/**` / `iterate/mainnet/**` | all | ✓ | ✓ | ✓ (env) | ✓ | +| Push tag `testnet-*` / `mainnet-*` | all | ✓ | ✓ | ✓ (tag prefix) | ✓ | +| `workflow_dispatch` `target-env=testnet\|mainnet` | all | ✓ | ✓ | ✓ (chosen) | ✓ | +| `workflow_dispatch` `pr-number=` | all | ✓ | ✓ | ✓ (PR's base) | ✓ | + +## Lifecycle: PR → main → deploy + +1. **Open a PR with base `main`.** CI runs the dry-run path: discover + the changed-tool matrix, build each image, but don't push to the + registry and don't touch the chain. Reviewers verify the build is + clean and tests pass. +1. **Merge to `main`.** Same as the PR run, but images now get pushed + to GCR (`gcr.io//nexus-tools/:sha-<7>`). The + image is "available" but no Cloud Run service points at it yet and + no FQN is registered. +1. **Promote to a deploy env.** Pick one of: + - **Tag deploy (canonical)** — tag a commit and push the tag: + + ~~~bash + git tag testnet-v0.1.0 + git push origin testnet-v0.1.0 + ~~~ + + Tag prefix selects the env (`testnet-*` → testnet, `mainnet-*` + → mainnet). The full chain runs against that exact commit, and + the tag is an immutable record of what was deployed. Best for + anything you'd want to roll back to or audit later. + - **Iterate branch** — push to `iterate/testnet/` (or + `iterate/mainnet/`). Fires the same pipeline. Best for + fast iteration on the deploy itself (debugging the register + step, tf-nexus-tools side, etc.) where you don't want a tag for + every attempt. + - **Manual one-off** — Actions → CI → "Run workflow". Pick + `target-env=testnet|mainnet`, or pass `pr-number=` to deploy a + specific PR's head against the PR's base env. Best for ad-hoc + deploys without cutting a branch or tag. + + Convention for mainnet promotions: tag should sit on a commit that + has already been deployed to testnet (i.e. is reachable from a + prior `testnet-*` tag). Not enforced in CI yet — keep it as a + review discipline. + +After the chosen trigger fires, the full pipeline runs: discover → +deploy → prepare → register → trigger-tf-apply. `ci-gate` only goes +green if the dispatched `tf-nexus-tools` terraform run also succeeded, +so a green pipeline means Cloud Run + ALB + DNS are reconciled and the +FQN URLs on chain point at the right endpoints. + +## What "full pipeline" does + +1. **discover** — globs `offchain/tools/*/tools.json` and computes a + per-tool content version from `git rev-parse HEAD:offchain/tools//` + (first 8 hex of sha256 → u32). +1. **deploy** — builds per-tool Docker images and pushes to + `gcr.io//nexus-tools/:sha-<7>`. Build and auth + are split so token expiry can't kill a slow build mid-push. +1. **prepare** — runs each container's `--meta` to extract the + ToolMeta JSON, renders a Cloud Run config blob to + `gs:////offchain/tools/-v.json`, + and generates a signed-HTTP keys secret in Secret Manager. The + `nexus_contracts_tag` is baked from `vars.NEXUS_TAG` into each blob + so it stays pinned to whichever Nexus version was current at + prepare time. +1. **register** — for each FQN: + - skips the CLI call if already on chain (snapshot from `nexus tool list --json`), + - otherwise pipes the ToolMeta to `nexus tool register offchain --from-meta -`, + - persists `owner_cap_over_tool` + `owner_cap_over_gas` to + `gs:////offchain/registration//.json`, + - registers signing keys (`nexus tool auth register-key`), + - reconciles the per-tool `toolkit-config` Secret Manager secret. + Pre-step: consolidates the deployer wallet's coins so a single coin + can cover the 1 SUI per-tx budget that heavy schemas require. +1. **trigger-tf-apply** — dispatches `Talus-Network/tf-nexus-tools`'s + `terraform.yml` (via `the-actions-org/workflow-dispatch`, which + works with fine-grained PATs). Terraform materializes the Cloud + Run services, internal ALB, and DNS records based on the per-tool + blobs in GCS. ci-gate inherits the dispatched run's conclusion, so + register-then-apply is one atomic gate. + +## Operational notes + +- **Deployer wallet** — `SUI_DEPLOYER_PK` (env-scoped repo secret) is a + `suiprivkey...` bech32 string. The register step imports it into the + sui keychain at the start of the job. The same secret will eventually + drive on-chain Move publishes for `onchain/` tools. +- **NEXUS_TAG** — env variable (testnet/mainnet). Each tool's deploy + pins to whichever value was current at prepare time; later bumps to + `NEXUS_TAG` only affect *new* deploys. +- **Stale Cloud Run services** — terraform iterates over whatever + config blobs are in `gs://...offchain/tools/`. If a stale blob from + a prior iteration is left over, terraform will try to create a + service for it. Either re-prepare to refresh the blob or + `gsutil rm` the stale one. +- **Costs** — internal ALB stack (URL map + proxy + per-region + forwarding rules) is independent from `tf-talus-nexus-v2`. Future + work may consolidate the two; for now they're kept separate so the + static legacy deploy and the dynamic pipeline can't influence each + other. diff --git a/codecov.yml b/codecov.yml index 92f6896..8557a58 100644 --- a/codecov.yml +++ b/codecov.yml @@ -23,14 +23,19 @@ coverage: threshold: 10% # Allow 10% drop for edge cases base: auto if_ci_failed: error - informational: false + # Report but don't block. Project-level check still gates the + # PR on absolute coverage drift; this one was noisy on patches + # that only touch trivial code (e.g. the fqn!() concat rewrite + # where every tool's fn fqn() body changed but is still a + # one-line constant return). + informational: true component_management: individual_components: - component_id: tools name: Tools paths: - - tools/** + - offchain/tools/** comment: layout: "header, diff, flags, components, files, footer" @@ -53,7 +58,7 @@ ignore: flags: unittests: paths: - - tools/ + - offchain/tools/ carryforward: true github_checks: diff --git a/justfile b/justfile index 957f5d7..51c983d 100644 --- a/justfile +++ b/justfile @@ -5,17 +5,17 @@ # # -# Commands concerning native Nexus Tools -mod tools 'tools/.just' +# Commands concerning native Nexus Tools (offchain workspace) +mod tools 'offchain/tools/.just' # Example invocations for the memory-memwal Tools bundle -mod memwal 'tools/memory-memwal/.just' +mod memwal 'offchain/tools/memory-memwal/.just' -# Pre-commit hooks +# Pre-commit hooks (still at repo root — they wrap git commit) mod pre-commit '.pre-commit/.just' -# Helpers -mod helpers 'helpers/helpers.just' +# Helpers (lives under the workspace) +mod helpers 'offchain/helpers/helpers.just' [private] _default: diff --git a/Cargo.lock b/offchain/Cargo.lock similarity index 99% rename from Cargo.lock rename to offchain/Cargo.lock index de1b13f..94e5889 100644 --- a/Cargo.lock +++ b/offchain/Cargo.lock @@ -925,6 +925,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "toml", ] [[package]] @@ -1381,6 +1382,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "toml", "url", ] @@ -2041,6 +2043,7 @@ dependencies = [ "strum", "strum_macros", "tokio", + "toml", ] [[package]] @@ -2072,7 +2075,9 @@ dependencies = [ "nexus-toolkit", "schemars", "serde", + "serde_json", "tokio", + "toml", ] [[package]] @@ -2107,6 +2112,7 @@ dependencies = [ "sha2 0.11.0", "thiserror 2.0.18", "tokio", + "toml", "url", "uuid", "zeroize", @@ -3680,6 +3686,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "toml", ] [[package]] @@ -3912,6 +3919,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "toml", ] [[package]] @@ -4435,6 +4443,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "toml", ] [[package]] diff --git a/Cargo.toml b/offchain/Cargo.toml similarity index 100% rename from Cargo.toml rename to offchain/Cargo.toml diff --git a/offchain/Dockerfile b/offchain/Dockerfile new file mode 100644 index 0000000..499da87 --- /dev/null +++ b/offchain/Dockerfile @@ -0,0 +1,77 @@ +# syntax=docker/dockerfile:1 +# Shared Dockerfile for all offchain Nexus tools. +# +# Builds a single tool binary identified by BINARY and TOOL_DIR build args +# (TOOL_DIR is the tool's path relative to the build context, e.g. +# "offchain/tools/http"), embeds TOOL_FQN_VERSION (computed by CI from the +# tool's subtree hash) so the tool's FQN strings carry the content-addressed +# version. +# +# Building via --manifest-path scopes cargo to the workspace member's +# Cargo.toml, sidestepping ambiguity when a tool crate's name collides with +# a transitive crates.io dep of the same name (e.g. our `http` tool vs the +# `http` crate pulled in by reqwest). + +FROM rust:1.93.1-slim-bookworm AS builder + +ARG BINARY +ARG TOOL_DIR +ARG TOOL_FQN_VERSION=1 + +# Surfaced as a cargo env var via build.rs -> env!("TOOL_FQN_VERSION"). +ENV TOOL_FQN_VERSION=${TOOL_FQN_VERSION} + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Copy the entire offchain workspace. Cargo's incremental build + the +# buildkit cache mounts keep this fast. +COPY offchain/ ./offchain/ + +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/app/offchain/target,sharing=locked \ + set -e && \ + cargo build --locked --profile release \ + --manifest-path "${TOOL_DIR}/Cargo.toml" \ + --bin "${BINARY}" && \ + cp "offchain/target/release/${BINARY}" "/app/${BINARY}" + +# ── Collect runtime libs for the distroless stage ───────────────────── +FROM debian:bookworm-slim AS lib-collector + +ARG BINARY +ARG TARGETARCH + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/${BINARY} /tmp/binary + +RUN set -e; \ + case "${TARGETARCH}" in \ + "amd64") LIB_DIR="/lib/x86_64-linux-gnu"; LINKER="/lib64/ld-linux-x86-64.so.2" ;; \ + "arm64") LIB_DIR="/lib/aarch64-linux-gnu"; LINKER="/lib/ld-linux-aarch64.so.1" ;; \ + *) echo "Unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + mkdir -p "/collected${LIB_DIR}" /collected/lib64 && \ + ldd /tmp/binary | awk '/=>/ { print $3 }' | while read -r lib; do \ + if [ -n "$lib" ] && [ -f "$lib" ]; then \ + cp -L "$lib" "/collected${LIB_DIR}/"; \ + fi \ + done && \ + cp -L "${LINKER}" "/collected${LINKER}" + +# ── Final distroless image ─────────────────────────────────────────── +FROM gcr.io/distroless/cc-debian12 + +ARG BINARY + +COPY --from=builder /app/${BINARY} /usr/local/bin/${BINARY} +COPY --from=lib-collector /collected/ / + +ENV PORT=8080 +EXPOSE 8080 diff --git a/deny.toml b/offchain/deny.toml similarity index 100% rename from deny.toml rename to offchain/deny.toml diff --git a/helpers/helpers.just b/offchain/helpers/helpers.just similarity index 98% rename from helpers/helpers.just rename to offchain/helpers/helpers.just index a81e61e..bd5ae07 100644 --- a/helpers/helpers.just +++ b/offchain/helpers/helpers.just @@ -119,4 +119,4 @@ get-nightly-version: #!/usr/bin/env bash # This file should contain the version of the nightly toolchain to use. - cat ../.nightly-version + cat ../../.nightly-version diff --git a/helpers/npm-install-g.sh b/offchain/helpers/npm-install-g.sh similarity index 55% rename from helpers/npm-install-g.sh rename to offchain/helpers/npm-install-g.sh index 5bc0f27..db8c74b 100755 --- a/helpers/npm-install-g.sh +++ b/offchain/helpers/npm-install-g.sh @@ -2,10 +2,11 @@ set -euo pipefail -# Go to the root of the git repository -cd $(git rev-parse --show-toplevel) +# Resolve the directory containing this script so the npm-install-g.txt +# lookup works regardless of the caller's PWD. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Install NPM stuff from helpers/npm-install-g.txt +# Install NPM stuff from the sibling helpers/npm-install-g.txt while read -r tool_at_version; do # Split the tool name and version tool=$(echo "$tool_at_version" | cut -d'@' -f1) @@ -16,4 +17,4 @@ while read -r tool_at_version; do else echo "Tool already installed: $tool_at_version" fi -done < helpers/npm-install-g.txt +done < "$SCRIPT_DIR/npm-install-g.txt" diff --git a/helpers/npm-install-g.txt b/offchain/helpers/npm-install-g.txt similarity index 100% rename from helpers/npm-install-g.txt rename to offchain/helpers/npm-install-g.txt diff --git a/rust-toolchain.toml b/offchain/rust-toolchain.toml similarity index 100% rename from rust-toolchain.toml rename to offchain/rust-toolchain.toml diff --git a/rustfmt.toml b/offchain/rustfmt.toml similarity index 100% rename from rustfmt.toml rename to offchain/rustfmt.toml diff --git a/tools/.just b/offchain/tools/.just similarity index 100% rename from tools/.just rename to offchain/tools/.just diff --git a/tools/exchanges-coinbase/Cargo.toml b/offchain/tools/exchanges-coinbase/Cargo.toml similarity index 84% rename from tools/exchanges-coinbase/Cargo.toml rename to offchain/tools/exchanges-coinbase/Cargo.toml index ab93616..5915d89 100644 --- a/tools/exchanges-coinbase/Cargo.toml +++ b/offchain/tools/exchanges-coinbase/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "exchanges-coinbase" +path = "src/main.rs" + [dependencies] chrono.workspace = true thiserror.workspace = true @@ -27,3 +31,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/exchanges-coinbase/README.md b/offchain/tools/exchanges-coinbase/README.md similarity index 100% rename from tools/exchanges-coinbase/README.md rename to offchain/tools/exchanges-coinbase/README.md diff --git a/offchain/tools/exchanges-coinbase/build.rs b/offchain/tools/exchanges-coinbase/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/exchanges-coinbase/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/exchanges-coinbase/src/coinbase_client.rs b/offchain/tools/exchanges-coinbase/src/coinbase_client.rs similarity index 100% rename from tools/exchanges-coinbase/src/coinbase_client.rs rename to offchain/tools/exchanges-coinbase/src/coinbase_client.rs diff --git a/tools/exchanges-coinbase/src/error.rs b/offchain/tools/exchanges-coinbase/src/error.rs similarity index 100% rename from tools/exchanges-coinbase/src/error.rs rename to offchain/tools/exchanges-coinbase/src/error.rs diff --git a/tools/exchanges-coinbase/src/main.rs b/offchain/tools/exchanges-coinbase/src/main.rs similarity index 100% rename from tools/exchanges-coinbase/src/main.rs rename to offchain/tools/exchanges-coinbase/src/main.rs diff --git a/tools/exchanges-coinbase/src/tools/get_product_stats.rs b/offchain/tools/exchanges-coinbase/src/tools/get_product_stats.rs similarity index 99% rename from tools/exchanges-coinbase/src/tools/get_product_stats.rs rename to offchain/tools/exchanges-coinbase/src/tools/get_product_stats.rs index 903cfcf..a8f6558 100644 --- a/tools/exchanges-coinbase/src/tools/get_product_stats.rs +++ b/offchain/tools/exchanges-coinbase/src/tools/get_product_stats.rs @@ -81,7 +81,10 @@ impl NexusTool for GetProductStats { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.exchanges.coinbase.get-product-stats@1") + fqn!(concat!( + "xyz.taluslabs.exchanges.coinbase.get-product-stats@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/exchanges-coinbase/src/tools/get_product_ticker.rs b/offchain/tools/exchanges-coinbase/src/tools/get_product_ticker.rs similarity index 99% rename from tools/exchanges-coinbase/src/tools/get_product_ticker.rs rename to offchain/tools/exchanges-coinbase/src/tools/get_product_ticker.rs index 54b1729..b018dfc 100644 --- a/tools/exchanges-coinbase/src/tools/get_product_ticker.rs +++ b/offchain/tools/exchanges-coinbase/src/tools/get_product_ticker.rs @@ -75,7 +75,10 @@ impl NexusTool for GetProductTicker { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.exchanges.coinbase.get-product-ticker@1") + fqn!(concat!( + "xyz.taluslabs.exchanges.coinbase.get-product-ticker@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/exchanges-coinbase/src/tools/get_spot_price.rs b/offchain/tools/exchanges-coinbase/src/tools/get_spot_price.rs similarity index 99% rename from tools/exchanges-coinbase/src/tools/get_spot_price.rs rename to offchain/tools/exchanges-coinbase/src/tools/get_spot_price.rs index f5d2383..796236e 100644 --- a/tools/exchanges-coinbase/src/tools/get_spot_price.rs +++ b/offchain/tools/exchanges-coinbase/src/tools/get_spot_price.rs @@ -83,7 +83,10 @@ impl NexusTool for GetSpotPrice { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.exchanges.coinbase.get-spot-price@1") + fqn!(concat!( + "xyz.taluslabs.exchanges.coinbase.get-spot-price@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/exchanges-coinbase/src/tools/mod.rs b/offchain/tools/exchanges-coinbase/src/tools/mod.rs similarity index 100% rename from tools/exchanges-coinbase/src/tools/mod.rs rename to offchain/tools/exchanges-coinbase/src/tools/mod.rs diff --git a/tools/exchanges-coinbase/src/tools/models.rs b/offchain/tools/exchanges-coinbase/src/tools/models.rs similarity index 100% rename from tools/exchanges-coinbase/src/tools/models.rs rename to offchain/tools/exchanges-coinbase/src/tools/models.rs diff --git a/offchain/tools/exchanges-coinbase/tools.json b/offchain/tools/exchanges-coinbase/tools.json new file mode 100644 index 0000000..845029b --- /dev/null +++ b/offchain/tools/exchanges-coinbase/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "exchanges-coinbase", + "command": "exchanges-coinbase", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/http/Cargo.toml b/offchain/tools/http/Cargo.toml similarity index 86% rename from tools/http/Cargo.toml rename to offchain/tools/http/Cargo.toml index 62a4645..b4154f7 100644 --- a/tools/http/Cargo.toml +++ b/offchain/tools/http/Cargo.toml @@ -10,6 +10,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "http" +path = "src/main.rs" + [dependencies] backon = "0.4" base64.workspace = true @@ -28,3 +32,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/http/README.md b/offchain/tools/http/README.md similarity index 100% rename from tools/http/README.md rename to offchain/tools/http/README.md diff --git a/offchain/tools/http/build.rs b/offchain/tools/http/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/http/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/http/src/errors.rs b/offchain/tools/http/src/errors.rs similarity index 100% rename from tools/http/src/errors.rs rename to offchain/tools/http/src/errors.rs diff --git a/tools/http/src/http.rs b/offchain/tools/http/src/http.rs similarity index 99% rename from tools/http/src/http.rs rename to offchain/tools/http/src/http.rs index a7037a9..3fb60a3 100644 --- a/tools/http/src/http.rs +++ b/offchain/tools/http/src/http.rs @@ -202,7 +202,10 @@ impl NexusTool for Http { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.http.generic@1") + fqn!(concat!( + "xyz.taluslabs.http.generic@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/http/src/http_client.rs b/offchain/tools/http/src/http_client.rs similarity index 100% rename from tools/http/src/http_client.rs rename to offchain/tools/http/src/http_client.rs diff --git a/tools/http/src/main.rs b/offchain/tools/http/src/main.rs similarity index 100% rename from tools/http/src/main.rs rename to offchain/tools/http/src/main.rs diff --git a/tools/http/src/models.rs b/offchain/tools/http/src/models.rs similarity index 100% rename from tools/http/src/models.rs rename to offchain/tools/http/src/models.rs diff --git a/tools/http/src/utils.rs b/offchain/tools/http/src/utils.rs similarity index 100% rename from tools/http/src/utils.rs rename to offchain/tools/http/src/utils.rs diff --git a/offchain/tools/http/tools.json b/offchain/tools/http/tools.json new file mode 100644 index 0000000..9be46f2 --- /dev/null +++ b/offchain/tools/http/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "http", + "command": "http", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/llm-openai-chat-completion/Cargo.toml b/offchain/tools/llm-openai-chat-completion/Cargo.toml similarity index 86% rename from tools/llm-openai-chat-completion/Cargo.toml rename to offchain/tools/llm-openai-chat-completion/Cargo.toml index 8e1ae5d..68718f0 100644 --- a/tools/llm-openai-chat-completion/Cargo.toml +++ b/offchain/tools/llm-openai-chat-completion/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "llm-openai-chat-completion" +path = "src/main.rs" + [dependencies] anyhow.workspace = true async-openai = "0.27" @@ -34,3 +38,7 @@ mockito.workspace = true portpicker.workspace = true rstest.workspace = true serial_test.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/llm-openai-chat-completion/README.md b/offchain/tools/llm-openai-chat-completion/README.md similarity index 100% rename from tools/llm-openai-chat-completion/README.md rename to offchain/tools/llm-openai-chat-completion/README.md diff --git a/offchain/tools/llm-openai-chat-completion/build.rs b/offchain/tools/llm-openai-chat-completion/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/llm-openai-chat-completion/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/llm-openai-chat-completion/src/main.rs b/offchain/tools/llm-openai-chat-completion/src/main.rs similarity index 99% rename from tools/llm-openai-chat-completion/src/main.rs rename to offchain/tools/llm-openai-chat-completion/src/main.rs index ebaefb9..5e40c4c 100644 --- a/tools/llm-openai-chat-completion/src/main.rs +++ b/offchain/tools/llm-openai-chat-completion/src/main.rs @@ -265,7 +265,10 @@ impl NexusTool for OpenaiChatCompletion { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.llm.openai.chat-completion@1") + fqn!(concat!( + "xyz.taluslabs.llm.openai.chat-completion@", + env!("TOOL_FQN_VERSION") + )) } fn timeout() -> std::time::Duration { diff --git a/tools/llm-openai-chat-completion/src/status.rs b/offchain/tools/llm-openai-chat-completion/src/status.rs similarity index 100% rename from tools/llm-openai-chat-completion/src/status.rs rename to offchain/tools/llm-openai-chat-completion/src/status.rs diff --git a/offchain/tools/llm-openai-chat-completion/tools.json b/offchain/tools/llm-openai-chat-completion/tools.json new file mode 100644 index 0000000..0252247 --- /dev/null +++ b/offchain/tools/llm-openai-chat-completion/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "llm-openai-chat-completion", + "command": "llm-openai-chat-completion", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/math/Cargo.toml b/offchain/tools/math/Cargo.toml similarity index 81% rename from tools/math/Cargo.toml rename to offchain/tools/math/Cargo.toml index 22890c6..69e5cfd 100644 --- a/tools/math/Cargo.toml +++ b/offchain/tools/math/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "math" +path = "src/main.rs" + [dependencies] schemars.workspace = true serde.workspace = true @@ -20,3 +24,7 @@ tokio.workspace = true # === Nexus deps === nexus-toolkit.workspace = true nexus-sdk.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/math/README.md b/offchain/tools/math/README.md similarity index 100% rename from tools/math/README.md rename to offchain/tools/math/README.md diff --git a/offchain/tools/math/build.rs b/offchain/tools/math/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/math/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/math/src/i64/add.rs b/offchain/tools/math/src/i64/add.rs similarity index 94% rename from tools/math/src/i64/add.rs rename to offchain/tools/math/src/i64/add.rs index 0bbd5b2..840e361 100644 --- a/tools/math/src/i64/add.rs +++ b/offchain/tools/math/src/i64/add.rs @@ -34,7 +34,10 @@ impl NexusTool for I64Add { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.add@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.add@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/i64/cmp.rs b/offchain/tools/math/src/i64/cmp.rs similarity index 94% rename from tools/math/src/i64/cmp.rs rename to offchain/tools/math/src/i64/cmp.rs index 71de02a..2521ced 100644 --- a/tools/math/src/i64/cmp.rs +++ b/offchain/tools/math/src/i64/cmp.rs @@ -36,7 +36,10 @@ impl NexusTool for I64Cmp { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.cmp@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.cmp@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/i64/mod.rs b/offchain/tools/math/src/i64/mod.rs similarity index 100% rename from tools/math/src/i64/mod.rs rename to offchain/tools/math/src/i64/mod.rs diff --git a/tools/math/src/i64/mul.rs b/offchain/tools/math/src/i64/mul.rs similarity index 94% rename from tools/math/src/i64/mul.rs rename to offchain/tools/math/src/i64/mul.rs index 5ccd528..fc75292 100644 --- a/tools/math/src/i64/mul.rs +++ b/offchain/tools/math/src/i64/mul.rs @@ -34,7 +34,10 @@ impl NexusTool for I64Mul { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.mul@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.mul@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/i64/sum.rs b/offchain/tools/math/src/i64/sum.rs similarity index 97% rename from tools/math/src/i64/sum.rs rename to offchain/tools/math/src/i64/sum.rs index 8d0408a..650e2ec 100644 --- a/tools/math/src/i64/sum.rs +++ b/offchain/tools/math/src/i64/sum.rs @@ -34,7 +34,10 @@ impl NexusTool for I64Sum { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.sum@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.sum@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/main.rs b/offchain/tools/math/src/main.rs similarity index 100% rename from tools/math/src/main.rs rename to offchain/tools/math/src/main.rs diff --git a/offchain/tools/math/tools.json b/offchain/tools/math/tools.json new file mode 100644 index 0000000..1f533b7 --- /dev/null +++ b/offchain/tools/math/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "math", + "command": "math", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/memory-memwal/.env.example b/offchain/tools/memory-memwal/.env.example similarity index 100% rename from tools/memory-memwal/.env.example rename to offchain/tools/memory-memwal/.env.example diff --git a/tools/memory-memwal/.just b/offchain/tools/memory-memwal/.just similarity index 100% rename from tools/memory-memwal/.just rename to offchain/tools/memory-memwal/.just diff --git a/tools/memory-memwal/Cargo.toml b/offchain/tools/memory-memwal/Cargo.toml similarity index 88% rename from tools/memory-memwal/Cargo.toml rename to offchain/tools/memory-memwal/Cargo.toml index f7ce1dc..9999f1f 100644 --- a/tools/memory-memwal/Cargo.toml +++ b/offchain/tools/memory-memwal/Cargo.toml @@ -34,3 +34,11 @@ zeroize.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" + +[[bin]] +name = "memory-memwal" +path = "src/main.rs" diff --git a/tools/memory-memwal/README.md b/offchain/tools/memory-memwal/README.md similarity index 100% rename from tools/memory-memwal/README.md rename to offchain/tools/memory-memwal/README.md diff --git a/offchain/tools/memory-memwal/build.rs b/offchain/tools/memory-memwal/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/memory-memwal/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/memory-memwal/src/analyze.rs b/offchain/tools/memory-memwal/src/analyze.rs similarity index 98% rename from tools/memory-memwal/src/analyze.rs rename to offchain/tools/memory-memwal/src/analyze.rs index f412ff9..d81cc98 100644 --- a/tools/memory-memwal/src/analyze.rs +++ b/offchain/tools/memory-memwal/src/analyze.rs @@ -52,7 +52,10 @@ impl NexusTool for AnalyzeAndRemember { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.analyze@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.analyze@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/memory-memwal/src/ask.rs b/offchain/tools/memory-memwal/src/ask.rs similarity index 98% rename from tools/memory-memwal/src/ask.rs rename to offchain/tools/memory-memwal/src/ask.rs index a58eb7e..f0968a1 100644 --- a/tools/memory-memwal/src/ask.rs +++ b/offchain/tools/memory-memwal/src/ask.rs @@ -75,7 +75,10 @@ impl NexusTool for AskMemory { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.ask@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.ask@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/memory-memwal/src/auth.rs b/offchain/tools/memory-memwal/src/auth.rs similarity index 100% rename from tools/memory-memwal/src/auth.rs rename to offchain/tools/memory-memwal/src/auth.rs diff --git a/tools/memory-memwal/src/client.rs b/offchain/tools/memory-memwal/src/client.rs similarity index 100% rename from tools/memory-memwal/src/client.rs rename to offchain/tools/memory-memwal/src/client.rs diff --git a/tools/memory-memwal/src/error.rs b/offchain/tools/memory-memwal/src/error.rs similarity index 100% rename from tools/memory-memwal/src/error.rs rename to offchain/tools/memory-memwal/src/error.rs diff --git a/tools/memory-memwal/src/forget.rs b/offchain/tools/memory-memwal/src/forget.rs similarity index 98% rename from tools/memory-memwal/src/forget.rs rename to offchain/tools/memory-memwal/src/forget.rs index c32d946..7b19088 100644 --- a/tools/memory-memwal/src/forget.rs +++ b/offchain/tools/memory-memwal/src/forget.rs @@ -53,7 +53,10 @@ impl NexusTool for ForgetMemories { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.forget@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.forget@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/memory-memwal/src/main.rs b/offchain/tools/memory-memwal/src/main.rs similarity index 59% rename from tools/memory-memwal/src/main.rs rename to offchain/tools/memory-memwal/src/main.rs index 3b06fe2..e28676a 100644 --- a/tools/memory-memwal/src/main.rs +++ b/offchain/tools/memory-memwal/src/main.rs @@ -25,12 +25,22 @@ fn main() { .format_timestamp_millis() .init(); - // dotenv + credential validation run single-threaded — `set_var` is - // unsound from a multi-threaded process. `main` is the only exit site. - client::load_dotenv_if_present(); - if let Err(reason) = client::validate_credentials_at_startup() { - log::error!("{} {reason}", client::ENV_PRIVATE_KEY); - std::process::exit(1); + // `--meta` is an introspection mode that emits the tool's FQN/URL/ + // schema list without standing up the runtime. It is invoked by the + // CI prepare step from a fresh Docker image with no env, so it must + // not require runtime credentials. Skip the credential validation + // path in this mode; nexus_toolkit's bootstrap! handles --meta and + // exits before any HTTP call is made. + let meta_only = std::env::args().any(|a| a == "--meta"); + + if !meta_only { + // dotenv + credential validation run single-threaded — `set_var` is + // unsound from a multi-threaded process. `main` is the only exit site. + client::load_dotenv_if_present(); + if let Err(reason) = client::validate_credentials_at_startup() { + log::error!("{} {reason}", client::ENV_PRIVATE_KEY); + std::process::exit(1); + } } tokio::runtime::Builder::new_multi_thread() diff --git a/tools/memory-memwal/src/recall.rs b/offchain/tools/memory-memwal/src/recall.rs similarity index 98% rename from tools/memory-memwal/src/recall.rs rename to offchain/tools/memory-memwal/src/recall.rs index 81d6a0c..d4e666d 100644 --- a/tools/memory-memwal/src/recall.rs +++ b/offchain/tools/memory-memwal/src/recall.rs @@ -79,7 +79,10 @@ impl NexusTool for RecallMemories { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.recall@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.recall@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/memory-memwal/src/remember.rs b/offchain/tools/memory-memwal/src/remember.rs similarity index 98% rename from tools/memory-memwal/src/remember.rs rename to offchain/tools/memory-memwal/src/remember.rs index 38cdb9e..ddbc968 100644 --- a/tools/memory-memwal/src/remember.rs +++ b/offchain/tools/memory-memwal/src/remember.rs @@ -52,7 +52,10 @@ impl NexusTool for RememberMemory { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.remember@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.remember@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/memory-memwal/src/remember_bulk.rs b/offchain/tools/memory-memwal/src/remember_bulk.rs similarity index 98% rename from tools/memory-memwal/src/remember_bulk.rs rename to offchain/tools/memory-memwal/src/remember_bulk.rs index 806024d..06dd433 100644 --- a/tools/memory-memwal/src/remember_bulk.rs +++ b/offchain/tools/memory-memwal/src/remember_bulk.rs @@ -62,7 +62,10 @@ impl NexusTool for RememberBulkMemories { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.remember_bulk@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.remember_bulk@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/memory-memwal/src/stats.rs b/offchain/tools/memory-memwal/src/stats.rs similarity index 98% rename from tools/memory-memwal/src/stats.rs rename to offchain/tools/memory-memwal/src/stats.rs index 74120bb..be5db1f 100644 --- a/tools/memory-memwal/src/stats.rs +++ b/offchain/tools/memory-memwal/src/stats.rs @@ -54,7 +54,10 @@ impl NexusTool for StatsForAccount { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.memory.memwal.stats@1") + fqn!(concat!( + "xyz.taluslabs.memory.memwal.stats@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/memory-memwal/tools.json b/offchain/tools/memory-memwal/tools.json new file mode 100644 index 0000000..8924f64 --- /dev/null +++ b/offchain/tools/memory-memwal/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "memory-memwal", + "command": "memory-memwal", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/social-twitter/Cargo.toml b/offchain/tools/social-twitter/Cargo.toml similarity index 84% rename from tools/social-twitter/Cargo.toml rename to offchain/tools/social-twitter/Cargo.toml index 0c08118..7d09c74 100644 --- a/tools/social-twitter/Cargo.toml +++ b/offchain/tools/social-twitter/Cargo.toml @@ -11,6 +11,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "social-twitter" +path = "src/main.rs" + [dependencies] oauth1-request = "0.6.1" base64 = "0.13.0" @@ -29,3 +33,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito = "1.7.0" + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/social-twitter/README.md b/offchain/tools/social-twitter/README.md similarity index 100% rename from tools/social-twitter/README.md rename to offchain/tools/social-twitter/README.md diff --git a/offchain/tools/social-twitter/build.rs b/offchain/tools/social-twitter/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/social-twitter/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/social-twitter/src/auth.rs b/offchain/tools/social-twitter/src/auth.rs similarity index 100% rename from tools/social-twitter/src/auth.rs rename to offchain/tools/social-twitter/src/auth.rs diff --git a/tools/social-twitter/src/direct_message/create_group_conversation.rs b/offchain/tools/social-twitter/src/direct_message/create_group_conversation.rs similarity index 99% rename from tools/social-twitter/src/direct_message/create_group_conversation.rs rename to offchain/tools/social-twitter/src/direct_message/create_group_conversation.rs index 7dd0dd0..0b32d11 100644 --- a/tools/social-twitter/src/direct_message/create_group_conversation.rs +++ b/offchain/tools/social-twitter/src/direct_message/create_group_conversation.rs @@ -66,7 +66,10 @@ impl NexusTool for CreateGroupDmConversation { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.create-group-conversation@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.create-group-conversation@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/get_conversation_messages.rs b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages.rs similarity index 98% rename from tools/social-twitter/src/direct_message/get_conversation_messages.rs rename to offchain/tools/social-twitter/src/direct_message/get_conversation_messages.rs index 962a30b..1d1d6a4 100644 --- a/tools/social-twitter/src/direct_message/get_conversation_messages.rs +++ b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages.rs @@ -160,7 +160,10 @@ impl NexusTool for GetConversationMessages { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-conversation-messages@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-conversation-messages@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs similarity index 99% rename from tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs rename to offchain/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs index 899b09f..64dbb86 100644 --- a/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs +++ b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs @@ -166,7 +166,10 @@ impl NexusTool for GetConversationMessagesById { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-conversation-messages-by-id@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-conversation-messages-by-id@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/mod.rs b/offchain/tools/social-twitter/src/direct_message/mod.rs similarity index 100% rename from tools/social-twitter/src/direct_message/mod.rs rename to offchain/tools/social-twitter/src/direct_message/mod.rs diff --git a/tools/social-twitter/src/direct_message/models.rs b/offchain/tools/social-twitter/src/direct_message/models.rs similarity index 100% rename from tools/social-twitter/src/direct_message/models.rs rename to offchain/tools/social-twitter/src/direct_message/models.rs diff --git a/tools/social-twitter/src/direct_message/send_direct_message.rs b/offchain/tools/social-twitter/src/direct_message/send_direct_message.rs similarity index 98% rename from tools/social-twitter/src/direct_message/send_direct_message.rs rename to offchain/tools/social-twitter/src/direct_message/send_direct_message.rs index 87ee63c..3a714d5 100644 --- a/tools/social-twitter/src/direct_message/send_direct_message.rs +++ b/offchain/tools/social-twitter/src/direct_message/send_direct_message.rs @@ -66,7 +66,10 @@ impl NexusTool for SendDirectMessage { // /:participant_id/dm_events fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.send-direct-message@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.send-direct-message@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs b/offchain/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs similarity index 99% rename from tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs rename to offchain/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs index 6b1bc53..853cfff 100644 --- a/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs +++ b/offchain/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs @@ -63,7 +63,10 @@ impl NexusTool for SendMessageToGroupConversation { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.send-message-to-group-conversation@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.send-message-to-group-conversation@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/error.rs b/offchain/tools/social-twitter/src/error.rs similarity index 100% rename from tools/social-twitter/src/error.rs rename to offchain/tools/social-twitter/src/error.rs diff --git a/tools/social-twitter/src/list/add_member.rs b/offchain/tools/social-twitter/src/list/add_member.rs similarity index 98% rename from tools/social-twitter/src/list/add_member.rs rename to offchain/tools/social-twitter/src/list/add_member.rs index 1b1f75d..b30c1d5 100644 --- a/tools/social-twitter/src/list/add_member.rs +++ b/offchain/tools/social-twitter/src/list/add_member.rs @@ -51,7 +51,10 @@ impl NexusTool for AddMember { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.add-member@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.add-member@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/create_list.rs b/offchain/tools/social-twitter/src/list/create_list.rs similarity index 99% rename from tools/social-twitter/src/list/create_list.rs rename to offchain/tools/social-twitter/src/list/create_list.rs index ac839e9..43fefe1 100644 --- a/tools/social-twitter/src/list/create_list.rs +++ b/offchain/tools/social-twitter/src/list/create_list.rs @@ -67,7 +67,10 @@ impl NexusTool for CreateList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.create-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.create-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/delete_list.rs b/offchain/tools/social-twitter/src/list/delete_list.rs similarity index 98% rename from tools/social-twitter/src/list/delete_list.rs rename to offchain/tools/social-twitter/src/list/delete_list.rs index 35e75f6..bc33477 100644 --- a/tools/social-twitter/src/list/delete_list.rs +++ b/offchain/tools/social-twitter/src/list/delete_list.rs @@ -57,7 +57,10 @@ impl NexusTool for DeleteList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.delete-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.delete-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_list.rs b/offchain/tools/social-twitter/src/list/get_list.rs similarity index 99% rename from tools/social-twitter/src/list/get_list.rs rename to offchain/tools/social-twitter/src/list/get_list.rs index fe36a2b..7a3a928 100644 --- a/tools/social-twitter/src/list/get_list.rs +++ b/offchain/tools/social-twitter/src/list/get_list.rs @@ -94,7 +94,10 @@ impl NexusTool for GetList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_list_members.rs b/offchain/tools/social-twitter/src/list/get_list_members.rs similarity index 99% rename from tools/social-twitter/src/list/get_list_members.rs rename to offchain/tools/social-twitter/src/list/get_list_members.rs index 6511085..51439f6 100644 --- a/tools/social-twitter/src/list/get_list_members.rs +++ b/offchain/tools/social-twitter/src/list/get_list_members.rs @@ -83,7 +83,10 @@ impl NexusTool for GetListMembers { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-list-members@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-list-members@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_list_tweets.rs b/offchain/tools/social-twitter/src/list/get_list_tweets.rs similarity index 99% rename from tools/social-twitter/src/list/get_list_tweets.rs rename to offchain/tools/social-twitter/src/list/get_list_tweets.rs index 4c40051..d7c89da 100644 --- a/tools/social-twitter/src/list/get_list_tweets.rs +++ b/offchain/tools/social-twitter/src/list/get_list_tweets.rs @@ -97,7 +97,10 @@ impl NexusTool for GetListTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-list-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-list-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_user_lists.rs b/offchain/tools/social-twitter/src/list/get_user_lists.rs similarity index 99% rename from tools/social-twitter/src/list/get_user_lists.rs rename to offchain/tools/social-twitter/src/list/get_user_lists.rs index 6fdeeb2..92f0466 100644 --- a/tools/social-twitter/src/list/get_user_lists.rs +++ b/offchain/tools/social-twitter/src/list/get_user_lists.rs @@ -85,7 +85,10 @@ impl NexusTool for GetUserLists { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-lists@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-lists@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/mod.rs b/offchain/tools/social-twitter/src/list/mod.rs similarity index 100% rename from tools/social-twitter/src/list/mod.rs rename to offchain/tools/social-twitter/src/list/mod.rs diff --git a/tools/social-twitter/src/list/models.rs b/offchain/tools/social-twitter/src/list/models.rs similarity index 100% rename from tools/social-twitter/src/list/models.rs rename to offchain/tools/social-twitter/src/list/models.rs diff --git a/tools/social-twitter/src/list/remove_member.rs b/offchain/tools/social-twitter/src/list/remove_member.rs similarity index 98% rename from tools/social-twitter/src/list/remove_member.rs rename to offchain/tools/social-twitter/src/list/remove_member.rs index 4c834d6..1ee6c1d 100644 --- a/tools/social-twitter/src/list/remove_member.rs +++ b/offchain/tools/social-twitter/src/list/remove_member.rs @@ -51,7 +51,10 @@ impl NexusTool for RemoveMember { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.remove-member@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.remove-member@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/update_list.rs b/offchain/tools/social-twitter/src/list/update_list.rs similarity index 99% rename from tools/social-twitter/src/list/update_list.rs rename to offchain/tools/social-twitter/src/list/update_list.rs index 0482775..6f90f27 100644 --- a/tools/social-twitter/src/list/update_list.rs +++ b/offchain/tools/social-twitter/src/list/update_list.rs @@ -61,7 +61,10 @@ impl NexusTool for UpdateList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.update-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.update-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/main.rs b/offchain/tools/social-twitter/src/main.rs similarity index 100% rename from tools/social-twitter/src/main.rs rename to offchain/tools/social-twitter/src/main.rs diff --git a/tools/social-twitter/src/media/mod.rs b/offchain/tools/social-twitter/src/media/mod.rs similarity index 100% rename from tools/social-twitter/src/media/mod.rs rename to offchain/tools/social-twitter/src/media/mod.rs diff --git a/tools/social-twitter/src/media/models.rs b/offchain/tools/social-twitter/src/media/models.rs similarity index 100% rename from tools/social-twitter/src/media/models.rs rename to offchain/tools/social-twitter/src/media/models.rs diff --git a/tools/social-twitter/src/media/upload_media.rs b/offchain/tools/social-twitter/src/media/upload_media.rs similarity index 99% rename from tools/social-twitter/src/media/upload_media.rs rename to offchain/tools/social-twitter/src/media/upload_media.rs index f499fd1..4665d08 100644 --- a/tools/social-twitter/src/media/upload_media.rs +++ b/offchain/tools/social-twitter/src/media/upload_media.rs @@ -132,7 +132,10 @@ impl NexusTool for UploadMedia { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.upload-media@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.upload-media@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/delete_tweet.rs b/offchain/tools/social-twitter/src/tweet/delete_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/delete_tweet.rs rename to offchain/tools/social-twitter/src/tweet/delete_tweet.rs index 7c7ca92..505579c 100644 --- a/tools/social-twitter/src/tweet/delete_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/delete_tweet.rs @@ -58,7 +58,10 @@ impl NexusTool for DeleteTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.delete-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.delete-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_mentioned_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_mentioned_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_mentioned_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_mentioned_tweets.rs index 2f4a8ed..ebb6345 100644 --- a/tools/social-twitter/src/tweet/get_mentioned_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_mentioned_tweets.rs @@ -138,7 +138,10 @@ impl NexusTool for GetMentionedTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-mentioned-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-mentioned-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_recent_search_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_recent_search_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_recent_search_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_recent_search_tweets.rs index 96e7494..c4f6c05 100644 --- a/tools/social-twitter/src/tweet/get_recent_search_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_recent_search_tweets.rs @@ -209,7 +209,10 @@ impl NexusTool for GetRecentSearchTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-recent-search-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-recent-search-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_recent_tweet_count.rs b/offchain/tools/social-twitter/src/tweet/get_recent_tweet_count.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_recent_tweet_count.rs rename to offchain/tools/social-twitter/src/tweet/get_recent_tweet_count.rs index ba24a28..3a163f1 100644 --- a/tools/social-twitter/src/tweet/get_recent_tweet_count.rs +++ b/offchain/tools/social-twitter/src/tweet/get_recent_tweet_count.rs @@ -124,7 +124,10 @@ impl NexusTool for GetRecentTweetCount { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-recent-tweet-count@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-recent-tweet-count@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_tweet.rs b/offchain/tools/social-twitter/src/tweet/get_tweet.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_tweet.rs rename to offchain/tools/social-twitter/src/tweet/get_tweet.rs index d4a494e..f17a1f7 100644 --- a/tools/social-twitter/src/tweet/get_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/get_tweet.rs @@ -168,7 +168,10 @@ impl NexusTool for GetTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_tweets.rs index c6101ca..c6ad6aa 100644 --- a/tools/social-twitter/src/tweet/get_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_tweets.rs @@ -117,7 +117,10 @@ impl NexusTool for GetTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_user_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_user_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_user_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_user_tweets.rs index 78be040..535253d 100644 --- a/tools/social-twitter/src/tweet/get_user_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_user_tweets.rs @@ -144,7 +144,10 @@ impl NexusTool for GetUserTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/like_tweet.rs b/offchain/tools/social-twitter/src/tweet/like_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/like_tweet.rs rename to offchain/tools/social-twitter/src/tweet/like_tweet.rs index 76b787e..0b84e48 100644 --- a/tools/social-twitter/src/tweet/like_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/like_tweet.rs @@ -56,7 +56,10 @@ impl NexusTool for LikeTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.like-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.like-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/mod.rs b/offchain/tools/social-twitter/src/tweet/mod.rs similarity index 100% rename from tools/social-twitter/src/tweet/mod.rs rename to offchain/tools/social-twitter/src/tweet/mod.rs diff --git a/tools/social-twitter/src/tweet/models.rs b/offchain/tools/social-twitter/src/tweet/models.rs similarity index 100% rename from tools/social-twitter/src/tweet/models.rs rename to offchain/tools/social-twitter/src/tweet/models.rs diff --git a/tools/social-twitter/src/tweet/post_tweet.rs b/offchain/tools/social-twitter/src/tweet/post_tweet.rs similarity index 99% rename from tools/social-twitter/src/tweet/post_tweet.rs rename to offchain/tools/social-twitter/src/tweet/post_tweet.rs index f3cc30b..3fe9fe2 100644 --- a/tools/social-twitter/src/tweet/post_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/post_tweet.rs @@ -94,7 +94,10 @@ impl NexusTool for PostTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.post-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.post-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/retweet_tweet.rs b/offchain/tools/social-twitter/src/tweet/retweet_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/retweet_tweet.rs rename to offchain/tools/social-twitter/src/tweet/retweet_tweet.rs index 4477448..48af3b0 100644 --- a/tools/social-twitter/src/tweet/retweet_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/retweet_tweet.rs @@ -63,7 +63,10 @@ impl NexusTool for RetweetTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.retweet-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.retweet-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/undo_retweet_tweet.rs b/offchain/tools/social-twitter/src/tweet/undo_retweet_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/undo_retweet_tweet.rs rename to offchain/tools/social-twitter/src/tweet/undo_retweet_tweet.rs index 43fb5d2..ca2d87f 100644 --- a/tools/social-twitter/src/tweet/undo_retweet_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/undo_retweet_tweet.rs @@ -60,7 +60,10 @@ impl NexusTool for UndoRetweetTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.undo-retweet-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.undo-retweet-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/unlike_tweet.rs b/offchain/tools/social-twitter/src/tweet/unlike_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/unlike_tweet.rs rename to offchain/tools/social-twitter/src/tweet/unlike_tweet.rs index 512cfae..8842872 100644 --- a/tools/social-twitter/src/tweet/unlike_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/unlike_tweet.rs @@ -60,7 +60,10 @@ impl NexusTool for UnlikeTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.unlike-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.unlike-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/twitter_client.rs b/offchain/tools/social-twitter/src/twitter_client.rs similarity index 100% rename from tools/social-twitter/src/twitter_client.rs rename to offchain/tools/social-twitter/src/twitter_client.rs diff --git a/tools/social-twitter/src/user/follow_user.rs b/offchain/tools/social-twitter/src/user/follow_user.rs similarity index 98% rename from tools/social-twitter/src/user/follow_user.rs rename to offchain/tools/social-twitter/src/user/follow_user.rs index 5e6fabe..fbd5408 100644 --- a/tools/social-twitter/src/user/follow_user.rs +++ b/offchain/tools/social-twitter/src/user/follow_user.rs @@ -63,7 +63,10 @@ impl NexusTool for FollowUser { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.follow-user@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.follow-user@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_user_by_id.rs b/offchain/tools/social-twitter/src/user/get_user_by_id.rs similarity index 99% rename from tools/social-twitter/src/user/get_user_by_id.rs rename to offchain/tools/social-twitter/src/user/get_user_by_id.rs index e722f30..1152cb9 100644 --- a/tools/social-twitter/src/user/get_user_by_id.rs +++ b/offchain/tools/social-twitter/src/user/get_user_by_id.rs @@ -144,7 +144,10 @@ impl NexusTool for GetUserById { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-by-id@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-by-id@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_user_by_username.rs b/offchain/tools/social-twitter/src/user/get_user_by_username.rs similarity index 99% rename from tools/social-twitter/src/user/get_user_by_username.rs rename to offchain/tools/social-twitter/src/user/get_user_by_username.rs index 5d96739..2441f4c 100644 --- a/tools/social-twitter/src/user/get_user_by_username.rs +++ b/offchain/tools/social-twitter/src/user/get_user_by_username.rs @@ -144,7 +144,10 @@ impl NexusTool for GetUserByUsername { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-by-username@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-by-username@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_users_by_id.rs b/offchain/tools/social-twitter/src/user/get_users_by_id.rs similarity index 99% rename from tools/social-twitter/src/user/get_users_by_id.rs rename to offchain/tools/social-twitter/src/user/get_users_by_id.rs index 5c9b1ed..7994c93 100644 --- a/tools/social-twitter/src/user/get_users_by_id.rs +++ b/offchain/tools/social-twitter/src/user/get_users_by_id.rs @@ -102,7 +102,10 @@ impl NexusTool for GetUsersById { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-users-by-id@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-users-by-id@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_users_by_username.rs b/offchain/tools/social-twitter/src/user/get_users_by_username.rs similarity index 99% rename from tools/social-twitter/src/user/get_users_by_username.rs rename to offchain/tools/social-twitter/src/user/get_users_by_username.rs index 160e92a..2f50e2c 100644 --- a/tools/social-twitter/src/user/get_users_by_username.rs +++ b/offchain/tools/social-twitter/src/user/get_users_by_username.rs @@ -75,7 +75,10 @@ impl NexusTool for GetUsersByUsername { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-users-by-username@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-users-by-username@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/mod.rs b/offchain/tools/social-twitter/src/user/mod.rs similarity index 100% rename from tools/social-twitter/src/user/mod.rs rename to offchain/tools/social-twitter/src/user/mod.rs diff --git a/tools/social-twitter/src/user/models.rs b/offchain/tools/social-twitter/src/user/models.rs similarity index 100% rename from tools/social-twitter/src/user/models.rs rename to offchain/tools/social-twitter/src/user/models.rs diff --git a/tools/social-twitter/src/user/unfollow_user.rs b/offchain/tools/social-twitter/src/user/unfollow_user.rs similarity index 98% rename from tools/social-twitter/src/user/unfollow_user.rs rename to offchain/tools/social-twitter/src/user/unfollow_user.rs index 30e4444..7edd8cd 100644 --- a/tools/social-twitter/src/user/unfollow_user.rs +++ b/offchain/tools/social-twitter/src/user/unfollow_user.rs @@ -60,7 +60,10 @@ impl NexusTool for UnfollowUser { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.unfollow-user@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.unfollow-user@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/social-twitter/tools.json b/offchain/tools/social-twitter/tools.json new file mode 100644 index 0000000..d5d37ca --- /dev/null +++ b/offchain/tools/social-twitter/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "social-twitter", + "command": "social-twitter", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/storage-walrus/Cargo.toml b/offchain/tools/storage-walrus/Cargo.toml similarity index 86% rename from tools/storage-walrus/Cargo.toml rename to offchain/tools/storage-walrus/Cargo.toml index a2f0fee..42acc1a 100644 --- a/tools/storage-walrus/Cargo.toml +++ b/offchain/tools/storage-walrus/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "walrus" +path = "src/main.rs" + [dependencies] schemars.workspace = true serde.workspace = true @@ -28,3 +32,7 @@ nexus-sdk = { workspace = true, features = ["walrus"] } [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/storage-walrus/README.md b/offchain/tools/storage-walrus/README.md similarity index 100% rename from tools/storage-walrus/README.md rename to offchain/tools/storage-walrus/README.md diff --git a/offchain/tools/storage-walrus/build.rs b/offchain/tools/storage-walrus/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/storage-walrus/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/storage-walrus/src/client.rs b/offchain/tools/storage-walrus/src/client.rs similarity index 100% rename from tools/storage-walrus/src/client.rs rename to offchain/tools/storage-walrus/src/client.rs diff --git a/tools/storage-walrus/src/main.rs b/offchain/tools/storage-walrus/src/main.rs similarity index 100% rename from tools/storage-walrus/src/main.rs rename to offchain/tools/storage-walrus/src/main.rs diff --git a/tools/storage-walrus/src/read_file.rs b/offchain/tools/storage-walrus/src/read_file.rs similarity index 98% rename from tools/storage-walrus/src/read_file.rs rename to offchain/tools/storage-walrus/src/read_file.rs index bba8351..d44f962 100644 --- a/tools/storage-walrus/src/read_file.rs +++ b/offchain/tools/storage-walrus/src/read_file.rs @@ -67,7 +67,10 @@ impl NexusTool for ReadFile { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.read-file@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.read-file@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/read_json.rs b/offchain/tools/storage-walrus/src/read_json.rs similarity index 99% rename from tools/storage-walrus/src/read_json.rs rename to offchain/tools/storage-walrus/src/read_json.rs index 9711831..5fe8ed6 100644 --- a/tools/storage-walrus/src/read_json.rs +++ b/offchain/tools/storage-walrus/src/read_json.rs @@ -96,7 +96,10 @@ impl NexusTool for ReadJson { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.read-json@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.read-json@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/upload_file.rs b/offchain/tools/storage-walrus/src/upload_file.rs similarity index 99% rename from tools/storage-walrus/src/upload_file.rs rename to offchain/tools/storage-walrus/src/upload_file.rs index fb06b6d..f45feeb 100644 --- a/tools/storage-walrus/src/upload_file.rs +++ b/offchain/tools/storage-walrus/src/upload_file.rs @@ -93,7 +93,10 @@ impl NexusTool for UploadFile { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.upload-file@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.upload-file@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/upload_json.rs b/offchain/tools/storage-walrus/src/upload_json.rs similarity index 99% rename from tools/storage-walrus/src/upload_json.rs rename to offchain/tools/storage-walrus/src/upload_json.rs index aa5f6a3..a3316d0 100644 --- a/tools/storage-walrus/src/upload_json.rs +++ b/offchain/tools/storage-walrus/src/upload_json.rs @@ -98,7 +98,10 @@ impl NexusTool for UploadJson { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.upload-json@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.upload-json@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/utils.rs b/offchain/tools/storage-walrus/src/utils.rs similarity index 100% rename from tools/storage-walrus/src/utils.rs rename to offchain/tools/storage-walrus/src/utils.rs diff --git a/tools/storage-walrus/src/verify_blob.rs b/offchain/tools/storage-walrus/src/verify_blob.rs similarity index 98% rename from tools/storage-walrus/src/verify_blob.rs rename to offchain/tools/storage-walrus/src/verify_blob.rs index 0016788..033590b 100644 --- a/tools/storage-walrus/src/verify_blob.rs +++ b/offchain/tools/storage-walrus/src/verify_blob.rs @@ -67,7 +67,10 @@ impl NexusTool for VerifyBlob { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.verify-blob@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.verify-blob@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/storage-walrus/tools.json b/offchain/tools/storage-walrus/tools.json new file mode 100644 index 0000000..c26f0f1 --- /dev/null +++ b/offchain/tools/storage-walrus/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "walrus", + "command": "walrus", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/templating-jinja/Cargo.toml b/offchain/tools/templating-jinja/Cargo.toml similarity index 76% rename from tools/templating-jinja/Cargo.toml rename to offchain/tools/templating-jinja/Cargo.toml index 34ae1a5..22cbdb7 100644 --- a/tools/templating-jinja/Cargo.toml +++ b/offchain/tools/templating-jinja/Cargo.toml @@ -10,6 +10,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "templating-jinja" +path = "src/main.rs" + [dependencies] minijinja.workspace = true schemars.workspace = true @@ -19,4 +23,8 @@ tokio.workspace = true # === Nexus deps === nexus-toolkit.workspace = true -nexus-sdk.workspace = true \ No newline at end of file +nexus-sdk.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" \ No newline at end of file diff --git a/tools/templating-jinja/README.md b/offchain/tools/templating-jinja/README.md similarity index 100% rename from tools/templating-jinja/README.md rename to offchain/tools/templating-jinja/README.md diff --git a/offchain/tools/templating-jinja/build.rs b/offchain/tools/templating-jinja/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/templating-jinja/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/templating-jinja/src/main.rs b/offchain/tools/templating-jinja/src/main.rs similarity index 100% rename from tools/templating-jinja/src/main.rs rename to offchain/tools/templating-jinja/src/main.rs diff --git a/tools/templating-jinja/src/template.rs b/offchain/tools/templating-jinja/src/template.rs similarity index 99% rename from tools/templating-jinja/src/template.rs rename to offchain/tools/templating-jinja/src/template.rs index 7587a53..95c6d76 100644 --- a/tools/templating-jinja/src/template.rs +++ b/offchain/tools/templating-jinja/src/template.rs @@ -67,7 +67,10 @@ impl NexusTool for TemplatingJinja { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.templating.jinja@1") + fqn!(concat!( + "xyz.taluslabs.templating.jinja@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/templating-jinja/tools.json b/offchain/tools/templating-jinja/tools.json new file mode 100644 index 0000000..986c0f0 --- /dev/null +++ b/offchain/tools/templating-jinja/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "templating-jinja", + "command": "templating-jinja", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/onchain/README.md b/onchain/README.md new file mode 100644 index 0000000..0481939 --- /dev/null +++ b/onchain/README.md @@ -0,0 +1,6 @@ +# onchain/ + +Reserved for onchain (Move / Sui package) tools and their CI. Empty +for now — see the offchain CI pipeline spec in +[docs/superpowers/specs/2026-05-19-offchain-tools-pipeline-design.md](../docs/superpowers/specs/2026-05-19-offchain-tools-pipeline-design.md) +for context. From 0bd2820c0a089d93925395b25d7fb7a687d4f9d7 Mon Sep 17 00:00:00 2001 From: tuky191 Date: Mon, 25 May 2026 17:49:44 +0200 Subject: [PATCH 3/3] payments-stripe: retrofit into offchain pipeline conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mike's PR #32 added payments-stripe at the legacy `tools/` path with per-tool deploy workflows. Main has since restructured (PR #33) so all tools live under `offchain/tools/` and share a single `offchain-tools.*` CI pipeline matrix. This branch realigns payments-stripe to the new layout so the CI picks it up automatically. Layout move: tools/payments-stripe/ → offchain/tools/payments-stripe/ Added (per the new conventions enforced by .pre-commit/14-check-tool-conventions): - tools.json (tool_name, command, environment) - build.rs (byte-identical to other tools; threads TOOL_FQN_VERSION env var via cargo:rustc-env) - Cargo.toml [[bin]] + [build-dependencies] sections Removed (legacy per-tool deploy artifacts replaced by the shared offchain-tools.{discover,deploy,prepare,register}.yml matrix): - tools/payments-stripe/deploy/{Dockerfile,cloud-run.*,register.sh} - tools/payments-stripe/paths.json (paths now come from each tool's --meta output) - .github/workflows/deploy-payments-stripe-{mainnet,testnet}.yml Source modifications: - 6 fqn!() macros across src/tools/*.rs threaded to the fqn!(concat!("...@", env!("TOOL_FQN_VERSION"))) form so the on-chain FQN version stays in sync with the content-hashed image tag CI produces. Verification: - cargo check -p payments-stripe → passes - .pre-commit/14-check-tool-conventions → ✅ Tool conventions OK --- .../deploy-payments-stripe-mainnet.yml | 89 ------------------- .../deploy-payments-stripe-testnet.yml | 71 --------------- offchain/Cargo.lock | 5 +- .../tools}/payments-stripe/AUDIT.md | 0 .../tools}/payments-stripe/Cargo.toml | 8 ++ .../tools}/payments-stripe/README.md | 0 offchain/tools/payments-stripe/build.rs | 50 +++++++++++ .../tools}/payments-stripe/src/error.rs | 0 .../tools}/payments-stripe/src/main.rs | 0 .../payments-stripe/src/stripe_client.rs | 0 .../src/tools/confirm_payment_intent.rs | 5 +- .../src/tools/create_customer.rs | 5 +- .../src/tools/create_payment_intent.rs | 5 +- .../payments-stripe/src/tools/get_balance.rs | 5 +- .../src/tools/get_payment_intent.rs | 5 +- .../payments-stripe/src/tools/list_charges.rs | 5 +- .../tools}/payments-stripe/src/tools/mod.rs | 0 .../payments-stripe/src/tools/models.rs | 0 offchain/tools/payments-stripe/tools.json | 7 ++ tools/payments-stripe/deploy/Dockerfile | 28 ------ .../deploy/cloud-run.mainnet.yaml | 61 ------------- .../deploy/cloud-run.testnet.yaml | 61 ------------- tools/payments-stripe/deploy/register.sh | 49 ---------- tools/payments-stripe/paths.json | 8 -- 24 files changed, 92 insertions(+), 375 deletions(-) delete mode 100644 .github/workflows/deploy-payments-stripe-mainnet.yml delete mode 100644 .github/workflows/deploy-payments-stripe-testnet.yml rename {tools => offchain/tools}/payments-stripe/AUDIT.md (100%) rename {tools => offchain/tools}/payments-stripe/Cargo.toml (84%) rename {tools => offchain/tools}/payments-stripe/README.md (100%) create mode 100644 offchain/tools/payments-stripe/build.rs rename {tools => offchain/tools}/payments-stripe/src/error.rs (100%) rename {tools => offchain/tools}/payments-stripe/src/main.rs (100%) rename {tools => offchain/tools}/payments-stripe/src/stripe_client.rs (100%) rename {tools => offchain/tools}/payments-stripe/src/tools/confirm_payment_intent.rs (97%) rename {tools => offchain/tools}/payments-stripe/src/tools/create_customer.rs (97%) rename {tools => offchain/tools}/payments-stripe/src/tools/create_payment_intent.rs (98%) rename {tools => offchain/tools}/payments-stripe/src/tools/get_balance.rs (96%) rename {tools => offchain/tools}/payments-stripe/src/tools/get_payment_intent.rs (97%) rename {tools => offchain/tools}/payments-stripe/src/tools/list_charges.rs (97%) rename {tools => offchain/tools}/payments-stripe/src/tools/mod.rs (100%) rename {tools => offchain/tools}/payments-stripe/src/tools/models.rs (100%) create mode 100644 offchain/tools/payments-stripe/tools.json delete mode 100644 tools/payments-stripe/deploy/Dockerfile delete mode 100644 tools/payments-stripe/deploy/cloud-run.mainnet.yaml delete mode 100644 tools/payments-stripe/deploy/cloud-run.testnet.yaml delete mode 100755 tools/payments-stripe/deploy/register.sh delete mode 100644 tools/payments-stripe/paths.json diff --git a/.github/workflows/deploy-payments-stripe-mainnet.yml b/.github/workflows/deploy-payments-stripe-mainnet.yml deleted file mode 100644 index 3d90565..0000000 --- a/.github/workflows/deploy-payments-stripe-mainnet.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: deploy-payments-stripe-mainnet - -on: - push: - tags: - - "v-payments-stripe-*" - -permissions: - contents: read - id-token: write - -env: - PROJECT_ID: talus-tools-mainnet - REGION: us-central1 - REPO: nexus-tools - CRATE: payments-stripe - SERVICE: payments-stripe-mainnet - -jobs: - guard: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Require testnet success on this commit - uses: actions/github-script@v7 - with: - script: | - const { owner, repo } = context.repo; - const ref = context.sha; - const { data } = await github.rest.checks.listForRef({ - owner, repo, ref, check_name: 'build-deploy-register' - }); - const ok = data.check_runs.some(c => - c.name === 'build-deploy-register' && c.conclusion === 'success' - ); - if (!ok) { - core.setFailed('Testnet deploy must succeed on this commit before mainnet.'); - } - - build-deploy-register: - needs: guard - runs-on: ubuntu-latest - environment: mainnet - steps: - - uses: actions/checkout@v4 - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_MAINNET }} - service_account: github-actions@${{ env.PROJECT_ID }}.iam.gserviceaccount.com - - - uses: google-github-actions/setup-gcloud@v2 - - - name: Configure Docker - run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet - - - name: Build and push image - env: - SHA: ${{ github.sha }} - run: | - IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/${CRATE}:${SHA}" - docker build -f tools/${CRATE}/deploy/Dockerfile -t "$IMAGE" . - docker push "$IMAGE" - - - name: Deploy to Cloud Run - env: - SHA: ${{ github.sha }} - run: | - export REGISTRY="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}" - envsubst < tools/${CRATE}/deploy/cloud-run.mainnet.yaml > /tmp/svc.yaml - gcloud run services replace /tmp/svc.yaml --region "$REGION" --quiet - URL=$(gcloud run services describe "$SERVICE" --region "$REGION" --format='value(status.url)') - echo "URL=$URL" >> "$GITHUB_ENV" - - - name: Smoke test - run: | - for p in $(jq -r '.[]' tools/${CRATE}/paths.json); do - curl -fsS "${URL}${p}/health" - curl -fsS "${URL}${p}/meta" | jq -e .fqn >/dev/null - done - - - name: Install Nexus CLI - run: | - curl -fsSL https://raw.githubusercontent.com/Talus-Network/homebrew-tap/main/install.sh | bash - echo "$HOME/.nexus/bin" >> "$GITHUB_PATH" - - - name: Register / update tools on Nexus mainnet - run: bash tools/${CRATE}/deploy/register.sh "$URL" mainnet diff --git a/.github/workflows/deploy-payments-stripe-testnet.yml b/.github/workflows/deploy-payments-stripe-testnet.yml deleted file mode 100644 index 59f7a38..0000000 --- a/.github/workflows/deploy-payments-stripe-testnet.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: deploy-payments-stripe-testnet - -on: - push: - branches: [main] - paths: - - "tools/payments-stripe/**" - - ".github/workflows/deploy-payments-stripe-testnet.yml" - workflow_dispatch: - -permissions: - contents: read - id-token: write - -env: - PROJECT_ID: talus-tools-testnet - REGION: us-central1 - REPO: nexus-tools - CRATE: payments-stripe - SERVICE: payments-stripe-testnet - -jobs: - build-deploy-register: - runs-on: ubuntu-latest - environment: testnet - steps: - - uses: actions/checkout@v4 - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_TESTNET }} - service_account: github-actions@${{ env.PROJECT_ID }}.iam.gserviceaccount.com - - - uses: google-github-actions/setup-gcloud@v2 - - - name: Configure Docker - run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet - - - name: Build and push image - env: - SHA: ${{ github.sha }} - run: | - IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/${CRATE}:${SHA}" - docker build -f tools/${CRATE}/deploy/Dockerfile -t "$IMAGE" . - docker push "$IMAGE" - - - name: Deploy to Cloud Run - env: - SHA: ${{ github.sha }} - run: | - export REGISTRY="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}" - envsubst < tools/${CRATE}/deploy/cloud-run.testnet.yaml > /tmp/svc.yaml - gcloud run services replace /tmp/svc.yaml --region "$REGION" --quiet - URL=$(gcloud run services describe "$SERVICE" --region "$REGION" --format='value(status.url)') - echo "URL=$URL" >> "$GITHUB_ENV" - - - name: Smoke test - run: | - for p in $(jq -r '.[]' tools/${CRATE}/paths.json); do - curl -fsS "${URL}${p}/health" - curl -fsS "${URL}${p}/meta" | jq -e .fqn >/dev/null - done - - - name: Install Nexus CLI - run: | - curl -fsSL https://raw.githubusercontent.com/Talus-Network/homebrew-tap/main/install.sh | bash - echo "$HOME/.nexus/bin" >> "$GITHUB_PATH" - - - name: Register / update tools on Nexus testnet - run: bash tools/${CRATE}/deploy/register.sh "$URL" testnet diff --git a/offchain/Cargo.lock b/offchain/Cargo.lock index 550ad52..0f6e73a 100644 --- a/offchain/Cargo.lock +++ b/offchain/Cargo.lock @@ -2568,12 +2568,13 @@ dependencies = [ "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", + "toml", ] [[package]] diff --git a/tools/payments-stripe/AUDIT.md b/offchain/tools/payments-stripe/AUDIT.md similarity index 100% rename from tools/payments-stripe/AUDIT.md rename to offchain/tools/payments-stripe/AUDIT.md diff --git a/tools/payments-stripe/Cargo.toml b/offchain/tools/payments-stripe/Cargo.toml similarity index 84% rename from tools/payments-stripe/Cargo.toml rename to offchain/tools/payments-stripe/Cargo.toml index 6c9f775..e953082 100644 --- a/tools/payments-stripe/Cargo.toml +++ b/offchain/tools/payments-stripe/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "payments-stripe" +path = "src/main.rs" + [dependencies] thiserror.workspace = true tokio.workspace = true @@ -26,3 +30,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/payments-stripe/README.md b/offchain/tools/payments-stripe/README.md similarity index 100% rename from tools/payments-stripe/README.md rename to offchain/tools/payments-stripe/README.md diff --git a/offchain/tools/payments-stripe/build.rs b/offchain/tools/payments-stripe/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/payments-stripe/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/payments-stripe/src/error.rs b/offchain/tools/payments-stripe/src/error.rs similarity index 100% rename from tools/payments-stripe/src/error.rs rename to offchain/tools/payments-stripe/src/error.rs diff --git a/tools/payments-stripe/src/main.rs b/offchain/tools/payments-stripe/src/main.rs similarity index 100% rename from tools/payments-stripe/src/main.rs rename to offchain/tools/payments-stripe/src/main.rs diff --git a/tools/payments-stripe/src/stripe_client.rs b/offchain/tools/payments-stripe/src/stripe_client.rs similarity index 100% rename from tools/payments-stripe/src/stripe_client.rs rename to offchain/tools/payments-stripe/src/stripe_client.rs diff --git a/tools/payments-stripe/src/tools/confirm_payment_intent.rs b/offchain/tools/payments-stripe/src/tools/confirm_payment_intent.rs similarity index 97% rename from tools/payments-stripe/src/tools/confirm_payment_intent.rs rename to offchain/tools/payments-stripe/src/tools/confirm_payment_intent.rs index bc91586..3b01790 100644 --- a/tools/payments-stripe/src/tools/confirm_payment_intent.rs +++ b/offchain/tools/payments-stripe/src/tools/confirm_payment_intent.rs @@ -63,7 +63,10 @@ impl NexusTool for ConfirmPaymentIntent { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.confirm-payment-intent@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.confirm-payment-intent@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/create_customer.rs b/offchain/tools/payments-stripe/src/tools/create_customer.rs similarity index 97% rename from tools/payments-stripe/src/tools/create_customer.rs rename to offchain/tools/payments-stripe/src/tools/create_customer.rs index 84c3413..cbf9afb 100644 --- a/tools/payments-stripe/src/tools/create_customer.rs +++ b/offchain/tools/payments-stripe/src/tools/create_customer.rs @@ -57,7 +57,10 @@ impl NexusTool for CreateCustomer { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.create-customer@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.create-customer@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/create_payment_intent.rs b/offchain/tools/payments-stripe/src/tools/create_payment_intent.rs similarity index 98% rename from tools/payments-stripe/src/tools/create_payment_intent.rs rename to offchain/tools/payments-stripe/src/tools/create_payment_intent.rs index f30fa57..0ddc107 100644 --- a/tools/payments-stripe/src/tools/create_payment_intent.rs +++ b/offchain/tools/payments-stripe/src/tools/create_payment_intent.rs @@ -73,7 +73,10 @@ impl NexusTool for CreatePaymentIntent { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.create-payment-intent@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.create-payment-intent@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/get_balance.rs b/offchain/tools/payments-stripe/src/tools/get_balance.rs similarity index 96% rename from tools/payments-stripe/src/tools/get_balance.rs rename to offchain/tools/payments-stripe/src/tools/get_balance.rs index afec74c..0c06785 100644 --- a/tools/payments-stripe/src/tools/get_balance.rs +++ b/offchain/tools/payments-stripe/src/tools/get_balance.rs @@ -50,7 +50,10 @@ impl NexusTool for GetBalance { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.get-balance@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.get-balance@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/get_payment_intent.rs b/offchain/tools/payments-stripe/src/tools/get_payment_intent.rs similarity index 97% rename from tools/payments-stripe/src/tools/get_payment_intent.rs rename to offchain/tools/payments-stripe/src/tools/get_payment_intent.rs index cd5a084..c059a6d 100644 --- a/tools/payments-stripe/src/tools/get_payment_intent.rs +++ b/offchain/tools/payments-stripe/src/tools/get_payment_intent.rs @@ -58,7 +58,10 @@ impl NexusTool for GetPaymentIntent { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.get-payment-intent@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.get-payment-intent@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/list_charges.rs b/offchain/tools/payments-stripe/src/tools/list_charges.rs similarity index 97% rename from tools/payments-stripe/src/tools/list_charges.rs rename to offchain/tools/payments-stripe/src/tools/list_charges.rs index 60177e0..3da217a 100644 --- a/tools/payments-stripe/src/tools/list_charges.rs +++ b/offchain/tools/payments-stripe/src/tools/list_charges.rs @@ -53,7 +53,10 @@ impl NexusTool for ListCharges { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.list-charges@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.list-charges@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/mod.rs b/offchain/tools/payments-stripe/src/tools/mod.rs similarity index 100% rename from tools/payments-stripe/src/tools/mod.rs rename to offchain/tools/payments-stripe/src/tools/mod.rs diff --git a/tools/payments-stripe/src/tools/models.rs b/offchain/tools/payments-stripe/src/tools/models.rs similarity index 100% rename from tools/payments-stripe/src/tools/models.rs rename to offchain/tools/payments-stripe/src/tools/models.rs diff --git a/offchain/tools/payments-stripe/tools.json b/offchain/tools/payments-stripe/tools.json new file mode 100644 index 0000000..bdcfd30 --- /dev/null +++ b/offchain/tools/payments-stripe/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "payments-stripe", + "command": "payments-stripe", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/payments-stripe/deploy/Dockerfile b/tools/payments-stripe/deploy/Dockerfile deleted file mode 100644 index 622fdd7..0000000 --- a/tools/payments-stripe/deploy/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# ---- Builder ---------------------------------------------------------------- -FROM rust:1.83-bookworm AS builder -WORKDIR /build - -RUN apt-get update && apt-get install -y --no-install-recommends \ - pkg-config libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* - -# Copy the whole workspace — `members = ["tools/*"]` requires every -# sibling crate to exist for cargo's resolver. -COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ -COPY tools tools -COPY helpers helpers - -RUN cargo +stable build --release --package payments-stripe - -# ---- Runtime ---------------------------------------------------------------- -FROM gcr.io/distroless/cc-debian12:nonroot AS runtime -WORKDIR /app - -COPY --from=builder /build/target/release/payments-stripe /app/payments-stripe - -ENV BIND_ADDR=0.0.0.0:8080 -EXPOSE 8080 - -USER nonroot -ENTRYPOINT ["/app/payments-stripe"] diff --git a/tools/payments-stripe/deploy/cloud-run.mainnet.yaml b/tools/payments-stripe/deploy/cloud-run.mainnet.yaml deleted file mode 100644 index 7970b80..0000000 --- a/tools/payments-stripe/deploy/cloud-run.mainnet.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Credential model: this service holds NO Stripe API credentials. -# The only secrets mounted here are: -# - nexus-toolkit-config-mainnet-payments-stripe (Tool's own Ed25519 signing key) -# - nexus-allowed-leaders-mainnet (Leader public keys, not secret) -# Stripe `api_key` arrives in the per-request Input struct, sourced by -# the Leader at DAG-execution time. Adding any other secretKeyRef fails -# the auditor's C10 check. -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: "payments-stripe-mainnet" - labels: - network: mainnet - tool-fqn: "xyz.taluslabs.payments.stripe" - annotations: - run.googleapis.com/launch-stage: GA -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - autoscaling.knative.dev/maxScale: "20" - run.googleapis.com/execution-environment: gen2 - spec: - serviceAccountName: "nexus-tools-mainnet@${PROJECT_ID}.iam.gserviceaccount.com" - timeoutSeconds: 30 - containerConcurrency: 80 - containers: - - name: tool - image: "${REGISTRY}/payments-stripe:${SHA}" - ports: - - name: http1 - containerPort: 8080 - env: - - name: BIND_ADDR - value: 0.0.0.0:8080 - - name: NEXUS_TOOLKIT_CONFIG_PATH - value: /etc/nexus/toolkit.json - - name: NEXUS_NETWORK - value: mainnet - volumeMounts: - - name: nexus-config - mountPath: /etc/nexus - readOnly: true - - name: allowed-leaders - mountPath: /etc/nexus/allowed_leaders - readOnly: true - resources: - limits: - cpu: "2" - memory: 1Gi - volumes: - - name: nexus-config - secret: - secretName: "nexus-toolkit-config-mainnet-payments-stripe" - - name: allowed-leaders - secret: - secretName: "nexus-allowed-leaders-mainnet" - traffic: - - percent: 100 - latestRevision: true diff --git a/tools/payments-stripe/deploy/cloud-run.testnet.yaml b/tools/payments-stripe/deploy/cloud-run.testnet.yaml deleted file mode 100644 index 59c22b0..0000000 --- a/tools/payments-stripe/deploy/cloud-run.testnet.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Credential model: this service holds NO Stripe API credentials. -# The only secrets mounted here are: -# - nexus-toolkit-config-testnet-payments-stripe (Tool's own Ed25519 signing key) -# - nexus-allowed-leaders-testnet (Leader public keys, not secret) -# Stripe `api_key` arrives in the per-request Input struct, sourced by -# the Leader at DAG-execution time. Adding any other secretKeyRef fails -# the auditor's C10 check. -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: "payments-stripe-testnet" - labels: - network: testnet - tool-fqn: "xyz.taluslabs.payments.stripe" - annotations: - run.googleapis.com/launch-stage: GA -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "0" - autoscaling.knative.dev/maxScale: "5" - run.googleapis.com/execution-environment: gen2 - spec: - serviceAccountName: "nexus-tools-testnet@${PROJECT_ID}.iam.gserviceaccount.com" - timeoutSeconds: 30 - containerConcurrency: 80 - containers: - - name: tool - image: "${REGISTRY}/payments-stripe:${SHA}" - ports: - - name: http1 - containerPort: 8080 - env: - - name: BIND_ADDR - value: 0.0.0.0:8080 - - name: NEXUS_TOOLKIT_CONFIG_PATH - value: /etc/nexus/toolkit.json - - name: NEXUS_NETWORK - value: testnet - volumeMounts: - - name: nexus-config - mountPath: /etc/nexus - readOnly: true - - name: allowed-leaders - mountPath: /etc/nexus/allowed_leaders - readOnly: true - resources: - limits: - cpu: "1" - memory: 512Mi - volumes: - - name: nexus-config - secret: - secretName: "nexus-toolkit-config-testnet-payments-stripe" - - name: allowed-leaders - secret: - secretName: "nexus-allowed-leaders-testnet" - traffic: - - percent: 100 - latestRevision: true diff --git a/tools/payments-stripe/deploy/register.sh b/tools/payments-stripe/deploy/register.sh deleted file mode 100755 index 65bfed5..0000000 --- a/tools/payments-stripe/deploy/register.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Idempotent registration of payments-stripe tools with Nexus. -# -# Usage: -# register.sh -# -# Reads tool paths from ../paths.json, fetches each tool's FQN from -# /meta, then registers (or updates) each FQN against the chosen Nexus -# network. - -set -euo pipefail - -URL="${1:?missing first arg: tool URL}" -NETWORK="${2:?missing second arg: testnet|mainnet}" - -case "$NETWORK" in - testnet|mainnet) ;; - *) echo "network must be 'testnet' or 'mainnet'"; exit 2 ;; -esac - -PATHS_FILE="$(dirname "$0")/../paths.json" -if [[ ! -f "$PATHS_FILE" ]]; then - echo "expected $PATHS_FILE to enumerate tool paths" >&2 - exit 2 -fi - -mapfile -t PATHS < <(jq -r '.[]' "$PATHS_FILE") - -for path in "${PATHS[@]}"; do - meta_url="${URL%/}${path}/meta" - meta="$(curl -fsS "$meta_url")" - fqn="$(echo "$meta" | jq -r .fqn)" - description="$(echo "$meta" | jq -r .description)" - - echo "registering $fqn at $URL$path on $NETWORK" - - if nexus tool list --network "$NETWORK" 2>/dev/null | grep -q "$fqn"; then - nexus tool update offchain \ - --network "$NETWORK" \ - --tool-fqn "$fqn" \ - --url "${URL%/}${path}" - else - nexus tool register offchain \ - --network "$NETWORK" \ - --tool-fqn "$fqn" \ - --url "${URL%/}${path}" \ - --description "$description" - fi -done diff --git a/tools/payments-stripe/paths.json b/tools/payments-stripe/paths.json deleted file mode 100644 index 28e0a40..0000000 --- a/tools/payments-stripe/paths.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - "/create-payment-intent", - "/get-payment-intent", - "/confirm-payment-intent", - "/create-customer", - "/get-balance", - "/list-charges" -]