diff --git a/CHANGELOG.md b/CHANGELOG.md index 0883900..cb74d8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,44 @@ ## Unreleased +### Added + +- **Lazy remote POD5 reading over range requests** (`remote` feature, off by + default). `escpod inspect` / `view` / `summary` now accept an `s3://`, + `gs://`, `az://`, or `https://` URL in place of a path and read it lazily: + opening transfers only the file tail and footer, and metadata commands then + fetch just the reads table. Inspecting a multi-GB object costs a few MB of + GETs instead of a full download. Credentials come from `object_store`'s + standard environment chain (`AWS_ACCESS_KEY_ID`, `AWS_REGION`, + `AWS_ENDPOINT` for MinIO and other S3-compatible services; note that an + `az://` URL carries no account name, so Azure also needs + `AZURE_STORAGE_ACCOUNT_NAME`). Build with `--features remote`; a binary + built without it rejects URLs with an explanatory error rather than + treating them as missing paths. + + Signal access still materializes the whole signal table, so signal-heavy + commands (`demux`, `resquiggle`, `repack`, `merge`) remain a poor fit for a + remote object — download it first. Per-batch lazy signal is future work. + Writing remotely is not supported (#83, phases 0–1). + +- **`ByteSource` abstraction** behind the reader (`MmapSource`, `MemorySource`, + and the gated `RemoteSource`). `Reader` no longer holds an `Mmap` directly; + it asks a source for byte ranges, which come back as refcounted `Bytes`. + The local path stays zero-copy — ranges are views over the mapping, never + copies. `Reader::from_source`, `Reader::file_size`, and + `Reader::source_description` are new; no existing API changed. + +### Fixed + +- **POD5 footer parsing now rejects corrupt footer lengths** instead of + computing an out-of-range slice offset. A negative length, or one that + would place the footer magic before the file's leading signature, is now a + clean `InvalidFooter` error. + +- **Two `escapepod-pod5` tests had a wrong relative path to the bundled test + POD5** (`../data/…` rather than `../../data/…`) and had been silently + skipping their assertions. They now run. + ## 0.6.2 (2026-07-22) ### Build / Tooling diff --git a/Cargo.lock b/Cargo.lock index 839162f..8605a26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,6 +331,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "atoi" version = "2.0.0" @@ -340,6 +351,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" @@ -382,6 +399,15 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -484,6 +510,7 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", + "serde", "windows-link", ] @@ -707,6 +734,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -778,15 +815,25 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", ] [[package]] @@ -801,6 +848,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "downcast-rs" version = "2.0.2" @@ -932,13 +990,17 @@ dependencies = [ "anyhow", "arrow", "byteorder", + "bytes", "criterion", "csv", "flatbuffers", "memmap2", + "object_store", "rayon", "tempfile", "thiserror 2.0.19", + "tokio", + "url", "uuid", "zstd", ] @@ -1054,30 +1116,113 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1085,8 +1230,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1108,11 +1255,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", ] [[package]] @@ -1177,12 +1345,41 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + [[package]] name = "hybrid-array" version = "0.4.13" @@ -1192,6 +1389,67 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1216,12 +1474,115 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1256,6 +1617,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1471,6 +1838,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -1486,6 +1859,12 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lzma-rust2" version = "0.15.8" @@ -1517,6 +1896,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.8.2" @@ -1564,6 +1953,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -1831,23 +2231,60 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "oorandom" -version = "11.1.5" +name = "object_store" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.5", + "reqwest", + "ring", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.19", + "tokio", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl" version = "0.10.80" @@ -2084,6 +2521,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2223,6 +2669,72 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.19", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.19", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.45" @@ -2251,10 +2763,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -2276,6 +2798,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2285,6 +2817,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rand_core" version = "0.10.1" @@ -2311,6 +2852,15 @@ dependencies = [ "rand 0.10.2", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -2375,6 +2925,49 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + [[package]] name = "ring" version = "0.17.14" @@ -2446,12 +3039,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ + "web-time", "zeroize", ] @@ -2602,6 +3217,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.11.0" @@ -2610,7 +3237,7 @@ checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.11.3", ] [[package]] @@ -2652,6 +3279,16 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "socks" version = "0.3.4" @@ -2675,6 +3312,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strength_reduce" version = "0.2.4" @@ -2725,6 +3368,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tabled" version = "0.21.0" @@ -2849,6 +3512,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2859,6 +3532,116 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -3131,6 +3914,12 @@ dependencies = [ "strength_reduce", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "twox-hash" version = "2.1.2" @@ -3211,12 +4000,30 @@ dependencies = [ "log", ] +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + [[package]] name = "utf8-zero" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -3272,6 +4079,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3309,6 +4125,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.125" @@ -3363,6 +4189,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -3679,6 +4518,12 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + [[package]] name = "xattr" version = "1.6.1" @@ -3689,6 +4534,29 @@ dependencies = [ "rustix", ] +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.52" @@ -3709,12 +4577,66 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zlib-rs" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index 3186a4c..c1456e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,10 @@ indicatif = "0.18" # I/O memmap2 = "0.9" numpy = "0.29" +# Lazy remote POD5 reading (opt-in via `remote`). `aws` also covers every +# S3-compatible endpoint (MinIO, Ceph); `http` covers plain HTTPS range GETs. +# rustls, not native-tls, so a static-musl release binary stays OpenSSL-free. +object_store = { version = "0.12", default-features = false, features = ["aws", "gcp", "azure", "http", "tls-webpki-roots"] } # Parquet I/O for fingerprint tables (workspace dep so both test fixtures # and the CLI share the same version; features are kept default-off at # the workspace level so non-parquet crates don't pay). @@ -47,7 +51,11 @@ rayon = "1" # from `ort` uses native-tls→openssl, which we deliberately avoid here. sha2 = "0.11" thiserror = "2" +# Only ever driven by a shared, process-wide runtime bridging object_store's +# async API into the synchronous Reader (see reader/remote.rs). +tokio = { version = "1", default-features = false, features = ["rt-multi-thread"] } ureq = { version = "3", default-features = false, features = ["rustls"] } +url = "2" # Structured logging tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/crates/escapepod-cli/Cargo.toml b/crates/escapepod-cli/Cargo.toml index bc91b05..b7eebca 100644 --- a/crates/escapepod-cli/Cargo.toml +++ b/crates/escapepod-cli/Cargo.toml @@ -113,6 +113,10 @@ cnn-detect = ["demux", "escapepod-demux/cnn-detect"] # preprocessing. Needs a CUDA-enabled libonnxruntime + visible GPU at runtime. cnn-gpu = ["cnn-detect", "escapepod-demux/cnn-gpu"] experimental = [] +# Read POD5 objects straight from S3/GCS/Azure/HTTPS instead of a local path, +# lazily over range requests (`escpod inspect s3://bucket/reads.pod5`). Opt-in +# because object_store + tokio are otherwise dead weight in a local-file build. +remote = ["signal", "escapepod-signal/remote"] # Download k-mer level models from the pinned nanoporetech/kmer_models commit # into the local cache (`escpod resquiggle models fetch`). Opt-in so the default # release binary pulls no TLS stack; implies `experimental` (the resquiggle diff --git a/crates/escapepod-cli/src/commands/inspect.rs b/crates/escapepod-cli/src/commands/inspect.rs index c660fa6..f76d856 100644 --- a/crates/escapepod-cli/src/commands/inspect.rs +++ b/crates/escapepod-cli/src/commands/inspect.rs @@ -1,8 +1,8 @@ //! Inspect command implementation. use crate::style; -use crate::util::resolve_pod5_inputs; -use escapepod_signal::{Reader, ReadsBatchView}; +use crate::util::{open_pod5, resolve_pod5_inputs}; +use escapepod_signal::ReadsBatchView; use std::path::PathBuf; use tracing::warn; @@ -27,7 +27,7 @@ pub fn summary(input: PathBuf) -> anyhow::Result<()> { let mut total_batches = 0usize; for file_path in &files { - let reader = match Reader::open(file_path) { + let reader = match open_pod5(file_path) { Ok(r) => r, Err(e) => { if is_directory { @@ -38,7 +38,7 @@ pub fn summary(input: PathBuf) -> anyhow::Result<()> { ); continue; } else { - return Err(e.into()); + return Err(e); } } }; @@ -134,7 +134,7 @@ pub fn reads(input: PathBuf) -> anyhow::Result<()> { println!("{}", "-".repeat(76)); for file_path in &files { - let reader = match Reader::open(file_path) { + let reader = match open_pod5(file_path) { Ok(r) => r, Err(e) => { if is_directory { @@ -145,7 +145,7 @@ pub fn reads(input: PathBuf) -> anyhow::Result<()> { ); continue; } else { - return Err(e.into()); + return Err(e); } } }; @@ -190,10 +190,10 @@ pub fn read(input: PathBuf, read_id: String) -> anyhow::Result<()> { let target_id: uuid::Uuid = read_id.parse()?; for file_path in &files { - let reader = match Reader::open(file_path) { + let reader = match open_pod5(file_path) { Ok(r) => r, Err(_) if is_directory => continue, - Err(e) => return Err(e.into()), + Err(e) => return Err(e), }; // Use the indexed by-id lookup so this is O(1) per file when a diff --git a/crates/escapepod-cli/src/commands/summary.rs b/crates/escapepod-cli/src/commands/summary.rs index 0fd32ac..4e69c29 100644 --- a/crates/escapepod-cli/src/commands/summary.rs +++ b/crates/escapepod-cli/src/commands/summary.rs @@ -3,9 +3,11 @@ //! Generates a comprehensive summary of POD5 file(s) with statistics and QC metrics. use crate::progress::create_progress_bar; -use crate::util::{format_bytes, format_duration_hours, format_number, resolve_pod5_inputs}; +use crate::util::{ + format_bytes, format_duration_hours, format_number, open_pod5, resolve_pod5_inputs, +}; use chrono::{TimeZone, Utc}; -use escapepod_signal::{Reader, ReadsBatchView, RunInfoData}; +use escapepod_signal::{ReadsBatchView, RunInfoData}; use owo_colors::OwoColorize; use serde::Serialize; use std::collections::HashMap; @@ -124,7 +126,7 @@ pub fn run(args: SummaryArgs) -> anyhow::Result<()> { ); } // Try to open the file, skip if corrupted - let reader = match Reader::open(path) { + let reader = match open_pod5(path) { Ok(r) => r, Err(e) => { corrupted_files.push(path.display().to_string()); @@ -143,8 +145,11 @@ pub fn run(args: SummaryArgs) -> anyhow::Result<()> { *old_version_files.entry(version.to_string()).or_insert(0) += 1; } - // Get file size - let size_bytes = fs::metadata(path).map(|m| m.len()).unwrap_or(0); + // Get file size. `fs::metadata` has nothing to stat for a remote + // object, so fall back to the size the reader already resolved. + let size_bytes = fs::metadata(path) + .map(|m| m.len()) + .unwrap_or_else(|_| reader.file_size()); // Get batch count (handle errors gracefully) let batch_count = reader.read_batch_count().unwrap_or(0); diff --git a/crates/escapepod-cli/src/util.rs b/crates/escapepod-cli/src/util.rs index 28e9dd0..3c2d727 100644 --- a/crates/escapepod-cli/src/util.rs +++ b/crates/escapepod-cli/src/util.rs @@ -13,11 +13,50 @@ use tracing::{info, warn}; use crate::style; +/// Whether an input path addresses a remote object. +/// +/// The scheme list lives in the pod5 layer and is compiled there +/// unconditionally, so this works — and [`open_pod5`] can explain itself — +/// even in a build without the `remote` feature. +pub fn path_is_remote_url(path: &Path) -> bool { + path.to_str().is_some_and(escapepod_signal::is_remote_url) +} + +/// Open a POD5 input that may be either a local path or a remote URL. +/// +/// Remote objects are read lazily over range requests, so metadata commands +/// transfer a few MB rather than the whole file. +pub fn open_pod5(input: &Path) -> anyhow::Result { + if path_is_remote_url(input) { + let url = input.to_string_lossy(); + #[cfg(feature = "remote")] + { + return Reader::open_url(&url).map_err(Into::into); + } + #[cfg(not(feature = "remote"))] + { + anyhow::bail!( + "Remote input {url} needs the `remote` feature; \ + this binary was built without it. Rebuild with \ + `cargo install escapepod-cli --features remote`, or download the file first." + ); + } + } + Reader::open(input).map_err(Into::into) +} + /// Resolve input path to a list of POD5 files. /// +/// - If the path is a remote URL, return it unchanged (no filesystem probing) /// - If path is a file, return it as a single-element vector /// - If path is a directory, find all *.pod5 files recursively pub fn resolve_pod5_inputs(path: &Path) -> anyhow::Result> { + // A URL names exactly one object; there is no directory to walk and + // `is_file()` would just report false for it. + if path_is_remote_url(path) { + return Ok(vec![path.to_path_buf()]); + } + if path.is_file() { return Ok(vec![path.to_path_buf()]); } @@ -199,8 +238,8 @@ pub enum OpenResult { /// /// In directory mode, file open errors result in a warning and `Skip`. /// In single-file mode, errors are propagated. -pub fn open_reader_with_warning(file_path: &PathBuf, is_directory: bool) -> OpenResult { - match Reader::open(file_path) { +pub fn open_reader_with_warning(file_path: &Path, is_directory: bool) -> OpenResult { + match open_pod5(file_path) { Ok(r) => OpenResult::Ok(r), Err(e) => { if is_directory { @@ -211,7 +250,7 @@ pub fn open_reader_with_warning(file_path: &PathBuf, is_directory: bool) -> Open ); OpenResult::Skip } else { - OpenResult::Err(e.into()) + OpenResult::Err(e) } } } diff --git a/crates/escapepod-pod5/Cargo.toml b/crates/escapepod-pod5/Cargo.toml index 0cc5c63..470a2ee 100644 --- a/crates/escapepod-pod5/Cargo.toml +++ b/crates/escapepod-pod5/Cargo.toml @@ -14,16 +14,27 @@ categories = ["parser-implementations", "science"] harness = false name = "io_hot_paths" +[features] +default = [] +# Lazy reading of POD5 objects over S3/GCS/Azure/HTTPS range requests. Off by +# default: it pulls in object_store + tokio, which a local-file build has no +# use for. +remote = ["dep:object_store", "dep:tokio", "dep:url"] + [dependencies] anyhow = { workspace = true } arrow = { workspace = true } byteorder = { workspace = true } +bytes = "1.9" csv = "1" flatbuffers = { workspace = true } memmap2 = { workspace = true } +object_store = { workspace = true, optional = true } rayon = { workspace = true } tempfile = "3" thiserror = { workspace = true } +tokio = { workspace = true, optional = true } +url = { workspace = true, optional = true } uuid = { workspace = true } zstd = { workspace = true } diff --git a/crates/escapepod-pod5/src/arrow_ipc.rs b/crates/escapepod-pod5/src/arrow_ipc.rs index 106ab24..1aa1d76 100644 --- a/crates/escapepod-pod5/src/arrow_ipc.rs +++ b/crates/escapepod-pod5/src/arrow_ipc.rs @@ -836,7 +836,7 @@ mod tests { #[test] fn test_parse_real_signal_table() { // Use bundled test data in data/drna - let test_file = std::path::Path::new("../data/drna/yeast_trna_reads.pod5"); + let test_file = std::path::Path::new("../../data/drna/yeast_trna_reads.pod5"); if !test_file.exists() { eprintln!("Skipping test - test file not found at {:?}", test_file); return; @@ -899,7 +899,7 @@ mod tests { /// (so the test degrades to garbage-only input rather than failing). fn load_real_signal_bytes() -> Option> { for candidate in [ - "../data/drna/yeast_trna_reads.pod5", + "../../data/drna/yeast_trna_reads.pod5", "../../ext/nanopore-dna-data/pod5/yeast_trna_reads.pod5", ] { let path = std::path::Path::new(candidate); diff --git a/crates/escapepod-pod5/src/error.rs b/crates/escapepod-pod5/src/error.rs index e152332..cf43217 100644 --- a/crates/escapepod-pod5/src/error.rs +++ b/crates/escapepod-pod5/src/error.rs @@ -28,6 +28,10 @@ pub enum Error { #[error("FlatBuffer error: {0}")] FlatBuffer(String), + /// Failure reading a POD5 object from object storage or over HTTP. + #[error("Remote read error: {0}")] + Remote(String), + /// Arrow IPC error. #[error("Arrow error: {0}")] Arrow(#[from] arrow::error::ArrowError), diff --git a/crates/escapepod-pod5/src/footer.rs b/crates/escapepod-pod5/src/footer.rs index bc47dea..8a1b4be 100644 --- a/crates/escapepod-pod5/src/footer.rs +++ b/crates/escapepod-pod5/src/footer.rs @@ -103,45 +103,97 @@ impl Footer { } } -/// Parse the footer from a POD5 file. -/// -/// The footer is located at the end of the file with the following structure: -/// - "FOOTER\0\0" magic (8 bytes) -/// - FlatBuffer data -/// - Footer length (8 bytes, little-endian i64) -/// - Section marker (16 bytes) -/// - Signature (8 bytes) -pub fn parse_footer(data: &[u8]) -> Result