diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8856fcc6..0aaf055b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,9 @@ jobs: libxkbcommon-dev \ libinput-dev \ libudev-dev \ - libseat-dev + libseat-dev \ + libclang-dev \ + libusb-1.0-0-dev - name: Install just uses: extractions/setup-just@v3 @@ -147,7 +149,9 @@ jobs: libxkbcommon-dev \ libinput-dev \ libudev-dev \ - libseat-dev + libseat-dev \ + libclang-dev \ + libusb-1.0-0-dev - name: Install cross if: ${{ matrix.use_cross }} @@ -161,7 +165,7 @@ jobs: if [ "${{ matrix.use_cross }}" = "true" ]; then cross build --target ${{ matrix.target }} else - just build-debug --target ${{ matrix.target }} + cargo build --target ${{ matrix.target }} fi flatpak-x86_64: diff --git a/Cargo.lock b/Cargo.lock index 8a6658a5..ba8b2e8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "aligned" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377e4c0ba83e4431b10df45c1d4666f178ea9c552cac93e60c3a88bf32785923" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" dependencies = [ "as-slice", ] @@ -277,9 +277,12 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +dependencies = [ + "rustversion", +] [[package]] name = "arg_enum_proc_macro" @@ -289,7 +292,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -450,7 +453,7 @@ dependencies = [ "futures-lite 2.6.1", "parking", "polling 3.11.0", - "rustix 1.1.2", + "rustix 1.1.3", "slab", "windows-sys 0.61.2", ] @@ -466,9 +469,9 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ "event-listener 5.4.1", "event-listener-strategy", @@ -500,14 +503,14 @@ checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" dependencies = [ "async-channel", "async-io 2.6.0", - "async-lock 3.4.1", + "async-lock 3.4.2", "async-signal", "async-task", "blocking", "cfg-if", "event-listener 5.4.1", "futures-lite 2.6.1", - "rustix 1.1.2", + "rustix 1.1.3", ] [[package]] @@ -518,7 +521,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -528,12 +531,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" dependencies = [ "async-io 2.6.0", - "async-lock 3.4.1", + "async-lock 3.4.2", "atomic-waker", "cfg-if", "futures-core", "futures-io", - "rustix 1.1.2", + "rustix 1.1.3", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -558,7 +561,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -575,7 +578,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -666,7 +669,7 @@ dependencies = [ "derive_utils", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -733,6 +736,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.114", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -882,9 +908,9 @@ checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "by_address" @@ -909,7 +935,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -952,7 +978,7 @@ checksum = "cb9f6e1368bd4621d2c86baa7e37de77a938adf5221e5dd3d6133340101b309e" dependencies = [ "bitflags 2.10.0", "polling 3.11.0", - "rustix 1.1.2", + "rustix 1.1.3", "slab", "tracing", ] @@ -976,7 +1002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" dependencies = [ "calloop 0.14.3", - "rustix 1.1.2", + "rustix 1.1.3", "wayland-backend", "wayland-client", ] @@ -994,6 +1020,7 @@ dependencies = [ "ctrlc", "dirs", "dng", + "freedepth", "futures", "gstreamer", "gstreamer-app", @@ -1001,6 +1028,7 @@ dependencies = [ "i18n-embed", "i18n-embed-fl", "image", + "las", "libc", "libcosmic", "naga 28.0.0", @@ -1016,6 +1044,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "v4l", "wgpu 27.0.1", "zbus 5.12.0", ] @@ -1031,9 +1060,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.48" +version = "1.2.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" +checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" dependencies = [ "find-msvc-tools", "jobserver", @@ -1047,11 +1076,20 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfg-expr" -version = "0.20.4" +version = "0.20.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9acd0bdbbf4b2612d09f52ba61da432140cb10930354079d0d53fafc12968726" +checksum = "21be0e1ce6cdb2ee7fff840f922fb04ead349e5cfb1e750b769132d44ce04720" dependencies = [ "smallvec", "target-lexicon", @@ -1088,6 +1126,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.54" @@ -1119,7 +1168,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1175,7 +1224,7 @@ dependencies = [ "bitflags 1.3.2", "block", "cocoa-foundation", - "core-foundation", + "core-foundation 0.9.4", "core-graphics", "foreign-types", "libc", @@ -1190,7 +1239,7 @@ checksum = "8c6234cbb2e4c785b456c0644748b1ac416dd045799740356f8363dfe00c93f7" dependencies = [ "bitflags 1.3.2", "block", - "core-foundation", + "core-foundation 0.9.4", "core-graphics-types", "libc", "objc", @@ -1212,7 +1261,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" dependencies = [ - "unicode-width 0.2.0", + "unicode-width 0.2.2", ] [[package]] @@ -1293,9 +1342,9 @@ dependencies = [ [[package]] name = "convert_case" -version = "0.7.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" dependencies = [ "unicode-segmentation", ] @@ -1310,6 +1359,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1323,7 +1382,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "core-graphics-types", "foreign-types", "libc", @@ -1336,7 +1395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "libc", ] @@ -1398,7 +1457,7 @@ version = "0.1.0" source = "git+https://github.com/pop-os/libcosmic.git#f6039597b72d3eefe2ee1d6528a04077982db238" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1531,7 +1590,7 @@ dependencies = [ "document-features", "mio", "parking_lot 0.12.5", - "rustix 1.1.2", + "rustix 1.1.3", "signal-hook", "signal-hook-mio", "winapi", @@ -1628,8 +1687,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1643,7 +1712,20 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.114", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.114", ] [[package]] @@ -1652,9 +1734,20 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1691,23 +1784,24 @@ dependencies = [ [[package]] name = "derive_more" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.111", + "rustc_version", + "syn 2.0.114", ] [[package]] @@ -1716,10 +1810,10 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae5c625eda104c228c06ecaf988d1c60e542176bd7a490e60eeda3493244c0c9" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1730,7 +1824,7 @@ checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1790,7 +1884,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1913,7 +2007,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1933,7 +2027,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2078,7 +2172,7 @@ checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2112,9 +2206,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" [[package]] name = "finl_unicode" @@ -2277,7 +2371,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2295,6 +2389,18 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "freedepth" +version = "0.1.0" +source = "git+https://github.com/FreddyFunk/freedepth?rev=aab2bbcc9c1c196751d8eb37a9b704e9b75683cb#aab2bbcc9c1c196751d8eb37a9b704e9b75683cb" +dependencies = [ + "bitflags 2.10.0", + "byteorder", + "nusb", + "thiserror 2.0.17", + "tracing", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2389,7 +2495,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2431,7 +2537,7 @@ dependencies = [ "g2poly", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2466,7 +2572,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix 1.1.2", + "rustix 1.1.3", "windows-link", ] @@ -2505,9 +2611,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f954a9e9159ec994f73a30a12b96a702dde78f5547bcb561174597924f7d4162" +checksum = "f5df2ba84018d80c213569363bdcd0c64e6933c67fe4c1d60ecf822971a3c35e" dependencies = [ "color_quant", "weezl", @@ -2515,9 +2621,9 @@ dependencies = [ [[package]] name = "gio-sys" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171ed2f6dd927abbe108cfd9eebff2052c335013f5879d55bab0dc1dee19b706" +checksum = "0071fe88dba8e40086c8ff9bbb62622999f49628344b1d1bf490a48a29d80f22" dependencies = [ "glib-sys", "gobject-sys", @@ -2545,9 +2651,9 @@ checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3" [[package]] name = "glib" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9dbecb1c33e483a98be4acfea2ab369e1c28f517c6eadb674537409c25c4b2" +checksum = "16de123c2e6c90ce3b573b7330de19be649080ec612033d397d72da265f1bd8b" dependencies = [ "bitflags 2.10.0", "futures-channel", @@ -2566,27 +2672,33 @@ dependencies = [ [[package]] name = "glib-macros" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "880e524e0085f3546cfb38532b2c202c0d64741d9977a6e4aa24704bfc9f19fb" +checksum = "cf59b675301228a696fe01c3073974643365080a76cc3ed5bc2cbc466ad87f17" dependencies = [ "heck 0.5.0", "proc-macro-crate 3.4.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "glib-sys" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09d3d0fddf7239521674e57b0465dfbd844632fec54f059f7f56112e3f927e1" +checksum = "2d95e1a3a19ae464a7286e14af9a90683c64d70c02532d88d87ce95056af3e6c" dependencies = [ "libc", "system-deps", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "glow" version = "0.13.1" @@ -2610,9 +2722,9 @@ dependencies = [ [[package]] name = "gobject-sys" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "538e41d8776173ec107e7b0f2aceced60abc368d7e1d81c1f0e2ecd35f59080d" +checksum = "2dca35da0d19a18f4575f3cb99fe1c9e029a2941af5662f326f738a21edaf294" dependencies = [ "glib-sys", "libc", @@ -2696,7 +2808,7 @@ dependencies = [ "num-integer", "num-rational", "option-operations", - "pastey 0.2.0", + "pastey 0.2.1", "pin-project-lite", "smallvec", "thiserror 2.0.17", @@ -2719,9 +2831,9 @@ dependencies = [ [[package]] name = "gstreamer-app-sys" -version = "0.24.0" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf1a3af017f9493c34ccc8439cbce5c48f6ddff6ec0514c23996b374ff25f9a" +checksum = "f7719cee28afda1a48ab1ee93769628bd0653d3c5be1923bce9a8a4550fcc980" dependencies = [ "glib-sys", "gstreamer-base-sys", @@ -2732,9 +2844,9 @@ dependencies = [ [[package]] name = "gstreamer-base" -version = "0.24.2" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71ff9b0bbc8041f0c6c8a53b206a6542f86c7d9fa8a7dff3f27d9c374d9f39b4" +checksum = "4dd15c7e37d306573766834a5cbdd8ee711265f217b060f40a9a8eda45298488" dependencies = [ "atomic_refcell", "cfg-if", @@ -2746,9 +2858,9 @@ dependencies = [ [[package]] name = "gstreamer-base-sys" -version = "0.24.2" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed78852b92db1459b8f4288f86e6530274073c20be2f94ba642cddaca08b00e" +checksum = "27a2eda2c61e13c11883bf19b290d07ea6b53d04fd8bfeb7af64b6006c6c9ee6" dependencies = [ "glib-sys", "gobject-sys", @@ -2759,9 +2871,9 @@ dependencies = [ [[package]] name = "gstreamer-sys" -version = "0.24.2" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24ae2930e683665832a19ef02466094b09d1f2da5673f001515ed5486aa9377" +checksum = "5d88630697e757c319e7bcec7b13919ba80492532dd3238481c1c4eee05d4904" dependencies = [ "cfg-if", "glib-sys", @@ -2788,9 +2900,9 @@ dependencies = [ [[package]] name = "gstreamer-video-sys" -version = "0.24.1" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d944b1492bdd7a72a02ae9a5da6e34a29194b8623d3bd02752590b06fb837a7" +checksum = "a00c28faad96cd40a7b7592433051199691b131b08f622ed5d51c54e049792d3" dependencies = [ "glib-sys", "gobject-sys", @@ -2908,6 +3020,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "i18n-config" version = "0.4.8" @@ -2958,7 +3079,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.114", "unic-langid", ] @@ -2972,7 +3093,7 @@ dependencies = [ "i18n-config", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2987,7 +3108,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.58.0", + "windows-core 0.62.2", ] [[package]] @@ -3272,9 +3393,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -3286,9 +3407,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" @@ -3342,7 +3463,7 @@ dependencies = [ "byteorder-lite", "color_quant", "exr", - "gif 0.14.0", + "gif 0.14.1", "image-webp", "moxcms", "num-traits", @@ -3353,7 +3474,7 @@ dependencies = [ "rgb", "tiff", "zune-core 0.5.0", - "zune-jpeg 0.5.5", + "zune-jpeg 0.5.8", ] [[package]] @@ -3428,15 +3549,15 @@ dependencies = [ [[package]] name = "instability" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6778b0196eefee7df739db78758e5cf9b37412268bfa5650bfeed028aed20d9c" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" dependencies = [ - "darling", + "darling 0.23.0", "indoc", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3456,7 +3577,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3478,6 +3599,16 @@ dependencies = [ "unic-langid", ] +[[package]] +name = "io-kit-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617ee6cf8e3f66f3b4ea67a4058564628cde41901316e19f559e14c7c72c5e7b" +dependencies = [ + "core-foundation-sys", + "mach2", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -3534,9 +3665,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jni" @@ -3694,12 +3825,43 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" +[[package]] +name = "las" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5fc2ade5beaea800d3e615ce9886abb8d325be0c2bf2c94553fa88677d1b30" +dependencies = [ + "byteorder", + "chrono", + "laz", + "log", + "num-traits", + "thiserror 2.0.17", + "uuid", +] + +[[package]] +name = "laz" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5b7251e6d7aee8f4263f35d61e5e40d22bb189ecfc4f9fbd550806fed98cce" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lebe" version = "0.5.3" @@ -3788,13 +3950,13 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ "bitflags 2.10.0", "libc", - "redox_syscall 0.5.18", + "redox_syscall 0.7.0", ] [[package]] @@ -3830,6 +3992,12 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a385b1be4e5c3e362ad2ffa73c392e53f031eaa5b7d648e64cd87f27f6063d7" +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -3859,9 +4027,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "loop9" @@ -3883,9 +4051,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" dependencies = [ "hashbrown 0.16.1", ] @@ -3952,6 +4120,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -4069,9 +4246,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "log", @@ -4081,9 +4258,9 @@ dependencies = [ [[package]] name = "moxcms" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80986bbbcf925ebd3be54c26613d861255284584501595cf418320c078945608" +checksum = "ac9557c559cd6fc9867e122e20d2cbefc9ca29d80d027a8e39310920ed2f0a97" dependencies = [ "num-traits", "pxfm", @@ -4338,7 +4515,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4400,7 +4577,7 @@ dependencies = [ "proc-macro-crate 3.4.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4412,6 +4589,23 @@ dependencies = [ "libc", ] +[[package]] +name = "nusb" +version = "0.2.1" +source = "git+https://github.com/FreddyFunk/nusb?branch=feature%2Fimplement-isochronous-transfers-for-linux#fef4a8337f7949a3a64303bd875f9714807e3617" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "futures-core", + "io-kit-sys", + "linux-raw-sys 0.9.4", + "log", + "once_cell", + "rustix 1.1.3", + "slab", + "windows-sys 0.60.2", +] + [[package]] name = "objc" version = "0.2.7" @@ -4718,19 +4912,20 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "option-operations" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b31ce827892359f23d3cd1cc4c75a6c241772bbd2db17a92dcf27cbefdf52689" +checksum = "aca39cf52b03268400c16eeb9b56382ea3c3353409309b63f5c8f0b1faf42754" dependencies = [ - "pastey 0.1.1", + "pastey 0.2.1", ] [[package]] name = "orbclient" -version = "0.3.49" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "247ad146e19b9437f8604c21f8652423595cf710ad108af40e77d3ae6e96b827" +checksum = "52ad2c6bae700b7aa5d1cc30c59bdd3a1c180b09dbaea51e2ae2b8e1cf211fdd" dependencies = [ + "libc", "libredox", ] @@ -4783,7 +4978,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4817,7 +5012,7 @@ dependencies = [ "by_address", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4888,9 +5083,9 @@ checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" [[package]] name = "pastey" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d6c094ee800037dff99e02cab0eaf3142826586742a270ab3d7a62656bd27a" +checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec" [[package]] name = "pathdiff" @@ -4898,6 +5093,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4906,9 +5107,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" dependencies = [ "memchr", "ucd-trie", @@ -4916,9 +5117,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" dependencies = [ "pest", "pest_generator", @@ -4926,22 +5127,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "pest_meta" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" dependencies = [ "pest", "sha2", @@ -5008,7 +5209,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5021,7 +5222,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5065,7 +5266,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5149,7 +5350,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.5.2", "pin-project-lite", - "rustix 1.1.2", + "rustix 1.1.3", "windows-sys 0.61.2", ] @@ -5161,9 +5362,9 @@ checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" [[package]] name = "portable-atomic-util" @@ -5204,6 +5405,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.114", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -5220,7 +5431,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.7", + "toml_edit 0.23.10+spec-1.0.0", ] [[package]] @@ -5242,14 +5453,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" dependencies = [ "unicode-ident", ] @@ -5262,7 +5473,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "version_check", "yansi", ] @@ -5283,14 +5494,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "pxfm" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3502d6155304a4173a5f2c34b52b7ed0dd085890326cb50fd625fdf39e86b3b" +checksum = "7186d3822593aa4393561d186d1393b3923e9d6163d3fbfd6e825e3e6cf3e6a8" dependencies = [ "num-traits", ] @@ -5312,18 +5523,18 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" dependencies = [ "proc-macro2", ] @@ -5401,9 +5612,9 @@ checksum = "c3d6831663a5098ea164f89cff59c6284e95f4e3c76ce9848d4529f5ccca9bde" [[package]] name = "rangemap" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbbbea733ec66275512d0b9694f34102e7d5406fdbe2ad8d21b28dce92887c" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" [[package]] name = "ratatui" @@ -5431,12 +5642,12 @@ dependencies = [ "indoc", "itertools 0.14.0", "kasuari", - "lru 0.16.2", + "lru 0.16.3", "strum", "thiserror 2.0.17", "unicode-segmentation", "unicode-truncate", - "unicode-width 0.2.0", + "unicode-width 0.2.2", ] [[package]] @@ -5487,7 +5698,7 @@ dependencies = [ "strum", "time", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.2", ] [[package]] @@ -5605,6 +5816,15 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -5777,7 +5997,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.111", + "syn 2.0.114", "walkdir", ] @@ -5803,6 +6023,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.28" @@ -5832,9 +6061,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags 2.10.0", "errno", @@ -5867,9 +6096,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "same-file" @@ -5907,9 +6136,15 @@ dependencies = [ [[package]] name = "self_cell" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16c2f82143577edb4921b71ede051dac62ca3c16084e918bf7b40c96ae10eb33" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" @@ -5938,7 +6173,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5963,14 +6198,14 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "serde_spanned" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" dependencies = [ "serde_core", ] @@ -6035,18 +6270,19 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "simd_helpers" @@ -6100,9 +6336,9 @@ checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "slotmap" -version = "1.0.7" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbff4acf519f630b3a3ddcfaea6c06b42174d9a44bc70c620e9ed1649d58b82a" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" dependencies = [ "version_check", ] @@ -6153,7 +6389,7 @@ dependencies = [ "log", "memmap2 0.9.9", "pkg-config", - "rustix 1.1.2", + "rustix 1.1.3", "thiserror 2.0.17", "wayland-backend", "wayland-client", @@ -6292,7 +6528,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6335,9 +6571,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -6352,7 +6588,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6373,7 +6609,7 @@ dependencies = [ "cfg-expr", "heck 0.5.0", "pkg-config", - "toml 0.9.8", + "toml 0.9.10+spec-1.1.0", "version-compare", ] @@ -6397,14 +6633,14 @@ checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand 2.3.0", "getrandom 0.3.4", "once_cell", - "rustix 1.1.2", + "rustix 1.1.3", "windows-sys 0.61.2", ] @@ -6506,7 +6742,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6517,7 +6753,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6655,14 +6891,14 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -6680,14 +6916,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.8" +version = "0.9.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" +checksum = "0825052159284a1a8b4d6c0c86cbc801f2da5afd2b225fa548c72f2e74002f48" dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime 0.7.3", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow 0.7.14", @@ -6701,9 +6937,9 @@ checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" [[package]] name = "toml_datetime" -version = "0.7.3" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] @@ -6721,30 +6957,30 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.7" +version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ "indexmap", - "toml_datetime 0.7.3", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "winnow 0.7.14", ] [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" dependencies = [ "winnow 0.7.14", ] [[package]] name = "toml_writer" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" [[package]] name = "tracing" @@ -6766,7 +7002,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6912,9 +7148,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-script" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f" +checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee" [[package]] name = "unicode-segmentation" @@ -6930,7 +7166,7 @@ checksum = "8fbf03860ff438702f3910ca5f28f8dac63c1c11e7efb5012b8b175493606330" dependencies = [ "itertools 0.13.0", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.2", ] [[package]] @@ -6947,9 +7183,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unicode-xid" @@ -6959,14 +7195,15 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -7027,6 +7264,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v4l" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fbfea44a46799d62c55323f3c55d06df722fbe577851d848d328a1041c3403" +dependencies = [ + "bitflags 1.3.2", + "libc", + "v4l2-sys-mit", +] + +[[package]] +name = "v4l2-sys-mit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6779878362b9bacadc7893eac76abe69612e8837ef746573c4a5239daf11990b" +dependencies = [ + "bindgen", +] + [[package]] name = "v_frame" version = "0.3.9" @@ -7141,7 +7398,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "wasm-bindgen-shared", ] @@ -7171,13 +7428,13 @@ dependencies = [ [[package]] name = "wayland-backend" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673a33c33048a5ade91a6b139580fa174e19fb0d23f396dca9fa15f2e1e49b35" +checksum = "fee64194ccd96bf648f42a65a7e589547096dfa702f7cadef84347b66ad164f9" dependencies = [ "cc", "downcast-rs", - "rustix 1.1.2", + "rustix 1.1.3", "scoped-tls", "smallvec", "wayland-sys", @@ -7185,12 +7442,12 @@ dependencies = [ [[package]] name = "wayland-client" -version = "0.31.11" +version = "0.31.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66a47e840dc20793f2264eb4b3e4ecb4b75d91c0dd4af04b456128e0bdd449d" +checksum = "b8e6faa537fbb6c186cb9f1d41f2f811a4120d1b57ec61f50da451a0c5122bec" dependencies = [ "bitflags 2.10.0", - "rustix 1.1.2", + "rustix 1.1.3", "wayland-backend", "wayland-scanner", ] @@ -7208,20 +7465,20 @@ dependencies = [ [[package]] name = "wayland-cursor" -version = "0.31.11" +version = "0.31.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447ccc440a881271b19e9989f75726d60faa09b95b0200a9b7eb5cc47c3eeb29" +checksum = "5864c4b5b6064b06b1e8b74ead4a98a6c45a285fe7a0e784d24735f011fdb078" dependencies = [ - "rustix 1.1.2", + "rustix 1.1.3", "wayland-client", "xcursor", ] [[package]] name = "wayland-protocols" -version = "0.32.9" +version = "0.32.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa790ed75fbfd71283bd2521a1cfdc022aabcc28bdcff00851f9e4ae88d9901" +checksum = "baeda9ffbcfc8cd6ddaade385eaf2393bd2115a69523c735f12242353c3df4f3" dependencies = [ "bitflags 2.10.0", "wayland-backend", @@ -7245,9 +7502,9 @@ dependencies = [ [[package]] name = "wayland-protocols-misc" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfe33d551eb8bffd03ff067a8b44bb963919157841a99957151299a6307d19c" +checksum = "791c58fdeec5406aa37169dd815327d1e47f334219b523444bc26d70ceb4c34e" dependencies = [ "bitflags 2.10.0", "wayland-backend", @@ -7258,9 +7515,9 @@ dependencies = [ [[package]] name = "wayland-protocols-plasma" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07a14257c077ab3279987c4f8bb987851bf57081b93710381daea94f2c2c032" +checksum = "aa98634619300a535a9a97f338aed9a5ff1e01a461943e8346ff4ae26007306b" dependencies = [ "bitflags 2.10.0", "wayland-backend", @@ -7271,9 +7528,9 @@ dependencies = [ [[package]] name = "wayland-protocols-wlr" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd94963ed43cf9938a090ca4f7da58eb55325ec8200c3848963e98dc25b78ec" +checksum = "e9597cdf02cf0c34cd5823786dce6b5ae8598f05c2daf5621b6e178d4f7345f3" dependencies = [ "bitflags 2.10.0", "wayland-backend", @@ -7285,9 +7542,9 @@ dependencies = [ [[package]] name = "wayland-scanner" -version = "0.31.7" +version = "0.31.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cb1e9dc49da91950bdfd8b848c49330536d9d1fb03d4bfec8cae50caa50ae3" +checksum = "5423e94b6a63e68e439803a3e153a9252d5ead12fd853334e2ad33997e3889e3" dependencies = [ "proc-macro2", "quick-xml", @@ -7296,22 +7553,22 @@ dependencies = [ [[package]] name = "wayland-server" -version = "0.31.10" +version = "0.31.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbd4f3aba6c9fba70445ad2a484c0ef0356c1a9459b1e8e435bedc1971a6222" +checksum = "9297ab90f8d1f597711d36455c5b1b2290eca59b8134485e377a296b80b118c9" dependencies = [ "bitflags 2.10.0", "downcast-rs", - "rustix 1.1.2", + "rustix 1.1.3", "wayland-backend", "wayland-scanner", ] [[package]] name = "wayland-sys" -version = "0.31.7" +version = "0.31.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34949b42822155826b41db8e5d0c1be3a2bd296c747577a43a3e6daefc296142" +checksum = "1e6dbfc3ac5ef974c92a2235805cc0114033018ae1290a72e474aa8b28cbbdfd" dependencies = [ "dlib", "log", @@ -7629,6 +7886,18 @@ dependencies = [ "log", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "widestring" version = "1.2.1" @@ -7741,10 +8010,23 @@ dependencies = [ "windows-implement 0.58.0", "windows-interface 0.58.0", "windows-result 0.2.0", - "windows-strings", + "windows-strings 0.1.0", "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-implement" version = "0.53.0" @@ -7753,7 +8035,7 @@ checksum = "942ac266be9249c84ca862f0a164a39533dc2f6f33dc98ec89c8da99b82ea0bd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7764,7 +8046,18 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] @@ -7775,7 +8068,7 @@ checksum = "da33557140a288fae4e1d5f8873aaf9eb6613a9cf82c3e070223ff177f598b60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7786,7 +8079,18 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] @@ -7813,6 +8117,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-strings" version = "0.1.0" @@ -7823,6 +8136,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -8134,7 +8456,7 @@ dependencies = [ "calloop 0.13.0", "cfg_aliases 0.2.1", "concurrent-queue", - "core-foundation", + "core-foundation 0.9.4", "core-graphics", "cursor-icon", "dpi", @@ -8223,7 +8545,7 @@ dependencies = [ "libc", "libloading", "once_cell", - "rustix 1.1.2", + "rustix 1.1.3", "x11rb-protocol", "xcursor", ] @@ -8349,7 +8671,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure", ] @@ -8398,7 +8720,7 @@ dependencies = [ "async-broadcast 0.7.2", "async-executor", "async-io 2.6.0", - "async-lock 3.4.1", + "async-lock 3.4.2", "async-process 2.5.0", "async-recursion", "async-task", @@ -8447,7 +8769,7 @@ dependencies = [ "proc-macro-crate 3.4.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "zbus_names 4.2.0", "zvariant 5.8.0", "zvariant_utils 3.2.1", @@ -8484,22 +8806,22 @@ checksum = "6df3dc4292935e51816d896edcd52aa30bc297907c26167fec31e2b0c6a32524" [[package]] name = "zerocopy" -version = "0.8.30" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c" +checksum = "1fabae64378cb18147bb18bca364e63bdbe72a0ffe4adf0addfec8aa166b2c56" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.30" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" +checksum = "c9c2d862265a8bb4471d87e033e730f536e2a285cc7cb05dbce09a2a97075f90" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8519,7 +8841,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure", ] @@ -8554,14 +8876,14 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "zmij" -version = "1.0.2" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4a4e8e9dc5c62d159f04fcdbe07f4c3fb710415aab4754bf11505501e3251d" +checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" [[package]] name = "zune-core" @@ -8595,9 +8917,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fb7703e32e9a07fb3f757360338b3a567a5054f21b5f52a666752e333d58e" +checksum = "e35aee689668bf9bd6f6f3a6c60bb29ba1244b3b43adfd50edd554a371da37d5" dependencies = [ "zune-core 0.5.0", ] @@ -8653,7 +8975,7 @@ dependencies = [ "proc-macro-crate 3.4.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "zvariant_utils 3.2.1", ] @@ -8677,6 +8999,6 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.111", + "syn 2.0.114", "winnow 0.7.14", ] diff --git a/Cargo.toml b/Cargo.toml index 6e31f73f..3dea9b21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,12 @@ description = "Camera application for the COSMICâ„¢ desktop" repository = "https://github.com/cosmic-utils/camera" authors = ["Frederic Laing "] +[features] +default = ["freedepth"] +# freedepth userspace library for Kinect depth cameras (fallback when kernel driver unavailable) +# Only works on x86_64 due to libclang/bindgen requirements +freedepth = ["dep:freedepth"] + [dependencies] futures = "0.3.31" i18n-embed = { version = "0.16.0", features = [ @@ -40,6 +46,13 @@ crossterm = "0.29" libc = "0.2.179" dng = "1.5" async-stream = "0.3" +v4l = "0.14" +las = { version = "0.9", features = ["laz"] } # LAZ point cloud export + +# freedepth is only available on x86_64 (requires libclang for v4l2-sys-mit bindgen) +# Enabled via the "freedepth" feature flag (on by default) +[target.'cfg(target_arch = "x86_64")'.dependencies] +freedepth = { git = "https://github.com/FreddyFunk/freedepth", rev = "aab2bbcc9c1c196751d8eb37a9b704e9b75683cb", optional = true } # Multi-device depth camera library # Separate wgpu for compute pipelines (independent from libcosmic's wgpu for UI) # This allows us to use the latest wgpu features like open_with_callback for low-priority queues diff --git a/i18n/en/camera.ftl b/i18n/en/camera.ftl index f7b814f3..79af1c36 100644 --- a/i18n/en/camera.ftl +++ b/i18n/en/camera.ftl @@ -10,6 +10,11 @@ git-description = Git commit {$hash} on {$date} mode-video = VIDEO mode-photo = PHOTO mode-virtual = VIRTUAL +mode-scene = SCENE + +# Scene view modes +scene-view-points = Points +scene-view-mesh = Mesh # Virtual camera virtual-camera-title = Virtual camera (experimental) @@ -53,6 +58,7 @@ device-info-card = Card device-info-driver = Driver device-info-path = Path device-info-real-path = Real Path +device-info-status = Status # Bitrate presets preset-low = Low @@ -65,6 +71,8 @@ initializing-camera = Initializing camera... # Format picker format-resolution = Resolution: format-framerate = Frame Rate: +format-depth = Depth Sensor: +depth-required-for-3d = 3D preview requires depth format - select Y10B format in settings # Status indicators indicator-res = RES @@ -119,6 +127,18 @@ tools-exposure = Exposure tools-color = Color tools-filter = Filter tools-theatre = Theatre +tools-kinect = Kinect + +# Kinect controls +kinect-title = Motor Controls +kinect-tilt = Tilt Angle +kinect-led = LED +kinect-led-off = Off +kinect-led-green = Green +kinect-led-red = Red +kinect-led-yellow = Yellow +kinect-led-blink-green = Blink Green +kinect-led-blink-red-yellow = Blink Red/Yellow # PTZ controls ptz-title = Camera Controls diff --git a/resources/51-kinect.rules b/resources/51-kinect.rules new file mode 100644 index 00000000..94fbf479 --- /dev/null +++ b/resources/51-kinect.rules @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-3.0-only +# +# udev rules for Microsoft Kinect +# Install to /etc/udev/rules.d/51-kinect.rules +# Then run: sudo udevadm control --reload-rules && sudo udevadm trigger +# +# Alternatively, add your user to the 'plugdev' group: +# sudo usermod -a -G plugdev $USER + +# Xbox 360 Kinect Camera +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02ae", MODE="0666", GROUP="plugdev" + +# Xbox 360 Kinect Motor +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02b0", MODE="0666", GROUP="plugdev" + +# Xbox 360 Kinect Audio +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02ad", MODE="0666", GROUP="plugdev" + +# Kinect for Windows Camera +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02bf", MODE="0666", GROUP="plugdev" + +# Kinect for Windows Audio +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02be", MODE="0666", GROUP="plugdev" + +# Kinect for Windows Audio (alternate PIDs) +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02c3", MODE="0666", GROUP="plugdev" +SUBSYSTEM=="usb", ATTR{idVendor}=="045e", ATTR{idProduct}=="02bb", MODE="0666", GROUP="plugdev" diff --git a/resources/button_icons/3d-depth.svg b/resources/button_icons/3d-depth.svg new file mode 100644 index 00000000..96737d6d --- /dev/null +++ b/resources/button_icons/3d-depth.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/button_icons/depth-off.svg b/resources/button_icons/depth-off.svg new file mode 100644 index 00000000..acc5b20b --- /dev/null +++ b/resources/button_icons/depth-off.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/resources/button_icons/depth-waves-off.svg b/resources/button_icons/depth-waves-off.svg new file mode 100644 index 00000000..df3fe5c8 --- /dev/null +++ b/resources/button_icons/depth-waves-off.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/button_icons/depth-waves.svg b/resources/button_icons/depth-waves.svg new file mode 100644 index 00000000..55f4c4a1 --- /dev/null +++ b/resources/button_icons/depth-waves.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/button_icons/depth.svg b/resources/button_icons/depth.svg new file mode 100644 index 00000000..6627fc1c --- /dev/null +++ b/resources/button_icons/depth.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/resources/button_icons/mesh.svg b/resources/button_icons/mesh.svg new file mode 100644 index 00000000..58b35fa6 --- /dev/null +++ b/resources/button_icons/mesh.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/button_icons/points.svg b/resources/button_icons/points.svg new file mode 100644 index 00000000..f6a920aa --- /dev/null +++ b/resources/button_icons/points.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/app/bottom_bar/mode_switcher.rs b/src/app/bottom_bar/mode_switcher.rs index 7e71c1fb..bd5e155f 100644 --- a/src/app/bottom_bar/mode_switcher.rs +++ b/src/app/bottom_bar/mode_switcher.rs @@ -80,6 +80,37 @@ impl AppModel { .push(styled_mode_button(photo_button, is_disabled)) .spacing(spacing.space_xxs); + // Show Scene button when camera provides depth data + let has_depth = self + .current_frame + .as_ref() + .map(|f| f.depth_data.is_some()) + .unwrap_or(false) + || self.kinect.is_device; + + if has_depth { + let scene_label = fl!("mode-scene"); + let scene_button = if is_disabled { + widget::button::text(scene_label).class(if self.mode == CameraMode::Scene { + cosmic::theme::Button::Suggested + } else { + cosmic::theme::Button::Text + }) + } else { + widget::button::text(scene_label) + .on_press(Message::SetMode(CameraMode::Scene)) + .class(if self.mode == CameraMode::Scene { + cosmic::theme::Button::Suggested + } else { + cosmic::theme::Button::Text + }) + }; + + row = row + .push(widget::horizontal_space().width(spacing.space_xs)) + .push(styled_mode_button(scene_button, is_disabled)); + } + // Only show Virtual button when the feature is enabled if self.config.virtual_camera_enabled { let virtual_label = fl!("mode-virtual"); diff --git a/src/app/camera_ops.rs b/src/app/camera_ops.rs index 541fc2f3..3e99b419 100644 --- a/src/app/camera_ops.rs +++ b/src/app/camera_ops.rs @@ -49,9 +49,9 @@ impl AppModel { // Determine what format would be selected in the new mode // Note: We don't use current format as fallback to avoid cross-contamination let would_select_format = match new_mode { - CameraMode::Photo | CameraMode::Virtual => { - // Photo/Virtual mode: saved settings > max resolution - // Virtual mode uses photo settings since it's similar behavior + CameraMode::Photo | CameraMode::Virtual | CameraMode::Scene => { + // Photo/Virtual/Scene mode: saved settings > max resolution + // Virtual and Scene modes use photo settings since they're similar behavior check_saved_settings(&self.config.photo_settings).or_else(|| { format_selection::select_max_resolution_format(&formats_for_new_mode) }) @@ -109,16 +109,17 @@ impl AppModel { }; // Store in per-camera settings based on current mode - // Virtual mode shares settings with Photo mode + // Virtual and Scene modes share settings with Photo mode let mode_name = match self.mode { - CameraMode::Photo | CameraMode::Virtual => { + CameraMode::Photo | CameraMode::Virtual | CameraMode::Scene => { self.config .photo_settings .insert(camera.path.clone(), format_settings); - if self.mode == CameraMode::Photo { - "Photo" - } else { - "Virtual" + match self.mode { + CameraMode::Photo => "Photo", + CameraMode::Virtual => "Virtual", + CameraMode::Scene => "Scene", + CameraMode::Video => unreachable!(), } } CameraMode::Video => { @@ -242,9 +243,11 @@ impl AppModel { self.available_formats = backend.get_formats(&device, mode == CameraMode::Video); // Format selection logic: both modes use saved settings, current format, or defaults - // Virtual mode uses the same format selection as Photo mode + // Virtual and Scene modes use the same format selection as Photo mode self.active_format = match mode { - CameraMode::Photo | CameraMode::Virtual => self.select_photo_format(&camera_path), + CameraMode::Photo | CameraMode::Virtual | CameraMode::Scene => { + self.select_photo_format(&camera_path) + } CameraMode::Video => self.select_video_format(&camera_path), }; diff --git a/src/app/camera_preview/widget.rs b/src/app/camera_preview/widget.rs index 4ae6da74..d493a55c 100644 --- a/src/app/camera_preview/widget.rs +++ b/src/app/camera_preview/widget.rs @@ -8,6 +8,7 @@ use crate::fl; use cosmic::Element; use cosmic::iced::{Background, Length}; use cosmic::widget; +use std::sync::Arc; use tracing::info; impl AppModel { @@ -53,6 +54,77 @@ impl AppModel { ); } + // Check if 3D preview mode is enabled and we have a rendered point cloud + if self.preview_3d.enabled { + if let Some((width, height, rgba_data)) = &self.preview_3d.rendered_preview { + // Create a CameraFrame from point cloud data for video widget + let point_cloud_frame = Arc::new(crate::backends::camera::types::CameraFrame { + width: *width, + height: *height, + data: Arc::from(rgba_data.as_slice()), + depth_data: None, + depth_width: 0, + depth_height: 0, + format: crate::backends::camera::types::PixelFormat::RGBA, + stride: *width * 4, + captured_at: std::time::Instant::now(), + video_timestamp: None, + }); + + // Use video widget to display point cloud (video_id=2 for 3D preview) + // Use Cover mode to fill the entire preview area + let video_elem = video_widget::video_widget( + point_cloud_frame, + 2, // Use separate video_id for 3D preview + VideoContentFit::Cover, + crate::app::state::FilterType::Standard, + 0.0, + false, // No mirror for 3D + None, // No crop + 1.0, // No zoom + false, // No scroll zoom + ); + + // Wrap in mouse_area for rotation control via drag and zoom via scroll + // Note: on_press doesn't provide position, so we start tracking on first move + let preview_with_mouse = widget::mouse_area( + widget::container(video_elem) + .width(Length::Fill) + .height(Length::Fill) + .align_x(cosmic::iced::alignment::Horizontal::Center) + .align_y(cosmic::iced::alignment::Vertical::Center), + ) + .on_press(Message::Preview3DMousePressed(0.0, 0.0)) + .on_move(|point: cosmic::iced::Point| { + Message::Preview3DMouseMoved(point.x, point.y) + }) + .on_release(Message::Preview3DMouseReleased) + .on_scroll(|delta: cosmic::iced::mouse::ScrollDelta| { + // Convert scroll delta to zoom amount + let scroll_amount = match delta { + cosmic::iced::mouse::ScrollDelta::Lines { y, .. } => y * 100.0, + cosmic::iced::mouse::ScrollDelta::Pixels { y, .. } => y, + }; + Message::Zoom3DPreview(scroll_amount) + }); + + return preview_with_mouse.into(); + } else { + // 3D mode enabled but no render yet - show loading + return widget::container(widget::text("Rendering 3D preview...").size(16)) + .width(Length::Fill) + .height(Length::Fill) + .align_x(cosmic::iced::alignment::Horizontal::Center) + .align_y(cosmic::iced::alignment::Vertical::Center) + .style(|theme: &cosmic::Theme| widget::container::Style { + background: Some(Background::Color(theme.cosmic().bg_color().into())), + text_color: Some(theme.cosmic().on_bg_color().into()), + ..Default::default() + }) + .into(); + } + } + // Use custom video widget with GPU primitive rendering // During transitions, use blur shader (video_id=1), otherwise normal shader (video_id=0) let should_blur = self.transition_state.should_blur(); @@ -68,11 +140,11 @@ impl AppModel { VideoContentFit::Contain }; - // Apply filters in Photo and Virtual modes (not in Video mode) + // Apply filters in Photo, Virtual, and Scene modes (not in Video mode) let filter_mode = match self.mode { - crate::app::state::CameraMode::Photo | crate::app::state::CameraMode::Virtual => { - self.selected_filter - } + crate::app::state::CameraMode::Photo + | crate::app::state::CameraMode::Virtual + | crate::app::state::CameraMode::Scene => self.selected_filter, crate::app::state::CameraMode::Video => crate::app::state::FilterType::Standard, }; // File sources should never be mirrored - they display content as-is diff --git a/src/app/controls/capture_button.rs b/src/app/controls/capture_button.rs index f12db728..710ec23f 100644 --- a/src/app/controls/capture_button.rs +++ b/src/app/controls/capture_button.rs @@ -48,11 +48,11 @@ impl AppModel { Color::from_rgb(0.2, 0.5, 0.9) // Blue for virtual mode } } - CameraMode::Photo => { + CameraMode::Photo | CameraMode::Scene => { if self.is_capturing { Color::from_rgb(0.7, 0.7, 0.7) // Gray when capturing } else { - Color::WHITE // White for photo mode + Color::WHITE // White for photo/scene mode } } } @@ -102,7 +102,7 @@ impl AppModel { // Normal interactive button widget::button::custom(button_inner) .on_press(match self.mode { - CameraMode::Photo => Message::Capture, + CameraMode::Photo | CameraMode::Scene => Message::Capture, CameraMode::Video => Message::ToggleRecording, CameraMode::Virtual => Message::ToggleVirtualCamera, }) diff --git a/src/app/format_picker/preferences.rs b/src/app/format_picker/preferences.rs index 0f9a8a92..4058d5fe 100644 --- a/src/app/format_picker/preferences.rs +++ b/src/app/format_picker/preferences.rs @@ -2,14 +2,26 @@ //! Format selection and preference logic -use crate::backends::camera::types::CameraFormat; +use crate::backends::camera::types::{CameraFormat, SensorType}; use crate::media::Codec; use tracing::info; +/// Filter formats to only include RGB (non-depth) formats +/// +/// Auto-selection always prefers RGB camera formats over depth sensor formats. +/// Depth data is captured automatically alongside RGB when available. +fn filter_rgb_formats(formats: &[CameraFormat]) -> Vec<&CameraFormat> { + formats + .iter() + .filter(|f| f.sensor_type == SensorType::Rgb) + .collect() +} + /// Select format with maximum resolution (for Photo mode) /// /// Photo mode: ALWAYS select maximum resolution, regardless of codec. /// The user wants the highest quality photo possible. +/// Always prefers RGB formats over depth sensor formats. /// /// Framerate preference: /// - Prefer 30-60 fps range (at least 30 fps, capped at 60 fps) @@ -19,20 +31,22 @@ use tracing::info; /// Raw formats: YUYV > UYVY > YUY2 > NV12 > YV12 > I420 /// Encoded formats: H.264 > HW-accelerated MJPEG > HW-accelerated > MJPEG > First pub fn select_max_resolution_format(formats: &[CameraFormat]) -> Option { - if formats.is_empty() { + // Filter to RGB formats only - depth formats are handled separately + let rgb_formats = filter_rgb_formats(formats); + if rgb_formats.is_empty() { return None; } - info!("Photo mode: selecting maximum resolution with optimal framerate"); + info!("Photo mode: selecting maximum resolution with optimal framerate (RGB only)"); - // Find max resolution (by total pixels) - let max_pixels = formats.iter().map(|f| f.width * f.height).max()?; + // Find max resolution (by total pixels) among RGB formats + let max_pixels = rgb_formats.iter().map(|f| f.width * f.height).max()?; - // Get all formats with max resolution - let max_res_formats: Vec<_> = formats + // Get all RGB formats with max resolution + let max_res_formats: Vec<_> = rgb_formats .iter() .filter(|f| f.width * f.height == max_pixels) - .cloned() + .map(|f| (*f).clone()) .collect(); // Filter to formats with 30-60 fps range (preferred range for photo mode) @@ -123,12 +137,19 @@ pub fn is_raw_format(pixel_format: &str) -> bool { /// Select format for first-time video mode usage /// Selects highest resolution with at least 25 fps, preferring highest framerate up to 60 fps +/// Always prefers RGB formats over depth sensor formats. pub fn select_first_time_video_format(formats: &[CameraFormat]) -> Option { use std::collections::HashMap; - // Group formats by resolution + // Filter to RGB formats only - depth formats are handled separately + let rgb_formats = filter_rgb_formats(formats); + if rgb_formats.is_empty() { + return None; + } + + // Group RGB formats by resolution let mut resolution_groups: HashMap<(u32, u32), Vec<&CameraFormat>> = HashMap::new(); - for format in formats { + for format in &rgb_formats { if let Some(fps) = format.framerate { // Only consider formats with at least 25 fps if fps >= 25 { @@ -141,8 +162,8 @@ pub fn select_first_time_video_format(formats: &[CameraFormat]) -> Option= 25 fps, fall back to any format - return formats.first().cloned(); + // No RGB formats with >= 25 fps, fall back to first RGB format + return rgb_formats.first().map(|f| (*f).clone()); } // Find the highest resolution (by pixel count) @@ -173,7 +194,7 @@ pub fn select_first_time_video_format(formats: &[CameraFormat]) -> Option { + let is_new = self.preview_3d.last_render_video_timestamp != Some(ts); + if is_new { + self.preview_3d.last_render_video_timestamp = Some(ts); + } + is_new + } + None => true, // No timestamp means always render (non-Kinect sources) + }; + + if video_is_new { + return self.handle_request_point_cloud_render(); + } + } + Task::none() } @@ -177,12 +235,15 @@ impl AppModel { self.update_framerate_options(); self.update_codec_options(); + // Update Kinect state if this is a Kinect device (may return async task) + let kinect_task = self.update_kinect_state(); + info!("Camera initialization complete, preview will start"); // Query exposure controls for the current camera - if let Some(device_path) = self.get_v4l2_device_path() { + let exposure_task = if let Some(device_path) = self.get_v4l2_device_path() { let path = device_path.clone(); - return Task::perform( + Task::perform( async move { let controls = crate::app::exposure_picker::query_exposure_controls(&path); let settings = @@ -198,27 +259,43 @@ impl AppModel { color_settings, )) }, - ); - } + ) + } else { + Task::none() + }; - Task::none() + Task::batch([kinect_task, exposure_task]) } pub(crate) fn handle_camera_list_changed( &mut self, new_cameras: Vec, ) -> Task> { + use crate::backends::camera::is_depth_camera; + info!( old_count = self.available_cameras.len(), new_count = new_cameras.len(), "Camera list changed (hotplug event)" ); + // Check if current camera is still available + // Special handling for Kinect: if native streaming is active, consider it available + // even if the path changed (V4L2 -> freedepth transition) let current_camera_still_available = if let Some(current) = self.available_cameras.get(self.current_camera_index) { - new_cameras + // Check by path and name + let exact_match = new_cameras .iter() - .any(|c| c.path == current.path && c.name == current.name) + .any(|c| c.path == current.path && c.name == current.name); + + // If native Kinect streaming is active and the current camera is a Kinect, + // check if there's any Kinect in the new list (path may have changed) + let kinect_still_available = self.kinect.streaming + && is_depth_camera(current) + && new_cameras.iter().any(|c| is_depth_camera(c)); + + exact_match || kinect_still_available } else { false }; @@ -235,6 +312,13 @@ impl AppModel { }) .collect(); + // If native Kinect streaming is active, update current_camera_index to point to the Kinect + if self.kinect.streaming { + if let Some(kinect_index) = new_cameras.iter().position(|c| is_depth_camera(c)) { + self.current_camera_index = kinect_index; + } + } + if !current_camera_still_available { // Stop virtual camera streaming if the camera used for streaming is disconnected if self.virtual_camera.is_streaming() { @@ -277,7 +361,10 @@ impl AppModel { self.current_camera_index = new_index; } } - Task::none() + + // Update Kinect state after hotplug - device might have been reconnected + // with new device_info or the Kinect might have been disconnected + self.update_kinect_state() } pub(crate) fn handle_start_camera_transition(&mut self) -> Task> { @@ -379,4 +466,222 @@ impl AppModel { |is_closed| cosmic::Action::App(Message::PrivacyCoverStatusChanged(is_closed)), )) } + + // ========================================================================= + // 3D Point Cloud Preview + // ========================================================================= + + /// Handle request to render point cloud from current frame + pub(crate) fn handle_request_point_cloud_render(&self) -> Task> { + // Only render if 3D preview is enabled + if !self.preview_3d.enabled { + return Task::none(); + } + + let Some(frame) = &self.current_frame else { + debug!("No frame available for point cloud render"); + return Task::none(); + }; + + // Get depth data - prefer current frame's depth data, fall back to stored latest + // Use actual depth dimensions, not RGB frame dimensions (which may differ for high-res modes) + let (depth_width, depth_height, depth_data) = if let Some(depth_data) = &frame.depth_data { + // Validate depth dimensions - if they're 0, use stored latest instead + if frame.depth_width > 0 && frame.depth_height > 0 { + ( + frame.depth_width, + frame.depth_height, + Arc::clone(depth_data), + ) + } else if let Some((w, h, _)) = &self.preview_3d.latest_depth_data { + // Use stored dimensions but current depth data + (*w, *h, Arc::clone(depth_data)) + } else { + // Fall back to Kinect defaults + (640, 480, Arc::clone(depth_data)) + } + } else if let Some((w, h, data)) = &self.preview_3d.latest_depth_data { + (*w, *h, Arc::clone(data)) + } else { + debug!("No depth data available for point cloud render"); + return Task::none(); + }; + + // Final validation + if depth_width == 0 || depth_height == 0 { + warn!("Invalid depth dimensions: {}x{}", depth_width, depth_height); + return Task::none(); + } + + // Clone RGB data for async task + let rgb_data = frame.data.clone(); + let rgb_width = frame.width; + let rgb_height = frame.height; + let (pitch, yaw) = self.preview_3d.rotation; + let zoom = self.preview_3d.zoom; + + info!( + rgb_width, + rgb_height, + rgb_bytes = rgb_data.len(), + depth_width, + depth_height, + depth_pixels = depth_data.len(), + frame_format = ?frame.format, + "Rendering point cloud" + ); + + // Check for resolution mismatch - if RGB and depth don't match, + // we need to use the depth dimensions and sample RGB accordingly + if rgb_width != depth_width || rgb_height != depth_height { + info!( + rgb = format!("{}x{}", rgb_width, rgb_height), + depth = format!("{}x{}", depth_width, depth_height), + "RGB/depth resolution mismatch - using depth dimensions" + ); + } + + // Use depth dimensions as the input base (640x480 for Kinect) + // The shader will sample RGB at scaled coordinates if needed + let input_width = depth_width; + let input_height = depth_height; + + // Render at higher resolution so 3D preview fills the available space + // Use the larger of RGB resolution or a reasonable minimum (1280x960) + // This prevents the preview from looking small/pixelated when scaled up + let output_width = rgb_width.max(1280); + let output_height = rgb_height.max(960); + + // Determine depth format based on source + // - Native Kinect backend provides depth in millimeters + // - V4L2 Y10B pipeline provides 10-bit disparity shifted to 16-bit + // Check if depth came from current frame (native Kinect) or from frame.depth_data + let depth_format = if self.kinect.is_device && frame.depth_data.is_some() { + // Native Kinect backend - depth is in millimeters + crate::shaders::DepthFormat::Millimeters + } else { + // V4L2 Y10B pipeline - depth is 10-bit disparity shifted to 16-bit + crate::shaders::DepthFormat::Disparity16 + }; + + // Get mirror setting + let mirror = self.config.mirror_preview; + + // Determine if we need to apply RGB-depth stereo registration + // Only apply when color data is from the RGB camera (not IR or depth tonemap) + // - If sensor_type is Ir or Depth: no registration (same sensor as depth) + // - If depth overlay is enabled: no registration (showing depth colormap, not RGB) + // - Only RGB sensor type WITHOUT depth overlay needs registration + // Registration tables are built for 640x480 RGB but can be scaled to higher resolutions + use crate::backends::camera::SensorType; + let sensor_type = self + .active_format + .as_ref() + .map(|f| f.sensor_type) + .unwrap_or(SensorType::Rgb); + let showing_depth_colormap = self.depth_viz.overlay_enabled; + let apply_rgb_registration = sensor_type == SensorType::Rgb && !showing_depth_colormap; + + // Get scene view mode (point cloud vs mesh) + let scene_view_mode = self.preview_3d.view_mode; + + // Get filter mode for scene rendering + use crate::app::state::FilterType; + let filter_mode = match self.selected_filter { + FilterType::Standard => 0u32, + FilterType::Mono => 1, + FilterType::Sepia => 2, + FilterType::Noir => 3, + FilterType::Vivid => 4, + FilterType::Cool => 5, + FilterType::Warm => 6, + FilterType::Fade => 7, + FilterType::Duotone => 8, + FilterType::Vignette => 9, + FilterType::Negative => 10, + FilterType::Posterize => 11, + FilterType::Solarize => 12, + FilterType::ChromaticAberration => 13, + FilterType::Pencil => 14, + }; + + info!( + depth_format = ?depth_format, + mirror, + apply_rgb_registration, + rgb_dim = format!("{}x{}", rgb_width, rgb_height), + depth_dim = format!("{}x{}", depth_width, depth_height), + ?sensor_type, + showing_depth_colormap, + ?scene_view_mode, + filter_mode, + "Rendering 3D scene" + ); + + Task::perform( + async move { + use crate::app::state::SceneViewMode; + let result = match scene_view_mode { + SceneViewMode::PointCloud => crate::shaders::render_point_cloud( + &rgb_data, + &depth_data, + rgb_width, + rgb_height, + input_width, + input_height, + output_width, + output_height, + pitch, + yaw, + zoom, + depth_format, + mirror, + apply_rgb_registration, + filter_mode, + ) + .await + .map(|r| (r.width, r.height, r.rgba)), + SceneViewMode::Mesh => { + // Fixed depth discontinuity threshold of 0.1 meters + const DEPTH_DISCONTINUITY_THRESHOLD: f32 = 0.1; + crate::shaders::render_mesh( + &rgb_data, + &depth_data, + rgb_width, + rgb_height, + input_width, + input_height, + output_width, + output_height, + pitch, + yaw, + zoom, + depth_format, + mirror, + apply_rgb_registration, + DEPTH_DISCONTINUITY_THRESHOLD, + filter_mode, + ) + .await + .map(|r| (r.width, r.height, r.rgba)) + } + }; + + match result { + Ok((width, height, rgba)) => Some((width, height, Arc::new(rgba))), + Err(e) => { + error!("Failed to render 3D scene: {}", e); + None + } + } + }, + |result| { + if let Some((width, height, data)) = result { + cosmic::Action::App(Message::PointCloudRendered(width, height, data)) + } else { + cosmic::Action::App(Message::Noop) + } + }, + ) + } } diff --git a/src/app/handlers/capture.rs b/src/app/handlers/capture.rs index bd4d8d7a..8abbc2c1 100644 --- a/src/app/handlers/capture.rs +++ b/src/app/handlers/capture.rs @@ -66,6 +66,11 @@ impl AppModel { /// Capture the current frame as a photo with the selected filter and zoom pub(crate) fn capture_photo(&mut self) -> Task> { + // Handle scene mode capture separately + if self.mode == CameraMode::Scene { + return self.capture_scene(); + } + // Use HDR+ burst mode if enabled in settings (Auto or fixed frame count) // No need to toggle the moon button - HDR+ is automatic when enabled if self.config.burst_mode_setting.is_enabled() { @@ -80,13 +85,24 @@ impl AppModel { info!("Capturing photo..."); self.is_capturing = true; + // Extract all values from frame before we need to call mutable methods let frame_arc = Arc::clone(frame); + let frame_width = frame.width; + let frame_height = frame.height; + let depth_info = frame.depth_data.as_ref().map(|depth_arc| { + crate::pipelines::photo::encoding::DepthDataInfo { + values: depth_arc.to_vec(), + width: frame_width, + height: frame_height, + } + }); + let save_dir = crate::app::get_photo_directory(); let filter_type = self.selected_filter; let zoom_level = self.zoom_level; // Calculate crop rectangle based on aspect ratio setting - let crop_rect = self.photo_aspect_ratio.crop_rect(frame.width, frame.height); + let crop_rect = self.photo_aspect_ratio.crop_rect(frame_width, frame_height); let crop_rect = if self.photo_aspect_ratio == crate::app::state::PhotoAspectRatio::Native { None } else { @@ -97,7 +113,7 @@ impl AppModel { let encoding_format: crate::pipelines::photo::EncodingFormat = self.config.photo_output_format.into(); - // Get camera metadata for DNG encoding (including exposure info) + // Get camera metadata for DNG encoding (including exposure info and depth data) let camera_metadata = self .available_cameras .get(self.current_camera_index) @@ -105,6 +121,7 @@ impl AppModel { let mut metadata = crate::pipelines::photo::CameraMetadata { camera_name: Some(cam.name.clone()), camera_driver: cam.device_info.as_ref().map(|info| info.driver.clone()), + depth_data: depth_info.clone(), ..Default::default() }; // Read exposure metadata from V4L2 device if available @@ -118,7 +135,13 @@ impl AppModel { } metadata }) - .unwrap_or_default(); + .unwrap_or_else(|| { + // Create metadata with just depth data if no camera info available + crate::pipelines::photo::CameraMetadata { + depth_data: depth_info, + ..Default::default() + } + }); let save_task = Task::perform( async move { @@ -483,29 +506,79 @@ impl AppModel { } pub(crate) fn handle_zoom_in(&mut self) -> Task> { - // Zoom in by 0.1x, max 10x - let new_zoom = (self.zoom_level + 0.1).min(10.0); - if (new_zoom - self.zoom_level).abs() > 0.001 { - self.zoom_level = new_zoom; - debug!(zoom = self.zoom_level, "Zoom in"); + // Try hardware zoom first if available + if self.available_exposure_controls.zoom_absolute.available { + let range = &self.available_exposure_controls.zoom_absolute; + if let Some(current) = self.get_v4l2_zoom_value() { + // Step by ~10% of range or at least 1 + let step = ((range.max - range.min) / 10).max(1); + let new_zoom = (current + step).min(range.max); + if new_zoom != current { + self.set_v4l2_zoom(new_zoom); + // Update zoom_level to reflect hardware zoom for display + // Map hardware range to 1.0-10.0 display range + let normalized = (new_zoom - range.min) as f32 / (range.max - range.min) as f32; + self.zoom_level = 1.0 + normalized * 9.0; + debug!( + hardware_zoom = new_zoom, + display_zoom = self.zoom_level, + "Hardware zoom in" + ); + } + } + } else { + // Fallback to shader zoom + let new_zoom = (self.zoom_level + 0.1).min(10.0); + if (new_zoom - self.zoom_level).abs() > 0.001 { + self.zoom_level = new_zoom; + debug!(zoom = self.zoom_level, "Shader zoom in"); + } } Task::none() } pub(crate) fn handle_zoom_out(&mut self) -> Task> { - // Zoom out by 0.1x, min 1.0x - let new_zoom = (self.zoom_level - 0.1).max(1.0); - if (new_zoom - self.zoom_level).abs() > 0.001 { - self.zoom_level = new_zoom; - debug!(zoom = self.zoom_level, "Zoom out"); + // Try hardware zoom first if available + if self.available_exposure_controls.zoom_absolute.available { + let range = &self.available_exposure_controls.zoom_absolute; + if let Some(current) = self.get_v4l2_zoom_value() { + // Step by ~10% of range or at least 1 + let step = ((range.max - range.min) / 10).max(1); + let new_zoom = (current - step).max(range.min); + if new_zoom != current { + self.set_v4l2_zoom(new_zoom); + // Update zoom_level to reflect hardware zoom for display + let normalized = (new_zoom - range.min) as f32 / (range.max - range.min) as f32; + self.zoom_level = 1.0 + normalized * 9.0; + debug!( + hardware_zoom = new_zoom, + display_zoom = self.zoom_level, + "Hardware zoom out" + ); + } + } + } else { + // Fallback to shader zoom + let new_zoom = (self.zoom_level - 0.1).max(1.0); + if (new_zoom - self.zoom_level).abs() > 0.001 { + self.zoom_level = new_zoom; + debug!(zoom = self.zoom_level, "Shader zoom out"); + } } Task::none() } pub(crate) fn handle_reset_zoom(&mut self) -> Task> { + // Reset hardware zoom if available + if self.available_exposure_controls.zoom_absolute.available { + let default = self.available_exposure_controls.zoom_absolute.default; + self.set_v4l2_zoom(default); + debug!(hardware_zoom = default, "Hardware zoom reset"); + } + // Always reset shader zoom if (self.zoom_level - 1.0).abs() > 0.001 { self.zoom_level = 1.0; - debug!("Zoom reset to 1.0"); + debug!("Shader zoom reset to 1.0"); } Task::none() } @@ -526,6 +599,137 @@ impl AppModel { Task::none() } + /// Capture a scene (depth + color + preview + point cloud + mesh) + pub(crate) fn capture_scene(&mut self) -> Task> { + let Some(frame) = &self.current_frame else { + info!("No frame available for scene capture"); + return Task::none(); + }; + + let Some((dw, dh, dd)) = &self.preview_3d.latest_depth_data else { + info!("No depth data available for scene capture"); + return Task::none(); + }; + + // Extract all data first before calling mutable methods + let rgb_data = frame.data.to_vec(); + let rgb_width = frame.width; + let rgb_height = frame.height; + let has_depth_data = frame.depth_data.is_some(); + let depth_data = dd.to_vec(); + let depth_width = *dw; + let depth_height = *dh; + + // Extract point cloud preview if available + let (preview_data, preview_width, preview_height) = + if let Some((pw, ph, pdata)) = &self.preview_3d.rendered_preview { + (Some(pdata.as_ref().clone()), *pw, *ph) + } else { + (None, 0, 0) + }; + + info!("Capturing scene..."); + self.is_capturing = true; + + let save_dir = crate::app::get_photo_directory(); + + // Get the encoding format from config + let image_format: crate::pipelines::photo::EncodingFormat = + self.config.photo_output_format.into(); + + // Determine depth format based on camera type + let depth_format = if self.kinect.is_device && has_depth_data { + // Kinect native depth is in millimeters + crate::shaders::DepthFormat::Millimeters + } else { + // V4L2 Y10B depth is disparity + crate::shaders::DepthFormat::Disparity16 + }; + let mirror = self.config.mirror_preview; + + let save_task = Task::perform( + async move { + use crate::pipelines::scene::{ + CameraIntrinsics, SceneCaptureConfig, capture_scene, + }; + + // Get registration data from the shader processor (same data used for preview) + let registration_data = + match crate::shaders::get_point_cloud_registration_data().await { + Ok(Some(shader_reg)) => { + // Calculate registration scale factors for high-res RGB + // Same logic as in point_cloud/processor.rs and mesh/processor.rs + let reg_scale_x = rgb_width as f32 / 640.0; + let reg_scale_y = reg_scale_x; // Same as X to maintain aspect ratio + let reg_y_offset = 0i32; // Top-aligned crop + + // Convert from shader RegistrationData to scene RegistrationData + Some(crate::pipelines::scene::RegistrationData { + registration_table: shader_reg.registration_table, + depth_to_rgb_shift: shader_reg.depth_to_rgb_shift, + target_offset: shader_reg.target_offset, + reg_scale_x, + reg_scale_y, + reg_y_offset, + }) + } + Ok(None) => { + tracing::warn!("No registration data available from shader processor"); + None + } + Err(e) => { + tracing::warn!("Failed to get registration data: {}", e); + None + } + }; + + let config = SceneCaptureConfig { + image_format, + intrinsics: CameraIntrinsics::default(), + depth_format, + mirror, + registration: registration_data, + }; + + capture_scene( + &rgb_data, + rgb_width, + rgb_height, + &depth_data, + depth_width, + depth_height, + preview_data.as_deref(), + preview_width, + preview_height, + save_dir, + config, + ) + .await + .map(|result| result.scene_dir.display().to_string()) + }, + |result| cosmic::Action::App(Message::SceneSaved(result)), + ); + + let animation_task = Self::delay_task(150, Message::ClearCaptureAnimation); + Task::batch([save_task, animation_task]) + } + + pub(crate) fn handle_scene_saved( + &mut self, + result: Result, + ) -> Task> { + match result { + Ok(path) => { + info!(path = %path, "Scene saved successfully"); + return Task::done(cosmic::Action::App(Message::RefreshGalleryThumbnail)); + } + Err(err) => { + error!(error = %err, "Failed to save scene"); + } + } + Task::none() + } + pub(crate) fn handle_clear_capture_animation(&mut self) -> Task> { self.is_capturing = false; Task::none() diff --git a/src/app/handlers/depth_camera.rs b/src/app/handlers/depth_camera.rs new file mode 100644 index 00000000..d76bcd5e --- /dev/null +++ b/src/app/handlers/depth_camera.rs @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Depth camera control handlers +//! +//! Handles depth camera device control including motor tilt, native streaming, +//! 3D preview frame polling, depth visualization settings, and calibration dialogs. +//! LED is automatically managed by freedepth based on device state. +//! +//! When freedepth is not available (non-x86_64 or feature disabled), stub +//! implementations are provided that return Task::none(). + +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use crate::backends::camera::{NativeDepthBackend, depth_device_index, rgb_to_rgba}; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use freedepth::{DepthRegistration, TILT_MAX_DEGREES, TILT_MIN_DEGREES}; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use std::time::{Duration, Instant}; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use tracing::{debug, info, warn}; + +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use crate::app::state::SceneViewMode; +use crate::app::state::{AppModel, Message}; +use cosmic::Task; + +/// Set up GPU shader registration data from device calibration +/// +/// This spawns an async task to update the point cloud and mesh processors +/// with the device-specific registration tables. Call this after the native +/// depth backend is initialized. +/// +/// Uses the generic DepthRegistration trait for device-agnostic access. +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub fn setup_shader_registration_data(registration: &dyn DepthRegistration) { + let reg_data = crate::shaders::RegistrationData { + registration_table: registration.registration_table_flat().to_vec(), + depth_to_rgb_shift: registration.depth_to_rgb_shift_table().to_vec(), + target_offset: registration.target_offset(), + }; + + tokio::spawn(async move { + if let Err(e) = crate::shaders::set_point_cloud_registration_data(®_data).await { + tracing::warn!("Failed to set point cloud registration data: {}", e); + } else { + tracing::info!("Point cloud registration data set from device calibration"); + } + if let Err(e) = crate::shaders::set_mesh_registration_data(®_data).await { + tracing::warn!("Failed to set mesh registration data: {}", e); + } else { + tracing::info!("Mesh registration data set from device calibration"); + } + }); +} + +// Full implementation when freedepth is available +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +impl AppModel { + /// Handle setting depth camera tilt angle (desired state) + /// + /// Updates the desired tilt angle immediately in the UI, then sends the command + /// to the motor via the global motor control interface. + pub(crate) fn handle_set_kinect_tilt(&mut self, degrees: i8) -> Task> { + if !self.kinect.is_device { + return Task::none(); + } + + // Clamp to valid range (from freedepth) + let degrees = degrees.clamp(TILT_MIN_DEGREES, TILT_MAX_DEGREES); + + // Update desired state immediately for responsive UI + self.kinect.tilt_angle = degrees; + + // Send command to motor via global motor control + use crate::backends::camera::motor_control::set_motor_tilt; + if let Err(e) = set_motor_tilt(degrees) { + tracing::warn!("Failed to set depth camera tilt: {}", e); + } + + Task::none() + } + + /// Update depth camera state when camera changes + /// + /// Call this after switching cameras to update is_depth_camera flag. + /// Note: freedepth initialization is deferred until motor picker is opened + /// to avoid conflicts with V4L2 streaming. + pub(crate) fn update_kinect_state(&mut self) -> Task> { + use crate::backends::camera::is_depth_camera; + + let was_depth_camera = self.kinect.is_device; + + if let Some(ref camera) = self.available_cameras.get(self.current_camera_index) { + // Debug: log device info for depth camera detection + debug!( + camera_name = %camera.name, + camera_path = %camera.path, + has_device_info = camera.device_info.is_some(), + driver = camera.device_info.as_ref().map(|i| i.driver.as_str()).unwrap_or("none"), + "Checking camera for depth camera" + ); + + self.kinect.is_device = is_depth_camera(camera); + + info!( + camera_name = %camera.name, + was_depth_camera = was_depth_camera, + is_depth_camera = self.kinect.is_device, + camera_index = self.current_camera_index, + "Updated depth camera state" + ); + + if self.kinect.is_device { + info!("Depth camera detected - motor controls available"); + } else if was_depth_camera { + info!("Switching away from depth camera device"); + } + } else { + info!( + was_depth_camera = was_depth_camera, + camera_index = self.current_camera_index, + num_cameras = self.available_cameras.len(), + "No camera at index - setting is_depth_camera to false" + ); + self.kinect.is_device = false; + } + Task::none() + } + + /// Check if the current camera is a depth sensor (not RGB) + pub(crate) fn is_current_camera_depth_sensor(&self) -> bool { + if let Some(camera) = self.available_cameras.get(self.current_camera_index) { + let name_lower = camera.name.to_lowercase(); + let path_lower = camera.path.to_lowercase(); + name_lower.contains("depth") || path_lower.contains("depth") + } else { + false + } + } + + /// Handle starting native depth camera streaming for simultaneous RGB+depth + pub(crate) fn handle_start_native_kinect_streaming(&mut self) -> Task> { + info!("Starting native depth camera streaming for simultaneous RGB+depth"); + + // Get device index from current camera + let device_index = self + .available_cameras + .get(self.current_camera_index) + .and_then(|d| depth_device_index(&d.path)) + .unwrap_or(0); + + // Create and start the native backend with default format + let mut backend = NativeDepthBackend::new(); + match backend.start(device_index) { + Ok(()) => { + // Set up registration data for depth-to-RGB alignment + if let Some(registration) = backend.get_registration() { + // Store calibration info for UI display (device-agnostic) + self.kinect.calibration_info = Some(registration.registration_summary()); + + // Store registration data for scene capture + self.kinect.registration_data = + Some(crate::pipelines::scene::RegistrationData { + registration_table: registration.registration_table_flat().to_vec(), + depth_to_rgb_shift: registration.depth_to_rgb_shift_table().to_vec(), + target_offset: registration.target_offset(), + reg_scale_x: 1.0, + reg_scale_y: 1.0, + reg_y_offset: 0, + }); + + // Set up GPU shader registration data + setup_shader_registration_data(registration); + } else { + warn!("No registration data available from depth camera backend"); + self.kinect.calibration_info = None; + self.kinect.registration_data = None; + } + + self.kinect.native_backend = Some(backend); + self.kinect.streaming = true; + info!("Native depth camera streaming started successfully"); + + // Start polling for frames + Task::done(cosmic::Action::App(Message::PollNativeKinectFrames)) + } + Err(e) => { + warn!(error = %e, "Failed to start native depth camera streaming"); + self.preview_3d.enabled = false; + Task::none() + } + } + } + + /// Handle stopping native depth camera streaming + pub(crate) fn handle_stop_native_kinect_streaming(&mut self) -> Task> { + info!("Stopping native depth camera streaming"); + if let Some(mut backend) = self.kinect.native_backend.take() { + backend.stop(); + } + self.kinect.streaming = false; + self.preview_3d.rendered_preview = None; + self.kinect.calibration_info = None; + self.kinect.registration_data = None; + Task::none() + } + + /// Handle polling for native depth camera frames + pub(crate) fn handle_poll_native_kinect_frames(&mut self) -> Task> { + if !self.kinect.streaming { + return Task::none(); + } + + let mut tasks = Vec::new(); + + if let Some(ref backend) = self.kinect.native_backend { + let mut video_frame_is_new = false; + + if let Some(video_frame) = backend.get_video_frame() { + video_frame_is_new = + self.preview_3d.last_render_video_timestamp != Some(video_frame.timestamp); + + let rgba_data = rgb_to_rgba(&video_frame.data); + + use crate::backends::camera::types::{CameraFrame, PixelFormat}; + let frame = CameraFrame { + width: video_frame.width, + height: video_frame.height, + stride: video_frame.width * 4, + data: rgba_data.into(), + format: PixelFormat::RGBA, + captured_at: std::time::Instant::now(), + depth_data: None, + depth_width: 0, + depth_height: 0, + video_timestamp: Some(video_frame.timestamp), + }; + + self.current_frame = Some(std::sync::Arc::new(frame)); + self.current_frame_is_file_source = false; + + if let Some(task) = self.transition_state.on_frame_received() { + tasks.push(task.map(cosmic::Action::App)); + } + + if video_frame_is_new { + self.preview_3d.last_render_video_timestamp = Some(video_frame.timestamp); + } + } + + if let Some(depth_frame) = backend.get_depth_frame() { + self.preview_3d.latest_depth_data = Some(( + depth_frame.width, + depth_frame.height, + std::sync::Arc::from(depth_frame.depth_mm.as_slice()), + )); + + if self.preview_3d.enabled && video_frame_is_new { + tasks.push(self.handle_request_point_cloud_render()); + } + } + } + + tasks.push( + Task::perform( + async { + tokio::time::sleep(std::time::Duration::from_millis(33)).await; + }, + |_| Message::PollNativeKinectFrames, + ) + .map(cosmic::Action::App), + ); + + Task::batch(tasks) + } + + // ===== Depth Visualization Handlers ===== + + /// Handle toggling depth colormap overlay + pub(crate) fn handle_toggle_depth_overlay(&mut self) -> Task> { + self.depth_viz.overlay_enabled = !self.depth_viz.overlay_enabled; + // Update shared state for depth processing pipeline + crate::shaders::depth::set_depth_colormap_enabled(self.depth_viz.overlay_enabled); + debug!( + enabled = self.depth_viz.overlay_enabled, + "Toggled depth overlay" + ); + Task::none() + } + + /// Handle toggling depth grayscale mode + pub(crate) fn handle_toggle_depth_grayscale(&mut self) -> Task> { + self.depth_viz.grayscale_mode = !self.depth_viz.grayscale_mode; + // Update shared state for depth processing pipeline + crate::shaders::depth::set_depth_grayscale_mode(self.depth_viz.grayscale_mode); + debug!( + enabled = self.depth_viz.grayscale_mode, + "Toggled depth grayscale mode" + ); + Task::none() + } + + // ===== Calibration Dialog Handlers ===== + + /// Handle showing the calibration dialog + pub(crate) fn handle_show_calibration_dialog(&mut self) -> Task> { + self.kinect.calibration_dialog_visible = true; + debug!("Showing calibration dialog"); + Task::none() + } + + /// Handle closing the calibration dialog + pub(crate) fn handle_close_calibration_dialog(&mut self) -> Task> { + self.kinect.calibration_dialog_visible = false; + debug!("Closing calibration dialog"); + Task::none() + } + + /// Handle starting calibration fetch from device + pub(crate) fn handle_start_calibration(&mut self) -> Task> { + // Close dialog and attempt to fetch calibration from device + self.kinect.calibration_dialog_visible = false; + info!("Starting calibration fetch from device"); + // The actual calibration fetch happens when the Kinect backend is initialized + // For now, we can re-trigger it by restarting the camera + // TODO: Implement direct calibration fetch without restart + Task::none() + } + + // ===== 3D Preview Handlers ===== + + /// Handle toggling the 3D preview mode + pub(crate) fn handle_toggle_3d_preview(&mut self) -> Task> { + self.preview_3d.enabled = !self.preview_3d.enabled; + info!( + enabled = self.preview_3d.enabled, + is_kinect = self.kinect.is_device, + has_depth = self.preview_3d.latest_depth_data.is_some(), + depth_pixels = self + .preview_3d + .latest_depth_data + .as_ref() + .map(|(_, _, d)| d.len()) + .unwrap_or(0), + "Toggled 3D preview" + ); + if self.preview_3d.enabled { + // Check if we need to start native Kinect streaming + // (when on RGB camera and no depth data available) + if self.kinect.is_device + && self.preview_3d.latest_depth_data.is_none() + && !self.is_current_camera_depth_sensor() + { + // Start native USB streaming for simultaneous RGB+depth + info!("Starting native Kinect streaming for 3D preview"); + return Task::done(cosmic::Action::App(Message::StartNativeKinectStreaming)); + } + // Trigger initial render when enabling 3D mode + self.handle_request_point_cloud_render() + } else { + // Stop native Kinect streaming if running + if self.kinect.streaming { + return Task::done(cosmic::Action::App(Message::StopNativeKinectStreaming)); + } + // Clear point cloud preview when disabling + self.preview_3d.rendered_preview = None; + Task::none() + } + } + + /// Handle toggling between point cloud and mesh view modes + pub(crate) fn handle_toggle_scene_view_mode(&mut self) -> Task> { + self.preview_3d.view_mode = match self.preview_3d.view_mode { + SceneViewMode::PointCloud => SceneViewMode::Mesh, + SceneViewMode::Mesh => SceneViewMode::PointCloud, + }; + info!(mode = ?self.preview_3d.view_mode, "Toggled scene view mode"); + // Trigger re-render with new mode + self.handle_request_point_cloud_render() + } + + /// Handle mouse press on 3D preview (start dragging) + pub(crate) fn handle_preview_3d_mouse_pressed( + &mut self, + _x: f32, + _y: f32, + ) -> Task> { + // Store current rotation as base for this drag (path independence) + self.preview_3d.base_rotation = self.preview_3d.rotation; + self.preview_3d.dragging = true; + // Don't set drag_start_pos here - on_press gives hardcoded (0,0) + // First mouse move will set the actual start position + self.preview_3d.drag_start_pos = None; + self.preview_3d.last_mouse_pos = None; + Task::none() + } + + /// Handle mouse move on 3D preview (update rotation while dragging) + pub(crate) fn handle_preview_3d_mouse_moved( + &mut self, + x: f32, + y: f32, + ) -> Task> { + if self.preview_3d.dragging { + // Check if this is the first move after press + if self.preview_3d.drag_start_pos.is_none() { + // First move - record actual start position + // (on_press gives hardcoded 0,0 so we capture real position here) + self.preview_3d.drag_start_pos = Some((x, y)); + return Task::none(); + } + + if let Some((start_x, start_y)) = self.preview_3d.drag_start_pos { + // Calculate delta from START position (path independence) + // This means dragging back to start returns to original view + let delta_x = x - start_x; + let delta_y = y - start_y; + let (base_pitch, base_yaw) = self.preview_3d.base_rotation; + + // Adjust rotation sensitivity (radians per pixel) + let sensitivity = 0.005; + // Invert pitch so dragging down tilts view up (natural) + // Clamp pitch to prevent flipping over + let new_pitch = (base_pitch - delta_y * sensitivity).clamp(-1.4, 1.4); + // Yaw rotates normally (drag right = rotate right) + let new_yaw = base_yaw + delta_x * sensitivity; + + self.preview_3d.rotation = (new_pitch, new_yaw); + } + + // Throttle render requests to ~60fps to prevent stuttering + // The GPU render blocks, so limiting requests prevents frame stacking + const RENDER_INTERVAL: Duration = Duration::from_millis(16); + let now = Instant::now(); + let should_render = self + .preview_3d + .last_render_request_time + .map(|last| now.duration_since(last) >= RENDER_INTERVAL) + .unwrap_or(true); + + if should_render { + self.preview_3d.last_render_request_time = Some(now); + return self.handle_request_point_cloud_render(); + } + return Task::none(); + } + Task::none() + } + + /// Handle mouse release on 3D preview (stop dragging) + pub(crate) fn handle_preview_3d_mouse_released(&mut self) -> Task> { + self.preview_3d.dragging = false; + self.preview_3d.drag_start_pos = None; + self.preview_3d.last_mouse_pos = None; + // Current rotation now becomes the "resting" rotation for next drag + // Re-render point cloud with final rotation + self.handle_request_point_cloud_render() + } + + /// Handle resetting 3D preview rotation and zoom to defaults + pub(crate) fn handle_reset_3d_preview_rotation(&mut self) -> Task> { + self.preview_3d.rotation = (0.0, 0.0); + self.preview_3d.base_rotation = (0.0, 0.0); + self.preview_3d.zoom = 0.0; // 0 = at sensor position (1x view) + debug!("Reset 3D preview rotation and zoom"); + // Re-render point cloud with new rotation + self.handle_request_point_cloud_render() + } + + /// Handle zooming the 3D preview (scroll wheel) + pub(crate) fn handle_zoom_3d_preview(&mut self, delta: f32) -> Task> { + // Zoom in/out based on scroll delta - "fly into scene" model + // Matches natural scrolling: scroll up = zoom in (like normal camera mode) + // preview_3d_zoom = camera Z position (0 = at sensor, positive = into scene, negative = behind) + // Scroll up (positive delta) = zoom in = move camera forward into scene + // Scroll down (negative delta) = zoom out = move camera back + // Range: -2.0 (2m behind sensor, zoomed out) to 3.0 (3m into scene, zoomed in) + let zoom_sensitivity = 0.002; + let new_zoom = (self.preview_3d.zoom + delta * zoom_sensitivity).clamp(-2.0, 3.0); + self.preview_3d.zoom = new_zoom; + debug!(zoom = new_zoom, "3D preview zoom changed"); + // Re-render point cloud with new zoom + self.handle_request_point_cloud_render() + } +} + +// Stub implementations when freedepth is disabled +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +impl AppModel { + pub(crate) fn handle_set_kinect_tilt(&mut self, _degrees: i8) -> Task> { + Task::none() + } + + pub(crate) fn update_kinect_state(&mut self) -> Task> { + self.kinect.is_device = false; + Task::none() + } + + pub(crate) fn is_current_camera_depth_sensor(&self) -> bool { + false + } + + pub(crate) fn handle_start_native_kinect_streaming(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_stop_native_kinect_streaming(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_poll_native_kinect_frames(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_toggle_depth_overlay(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_toggle_depth_grayscale(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_show_calibration_dialog(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_close_calibration_dialog(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_start_calibration(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_toggle_3d_preview(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_toggle_scene_view_mode(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_preview_3d_mouse_pressed( + &mut self, + _x: f32, + _y: f32, + ) -> Task> { + Task::none() + } + + pub(crate) fn handle_preview_3d_mouse_moved( + &mut self, + _x: f32, + _y: f32, + ) -> Task> { + Task::none() + } + + pub(crate) fn handle_preview_3d_mouse_released(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_reset_3d_preview_rotation(&mut self) -> Task> { + Task::none() + } + + pub(crate) fn handle_zoom_3d_preview(&mut self, _delta: f32) -> Task> { + Task::none() + } +} diff --git a/src/app/handlers/format.rs b/src/app/handlers/format.rs index fb247737..52b089e6 100644 --- a/src/app/handlers/format.rs +++ b/src/app/handlers/format.rs @@ -56,6 +56,20 @@ impl AppModel { self.mode = mode; self.zoom_level = 1.0; // Reset zoom when switching modes + + // Handle 3D preview based on Scene mode + if mode == CameraMode::Scene { + // Enable 3D preview when entering Scene mode + self.preview_3d.enabled = true; + self.preview_3d.rotation = (0.0, 0.0); + self.preview_3d.base_rotation = (0.0, 0.0); + self.preview_3d.zoom = 0.0; + } else if self.preview_3d.enabled { + // Disable 3D preview when leaving Scene mode + self.preview_3d.enabled = false; + self.preview_3d.rendered_preview = None; + } + self.switch_camera_or_mode(self.current_camera_index, mode); // When switching to Virtual mode with a file source, restore the file source preview diff --git a/src/app/handlers/mod.rs b/src/app/handlers/mod.rs index 597e7612..3150c134 100644 --- a/src/app/handlers/mod.rs +++ b/src/app/handlers/mod.rs @@ -8,6 +8,7 @@ pub mod camera; pub mod capture; pub mod color; +pub mod depth_camera; pub mod exposure; pub mod format; pub mod system; diff --git a/src/app/mod.rs b/src/app/mod.rs index 0b454581..1006afb7 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -58,8 +58,8 @@ use cosmic::widget::{self, about::About}; use cosmic::{Element, Task}; pub use state::{ AppFlags, AppModel, BurstModeStage, BurstModeState, CameraMode, ContextPage, FileSource, - FilterType, Message, PhotoAspectRatio, PhotoTimerSetting, RecordingState, TheatreState, - VirtualCameraState, + FilterType, Message, PhotoAspectRatio, PhotoTimerSetting, RecordingState, SceneViewMode, + TheatreState, VirtualCameraState, }; use std::sync::Arc; use tracing::{error, info, warn}; @@ -327,6 +327,12 @@ impl cosmic::Application for AppModel { last_qr_detection_time: None, // Privacy cover detection privacy_cover_closed: false, + // Kinect state (device, motor, calibration, native backend) + kinect: crate::app::state::KinectState::default(), + // Depth visualization settings + depth_viz: crate::app::state::DepthVisualizationState::default(), + // 3D preview state (rotation, zoom, rendering) + preview_3d: crate::app::state::Preview3DState::default(), }; // Make context drawer overlay the content instead of reserving space @@ -489,6 +495,12 @@ impl cosmic::Application for AppModel { return Task::none(); } + // Close motor picker if open + if self.motor_picker_visible { + self.motor_picker_visible = false; + return Task::none(); + } + // Close tools menu if open if self.tools_menu_visible { self.tools_menu_visible = false; @@ -517,7 +529,7 @@ impl cosmic::Application for AppModel { /// Register subscriptions for this application. fn subscription(&self) -> Subscription { - use cosmic::iced::futures::{SinkExt, StreamExt}; + use cosmic::iced::futures::SinkExt; let config_sub = self .core() @@ -637,8 +649,13 @@ impl cosmic::Application for AppModel { } // Create camera pipeline using PipeWire backend + // For Y10B depth formats, use V4L2 direct capture with GPU unpacking use crate::backends::camera::pipewire::PipeWirePipeline; - use crate::backends::camera::types::{CameraDevice, CameraFormat}; + use crate::backends::camera::types::{ + CameraDevice, CameraFormat, SensorType, + }; + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + use crate::backends::camera::types::{CameraFrame, PixelFormat}; let (sender, mut receiver) = cosmic::iced::futures::channel::mpsc::channel(100); @@ -658,120 +675,546 @@ impl cosmic::Application for AppModel { .and_then(|c| c.device_info.clone()), }; + let pixel_format_str = pixel_format.unwrap_or("MJPEG"); + let is_depth_format = pixel_format_str == "Y10B"; + let sensor_type = if is_depth_format { + SensorType::Depth + } else { + SensorType::Rgb + }; let format = CameraFormat { width: width.unwrap_or(640), height: height.unwrap_or(480), framerate, hardware_accelerated: true, // Assume HW acceleration available - pixel_format: pixel_format.unwrap_or("MJPEG").to_string(), + pixel_format: pixel_format_str.to_string(), + sensor_type, }; - let pipeline_opt = match PipeWirePipeline::new(&device, &format, sender) - { - Ok(pipeline) => { - info!("Pipeline created successfully"); - Some(pipeline) + // For depth camera devices, use native freedepth backend (bypasses V4L2) + // For depth formats (Y10B), use V4L2 direct capture with GPU processing + // For regular formats, use GStreamer via PipeWire + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + use crate::backends::camera::v4l2_depth::V4l2DepthPipeline; + use crate::backends::camera::{ + NativeDepthBackend, V4l2KernelDepthBackend, is_depth_native_device, + is_kernel_depth_device, + }; + + // Enum to hold active pipeline - fields are used for ownership semantics + // (keeping pipeline alive to continue capture) + #[allow(dead_code)] + enum ActivePipeline { + GStreamer(PipeWirePipeline), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Depth(V4l2DepthPipeline), + DepthCamera(NativeDepthBackend), + KernelDepth(V4l2KernelDepthBackend), + } + + // Check if this is a kernel depth device first (highest priority) + let is_kernel_depth = is_kernel_depth_device(&device.path); + // Then check for freedepth device (only when kernel driver not present) + let is_depth_cam = + !is_kernel_depth && is_depth_native_device(&device.path); + if is_kernel_depth { + info!(device = %device.name, path = %device.path, "Using V4L2 kernel depth backend"); + } else if is_depth_cam { + info!(device = %device.name, path = %device.path, "Using native depth camera backend (freedepth)"); + } + + let pipeline_opt: Option = if is_kernel_depth { + // Kernel depth camera backend - uses V4L2 with depth controls + use crate::backends::camera::CameraBackend; + let mut kernel_backend = V4l2KernelDepthBackend::new(); + match kernel_backend.initialize(&device, &format) { + Ok(()) => { + info!("V4L2 kernel depth pipeline started successfully"); + Some(ActivePipeline::KernelDepth(kernel_backend)) + } + Err(e) => { + error!(error = %e, "Failed to start kernel depth backend"); + None + } } - Err(e) => { - error!(error = %e, "Failed to initialize pipeline"); - None + } else if is_depth_cam { + // Native depth camera backend - bypasses V4L2 entirely + // Use initialize() which properly sets depth_only_mode for Y10B format + use crate::backends::camera::CameraBackend; + let mut depth_backend = NativeDepthBackend::new(); + match depth_backend.initialize(&device, &format) { + Ok(()) => { + // Set up registration data for depth-to-RGB alignment + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + if let Some(registration) = depth_backend.get_registration() + { + // Use shared helper for GPU shader registration + crate::app::handlers::depth_camera::setup_shader_registration_data(registration); + } + info!("Native Kinect pipeline started successfully"); + Some(ActivePipeline::DepthCamera(depth_backend)) + } + Err(e) => { + error!(error = %e, "Failed to start native Kinect backend"); + None + } + } + } else { + // Check for Y10B depth format (only with freedepth) + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + if is_depth_format { + info!("Creating V4L2 depth pipeline for Y10B format"); + use crate::shaders::depth::{ + is_depth_colormap_enabled, is_depth_only_mode, + unpack_y10b_gpu, + }; + + // Create depth frame channel + let (depth_sender, mut depth_receiver) = + cosmic::iced::futures::channel::mpsc::channel(10); + + // Create V4L2 depth pipeline + match V4l2DepthPipeline::new(&device, &format, depth_sender) { + Ok(depth_pipeline) => { + info!("V4L2 depth pipeline created successfully"); + + // Spawn task to process depth frames with GPU shader + let depth_width = format.width; + let depth_height = format.height; + let mut frame_sender_clone = sender.clone(); + + tokio::spawn(async move { + use futures::StreamExt; + info!("Depth frame processing task started"); + + while let Some(depth_frame) = + depth_receiver.next().await + { + // Process with GPU shader + // Check visualization modes from shared state + let use_colormap = is_depth_colormap_enabled(); + let depth_only = is_depth_only_mode(); + match unpack_y10b_gpu( + &depth_frame.raw_data, + depth_width, + depth_height, + use_colormap, + depth_only, + ) + .await + { + Ok(result) => { + // Create CameraFrame with RGBA preview and depth data + let camera_frame = CameraFrame { + width: result.width, + height: result.height, + data: std::sync::Arc::from( + result + .rgba_preview + .into_boxed_slice(), + ), + format: PixelFormat::Depth16, + stride: result.width * 4, + captured_at: depth_frame + .captured_at, + depth_data: Some( + std::sync::Arc::from( + result + .depth_u16 + .into_boxed_slice(), + ), + ), + depth_width: result.width, + depth_height: result.height, + video_timestamp: None, + }; + + // Send to preview channel + if frame_sender_clone + .try_send(camera_frame) + .is_err() + { + tracing::debug!( + "Preview channel full, dropping depth frame" + ); + } + } + Err(e) => { + tracing::warn!(error = %e, "Failed to unpack Y10B frame with GPU"); + } + } + } + + info!("Depth frame processing task ended"); + }); + + // Keep depth_pipeline alive so capture thread continues + Some(ActivePipeline::Depth(depth_pipeline)) + } + Err(e) => { + error!(error = %e, "Failed to create V4L2 depth pipeline"); + None + } + } + } else { + // Regular format - use GStreamer via PipeWire + match PipeWirePipeline::new(&device, &format, sender.clone()) { + Ok(pipeline) => { + info!("Pipeline created successfully"); + Some(ActivePipeline::GStreamer(pipeline)) + } + Err(e) => { + error!(error = %e, "Failed to initialize pipeline"); + None + } + } + } + + // When freedepth is not available, use GStreamer + #[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] + { + // Regular format - use GStreamer via PipeWire + match PipeWirePipeline::new(&device, &format, sender) { + Ok(pipeline) => { + info!("Pipeline created successfully"); + Some(ActivePipeline::GStreamer(pipeline)) + } + Err(e) => { + error!(error = %e, "Failed to initialize pipeline"); + None + } + } } }; if let Some(pipeline) = pipeline_opt { info!("Waiting for frames from pipeline..."); - // Keep pipeline alive and forward frames - loop { - // Check cancel flag first (set when switching cameras/modes) - if cancel_flag.load(std::sync::atomic::Ordering::Acquire) { - info!( - "Cancel flag set - PipeWire subscription being cancelled" - ); - break; - } - // Check if subscription is still active before processing next frame - if output.is_closed() { - info!( - "Output channel closed - PipeWire subscription being cancelled" - ); - break; - } + // For kernel depth devices, poll frames directly via V4L2 + if let ActivePipeline::KernelDepth(ref kernel_backend) = pipeline { + info!("Starting kernel depth frame polling loop"); + loop { + // Check cancel flag + if cancel_flag.load(std::sync::atomic::Ordering::Acquire) { + info!( + "Cancel flag set - kernel depth subscription being cancelled" + ); + break; + } + + // Check if output is closed + if output.is_closed() { + info!( + "Output channel closed - kernel depth subscription being cancelled" + ); + break; + } - // Wait for next frame with a timeout to periodically check cancellation - // Use 16ms timeout (~60fps) to reduce frame delivery jitter - match tokio::time::timeout( - tokio::time::Duration::from_millis(16), - receiver.next(), - ) - .await - { - Ok(Some(frame)) => { + // Poll for frame via CameraBackend trait + if let Some(frame) = kernel_backend.get_frame() { frame_count += 1; - // Calculate frame latency (time from capture to subscription delivery) - let latency_us = - frame.captured_at.elapsed().as_micros(); if frame_count % 30 == 0 { info!( frame = frame_count, width = frame.width, height = frame.height, - latency_ms = latency_us as f64 / 1000.0, - "Received frame from pipeline" + "Received kernel depth frame" ); } - // Warn if latency exceeds 2 frame periods (>33ms at 60fps) - if latency_us > 33_000 { - tracing::warn!( + // Send frame to UI + let _ = output + .try_send(Message::CameraFrame(Arc::new(frame))); + } + + // Small sleep to avoid busy-waiting (~30fps) + tokio::time::sleep(tokio::time::Duration::from_millis(16)) + .await; + } + } + + // For freedepth Kinect, poll frames directly instead of using a channel + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + if let ActivePipeline::DepthCamera(ref depth_backend) = pipeline { + use crate::shaders::depth::{ + depth_mm_to_rgba, is_depth_colormap_enabled, + is_depth_grayscale_mode, is_depth_only_mode, rgb_to_rgba, + }; + + info!("Starting Kinect frame polling loop"); + loop { + // Check cancel flag + if cancel_flag.load(std::sync::atomic::Ordering::Acquire) { + info!( + "Cancel flag set - Kinect subscription being cancelled" + ); + break; + } + + // Check if output is closed + if output.is_closed() { + info!( + "Output channel closed - Kinect subscription being cancelled" + ); + break; + } + + // Poll for video frame + if let Some(video_frame) = depth_backend.get_video_frame() { + frame_count += 1; + + // Get depth frame for depth overlay or 3D preview + let depth_frame = depth_backend.get_depth_frame(); + + // Check if depth colormap should be shown instead of RGB + // Show depth when: UI toggle enabled, OR Y10B format selected (depth_only_mode) + let use_colormap = is_depth_colormap_enabled(); + let depth_only = is_depth_only_mode(); + let format_is_depth = + depth_backend.is_depth_only_mode(); + + // Decide frame data: depth colormap or RGB + let grayscale = is_depth_grayscale_mode(); + let (rgba_data, frame_width, frame_height) = + if (use_colormap || depth_only || format_is_depth) + && depth_frame.is_some() + { + let df = depth_frame.as_ref().unwrap(); + let visualization = depth_mm_to_rgba( + &df.depth_mm, + df.width, + df.height, + depth_only || format_is_depth, + grayscale, + ); + (visualization, df.width, df.height) + } else { + // Normal RGB preview + ( + rgb_to_rgba(&video_frame.data), + video_frame.width, + video_frame.height, + ) + }; + + // Get depth dimensions before consuming depth_frame + let (depth_width, depth_height) = depth_frame + .as_ref() + .map(|d| (d.width, d.height)) + .unwrap_or((0, 0)); + + let frame = CameraFrame { + width: frame_width, + height: frame_height, + data: std::sync::Arc::from( + rgba_data.into_boxed_slice(), + ), + format: PixelFormat::RGBA, + stride: frame_width * 4, + captured_at: std::time::Instant::now(), + depth_data: depth_frame.map(|d| { + std::sync::Arc::from( + d.depth_mm.into_boxed_slice(), + ) + }), + depth_width, + depth_height, + video_timestamp: Some(video_frame.timestamp), + }; + + if frame_count % 30 == 0 { + info!( frame = frame_count, - latency_ms = latency_us as f64 / 1000.0, - "High frame latency detected - possible stuttering" + width = frame.width, + height = frame.height, + has_depth = frame.depth_data.is_some(), + depth_pixels = frame + .depth_data + .as_ref() + .map(|d| d.len()) + .unwrap_or(0), + use_colormap = use_colormap, + depth_only = depth_only, + "Received Kinect frame" ); } - // Use try_send to avoid blocking the subscription when UI is busy - // Dropping frames is fine for live preview - we want the latest frame - match output - .try_send(Message::CameraFrame(Arc::new(frame))) - { - Ok(_) => { - if frame_count % 30 == 0 { - info!( - frame = frame_count, - "Frame forwarded to UI" - ); - } + // Send frame to UI + let _ = output + .try_send(Message::CameraFrame(Arc::new(frame))); + } + + // Small sleep to avoid busy-waiting (~30fps) + tokio::time::sleep(tokio::time::Duration::from_millis(16)) + .await; + } + } else { + use cosmic::iced::futures::StreamExt; + // GStreamer or Depth pipeline - use channel-based receiver + loop { + // Check cancel flag first (set when switching cameras/modes) + if cancel_flag.load(std::sync::atomic::Ordering::Acquire) { + info!( + "Cancel flag set - PipeWire subscription being cancelled" + ); + break; + } + + // Check if subscription is still active before processing next frame + if output.is_closed() { + info!( + "Output channel closed - PipeWire subscription being cancelled" + ); + break; + } + + // Wait for next frame with a timeout to periodically check cancellation + // Use 16ms timeout (~60fps) to reduce frame delivery jitter + match tokio::time::timeout( + tokio::time::Duration::from_millis(16), + receiver.next(), + ) + .await + { + Ok(Some(frame)) => { + frame_count += 1; + // Calculate frame latency (time from capture to subscription delivery) + let latency_us = + frame.captured_at.elapsed().as_micros(); + + if frame_count % 30 == 0 { + info!( + frame = frame_count, + width = frame.width, + height = frame.height, + latency_ms = latency_us as f64 / 1000.0, + "Received frame from pipeline" + ); } - Err(e) => { - // Always log dropped frames for diagnostics + + // Warn if latency exceeds 2 frame periods (>33ms at 60fps) + if latency_us > 33_000 { tracing::warn!( frame = frame_count, - error = ?e, - "Frame dropped (UI channel full) - stuttering likely" + latency_ms = latency_us as f64 / 1000.0, + "High frame latency detected - possible stuttering" ); - // Check if channel is closed (subscription cancelled) - if e.is_disconnected() { - info!( - "Output channel disconnected - PipeWire subscription being cancelled" + } + + // Use try_send to avoid blocking the subscription when UI is busy + // Dropping frames is fine for live preview - we want the latest frame + match output + .try_send(Message::CameraFrame(Arc::new(frame))) + { + Ok(_) => { + if frame_count % 30 == 0 { + info!( + frame = frame_count, + "Frame forwarded to UI" + ); + } + } + Err(e) => { + // Always log dropped frames for diagnostics + tracing::warn!( + frame = frame_count, + error = ?e, + "Frame dropped (UI channel full) - stuttering likely" ); - break; + // Check if channel is closed (subscription cancelled) + if e.is_disconnected() { + info!( + "Output channel disconnected - PipeWire subscription being cancelled" + ); + break; + } } } } + Ok(None) => { + info!("PipeWire pipeline frame stream ended"); + break; + } + Err(_) => { + // Timeout - continue loop to check if channel is closed + continue; + } } - Ok(None) => { - info!("PipeWire pipeline frame stream ended"); + } + } + + // Non-freedepth: GStreamer pipeline only + #[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] + { + use cosmic::iced::futures::StreamExt; + // GStreamer pipeline - use channel-based receiver + loop { + // Check cancel flag first + if cancel_flag.load(std::sync::atomic::Ordering::Acquire) { + info!("Cancel flag set - subscription being cancelled"); break; } - Err(_) => { - // Timeout - continue loop to check if channel is closed - continue; + + if output.is_closed() { + info!( + "Output channel closed - subscription being cancelled" + ); + break; + } + + match tokio::time::timeout( + tokio::time::Duration::from_millis(16), + receiver.next(), + ) + .await + { + Ok(Some(frame)) => { + frame_count += 1; + let latency_us = + frame.captured_at.elapsed().as_micros(); + + if frame_count % 30 == 0 { + info!( + frame = frame_count, + width = frame.width, + height = frame.height, + latency_ms = latency_us as f64 / 1000.0, + "Received frame from pipeline" + ); + } + + match output + .try_send(Message::CameraFrame(Arc::new(frame))) + { + Ok(()) => { + if frame_count % 30 == 0 { + tracing::debug!( + frame = frame_count, + "Frame forwarded to UI" + ); + } + } + Err(e) => { + tracing::warn!(frame = frame_count, error = ?e, "Frame dropped"); + if e.is_disconnected() { + info!("Output channel disconnected"); + break; + } + } + } + } + Ok(None) => { + info!("Pipeline frame stream ended"); + break; + } + Err(_) => { + continue; + } } } } - info!("Cleaning up PipeWire pipeline"); + info!("Cleaning up pipeline"); // Pipeline will be dropped here, stopping the camera drop(pipeline); } else { diff --git a/src/app/motor_picker.rs b/src/app/motor_picker.rs index bcde8000..cb4faa1a 100644 --- a/src/app/motor_picker.rs +++ b/src/app/motor_picker.rs @@ -4,9 +4,11 @@ //! //! Provides a floating overlay for camera motor controls including: //! - V4L2 pan/tilt/zoom controls (for PTZ cameras) +//! - Depth camera tilt control (via freedepth) use crate::app::state::{AppModel, Message}; use crate::backends::camera::v4l2_controls; +use crate::backends::camera::{TILT_MAX_DEGREES, TILT_MIN_DEGREES}; use crate::constants::ui::OVERLAY_BACKGROUND_ALPHA; use crate::fl; use cosmic::Element; @@ -37,9 +39,9 @@ fn picker_panel_style(theme: &cosmic::Theme) -> widget::container::Style { } impl AppModel { - /// Check if any motor controls are available (V4L2 PTZ) + /// Check if any motor controls are available (V4L2 PTZ or Kinect) pub fn has_motor_controls(&self) -> bool { - self.available_exposure_controls.has_any_ptz() + self.kinect.is_device || self.available_exposure_controls.has_any_ptz() } /// Build the motor controls picker overlay @@ -63,6 +65,28 @@ impl AppModel { .align_y(cosmic::iced::Alignment::Center); column = column.push(title_row); + // Depth camera tilt control + if self.kinect.is_device { + let tilt_value = self.kinect.tilt_angle as i32; + let tilt_min = TILT_MIN_DEGREES as i32; + let tilt_max = TILT_MAX_DEGREES as i32; + let tilt_row = widget::row() + .push(widget::text::body(fl!("kinect-tilt")).width(Length::Fixed(60.0))) + .push( + widget::slider(tilt_min..=tilt_max, tilt_value, |val| { + Message::SetKinectTilt(val as i8) + }) + .width(Length::Fixed(140.0)), + ) + .push( + widget::text::body(format!("{}°", self.kinect.tilt_angle)) + .width(Length::Fixed(40.0)), + ) + .spacing(spacing.space_xs) + .align_y(cosmic::iced::Alignment::Center); + column = column.push(tilt_row); + } + // V4L2 Pan control if self.available_exposure_controls.pan_absolute.available { let range = &self.available_exposure_controls.pan_absolute; @@ -208,6 +232,8 @@ impl AppModel { } }); } + + // Note: Kinect tilt reset is handled by Message::ResetPanTilt handler in update.rs } } diff --git a/src/app/settings/view.rs b/src/app/settings/view.rs index 8fe9622b..391cdcd9 100644 --- a/src/app/settings/view.rs +++ b/src/app/settings/view.rs @@ -242,7 +242,13 @@ impl AppModel { info_column = info_column.push(info_row(fl!("device-info-card"), &info.card)); } if !info.driver.is_empty() { - info_column = info_column.push(info_row(fl!("device-info-driver"), &info.driver)); + // Show freedepth when native Kinect streaming is active + let driver_display = if self.kinect.streaming { + "freedepth (native USB)" + } else { + &info.driver + }; + info_column = info_column.push(info_row(fl!("device-info-driver"), driver_display)); } if !info.path.is_empty() { info_column = info_column.push(info_row(fl!("device-info-path"), &info.path)); @@ -251,6 +257,11 @@ impl AppModel { info_column = info_column.push(info_row(fl!("device-info-real-path"), &info.real_path)); } + // Show streaming status when native Kinect streaming is active + if self.kinect.streaming { + info_column = + info_column.push(info_row(fl!("device-info-status"), "RGB + Depth streaming")); + } } else { info_column = info_column.push(widget::text("No device information available").size(12)); diff --git a/src/app/state.rs b/src/app/state.rs index 001af32a..6effd2c9 100644 --- a/src/app/state.rs +++ b/src/app/state.rs @@ -496,6 +496,129 @@ impl Default for BurstModeState { } } +/// Kinect device state +/// +/// Consolidates all Kinect-specific state including device detection, +/// motor control, calibration, and native backend streaming. +#[derive(Default)] +pub struct KinectState { + /// Whether the current camera is a Kinect device + pub is_device: bool, + /// Current tilt angle (see freedepth::TILT_MIN_DEGREES/TILT_MAX_DEGREES) + pub tilt_angle: i8, + /// Path to the Kinect depth device (found when 3D mode enabled on RGB) + pub depth_device_path: Option, + /// Native depth camera backend for simultaneous RGB+depth streaming + pub native_backend: Option, + /// Whether native Kinect streaming is active + pub streaming: bool, + /// Current calibration info from depth device (for display) + /// Uses generic RegistrationSummary for device-agnostic access + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + pub calibration_info: Option, + #[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] + pub calibration_info: Option<()>, + /// Whether calibration dialog is visible + pub calibration_dialog_visible: bool, + /// Registration data for depth-to-RGB alignment (used by scene capture) + pub registration_data: Option, +} + +impl KinectState { + /// Check if native backend should be shut down + pub fn shutdown_backend(&mut self) { + if self.native_backend.is_some() { + tracing::info!("KinectState: shutting down native Kinect backend"); + self.native_backend = None; // Drop will handle LED cleanup + self.streaming = false; + } + } +} + +/// 3D preview state for depth camera visualization +/// +/// Consolidates all state related to 3D point cloud/mesh rendering +/// including rotation, zoom, and render caching. +#[derive(Clone)] +pub struct Preview3DState { + /// Whether to show 3D preview (for depth cameras) + pub enabled: bool, + /// Scene view mode (point cloud or mesh) + pub view_mode: SceneViewMode, + /// Current rotation angles (pitch, yaw in radians) + pub rotation: (f32, f32), + /// Base rotation when drag started (for path independence) + pub base_rotation: (f32, f32), + /// Whether the mouse is currently dragging to rotate + pub dragging: bool, + /// Mouse position when drag started + pub drag_start_pos: Option<(f32, f32)>, + /// Last mouse position during drag + pub last_mouse_pos: Option<(f32, f32)>, + /// Zoom level (1.0 = default, higher = closer) + pub zoom: f32, + /// Rendered point cloud RGBA data (width, height, data) + pub rendered_preview: Option<(u32, u32, Arc>)>, + /// Most recent depth data (width, height, depth_u16_data) + pub latest_depth_data: Option<(u32, u32, Arc<[u16]>)>, + /// Last video frame timestamp rendered in scene view + pub last_render_video_timestamp: Option, + /// Last time a scene render was requested (for throttling) + pub last_render_request_time: Option, +} + +impl Default for Preview3DState { + fn default() -> Self { + Self { + enabled: false, + view_mode: SceneViewMode::default(), + rotation: (0.0, 0.0), + base_rotation: (0.0, 0.0), + dragging: false, + drag_start_pos: None, + last_mouse_pos: None, + zoom: 1.0, + rendered_preview: None, + latest_depth_data: None, + last_render_video_timestamp: None, + last_render_request_time: None, + } + } +} + +impl Preview3DState { + /// Reset rotation to default view + pub fn reset_rotation(&mut self) { + self.rotation = (0.0, 0.0); + self.base_rotation = (0.0, 0.0); + } + + /// Start drag operation + pub fn start_drag(&mut self, x: f32, y: f32) { + self.dragging = true; + self.drag_start_pos = Some((x, y)); + self.base_rotation = self.rotation; + } + + /// End drag operation + pub fn end_drag(&mut self) { + self.dragging = false; + self.drag_start_pos = None; + self.last_mouse_pos = None; + } +} + +/// Depth visualization settings +/// +/// Controls how depth data is displayed in the camera preview. +#[derive(Debug, Clone, Default)] +pub struct DepthVisualizationState { + /// Whether to show depth overlay on camera preview + pub overlay_enabled: bool, + /// Whether to use grayscale instead of colormap + pub grayscale_mode: bool, +} + /// The application model stores app-specific state used to describe its interface and /// drive its logic. pub struct AppModel { @@ -649,6 +772,25 @@ pub struct AppModel { // ===== Privacy Cover Detection ===== /// Whether the camera privacy cover is closed (blocking the camera) pub privacy_cover_closed: bool, + + // ===== Kinect State ===== + /// Kinect device state (detection, motor, calibration, native streaming) + pub kinect: KinectState, + + // ===== Depth Visualization ===== + /// Depth visualization settings (overlay, grayscale mode) + pub depth_viz: DepthVisualizationState, + + // ===== 3D Preview ===== + /// 3D preview state (rotation, zoom, rendering) + pub preview_3d: Preview3DState, +} + +impl Drop for AppModel { + fn drop(&mut self) { + // Ensure Kinect native backend is shut down (LED will be turned off) + self.kinect.shutdown_backend(); + } } /// State for smooth blur transitions when changing camera settings @@ -671,6 +813,18 @@ pub enum CameraMode { Video, /// Virtual camera mode - streams filtered video to a PipeWire virtual camera Virtual, + /// Scene mode - 3D point cloud view for depth cameras (Kinect, RealSense, etc.) + Scene, +} + +/// Scene mode view type (point cloud vs mesh) +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SceneViewMode { + /// Point cloud - individual points rendered + PointCloud, + /// Mesh - triangulated surface with RGB texture (default) + #[default] + Mesh, } /// File source for virtual camera streaming @@ -1038,6 +1192,16 @@ pub enum Message { /// Reset pan/tilt to center position ResetPanTilt, + // ===== Depth Camera Controls ===== + /// Set depth camera tilt angle (see freedepth::TILT_MIN_DEGREES/TILT_MAX_DEGREES) + SetKinectTilt(i8), + /// Kinect state updated (tilt) - response from async operation + KinectStateUpdated(i8), + /// Kinect control operation failed + KinectControlFailed(String), + /// Kinect controller initialized (called after lazy init when motor picker opens) + KinectInitialized(Result), + // ===== Format Selection ===== /// Switch between Photo/Video mode SetMode(CameraMode), @@ -1097,6 +1261,8 @@ pub enum Message { ResetZoom, /// Photo was saved successfully with the given file path PhotoSaved(Result), + /// Scene was captured successfully with the scene directory path + SceneSaved(Result), /// Clear capture animation after brief delay ClearCaptureAnimation, /// Toggle video recording @@ -1212,6 +1378,54 @@ pub enum Message { /// Privacy cover status changed (true = cover closed/camera blocked) PrivacyCoverStatusChanged(bool), + // ===== Depth Visualization ===== + /// Toggle depth overlay visibility + ToggleDepthOverlay, + /// Toggle grayscale depth mode (grayscale instead of colormap) + ToggleDepthGrayscale, + + // ===== Calibration ===== + /// Show calibration status dialog (explains current calibration state) + ShowCalibrationDialog, + /// Close calibration dialog + CloseCalibrationDialog, + /// Start calibration procedure + StartCalibration, + + // ===== 3D Preview ===== + /// Toggle 3D point cloud preview (for depth cameras) + Toggle3DPreview, + /// Toggle scene view mode (point cloud vs mesh) + ToggleSceneViewMode, + /// 3D preview mouse pressed (start dragging) + Preview3DMousePressed(f32, f32), + /// 3D preview mouse moved (update rotation while dragging) + Preview3DMouseMoved(f32, f32), + /// 3D preview mouse released (stop dragging) + Preview3DMouseReleased, + /// Reset 3D preview rotation to default view + Reset3DPreviewRotation, + /// Zoom 3D preview (delta: positive = zoom in, negative = zoom out) + Zoom3DPreview(f32), + /// Point cloud preview rendered (width, height, rgba_data) + PointCloudRendered(u32, u32, Arc>), + /// Request point cloud render (triggered on rotation change while 3D mode active) + RequestPointCloudRender, + /// Secondary depth frame received (for 3D preview on RGB camera) + SecondaryDepthFrame(u32, u32, Arc<[u16]>), + + // ===== Native Kinect Streaming ===== + /// Start native Kinect streaming (bypasses V4L2 for simultaneous RGB+depth) + StartNativeKinectStreaming, + /// Stop native Kinect streaming + StopNativeKinectStreaming, + /// Native Kinect streaming started successfully + NativeKinectStreamingStarted, + /// Native Kinect streaming failed to start + NativeKinectStreamingFailed(String), + /// Poll native Kinect backend for new frames + PollNativeKinectFrames, + /// No-op message for async tasks that don't need a response Noop, } diff --git a/src/app/ui.rs b/src/app/ui.rs index 1a29872e..31a8d1cf 100644 --- a/src/app/ui.rs +++ b/src/app/ui.rs @@ -5,15 +5,15 @@ //! This module contains shared UI utilities that are used by multiple view modules. use crate::app::state::AppModel; -use crate::backends::camera::types::CameraFormat; +use crate::backends::camera::types::{CameraFormat, SensorType}; use crate::constants; use std::collections::HashMap; impl AppModel { - /// Group formats by resolution label and return sorted list with best resolution for each label + /// Group RGB formats by resolution label and return sorted list with best resolution for each label /// /// This helper is used by the format picker to organize formats by resolution categories - /// (SD, HD, 720p, 4K, etc.). + /// (SD, HD, 720p, 4K, etc.). Only includes RGB camera formats, not depth sensor formats. /// /// Returns: /// - A sorted list of (label, width) pairs representing unique resolution categories @@ -25,12 +25,18 @@ impl AppModel { HashMap>, ) { // Group formats by their label (SD, HD, 4K, etc.) and pick the highest resolution for each + // Only include RGB formats, not depth sensor formats let mut label_to_best_format: HashMap< &'static str, (u32, u32, Vec<(usize, &CameraFormat)>), > = HashMap::new(); for (idx, fmt) in self.available_formats.iter().enumerate() { + // Skip depth sensor formats - they are shown separately + if fmt.sensor_type == SensorType::Depth { + continue; + } + if let Some(label) = constants::get_resolution_label(fmt.width) { let resolution_score = fmt.width * fmt.height; diff --git a/src/app/update.rs b/src/app/update.rs index f305f498..1fb6e9d7 100644 --- a/src/app/update.rs +++ b/src/app/update.rs @@ -50,6 +50,24 @@ impl AppModel { self.exposure_picker_visible = false; self.color_picker_visible = false; self.tools_menu_visible = false; + + // Get initial tilt from motor control if available + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + if self.kinect.is_device { + use crate::backends::camera::motor_control::{ + get_motor_tilt, is_motor_available, + }; + if is_motor_available() { + match get_motor_tilt() { + Ok(tilt) => { + self.kinect.tilt_angle = tilt; + } + Err(e) => { + tracing::warn!("Failed to get Kinect tilt: {}", e); + } + } + } + } } Task::none() } @@ -71,6 +89,10 @@ impl AppModel { } Message::ResetPanTilt => { self.reset_pan_tilt(); + // Also reset Kinect tilt if it's a Kinect device + if self.kinect.is_device { + return self.handle_set_kinect_tilt(0); + } Task::none() } @@ -179,6 +201,7 @@ impl AppModel { Message::ZoomOut => self.handle_zoom_out(), Message::ResetZoom => self.handle_reset_zoom(), Message::PhotoSaved(result) => self.handle_photo_saved(result), + Message::SceneSaved(result) => self.handle_scene_saved(result), Message::ClearCaptureAnimation => self.handle_clear_capture_animation(), Message::ToggleRecording => self.handle_toggle_recording(), Message::RecordingStarted(path) => self.handle_recording_started(path), @@ -265,6 +288,79 @@ impl AppModel { self.handle_privacy_cover_status_changed(is_closed) } + // ===== Depth Visualization ===== + Message::ToggleDepthOverlay => self.handle_toggle_depth_overlay(), + Message::ToggleDepthGrayscale => self.handle_toggle_depth_grayscale(), + + // ===== Calibration ===== + Message::ShowCalibrationDialog => self.handle_show_calibration_dialog(), + Message::CloseCalibrationDialog => self.handle_close_calibration_dialog(), + Message::StartCalibration => self.handle_start_calibration(), + + // ===== 3D Preview ===== + Message::Toggle3DPreview => self.handle_toggle_3d_preview(), + Message::ToggleSceneViewMode => self.handle_toggle_scene_view_mode(), + Message::Preview3DMousePressed(x, y) => self.handle_preview_3d_mouse_pressed(x, y), + Message::Preview3DMouseMoved(x, y) => self.handle_preview_3d_mouse_moved(x, y), + Message::Preview3DMouseReleased => self.handle_preview_3d_mouse_released(), + Message::Reset3DPreviewRotation => self.handle_reset_3d_preview_rotation(), + Message::Zoom3DPreview(delta) => self.handle_zoom_3d_preview(delta), + Message::PointCloudRendered(width, height, data) => { + self.preview_3d.rendered_preview = Some((width, height, data)); + Task::none() + } + Message::SecondaryDepthFrame(width, height, depth_data) => { + // Store depth data (unused - secondary capture was never implemented) + self.preview_3d.latest_depth_data = Some((width, height, depth_data)); + info!(width, height, "Received depth frame"); + self.handle_request_point_cloud_render() + } + Message::RequestPointCloudRender => self.handle_request_point_cloud_render(), + + // ===== Kinect Controls ===== + Message::SetKinectTilt(degrees) => self.handle_set_kinect_tilt(degrees), + Message::KinectStateUpdated(_tilt) => { + // Tilt is managed as "desired state" in UI to avoid flickering from motor feedback + // LED is automatically managed by freedepth + Task::none() + } + Message::KinectControlFailed(error) => { + tracing::warn!(error = %error, "Kinect control failed"); + Task::none() + } + Message::KinectInitialized(result) => { + match result { + Ok(tilt) => { + tracing::info!(tilt, "Kinect controller initialized"); + self.kinect.tilt_angle = tilt; + } + Err(error) => { + tracing::warn!(error = %error, "Failed to initialize Kinect controller"); + } + } + Task::none() + } + + // ===== Native Kinect Streaming ===== + Message::StartNativeKinectStreaming => self.handle_start_native_kinect_streaming(), + Message::StopNativeKinectStreaming => self.handle_stop_native_kinect_streaming(), + + Message::NativeKinectStreamingStarted => { + info!("Native Kinect streaming started"); + self.kinect.streaming = true; + // Start polling for frames + Task::done(cosmic::Action::App(Message::PollNativeKinectFrames)) + } + + Message::NativeKinectStreamingFailed(error) => { + warn!(error = %error, "Native Kinect streaming failed"); + self.kinect.streaming = false; + self.preview_3d.enabled = false; + Task::none() + } + + Message::PollNativeKinectFrames => self.handle_poll_native_kinect_frames(), + Message::Noop => Task::none(), } } diff --git a/src/app/view.rs b/src/app/view.rs index 33ad5fa7..0ff18eae 100644 --- a/src/app/view.rs +++ b/src/app/view.rs @@ -15,6 +15,12 @@ use crate::app::video_widget::VideoContentFit; use crate::constants::resolution_thresholds; use crate::constants::ui::{self, OVERLAY_BACKGROUND_ALPHA}; use crate::fl; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use crate::shaders::depth::{DEPTH_MAX_MM_U16, DEPTH_MIN_MM_U16}; +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +const DEPTH_MAX_MM_U16: u16 = 10000; +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +const DEPTH_MIN_MM_U16: u16 = 400; use cosmic::Element; use cosmic::iced::{Alignment, Background, Color, Length}; use cosmic::widget::{self, icon}; @@ -47,8 +53,16 @@ const TOOLS_GRID_ICON: &[u8] = include_bytes!("../../resources/button_icons/tool const MOON_ICON: &[u8] = include_bytes!("../../resources/button_icons/moon.svg"); /// Moon off icon SVG (burst mode disabled, with strike-through) const MOON_OFF_ICON: &[u8] = include_bytes!("../../resources/button_icons/moon-off.svg"); +/// Depth visualization icon SVG (for depth cameras like Kinect) +const DEPTH_ICON: &[u8] = include_bytes!("../../resources/button_icons/depth-waves.svg"); +/// Depth visualization off icon SVG +const DEPTH_OFF_ICON: &[u8] = include_bytes!("../../resources/button_icons/depth-waves-off.svg"); /// Camera tilt/motor control icon SVG const CAMERA_TILT_ICON: &[u8] = include_bytes!("../../resources/button_icons/camera-tilt.svg"); +/// Mesh view icon SVG (triangulated surface) +const MESH_ICON: &[u8] = include_bytes!("../../resources/button_icons/mesh.svg"); +/// Points view icon SVG (point cloud) +const POINTS_ICON: &[u8] = include_bytes!("../../resources/button_icons/points.svg"); /// Burst mode progress bar dimensions const BURST_MODE_PROGRESS_BAR_WIDTH: f32 = 200.0; @@ -200,8 +214,8 @@ impl AppModel { camera_preview }; - // Check if zoom label should be shown (only in Photo mode) - let show_zoom_label = self.mode == CameraMode::Photo; + // Check if zoom label should be shown (Photo mode or 3D preview mode) + let show_zoom_label = self.mode == CameraMode::Photo || self.preview_3d.enabled; // Capture button area - changes based on recording/streaming state and video file selection // Check if we have video file controls (play/pause button for video file sources) @@ -321,6 +335,8 @@ impl AppModel { camera_preview, // QR overlay (custom widget calculates positions at render time) self.build_qr_overlay(), + // Depth legend overlay (bottom right, when depth overlay enabled) + self.build_depth_legend(), // Privacy cover warning overlay (centered) self.build_privacy_warning(), // Top bar aligned to top (no extra padding - row has its own padding) @@ -343,6 +359,7 @@ impl AppModel { cosmic::iced::widget::stack![ camera_preview, self.build_qr_overlay(), + self.build_depth_legend(), self.build_privacy_warning() ] .width(Length::Fill) @@ -356,6 +373,8 @@ impl AppModel { camera_preview, // QR overlay (custom widget calculates positions at render time) self.build_qr_overlay(), + // Depth legend overlay (bottom right, when depth overlay enabled) + self.build_depth_legend(), // Privacy cover warning overlay (centered) self.build_privacy_warning(), widget::container(top_bar) @@ -421,6 +440,11 @@ impl AppModel { main_stack = main_stack.push(self.build_tools_menu()); } + // Add calibration dialog overlay if visible + if self.kinect.calibration_dialog_visible { + main_stack = main_stack.push(self.build_calibration_dialog()); + } + // Wrap everything in a themed background container widget::container(main_stack) .width(Length::Fill) @@ -598,6 +622,70 @@ impl AppModel { row = row.push(widget::Space::new(Length::Fixed(5.0), Length::Shrink)); } + // Depth visualization toggle (shows when camera has depth data) + let has_depth = self + .current_frame + .as_ref() + .map(|f| f.depth_data.is_some()) + .unwrap_or(false); + if has_depth { + let depth_icon_bytes = if self.depth_viz.overlay_enabled { + DEPTH_ICON + } else { + DEPTH_OFF_ICON + }; + let depth_icon = widget::icon::from_svg_bytes(depth_icon_bytes).symbolic(true); + + if is_disabled { + row = row.push( + widget::container(widget::icon(depth_icon).size(20)) + .style(|_theme| widget::container::Style { + text_color: Some(Color::from_rgba(1.0, 1.0, 1.0, 0.3)), + ..Default::default() + }) + .padding([4, 8]), + ); + } else { + row = row.push(overlay_icon_button( + depth_icon, + Some(Message::ToggleDepthOverlay), + self.depth_viz.overlay_enabled, + )); + } + + // 5px spacing + row = row.push(widget::Space::new(Length::Fixed(5.0), Length::Shrink)); + } + + // Scene view mode toggle (only in Scene mode) + // Switches between point cloud and mesh rendering + if self.mode == CameraMode::Scene && self.preview_3d.enabled { + use crate::app::state::SceneViewMode; + let is_mesh = self.preview_3d.view_mode == SceneViewMode::Mesh; + let view_icon_bytes = if is_mesh { MESH_ICON } else { POINTS_ICON }; + let view_icon = widget::icon::from_svg_bytes(view_icon_bytes).symbolic(true); + + if is_disabled { + row = row.push( + widget::container(widget::icon(view_icon).size(20)) + .style(|_theme| widget::container::Style { + text_color: Some(Color::from_rgba(1.0, 1.0, 1.0, 0.3)), + ..Default::default() + }) + .padding([4, 8]), + ); + } else { + row = row.push(overlay_icon_button( + view_icon, + Some(Message::ToggleSceneViewMode), + false, // Never highlight - toggle state shown by icon change + )); + } + } + + // 5px spacing + row = row.push(widget::Space::new(Length::Fixed(5.0), Length::Shrink)); + // Tools menu button (opens overlay with timer, aspect ratio, exposure, filter, theatre) // Highlight when tools menu is open or any tool setting is non-default let tools_active = self.tools_menu_visible || self.has_non_default_tool_settings(); @@ -728,28 +816,82 @@ impl AppModel { /// Build zoom level button for display above capture button /// - /// Shows current zoom level (1x, 1.3x, 2x, etc.) in Photo mode. - /// Click to reset zoom to 1.0. + /// Shows current zoom level (1x, 1.3x, 2x, etc.) in Photo mode or 3D preview mode. + /// In 3D mode, also shows a rotation reset button. + /// Click zoom to reset to 1.0. fn build_zoom_label(&self) -> Element<'_, Message> { - let zoom_text = if self.zoom_level >= 10.0 { - "10x".to_string() - } else if (self.zoom_level - self.zoom_level.round()).abs() < 0.05 { - format!("{}x", self.zoom_level.round() as u32) - } else { - format!("{:.1}x", self.zoom_level) - }; + let spacing = cosmic::theme::spacing(); - let is_zoomed = (self.zoom_level - 1.0).abs() > 0.01; + if self.preview_3d.enabled { + // 3D preview mode: "fly into scene" zoom model + // preview_3d_zoom = 0 means camera at sensor (1x view) + // Positive values = camera moved into scene = zoomed in + // Negative values = camera moved back = zoomed out + let display_zoom = if self.preview_3d.zoom >= 0.0 { + 1.0 + self.preview_3d.zoom // 0=1x, 1=2x, 2=3x + } else { + 1.0 / (1.0 - self.preview_3d.zoom) // -1=0.5x, -2=0.33x + }; + let zoom_text = if display_zoom >= 4.0 { + format!("{:.0}x", display_zoom) + } else if display_zoom < 1.0 { + format!("{:.1}x", display_zoom) + } else if (display_zoom - display_zoom.round()).abs() < 0.05 { + format!("{}x", display_zoom.round() as u32) + } else { + format!("{:.1}x", display_zoom) + }; - // Use text button style - Suggested when zoomed, Standard when at 1x - widget::button::text(zoom_text) - .on_press(Message::ResetZoom) - .class(if is_zoomed { - cosmic::theme::Button::Suggested + let is_zoomed = self.preview_3d.zoom.abs() > 0.01; + let (pitch, yaw) = self.preview_3d.rotation; + let is_rotated = pitch.abs() > 0.01 || yaw.abs() > 0.01; + + // Zoom reset button + let zoom_button = widget::button::text(zoom_text) + .on_press(Message::Reset3DPreviewRotation) + .class(if is_zoomed { + cosmic::theme::Button::Suggested + } else { + cosmic::theme::Button::Standard + }); + + // Rotation reset button (shows when rotated) + let reset_button = widget::button::text("⟲") + .on_press(Message::Reset3DPreviewRotation) + .class(if is_rotated { + cosmic::theme::Button::Suggested + } else { + cosmic::theme::Button::Standard + }); + + widget::row() + .push(reset_button) + .push(zoom_button) + .spacing(spacing.space_xxs) + .align_y(Alignment::Center) + .into() + } else { + // Normal Photo mode zoom + let zoom_text = if self.zoom_level >= 10.0 { + "10x".to_string() + } else if (self.zoom_level - self.zoom_level.round()).abs() < 0.05 { + format!("{}x", self.zoom_level.round() as u32) } else { - cosmic::theme::Button::Standard - }) - .into() + format!("{:.1}x", self.zoom_level) + }; + + let is_zoomed = (self.zoom_level - 1.0).abs() > 0.01; + + // Use text button style - Suggested when zoomed, Standard when at 1x + widget::button::text(zoom_text) + .on_press(Message::ResetZoom) + .class(if is_zoomed { + cosmic::theme::Button::Suggested + } else { + cosmic::theme::Button::Standard + }) + .into() + } } /// Build the QR code overlay layer @@ -1281,4 +1423,294 @@ impl AppModel { .align_y(cosmic::iced::alignment::Vertical::Center) .into() } + + /// Build the depth legend overlay + /// + /// Shows a horizontal gradient bar with depth values in mm. + /// Uses the turbo colormap (blue=near to red=far). + /// Positioned in bottom right, limited to capture button height. + fn build_depth_legend(&self) -> Element<'_, Message> { + // Only show when depth overlay is enabled and we have depth data + if !self.depth_viz.overlay_enabled { + return widget::Space::new(Length::Fill, Length::Fill).into(); + } + + let has_depth = self + .current_frame + .as_ref() + .map(|f| f.depth_data.is_some()) + .unwrap_or(false); + + if !has_depth { + return widget::Space::new(Length::Fill, Length::Fill).into(); + } + + let spacing = cosmic::theme::spacing(); + + // Kinect depth range from shared constants + let min_depth_mm = DEPTH_MIN_MM_U16; + let max_depth_mm = DEPTH_MAX_MM_U16; + + // Build the legend content: gradient bar with labels + // Height limited to capture button size (60px) + let legend_height = ui::CAPTURE_BUTTON_OUTER; + let bar_height = 12.0_f32; + let label_size = 10_u16; + + // Create gradient bar using a row of colored segments + let num_segments = 40; + let segment_width = 4.0_f32; + let mut gradient_row = widget::row().spacing(0); + + // Turbo colormap gradient stops (used when not in grayscale mode) + let turbo_colors: [(f32, Color); 7] = [ + (0.0, Color::from_rgb(0.18995, 0.07176, 0.23217)), // Dark blue/purple + (0.17, Color::from_rgb(0.12178, 0.38550, 0.90354)), // Blue + (0.33, Color::from_rgb(0.09859, 0.70942, 0.66632)), // Cyan + (0.5, Color::from_rgb(0.50000, 0.85810, 0.27671)), // Green/yellow + (0.67, Color::from_rgb(0.91567, 0.85024, 0.09695)), // Yellow + (0.83, Color::from_rgb(0.99214, 0.50000, 0.07763)), // Orange + (1.0, Color::from_rgb(0.72340, 0.10000, 0.08125)), // Red + ]; + + let use_grayscale = self.depth_viz.grayscale_mode; + + for i in 0..num_segments { + let t = i as f32 / (num_segments - 1) as f32; + + // Use grayscale or turbo colormap based on mode + // Grayscale: near (t=0) = bright, far (t=1) = dark (matches shader) + let color = if use_grayscale { + let gray = 1.0 - t; // Invert: near=bright, far=dark + Color::from_rgb(gray, gray, gray) + } else { + Self::interpolate_turbo(t, &turbo_colors) + }; + + gradient_row = gradient_row.push( + widget::container(widget::Space::new( + Length::Fixed(segment_width), + Length::Fixed(bar_height), + )) + .style(move |_| widget::container::Style { + background: Some(Background::Color(color)), + ..Default::default() + }), + ); + } + + // Labels row: Near (blue) on left, Far (red) on right + // Matches turbo colormap: t=0 (left) = blue = near, t=1 (right) = red = far + let labels_row = widget::row() + .push(widget::text::caption(format!("{}mm", min_depth_mm)).size(label_size)) + .push(widget::Space::new(Length::Fill, Length::Shrink)) + .push(widget::text::caption(format!("{}mm", max_depth_mm)).size(label_size)) + .width(Length::Fixed(segment_width * num_segments as f32)); + + // Toggle button for grayscale mode + let grayscale_toggle = widget::row() + .push( + widget::checkbox("Grayscale", self.depth_viz.grayscale_mode) + .on_toggle(|_| Message::ToggleDepthGrayscale) + .size(14) + .text_size(label_size), + ) + .width(Length::Fixed(segment_width * num_segments as f32)); + + // Combine into a column: labels on top, gradient bar, then grayscale toggle + let legend_content = widget::column() + .push(labels_row) + .push(gradient_row) + .push(grayscale_toggle) + .spacing(2) + .align_x(cosmic::iced::Alignment::Center); + + // Semi-transparent container for the legend + let legend_container = widget::container(legend_content) + .padding([spacing.space_xxs, spacing.space_xs]) + .style(|theme: &cosmic::Theme| { + let cosmic = theme.cosmic(); + let bg = cosmic.bg_color(); + widget::container::Style { + background: Some(Background::Color(Color::from_rgba( + bg.red, + bg.green, + bg.blue, + OVERLAY_BACKGROUND_ALPHA, + ))), + border: cosmic::iced::Border { + radius: cosmic.corner_radii.radius_s.into(), + ..Default::default() + }, + ..Default::default() + } + }); + + // Position in bottom right corner + // Bottom padding: space_s (bottom bar margin) + capture button height + xxs gap + let bottom_padding = spacing.space_s as f32 + legend_height + spacing.space_xxs as f32; + + widget::container( + widget::row() + .push(widget::Space::new(Length::Fill, Length::Shrink)) + .push(legend_container), + ) + .width(Length::Fill) + .height(Length::Fill) + .align_y(cosmic::iced::alignment::Vertical::Bottom) + .padding([0.0, spacing.space_s as f32, bottom_padding, 0.0]) + .into() + } + + /// Build the calibration dialog overlay + /// + /// Shows calibration status and prompts user to calibrate if using defaults. + fn build_calibration_dialog(&self) -> Element<'_, Message> { + let spacing = cosmic::theme::spacing(); + + // Get calibration info + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + let (status_text, detail_text, has_device_calibration) = + if let Some(ref calib) = self.kinect.calibration_info { + if calib.from_device { + ( + "Device Calibration Active", + format!( + "Factory calibration loaded from device EEPROM.\n\n\ + Reference distance: {:.0}mm\n\ + IR-RGB baseline: {:.1}mm", + calib.reference_distance_mm, calib.stereo_baseline_mm + ), + true, + ) + } else { + ( + "Using Default Calibration", + "Factory calibration could not be read from the device.\n\ + Depth-to-color alignment may be inaccurate.\n\n\ + This can happen when:\n\ + • USB permission issues (try running as root)\n\ + • Device not fully initialized\n\ + • Using V4L2 mode instead of native Kinect\n\n\ + Try restarting the camera or switching to native mode." + .to_string(), + false, + ) + } + } else { + ( + "No Calibration Data", + "No depth camera is currently active.".to_string(), + false, + ) + }; + #[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] + let (status_text, detail_text, has_device_calibration) = ( + "No Calibration Data", + "freedepth feature not enabled.".to_string(), + false, + ); + + // Icon - check mark for device calibration, warning for default + let status_icon = if has_device_calibration { + icon::from_name("emblem-ok-symbolic").symbolic(true) + } else { + icon::from_name("dialog-warning-symbolic").symbolic(true) + }; + + // Build dialog content + let content = widget::column() + .push( + widget::icon(status_icon.into()) + .size(48) + .class(if has_device_calibration { + cosmic::theme::Svg::Default + } else { + cosmic::theme::Svg::Custom(std::rc::Rc::new(|theme: &cosmic::Theme| { + cosmic::widget::svg::Style { + color: Some(theme.cosmic().destructive_color().into()), + } + })) + }), + ) + .push( + widget::text(status_text) + .size(20) + .font(cosmic::font::bold()), + ) + .push(widget::text(detail_text).size(14)) + .push(widget::Space::new(Length::Shrink, Length::Fixed(16.0))) + .push( + widget::button::text("Close") + .on_press(Message::CloseCalibrationDialog) + .class(cosmic::theme::Button::Standard), + ) + .spacing(spacing.space_s) + .align_x(Alignment::Center); + + // Dialog container with semi-transparent background + let dialog_box = + widget::container(content) + .padding(spacing.space_m) + .style(|theme: &cosmic::Theme| { + let cosmic = theme.cosmic(); + let bg = cosmic.bg_color(); + widget::container::Style { + background: Some(Background::Color(Color::from_rgba( + bg.red, bg.green, bg.blue, 0.95, + ))), + border: cosmic::iced::Border { + radius: cosmic.corner_radii.radius_m.into(), + ..Default::default() + }, + ..Default::default() + } + }); + + // Dark overlay behind dialog to dim the background + let overlay = widget::mouse_area( + widget::container(dialog_box) + .width(Length::Fill) + .height(Length::Fill) + .align_x(cosmic::iced::alignment::Horizontal::Center) + .align_y(cosmic::iced::alignment::Vertical::Center), + ) + .on_press(Message::CloseCalibrationDialog); + + widget::container(overlay) + .width(Length::Fill) + .height(Length::Fill) + .style(|_theme| widget::container::Style { + background: Some(Background::Color(Color::from_rgba(0.0, 0.0, 0.0, 0.5))), + ..Default::default() + }) + .into() + } + + /// Interpolate turbo colormap from pre-defined color stops + fn interpolate_turbo(t: f32, colors: &[(f32, Color); 7]) -> Color { + let t = t.clamp(0.0, 1.0); + + // Find the two color stops to interpolate between + let mut i = 0; + while i < colors.len() - 1 && colors[i + 1].0 < t { + i += 1; + } + + if i >= colors.len() - 1 { + return colors[colors.len() - 1].1; + } + + let (t0, c0) = colors[i]; + let (t1, c1) = colors[i + 1]; + + // Linear interpolation between the two stops + let factor = if t1 > t0 { (t - t0) / (t1 - t0) } else { 0.0 }; + + Color::from_rgb( + c0.r + (c1.r - c0.r) * factor, + c0.g + (c1.g - c0.g) * factor, + c0.b + (c1.b - c0.b) * factor, + ) + } } diff --git a/src/backends/camera/depth_controller.rs b/src/backends/camera/depth_controller.rs new file mode 100644 index 00000000..37d3e4f4 --- /dev/null +++ b/src/backends/camera/depth_controller.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(all(target_arch = "x86_64", feature = "freedepth"))] + +//! Depth camera device controller +//! +//! Manages freedepth integration for depth cameras, providing: +//! - Device detection (by V4L2 driver name) +//! - Depth-to-mm conversion (for V4L2 mode) +//! +//! Note: Motor control is handled by motor_control.rs + +use tracing::info; + +use super::types::CameraDevice; + +// ============================================================================= +// Device Detection +// ============================================================================= + +/// Check if a camera device is a depth camera supported by freedepth +/// +/// Detection is based on: +/// - Device path prefix (freedepth-enumerated devices) +/// - V4L2 driver name (kernel drivers) +/// - Device name patterns +pub fn is_depth_camera(device: &CameraDevice) -> bool { + // Check for freedepth-enumerated device (path starts with depth path prefix) + if device + .path + .starts_with(super::depth_native::DEPTH_PATH_PREFIX) + { + return true; + } + + if let Some(ref info) = device.device_info { + // Check for known depth camera kernel drivers + // "gspca_kinect" and "kinect" are the kernel drivers for Xbox 360 Kinect + // "freedepth" is used when enumerated via freedepth + if info.driver == "gspca_kinect" || info.driver == "kinect" || info.driver == "freedepth" { + return true; + } + } + // Also check by name as fallback for depth cameras + let name_lower = device.name.to_lowercase(); + name_lower.contains("kinect") || name_lower.contains("xbox nui") +} + +// ============================================================================= +// Depth Conversion for V4L2/PipeWire mode +// ============================================================================= + +/// Global depth converter using default calibration (for V4L2 mode) +/// +/// Note: In V4L2 mode, we cannot access the USB device to fetch device-specific +/// calibration because the kernel driver holds the device. Using default +/// calibration values provides reasonable accuracy for most Kinect sensors. +static V4L2_DEPTH_CONVERTER: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn get_v4l2_depth_converter() -> &'static freedepth::DepthToMm { + V4L2_DEPTH_CONVERTER.get_or_init(|| freedepth::DepthToMm::with_defaults()) +} + +/// Depth camera controller for V4L2/PipeWire mode +/// +/// Uses default calibration values since we cannot access the USB device +/// while the kernel driver is active. +pub struct DepthController; + +impl DepthController { + /// Initialize is a no-op now - depth conversion uses default calibration + pub fn initialize() -> Result<(), String> { + // Pre-initialize the converter + let _ = get_v4l2_depth_converter(); + info!("Depth converter initialized with default calibration"); + Ok(()) + } + + /// Shutdown is a no-op + pub fn shutdown() { + // Nothing to do + } + + /// Always returns true since we use default calibration + pub fn is_initialized() -> bool { + true + } + + /// Convert raw 11-bit depth values to millimeters + /// + /// Takes raw depth values and converts them to millimeters + /// using default calibration data. + pub fn convert_depth_to_mm(raw_depth: &[u16]) -> Option> { + let converter = get_v4l2_depth_converter(); + + Some( + raw_depth + .iter() + .map(|&raw| { + // If the value was left-shifted by 6 (from GPU), shift it back + // Values from GPU processor are: 10-bit << 6 + let raw_10bit = raw >> 6; + // Convert to mm (calibration expects raw 11-bit values 0-2047) + // For 10-bit input, scale up to 11-bit range + let raw_11bit = (raw_10bit as u32 * 2) as u16; + converter.convert(raw_11bit.min(2047)) + }) + .collect(), + ) + } + + /// Convert a single raw depth value to millimeters + pub fn convert_single_depth(raw_10bit: u16) -> Option { + let converter = get_v4l2_depth_converter(); + + // Scale 10-bit to 11-bit and convert + let raw_11bit = ((raw_10bit as u32) * 2).min(2047) as u16; + Some(converter.convert(raw_11bit)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_depth_camera() { + use super::super::types::DeviceInfo; + + // Test with gspca_kinect driver (depth camera kernel driver) + let depth_device = CameraDevice { + name: "Some Camera".to_string(), + path: "pipewire-123".to_string(), + metadata_path: None, + device_info: Some(DeviceInfo { + card: "Depth Camera".to_string(), + driver: "gspca_kinect".to_string(), + path: "/dev/video0".to_string(), + real_path: "/dev/video0".to_string(), + }), + }; + assert!(is_depth_camera(&depth_device)); + + // Test with different driver (not a depth camera) + let other_device = CameraDevice { + name: "Regular Webcam".to_string(), + path: "pipewire-456".to_string(), + metadata_path: None, + device_info: Some(DeviceInfo { + card: "Webcam".to_string(), + driver: "uvcvideo".to_string(), + path: "/dev/video2".to_string(), + real_path: "/dev/video2".to_string(), + }), + }; + assert!(!is_depth_camera(&other_device)); + + // Test with name-based detection + let depth_by_name = CameraDevice { + name: "Xbox NUI Kinect Camera".to_string(), + path: "pipewire-789".to_string(), + metadata_path: None, + device_info: None, + }; + assert!(is_depth_camera(&depth_by_name)); + + // Test with Xbox NUI name + let xbox_nui = CameraDevice { + name: "Xbox NUI Camera".to_string(), + path: "pipewire-101".to_string(), + metadata_path: None, + device_info: None, + }; + assert!(is_depth_camera(&xbox_nui)); + + // Test with kinect driver name + let kinect_driver = CameraDevice { + name: "Some Camera".to_string(), + path: "pipewire-102".to_string(), + metadata_path: None, + device_info: Some(DeviceInfo { + card: "Camera".to_string(), + driver: "kinect".to_string(), + path: "/dev/video11".to_string(), + real_path: "/dev/video11".to_string(), + }), + }; + assert!(is_depth_camera(&kinect_driver)); + } +} diff --git a/src/backends/camera/depth_native.rs b/src/backends/camera/depth_native.rs new file mode 100644 index 00000000..e28ad873 --- /dev/null +++ b/src/backends/camera/depth_native.rs @@ -0,0 +1,870 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(all(target_arch = "x86_64", feature = "freedepth"))] + +//! Native depth camera backend for simultaneous RGB and depth streaming +//! +//! This backend uses freedepth's KinectStreamer to bypass V4L2 and +//! stream both RGB video and depth data simultaneously. +//! +//! # Key Features +//! +//! - Direct USB isochronous streaming via nusb (not V4L2) +//! - Simultaneous RGB + depth capture at 30fps +//! - Automatic kernel driver unbind/rebind +//! - Full calibration support for accurate depth values +//! +//! # Architecture +//! +//! Depth camera devices are identified by path prefix followed by the +//! freedepth device index. When a depth camera is selected, this backend +//! is used instead of PipeWire/GStreamer. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use std::sync::mpsc::Receiver; + +use freedepth::{ + DepthFormat, DepthFrame, DepthRegistration, KinectStreamer, Resolution, VideoFormat, VideoFrame, +}; +use tracing::{debug, info, warn}; + +use super::CameraBackend; +use super::format_converters::{ + self, DepthVisualizationOptions, ir_8bit_to_rgb, ir_10bit_to_rgb, ir_10bit_unpacked_to_rgb, +}; +use super::types::*; + +/// Path prefix for depth camera devices to distinguish from PipeWire cameras +pub const DEPTH_PATH_PREFIX: &str = "kinect:"; + +/// Enumerate depth cameras via freedepth +/// +/// Returns camera devices for each connected depth sensor. +/// These devices use the native USB backend, bypassing V4L2 entirely. +pub fn enumerate_depth_cameras() -> Vec { + let devices = match freedepth::enumerate_devices() { + Ok(d) => d, + Err(e) => { + debug!("Failed to enumerate depth cameras: {}", e); + return Vec::new(); + } + }; + + devices + .iter() + .map(|dev| { + let name = dev.name.clone(); + let path = format!("{}{}", DEPTH_PATH_PREFIX, dev.index); + let serial = dev + .id + .serial + .clone() + .unwrap_or_else(|| "unknown".to_string()); + + info!( + name = %name, + path = %path, + serial = %serial, + "Found depth camera via freedepth" + ); + + CameraDevice { + name, + path, + metadata_path: Some(serial), + device_info: Some(DeviceInfo { + card: dev.id.family.to_string(), + driver: "freedepth".to_string(), + path: format!("bus {} addr {}", dev.id.bus, dev.id.address), + real_path: format!("usb:{}:{}", dev.id.bus, dev.id.address), + }), + } + }) + .collect() +} + +/// Check if a device path refers to a depth camera (native backend) +pub fn is_depth_native_device(path: &str) -> bool { + path.starts_with(DEPTH_PATH_PREFIX) +} + +/// Extract the depth camera device index from a path +pub fn depth_device_index(path: &str) -> Option { + path.strip_prefix(DEPTH_PATH_PREFIX)?.parse().ok() +} + +/// Get supported formats for a depth camera device +/// +/// Returns native formats supported by the depth sensor. +/// Only exposes formats the camera hardware natively outputs: +/// - Bayer: Raw sensor data (640x480 @ 30fps, 1280x1024 @ 10fps) +/// - RGB: Demosaiced Bayer (software conversion, commonly expected output) +/// - YUV: ISP-processed UYVY (640x480 @ 15fps only) +/// - IR 10-bit packed: Native IR sensor output (640x488 @ 30fps, 1280x1024 @ 10fps) +/// - Depth: 11-bit depth data (640x480 @ 30fps) +/// +/// Note: IR 8-bit is NOT included as it's a software conversion from 10-bit packed. +/// Note: IR shares the video endpoint with Bayer/YUV - can only stream one at a time. +pub fn get_depth_formats(_device: &CameraDevice) -> Vec { + vec![ + // Bayer GRBG: 640x480 @ 30fps (raw sensor data, requires demosaicing) + CameraFormat { + width: 640, + height: 480, + framerate: Some(30), + hardware_accelerated: true, + pixel_format: "GRBG".to_string(), + sensor_type: SensorType::Rgb, + }, + // Bayer GRBG: 1280x1024 @ 10fps (high-res raw sensor data) + CameraFormat { + width: 1280, + height: 1024, + framerate: Some(10), + hardware_accelerated: true, + pixel_format: "GRBG".to_string(), + sensor_type: SensorType::Rgb, + }, + // RGB demosaiced: 640x480 @ 30fps + CameraFormat { + width: 640, + height: 480, + framerate: Some(30), + hardware_accelerated: true, + pixel_format: "RGB3".to_string(), + sensor_type: SensorType::Rgb, + }, + // RGB demosaiced: 1280x1024 @ 10fps + CameraFormat { + width: 1280, + height: 1024, + framerate: Some(10), + hardware_accelerated: true, + pixel_format: "RGB3".to_string(), + sensor_type: SensorType::Rgb, + }, + // YUV UYVY: 640x480 @ 15fps (ISP processed, better colors) + CameraFormat { + width: 640, + height: 480, + framerate: Some(15), + hardware_accelerated: true, + pixel_format: "UYVY".to_string(), + sensor_type: SensorType::Rgb, + }, + // IR 10-bit packed: 640x488 @ 30fps (note: height is 488) + CameraFormat { + width: 640, + height: 488, + framerate: Some(30), + hardware_accelerated: true, + pixel_format: "IR10".to_string(), + sensor_type: SensorType::Ir, + }, + // IR 10-bit packed: 1280x1024 @ 10fps (high-res requires firmware workaround) + CameraFormat { + width: 1280, + height: 1024, + framerate: Some(10), + hardware_accelerated: true, + pixel_format: "IR10".to_string(), + sensor_type: SensorType::Ir, + }, + // Depth 11-bit: 640x480 @ 30fps + CameraFormat { + width: 640, + height: 480, + framerate: Some(30), + hardware_accelerated: true, + pixel_format: "Y10B".to_string(), + sensor_type: SensorType::Depth, + }, + ] +} + +/// Native depth camera backend state +/// +/// Note: The frame receivers are NOT stored here because `mpsc::Receiver` is not `Sync`. +/// They are moved directly to the frame processing thread in `start()`. +pub struct NativeDepthBackend { + /// The freedepth streamer + streamer: Option, + /// Running flag + running: Arc, + /// Depth-only mode flag (when Y10B format is selected) + depth_only_mode: Arc, + /// Last video frame (for preview) + last_video_frame: Arc>>, + /// Last depth frame (for 3D preview) + last_depth_frame: Arc>>, + /// Frame processing thread + frame_thread: Option>, + /// Currently active device (for CameraBackend trait) + current_device: Option, + /// Currently active format (for CameraBackend trait) + current_format: Option, + /// Depth registration data for depth-to-RGB alignment (from device calibration) + /// Uses trait object for device-agnostic access + registration: Option>, +} + +impl NativeDepthBackend { + /// Create a new native depth camera backend + pub fn new() -> Self { + Self { + streamer: None, + running: Arc::new(AtomicBool::new(false)), + depth_only_mode: Arc::new(AtomicBool::new(false)), + last_video_frame: Arc::new(Mutex::new(None)), + last_depth_frame: Arc::new(Mutex::new(None)), + frame_thread: None, + current_device: None, + current_format: None, + registration: None, + } + } + + /// Initialize and start streaming with default settings + /// + /// This will unbind the kernel driver and start native USB streaming. + /// Uses Bayer format at Medium resolution by default. + /// + /// # Arguments + /// * `device_index` - The depth camera device index + pub fn start(&mut self, device_index: usize) -> Result<(), String> { + self.start_with_format(device_index, VideoFormat::Bayer, Resolution::Medium) + } + + /// Initialize and start streaming with specific format and resolution + /// + /// # Arguments + /// * `device_index` - The depth camera device index + /// * `video_format` - The video format to use + /// * `resolution` - The resolution to use (Medium or High) + fn start_with_format( + &mut self, + device_index: usize, + video_format: VideoFormat, + resolution: Resolution, + ) -> Result<(), String> { + if self.running.load(Ordering::SeqCst) { + return Err("Already running".to_string()); + } + + info!( + device = device_index, + format = ?video_format, + resolution = ?resolution, + "Starting native depth camera backend" + ); + + // Create the streamer (unbinds kernel driver) + let mut streamer = KinectStreamer::new(device_index) + .map_err(|e| format!("Failed to create depth camera streamer: {}", e))?; + + // Start streaming with selected video format and resolution + let (video_rx, depth_rx) = streamer + .start(video_format, resolution, DepthFormat::Depth11Bit) + .map_err(|e| format!("Failed to start streaming: {}", e))?; + + // Fetch device-specific registration data for depth-to-RGB alignment + let registration = streamer.create_depth_registration(); + + // Extract the calibrated depth-to-mm converter before boxing + // (uses Kinect-specific method, but the trait provides generic GPU data access) + let depth_converter = Arc::new(registration.depth_to_mm().clone()); + + // Log using trait methods for device-agnostic access + let summary = registration.registration_summary(); + info!( + "Fetched depth registration with target_offset={}, from_device={}", + registration.target_offset(), + summary.from_device + ); + + // Store as trait object for generic access + self.registration = Some(Box::new(registration)); + + // Register USB device for global motor control access + if let Some(usb) = streamer.usb_device() { + super::motor_control::set_motor_usb_device(usb); + } + + self.streamer = Some(streamer); + self.running.store(true, Ordering::SeqCst); + + // Start frame processing thread + // Note: receivers are moved directly to the thread (not stored in struct) + // because mpsc::Receiver is not Sync + let running = Arc::clone(&self.running); + let last_video = Arc::clone(&self.last_video_frame); + let last_depth = Arc::clone(&self.last_depth_frame); + + let thread = thread::spawn(move || { + frame_processing_thread( + running, + video_rx, + depth_rx, + last_video, + last_depth, + depth_converter, + ); + }); + + self.frame_thread = Some(thread); + + info!("Native depth camera backend started"); + Ok(()) + } + + /// Stop streaming and rebind the kernel driver + pub fn stop(&mut self) { + if !self.running.load(Ordering::SeqCst) { + return; + } + + info!("Stopping native depth camera backend"); + self.running.store(false, Ordering::SeqCst); + + // Wait for frame thread to finish + if let Some(thread) = self.frame_thread.take() { + let _ = thread.join(); + } + + // Clear global motor control reference + super::motor_control::clear_motor_usb_device(); + + // Stop streamer (this rebinds the driver) + if let Some(mut streamer) = self.streamer.take() { + streamer.stop(); + if let Err(e) = streamer.rebind_driver() { + warn!("Failed to rebind kernel driver: {}", e); + } + } + + // Clear frame data + *self.last_video_frame.lock().unwrap() = None; + *self.last_depth_frame.lock().unwrap() = None; + + info!("Native depth camera backend stopped"); + } + + /// Check if the backend is running + pub fn is_running(&self) -> bool { + self.running.load(Ordering::SeqCst) + } + + /// Get the latest video frame (for preview) + /// + /// In depth-only mode (Y10B format), this returns a colormap visualization + /// of the depth data instead of the RGB video. + pub fn get_video_frame(&self) -> Option { + if self.depth_only_mode.load(Ordering::Relaxed) { + // In depth-only mode, convert depth to RGB visualization + use crate::shaders::depth::{is_depth_grayscale_mode, is_depth_only_mode}; + + let depth_frame = self.last_depth_frame.lock().ok()?.clone()?; + let viz_options = DepthVisualizationOptions { + grayscale: is_depth_grayscale_mode(), + quantize: is_depth_only_mode(), // Use quantization in depth-only mode + ..DepthVisualizationOptions::kinect() + }; + let rgb_data = format_converters::depth_to_rgb( + &depth_frame.depth_mm, + depth_frame.width, + depth_frame.height, + &viz_options, + ); + Some(VideoFrameData { + width: depth_frame.width, + height: depth_frame.height, + data: rgb_data, + timestamp: depth_frame.timestamp, + }) + } else { + self.last_video_frame.lock().ok()?.clone() + } + } + + /// Get the latest depth frame (for 3D preview) + pub fn get_depth_frame(&self) -> Option { + self.last_depth_frame.lock().ok()?.clone() + } + + /// Check if in depth-only mode + pub fn is_depth_only_mode(&self) -> bool { + self.depth_only_mode.load(Ordering::Relaxed) + } + + /// Check if depth data is available + pub fn has_depth(&self) -> bool { + self.last_depth_frame + .lock() + .map(|d| d.is_some()) + .unwrap_or(false) + } + + // ========================================================================= + // Motor Control + // ========================================================================= + + /// Set the tilt angle in degrees (see freedepth::TILT_MIN_DEGREES/TILT_MAX_DEGREES) + pub fn set_tilt(&self, degrees: i8) -> Result<(), String> { + let streamer = self.streamer.as_ref().ok_or("Depth camera not running")?; + streamer + .set_tilt(degrees) + .map_err(|e| format!("Failed to set tilt: {}", e)) + } + + /// Get the current tilt angle in degrees + pub fn get_tilt(&self) -> Result { + let streamer = self.streamer.as_ref().ok_or("Depth camera not running")?; + streamer + .get_tilt() + .map_err(|e| format!("Failed to get tilt: {}", e)) + } + + /// Get the full motor/accelerometer state + pub fn get_tilt_state(&self) -> Result { + let streamer = self.streamer.as_ref().ok_or("Depth camera not running")?; + streamer + .get_tilt_state() + .map_err(|e| format!("Failed to get tilt state: {}", e)) + } + + /// Get the USB device Arc for shared motor control access + /// + /// Returns an Arc to the USB device that can be used for motor control + /// even when the backend is owned elsewhere. + pub fn usb_device(&self) -> Option>> { + self.streamer.as_ref()?.usb_device() + } + + /// Get the depth registration data + /// + /// Returns the registration data fetched from the device calibration, + /// which can be used for accurate depth-to-RGB alignment. + /// Uses the generic DepthRegistration trait for device-agnostic access. + pub fn get_registration(&self) -> Option<&dyn DepthRegistration> { + self.registration.as_ref().map(|r| r.as_ref()) + } +} + +impl Default for NativeDepthBackend { + fn default() -> Self { + Self::new() + } +} + +impl Drop for NativeDepthBackend { + fn drop(&mut self) { + self.stop(); + } +} + +/// Frame processing thread +fn frame_processing_thread( + running: Arc, + video_rx: Receiver, + depth_rx: Receiver, + last_video: Arc>>, + last_depth: Arc>>, + depth_converter: Arc, +) { + info!("Frame processing thread started (using device-calibrated depth converter)"); + + let mut video_count = 0u64; + let mut depth_count = 0u64; + + while running.load(Ordering::Relaxed) { + // Process video frames + match video_rx.try_recv() { + Ok(frame) => { + video_count += 1; + if let Some(processed) = process_video_frame(&frame) { + if let Ok(mut guard) = last_video.lock() { + *guard = Some(processed); + } + } + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + Err(std::sync::mpsc::TryRecvError::Disconnected) => { + debug!("Video channel disconnected"); + break; + } + } + + // Process depth frames + match depth_rx.try_recv() { + Ok(frame) => { + depth_count += 1; + if let Some(processed) = process_depth_frame(&frame, &depth_converter) { + if let Ok(mut guard) = last_depth.lock() { + *guard = Some(processed); + } + } + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + Err(std::sync::mpsc::TryRecvError::Disconnected) => { + debug!("Depth channel disconnected"); + break; + } + } + + // Small sleep to avoid busy-waiting + thread::sleep(Duration::from_micros(100)); + } + + info!( + video_frames = video_count, + depth_frames = depth_count, + "Frame processing thread ended" + ); +} + +/// Process a video frame from freedepth format to RGB +/// +/// For YUV formats, uses GPU-accelerated conversion when available. +/// For IR formats, unpacks data and converts to grayscale RGB. +fn process_video_frame(frame: &VideoFrame) -> Option { + // Handle different video formats + // Note: VideoFormat::Rgb from Kinect is actually Bayer data that needs demosaicing. + // The Kinect hardware doesn't have native RGB output - all formats go through + // Bayer pattern data, and RGB conversion happens in software. + let rgb_data = match frame.format { + VideoFormat::Rgb | VideoFormat::Bayer => { + // Both formats are actually Bayer data from the hardware - needs demosaicing + // The Kinect doesn't have native RGB output, so we always demosaic + let pixels = (frame.width * frame.height) as usize; + let mut rgb = vec![0u8; pixels * 3]; + freedepth::convert_bayer_to_rgb(&frame.data, &mut rgb, frame.width, frame.height); + rgb + } + VideoFormat::YuvRaw => { + // YUV format - try GPU conversion first, fall back to CPU + match try_gpu_yuv_to_rgb(&frame.data, frame.width, frame.height) { + Some(rgb) => rgb, + None => { + // GPU not available, use CPU conversion + frame.yuv_to_rgb()? + } + } + } + VideoFormat::YuvRgb => { + // Already converted to RGB by freedepth + frame.data.clone() + } + VideoFormat::Ir8Bit => { + // 8-bit grayscale - simply expand to RGB (1 byte per pixel input) + ir_8bit_to_rgb(&frame.data, frame.width, frame.height) + } + VideoFormat::Ir10BitPacked => { + // 10-bit packed IR - unpack using freedepth and convert to grayscale RGB + let unpacked = freedepth::unpack_10bit_ir(&frame.data, frame.width, frame.height); + ir_10bit_unpacked_to_rgb(&unpacked, frame.width, frame.height) + } + VideoFormat::Ir10Bit => { + // 10-bit unpacked IR (stored as u16) - convert to grayscale RGB + ir_10bit_to_rgb(&frame.data, frame.width, frame.height) + } + }; + + Some(VideoFrameData { + width: frame.width, + height: frame.height, + data: rgb_data, + timestamp: frame.timestamp, + }) +} + +/// Try GPU-accelerated YUV to RGB conversion +/// +/// Returns RGB data (3 bytes per pixel) on success, None if GPU not available. +/// Uses UYVY format which is what the Kinect outputs. +fn try_gpu_yuv_to_rgb(yuv_data: &[u8], width: u32, height: u32) -> Option> { + use crate::shaders::{YuvFormat, convert_yuv_to_rgba_gpu}; + + // Use pollster to block on the async GPU conversion + // This is safe because we're in a dedicated frame processing thread + // Kinect uses UYVY format (U, Y0, V, Y1) per formats.rs + let rgba = pollster::block_on(async { + convert_yuv_to_rgba_gpu(yuv_data, width, height, YuvFormat::Uyvy).await + }) + .ok()?; + + // Convert RGBA to RGB (drop alpha channel) + let pixels = (width * height) as usize; + let mut rgb = Vec::with_capacity(pixels * 3); + for i in 0..pixels { + rgb.push(rgba[i * 4]); // R + rgb.push(rgba[i * 4 + 1]); // G + rgb.push(rgba[i * 4 + 2]); // B + } + + Some(rgb) +} + +/// Process a depth frame from freedepth format to mm values +/// +/// Uses the device-calibrated depth converter for accurate raw-to-mm conversion. +fn process_depth_frame( + frame: &DepthFrame, + converter: &freedepth::DepthToMm, +) -> Option { + // Get depth as u16 slice + let depth_raw = frame.as_u16()?; + + // Convert raw 11-bit disparity values to millimeters using device-calibrated lookup table + let mut depth_mm = vec![0u16; depth_raw.len()]; + converter.convert_frame(depth_raw, &mut depth_mm); + + Some(DepthFrameData { + width: frame.width, + height: frame.height, + depth_mm, + timestamp: frame.timestamp, + }) +} + +/// Check if native depth camera backend can be used +/// +/// Returns true if: +/// - A depth camera device is connected +/// - We can likely unbind the kernel driver +pub fn can_use_native_backend() -> bool { + match freedepth::enumerate_devices() { + Ok(devices) => !devices.is_empty(), + Err(_) => false, + } +} + +// Re-export rgb_to_rgba from centralized format converters +pub use super::format_converters::rgb_to_rgba; + +impl CameraBackend for NativeDepthBackend { + fn enumerate_cameras(&self) -> Vec { + enumerate_depth_cameras() + } + + fn get_formats(&self, device: &CameraDevice, _video_mode: bool) -> Vec { + get_depth_formats(device) + } + + fn initialize(&mut self, device: &CameraDevice, format: &CameraFormat) -> BackendResult<()> { + info!( + device = %device.name, + format = %format, + "Initializing native depth camera backend" + ); + + // Shutdown any existing stream + if self.is_running() { + self.stop(); + } + + // Extract device index from path + let device_index = depth_device_index(&device.path).ok_or_else(|| { + BackendError::DeviceNotFound(format!( + "Invalid depth camera device path: {}", + device.path + )) + })?; + + // Map FourCC pixel_format to freedepth VideoFormat and Resolution + // Resolution is determined by width: 640 = Medium, 1280 = High + let resolution = if format.width >= 1280 { + Resolution::High + } else { + Resolution::Medium + }; + + let (video_format, depth_only) = match format.pixel_format.as_str() { + // Bayer raw (requires demosaicing) + "GRBG" => (VideoFormat::Bayer, false), + // RGB demosaiced (same as Bayer at hardware level, demosaiced in software) + "RGB3" => (VideoFormat::Rgb, false), + // YUV UYVY (ISP processed) - only Medium resolution supported + "UYVY" => { + if resolution == Resolution::High { + return Err(BackendError::FormatNotSupported( + "YUV format only supports 640x480 @ 15fps".to_string(), + )); + } + (VideoFormat::YuvRaw, false) + } + // IR 10-bit packed + "IR10" => (VideoFormat::Ir10BitPacked, false), + // Depth-only mode (use Bayer internally but display depth visualization) + "Y10B" => (VideoFormat::Bayer, true), + _ => { + return Err(BackendError::FormatNotSupported(format!( + "Unknown depth camera format: {} at {}x{}", + format.pixel_format, format.width, format.height + ))); + } + }; + + // Set depth-only mode flag + self.depth_only_mode.store(depth_only, Ordering::SeqCst); + + // Store device and format + self.current_device = Some(device.clone()); + self.current_format = Some(format.clone()); + + // Start streaming with selected format and resolution + self.start_with_format(device_index, video_format, resolution) + .map_err(|e| { + BackendError::InitializationFailed(format!( + "Failed to start depth camera streaming: {}", + e + )) + })?; + + info!("Native depth camera backend initialized successfully"); + Ok(()) + } + + fn shutdown(&mut self) -> BackendResult<()> { + info!("Shutting down native depth camera backend"); + self.stop(); + self.current_device = None; + self.current_format = None; + Ok(()) + } + + fn is_initialized(&self) -> bool { + self.is_running() && self.current_device.is_some() + } + + fn recover(&mut self) -> BackendResult<()> { + info!("Attempting to recover native depth camera backend"); + let device = self + .current_device + .clone() + .ok_or_else(|| BackendError::Other("No device to recover".to_string()))?; + let format = self + .current_format + .clone() + .ok_or_else(|| BackendError::Other("No format to recover".to_string()))?; + + self.stop(); + self.initialize(&device, &format) + } + + fn switch_camera(&mut self, device: &CameraDevice) -> BackendResult<()> { + info!(device = %device.name, "Switching to new depth camera device"); + + let formats = self.get_formats(device, false); + if formats.is_empty() { + return Err(BackendError::FormatNotSupported( + "No formats available for device".to_string(), + )); + } + + let format = formats + .first() + .cloned() + .ok_or_else(|| BackendError::Other("Failed to select format".to_string()))?; + + self.initialize(device, &format) + } + + fn apply_format(&mut self, format: &CameraFormat) -> BackendResult<()> { + info!(format = %format, "Applying new format to depth camera"); + + let device = self + .current_device + .clone() + .ok_or_else(|| BackendError::Other("No active device".to_string()))?; + + self.initialize(&device, format) + } + + fn capture_photo(&self) -> BackendResult { + debug!("Capturing photo from native depth camera backend"); + + // Get the latest video frame + let video_frame = self + .get_video_frame() + .ok_or_else(|| BackendError::Other("No video frame available".to_string()))?; + + // Convert RGB to RGBA + let rgba_data = rgb_to_rgba(&video_frame.data); + + // Get depth frame and its dimensions + let depth_frame = self.get_depth_frame(); + let (depth_width, depth_height, depth_data) = match &depth_frame { + Some(d) => ( + d.width, + d.height, + Some(Arc::from(d.depth_mm.clone().into_boxed_slice())), + ), + None => (0, 0, None), + }; + + Ok(CameraFrame { + width: video_frame.width, + height: video_frame.height, + data: Arc::from(rgba_data.into_boxed_slice()), + format: PixelFormat::RGBA, + stride: video_frame.width * 4, + captured_at: Instant::now(), + depth_data, + depth_width, + depth_height, + video_timestamp: Some(video_frame.timestamp), + }) + } + + fn start_recording(&mut self, _output_path: PathBuf) -> BackendResult<()> { + // Video recording not yet implemented for native depth cameras + Err(BackendError::Other( + "Video recording not yet implemented for native depth camera backend".to_string(), + )) + } + + fn stop_recording(&mut self) -> BackendResult { + Err(BackendError::NoRecordingInProgress) + } + + fn is_recording(&self) -> bool { + false + } + + fn get_preview_receiver(&self) -> Option { + // Preview is handled via polling (get_video_frame) + None + } + + fn backend_type(&self) -> CameraBackendType { + CameraBackendType::PipeWire // Report as PipeWire for compatibility + } + + fn is_available(&self) -> bool { + can_use_native_backend() + } + + fn current_device(&self) -> Option<&CameraDevice> { + self.current_device.as_ref() + } + + fn current_format(&self) -> Option<&CameraFormat> { + self.current_format.as_ref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_backend_creation() { + let backend = NativeDepthBackend::new(); + assert!(!backend.is_running()); + } +} diff --git a/src/backends/camera/format_converters.rs b/src/backends/camera/format_converters.rs new file mode 100644 index 00000000..45353838 --- /dev/null +++ b/src/backends/camera/format_converters.rs @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: GPL-3.0-only +//! Unified pixel format conversion utilities for depth camera backends +//! +//! This module consolidates all pixel format conversion functions used by +//! the various depth camera backends (native freedepth, V4L2 kernel, V4L2 raw). + +/// Convert UYVY (YUV 4:2:2) to RGBA +/// +/// UYVY format: U0 Y0 V0 Y1 - each 4-byte group encodes 2 pixels. +/// Uses BT.601 coefficients for YUV to RGB conversion. +pub fn uyvy_to_rgba(data: &[u8], width: u32, height: u32) -> Vec { + let pixel_count = (width * height) as usize; + let mut rgba = Vec::with_capacity(pixel_count * 4); + + // UYVY: U0 Y0 V0 Y1 - processes 2 pixels at a time + for chunk in data.chunks_exact(4) { + let u = chunk[0] as f32 - 128.0; + let y0 = chunk[1] as f32; + let v = chunk[2] as f32 - 128.0; + let y1 = chunk[3] as f32; + + // Convert YUV to RGB (BT.601) + for y in [y0, y1] { + let r = (y + 1.402 * v).clamp(0.0, 255.0) as u8; + let g = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8; + let b = (y + 1.772 * u).clamp(0.0, 255.0) as u8; + + rgba.push(r); + rgba.push(g); + rgba.push(b); + rgba.push(255); + + if rgba.len() >= pixel_count * 4 { + break; + } + } + } + + rgba +} + +/// Convert Bayer GRBG to RGBA using simple nearest-neighbor demosaic +/// +/// Bayer pattern (GRBG): +/// ```text +/// G R +/// B G +/// ``` +/// Each 2x2 block produces 4 pixels with the same RGB values. +pub fn grbg_to_rgba(data: &[u8], width: u32, height: u32) -> Vec { + let w = width as usize; + let h = height as usize; + let mut rgba = vec![0u8; w * h * 4]; + + // Simple demosaic: for each 2x2 Bayer block + for y in (0..h.saturating_sub(1)).step_by(2) { + for x in (0..w.saturating_sub(1)).step_by(2) { + let g0 = data[y * w + x] as u32; + let r = data[y * w + x + 1] as u32; + let b = data[(y + 1) * w + x] as u32; + let g1 = data[(y + 1) * w + x + 1] as u32; + let g = ((g0 + g1) / 2) as u8; + + // Apply same color to all 4 pixels in block + for dy in 0..2 { + for dx in 0..2 { + let idx = ((y + dy) * w + (x + dx)) * 4; + rgba[idx] = r as u8; + rgba[idx + 1] = g; + rgba[idx + 2] = b as u8; + rgba[idx + 3] = 255; + } + } + } + } + + rgba +} + +/// Unpack Y10B (10-bit packed) depth data to 16-bit values +/// +/// Y10B packs 4 10-bit values into 5 bytes: +/// ```text +/// [A9:A2][B9:B2][C9:C2][D9:D2][D1:D0,C1:C0,B1:B0,A1:A0] +/// ``` +/// +/// Returns raw 10-bit values (0-1023 range). +pub fn unpack_y10b(data: &[u8], width: u32, height: u32) -> Vec { + let pixel_count = (width * height) as usize; + let mut output = Vec::with_capacity(pixel_count); + + for chunk in data.chunks_exact(5) { + if output.len() >= pixel_count { + break; + } + + let a = ((chunk[0] as u16) << 2) | ((chunk[4] as u16) & 0x03); + let b = ((chunk[1] as u16) << 2) | (((chunk[4] as u16) >> 2) & 0x03); + let c = ((chunk[2] as u16) << 2) | (((chunk[4] as u16) >> 4) & 0x03); + let d = ((chunk[3] as u16) << 2) | (((chunk[4] as u16) >> 6) & 0x03); + + output.push(a); + if output.len() < pixel_count { + output.push(b); + } + if output.len() < pixel_count { + output.push(c); + } + if output.len() < pixel_count { + output.push(d); + } + } + + output +} + +/// Unpack Y10B and scale to 16-bit range +/// +/// Same as `unpack_y10b` but shifts values left by 6 bits to use full 16-bit range. +/// This is useful when the 10-bit values need to be treated as 16-bit depth. +pub fn unpack_y10b_to_u16(data: &[u8], width: u32, height: u32) -> Vec { + unpack_y10b(data, width, height) + .into_iter() + .map(|v| v << 6) + .collect() +} + +/// Depth visualization options +#[derive(Debug, Clone, Copy, Default)] +pub struct DepthVisualizationOptions { + /// Use grayscale instead of colormap (near=bright, far=dark) + pub grayscale: bool, + /// Quantize depth into bands for smoother visualization + pub quantize: bool, + /// Number of quantization bands (default: 32) + pub quantize_bands: u32, + /// Minimum depth in mm (values below are clamped) + pub min_depth_mm: u16, + /// Maximum depth in mm (values above are clamped) + pub max_depth_mm: u16, + /// Value that indicates invalid depth (rendered as black) + pub invalid_value: u16, +} + +impl DepthVisualizationOptions { + /// Create options for Kinect sensor (freedepth usable range) + pub fn kinect() -> Self { + Self { + grayscale: false, + quantize: false, + quantize_bands: 32, + min_depth_mm: 500, // DEPTH_MIN_USABLE_MM + max_depth_mm: 4000, // DEPTH_MAX_USABLE_MM + invalid_value: 8191, // DEPTH_INVALID_THRESHOLD_MM + } + } + + /// Create options for generic depth sensor with auto-ranging + pub fn auto_range() -> Self { + Self { + grayscale: true, + quantize: false, + quantize_bands: 32, + min_depth_mm: 0, + max_depth_mm: 0, // 0 = auto-detect from data + invalid_value: 0, + } + } +} + +/// Turbo colormap: perceptually uniform rainbow (blue=near, red=far) +/// +/// Based on the Google Turbo colormap. +fn turbo(t: f32) -> [u8; 3] { + let r = (0.13572138 + + t * (4.6153926 + t * (-42.66032 + t * (132.13108 + t * (-152.54825 + t * 59.28144))))) + .clamp(0.0, 1.0); + let g = (0.09140261 + + t * (2.19418 + t * (4.84296 + t * (-14.18503 + t * (4.27805 + t * 2.53377))))) + .clamp(0.0, 1.0); + let b = (0.1066733 + + t * (12.64194 + t * (-60.58204 + t * (109.99648 + t * (-82.52904 + t * 20.43388))))) + .clamp(0.0, 1.0); + [(r * 255.0) as u8, (g * 255.0) as u8, (b * 255.0) as u8] +} + +/// Convert 16-bit depth values to RGB visualization +/// +/// This is the unified depth visualization function that supports: +/// - Grayscale mode (near=bright, far=dark) +/// - Turbo colormap (blue=near, red=far) +/// - Optional band quantization +/// - Auto-ranging or fixed range +pub fn depth_to_rgb( + depth: &[u16], + width: u32, + height: u32, + options: &DepthVisualizationOptions, +) -> Vec { + let pixel_count = (width * height) as usize; + let mut rgb = Vec::with_capacity(pixel_count * 3); + + // Determine range (auto-detect if max_depth_mm is 0) + let (min_depth, max_depth) = if options.max_depth_mm == 0 { + // Auto-range: find min/max from data (excluding invalid) + let mut min_d = u16::MAX; + let mut max_d = 0u16; + for &d in depth.iter().take(pixel_count) { + if d != 0 && d != options.invalid_value && d < 10000 { + min_d = min_d.min(d); + max_d = max_d.max(d); + } + } + if max_d <= min_d { + (0u16, 4000u16) // Default fallback + } else { + (min_d, max_d) + } + } else { + (options.min_depth_mm, options.max_depth_mm) + }; + + let range = (max_depth - min_depth) as f32; + + for &d in depth.iter().take(pixel_count) { + if d == 0 || d == options.invalid_value || d >= 10000 { + // Invalid depth - black + rgb.extend_from_slice(&[0, 0, 0]); + } else { + // Normalize to 0.0-1.0 range + let mut t = (d.saturating_sub(min_depth) as f32) / range; + t = t.clamp(0.0, 1.0); + + // Quantize to bands for smoother visualization + if options.quantize && options.quantize_bands > 0 { + let bands = options.quantize_bands as f32; + t = (t * bands).floor() / bands; + } + + if options.grayscale { + // Grayscale: near=bright, far=dark (invert t) + let gray = ((1.0 - t) * 255.0) as u8; + rgb.extend_from_slice(&[gray, gray, gray]); + } else { + // Colormap: turbo (blue=near, red=far) + let color = turbo(t); + rgb.extend_from_slice(&color); + } + } + } + + rgb +} + +/// Convert 16-bit depth values to RGBA visualization +/// +/// Same as `depth_to_rgb` but outputs RGBA with alpha=255. +pub fn depth_to_rgba( + depth: &[u16], + width: u32, + height: u32, + options: &DepthVisualizationOptions, +) -> Vec { + let rgb = depth_to_rgb(depth, width, height, options); + let mut rgba = Vec::with_capacity(rgb.len() / 3 * 4); + + for chunk in rgb.chunks_exact(3) { + rgba.push(chunk[0]); + rgba.push(chunk[1]); + rgba.push(chunk[2]); + rgba.push(255); + } + + rgba +} + +/// Simple depth to grayscale conversion (upper 8 bits) +/// +/// Fast conversion for preview when full visualization isn't needed. +pub fn depth_to_grayscale(depth: &[u16]) -> Vec { + depth.iter().map(|&d| (d >> 8) as u8).collect() +} + +/// Convert RGB to RGBA by adding alpha=255 +pub fn rgb_to_rgba(rgb: &[u8]) -> Vec { + let mut rgba = Vec::with_capacity(rgb.len() / 3 * 4); + for chunk in rgb.chunks_exact(3) { + rgba.push(chunk[0]); + rgba.push(chunk[1]); + rgba.push(chunk[2]); + rgba.push(255); + } + rgba +} + +/// Convert IR 8-bit grayscale data to RGB +/// +/// Simply expands each grayscale byte to RGB triplet. +pub fn ir_8bit_to_rgb(data: &[u8], width: u32, height: u32) -> Vec { + let pixel_count = (width * height) as usize; + let mut rgb = Vec::with_capacity(pixel_count * 3); + + for &gray in data.iter().take(pixel_count) { + rgb.extend_from_slice(&[gray, gray, gray]); + } + + // Handle missing pixels (if any) + while rgb.len() < pixel_count * 3 { + rgb.extend_from_slice(&[0, 0, 0]); + } + + rgb +} + +/// Convert IR 10-bit unpacked data (u16 little-endian bytes) to RGB grayscale +/// +/// The data is stored as little-endian u16 values (10-bit in lower bits). +pub fn ir_10bit_to_rgb(data: &[u8], width: u32, height: u32) -> Vec { + let pixel_count = (width * height) as usize; + let mut rgb = Vec::with_capacity(pixel_count * 3); + + // Interpret data as u16 (little-endian) + for chunk in data.chunks_exact(2).take(pixel_count) { + let val = u16::from_le_bytes([chunk[0], chunk[1]]); + let gray = (val >> 2) as u8; // 10-bit to 8-bit + rgb.extend_from_slice(&[gray, gray, gray]); + } + + // Handle missing pixels (if any) + while rgb.len() < pixel_count * 3 { + rgb.extend_from_slice(&[0, 0, 0]); + } + + rgb +} + +/// Convert already-unpacked IR 10-bit values (u16) to RGB grayscale +/// +/// Takes unpacked 10-bit values (0-1023) and converts to 8-bit grayscale RGB. +pub fn ir_10bit_unpacked_to_rgb(unpacked: &[u16], width: u32, height: u32) -> Vec { + let pixel_count = (width * height) as usize; + let mut rgb = Vec::with_capacity(pixel_count * 3); + + for &val in unpacked.iter().take(pixel_count) { + let gray = (val >> 2) as u8; // 10-bit to 8-bit + rgb.extend_from_slice(&[gray, gray, gray]); + } + + // Handle any missing pixels + while rgb.len() < pixel_count * 3 { + rgb.extend_from_slice(&[0, 0, 0]); + } + + rgb +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_y10b_unpacking() { + // Kernel Y10B format: [A9:A2][B9:B2][C9:C2][D9:D2][D1:D0,C1:C0,B1:B0,A1:A0] + // Test data: 4 pixels A=1023, B=512, C=256, D=0 + // Byte 0 = A[9:2] = 1023 >> 2 = 255 + // Byte 1 = B[9:2] = 512 >> 2 = 128 + // Byte 2 = C[9:2] = 256 >> 2 = 64 + // Byte 3 = D[9:2] = 0 + // Byte 4 = (D[1:0]<<6) | (C[1:0]<<4) | (B[1:0]<<2) | A[1:0] + // = (0<<6) | (0<<4) | (0<<2) | 3 = 3 + let raw_data = vec![255u8, 128, 64, 0, 3]; + let depth = unpack_y10b(&raw_data, 2, 2); + + assert_eq!(depth[0], 1023); + assert_eq!(depth[1], 512); + assert_eq!(depth[2], 256); + assert_eq!(depth[3], 0); + } + + #[test] + fn test_y10b_to_u16() { + // Same kernel Y10B format data as above + let raw_data = vec![255u8, 128, 64, 0, 3]; + let depth = unpack_y10b_to_u16(&raw_data, 2, 2); + + // Values are shifted left by 6 to use full 16-bit range + assert_eq!(depth[0], 1023 << 6); + assert_eq!(depth[1], 512 << 6); + assert_eq!(depth[2], 256 << 6); + assert_eq!(depth[3], 0); + } + + #[test] + fn test_depth_to_rgba_grayscale() { + let depth = vec![0xFFFF, 0x8000, 0x0000]; + let options = DepthVisualizationOptions { + grayscale: true, + max_depth_mm: 0, // Auto-range + ..Default::default() + }; + let rgba = depth_to_rgba(&depth, 3, 1, &options); + + assert_eq!(rgba.len(), 12); + // Alpha should always be 255 + assert_eq!(rgba[3], 255); + assert_eq!(rgba[7], 255); + assert_eq!(rgba[11], 255); + } + + #[test] + fn test_uyvy_to_rgba() { + // Pure white in YUV (Y=255, U=128, V=128) + let uyvy = vec![128u8, 255, 128, 255]; + let rgba = uyvy_to_rgba(&uyvy, 2, 1); + + assert_eq!(rgba.len(), 8); + // Both pixels should be near white + assert!(rgba[0] > 250); // R + assert!(rgba[1] > 250); // G + assert!(rgba[2] > 250); // B + assert_eq!(rgba[3], 255); // A + } + + #[test] + fn test_rgb_to_rgba() { + let rgb = vec![255, 128, 64, 0, 0, 0]; + let rgba = rgb_to_rgba(&rgb); + + assert_eq!(rgba.len(), 8); + assert_eq!(rgba[0..4], [255, 128, 64, 255]); + assert_eq!(rgba[4..8], [0, 0, 0, 255]); + } + + #[test] + fn test_turbo_colormap() { + // Test that colormap changes across the range (t=0 to t=1) + let start = turbo(0.0); + let mid = turbo(0.5); + let end = turbo(1.0); + + // Colors should be different at different t values + assert_ne!(start, mid); + assert_ne!(mid, end); + assert_ne!(start, end); + + // End (t=1) should have higher red than start (t=0) + assert!(end[0] > start[0]); + } +} diff --git a/src/backends/camera/frame_loop.rs b/src/backends/camera/frame_loop.rs new file mode 100644 index 00000000..4d10d971 --- /dev/null +++ b/src/backends/camera/frame_loop.rs @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-3.0-only +//! Thread lifecycle management for capture loops +//! +//! This module provides a standardized way to manage capture loop threads +//! across different depth camera backends, reducing code duplication and +//! ensuring consistent thread lifecycle handling. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread::{self, JoinHandle}; +use tracing::{debug, info, warn}; + +/// Action returned by the capture loop callback to control loop behavior +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LoopAction { + /// Continue running the loop + Continue, + /// Stop the loop gracefully + Stop, +} + +/// Controller for a capture loop running in a separate thread +/// +/// This provides a standardized interface for starting, stopping, and +/// managing the lifecycle of capture loop threads. +/// +/// # Example +/// +/// ```ignore +/// let controller = CaptureLoopController::start("depth-capture", || { +/// // Capture and process a frame +/// match capture_frame() { +/// Ok(frame) => { +/// process_frame(frame); +/// LoopAction::Continue +/// } +/// Err(e) => { +/// warn!("Capture error: {}", e); +/// LoopAction::Continue // Keep trying +/// } +/// } +/// }); +/// +/// // Later, stop the loop +/// controller.stop(); +/// ``` +pub struct CaptureLoopController { + /// Thread handle for joining + thread_handle: Option>, + /// Signal to stop the loop + stop_signal: Arc, + /// Name for logging + name: String, +} + +impl CaptureLoopController { + /// Start a new capture loop in a separate thread + /// + /// The provided closure is called repeatedly until it returns `LoopAction::Stop` + /// or the controller's `stop()` method is called. + /// + /// # Arguments + /// + /// * `name` - A descriptive name for the loop (used in logging) + /// * `loop_fn` - A closure that performs one iteration of the capture loop + /// + /// # Returns + /// + /// A controller that can be used to stop the loop and wait for it to finish. + pub fn start(name: &str, mut loop_fn: F) -> Self + where + F: FnMut() -> LoopAction + Send + 'static, + { + let stop_signal = Arc::new(AtomicBool::new(false)); + let stop_signal_clone = Arc::clone(&stop_signal); + let name_clone = name.to_string(); + + info!(name = %name, "Starting capture loop"); + + let thread_handle = thread::spawn(move || { + debug!(name = %name_clone, "Capture loop thread started"); + + loop { + // Check stop signal first + if stop_signal_clone.load(Ordering::SeqCst) { + debug!(name = %name_clone, "Stop signal received"); + break; + } + + // Execute one iteration + match loop_fn() { + LoopAction::Continue => {} + LoopAction::Stop => { + debug!(name = %name_clone, "Loop requested stop"); + break; + } + } + } + + info!(name = %name_clone, "Capture loop thread exiting"); + }); + + Self { + thread_handle: Some(thread_handle), + stop_signal, + name: name.to_string(), + } + } + + /// Start a capture loop with initialization + /// + /// The `init_fn` is called once at the start of the thread to set up + /// resources. If initialization fails, the thread exits immediately. + /// + /// # Arguments + /// + /// * `name` - A descriptive name for the loop + /// * `init_fn` - Initialization closure, returns Ok(state) or Err(message) + /// * `loop_fn` - Loop closure that receives the state and returns LoopAction + pub fn start_with_init(name: &str, init_fn: I, mut loop_fn: F) -> Self + where + S: Send + 'static, + I: FnOnce() -> Result + Send + 'static, + F: FnMut(&mut S) -> LoopAction + Send + 'static, + { + let stop_signal = Arc::new(AtomicBool::new(false)); + let stop_signal_clone = Arc::clone(&stop_signal); + let name_clone = name.to_string(); + + info!(name = %name, "Starting capture loop with initialization"); + + let thread_handle = thread::spawn(move || { + debug!(name = %name_clone, "Capture loop thread started, initializing..."); + + // Run initialization + let mut state = match init_fn() { + Ok(s) => { + debug!(name = %name_clone, "Initialization successful"); + s + } + Err(e) => { + warn!(name = %name_clone, error = %e, "Initialization failed"); + return; + } + }; + + loop { + if stop_signal_clone.load(Ordering::SeqCst) { + debug!(name = %name_clone, "Stop signal received"); + break; + } + + match loop_fn(&mut state) { + LoopAction::Continue => {} + LoopAction::Stop => { + debug!(name = %name_clone, "Loop requested stop"); + break; + } + } + } + + info!(name = %name_clone, "Capture loop thread exiting"); + }); + + Self { + thread_handle: Some(thread_handle), + stop_signal, + name: name.to_string(), + } + } + + /// Check if the loop is still running + pub fn is_running(&self) -> bool { + self.thread_handle + .as_ref() + .map(|h| !h.is_finished()) + .unwrap_or(false) + } + + /// Get a clone of the stop signal for external use + /// + /// This can be passed to capture functions that need to check + /// for stop requests within long-running operations. + pub fn stop_signal(&self) -> Arc { + Arc::clone(&self.stop_signal) + } + + /// Signal the loop to stop (non-blocking) + /// + /// This sets the stop signal but doesn't wait for the thread to finish. + /// Use `stop()` or `stop_and_wait()` if you need to wait. + pub fn request_stop(&self) { + debug!(name = %self.name, "Requesting capture loop stop"); + self.stop_signal.store(true, Ordering::SeqCst); + } + + /// Stop the loop and wait for the thread to finish + /// + /// This is the preferred way to stop a capture loop as it ensures + /// clean shutdown before returning. + pub fn stop(&mut self) { + self.request_stop(); + self.join(); + } + + /// Wait for the thread to finish without sending stop signal + /// + /// Useful if the loop stops itself via `LoopAction::Stop`. + pub fn join(&mut self) { + if let Some(handle) = self.thread_handle.take() { + debug!(name = %self.name, "Waiting for capture loop thread to finish"); + if let Err(e) = handle.join() { + warn!(name = %self.name, "Capture loop thread panicked: {:?}", e); + } else { + debug!(name = %self.name, "Capture loop thread finished"); + } + } + } +} + +impl Drop for CaptureLoopController { + fn drop(&mut self) { + if self.thread_handle.is_some() { + debug!(name = %self.name, "CaptureLoopController dropped, stopping loop"); + self.stop(); + } + } +} + +/// Builder for creating capture loops with common configuration +pub struct CaptureLoopBuilder { + name: String, +} + +impl CaptureLoopBuilder { + /// Create a new builder with the given loop name + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + } + } + + /// Start the capture loop with the given closure + pub fn start(self, loop_fn: F) -> CaptureLoopController + where + F: FnMut() -> LoopAction + Send + 'static, + { + CaptureLoopController::start(&self.name, loop_fn) + } + + /// Start the capture loop with initialization + pub fn start_with_init(self, init_fn: I, loop_fn: F) -> CaptureLoopController + where + S: Send + 'static, + I: FnOnce() -> Result + Send + 'static, + F: FnMut(&mut S) -> LoopAction + Send + 'static, + { + CaptureLoopController::start_with_init(&self.name, init_fn, loop_fn) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicU32; + use std::time::Duration; + + #[test] + fn test_basic_loop() { + let counter = Arc::new(AtomicU32::new(0)); + let counter_clone = Arc::clone(&counter); + + let mut controller = CaptureLoopController::start("test-loop", move || { + let count = counter_clone.fetch_add(1, Ordering::SeqCst); + if count >= 10 { + LoopAction::Stop + } else { + LoopAction::Continue + } + }); + + // Wait for loop to finish itself + controller.join(); + + assert_eq!(counter.load(Ordering::SeqCst), 11); // 0-10 inclusive + } + + #[test] + fn test_stop_signal() { + let counter = Arc::new(AtomicU32::new(0)); + let counter_clone = Arc::clone(&counter); + + let mut controller = CaptureLoopController::start("test-loop", move || { + counter_clone.fetch_add(1, Ordering::SeqCst); + thread::sleep(Duration::from_millis(10)); + LoopAction::Continue + }); + + // Let it run a bit + thread::sleep(Duration::from_millis(50)); + + // Stop and verify it ran at least once + controller.stop(); + assert!(counter.load(Ordering::SeqCst) > 0); + } + + #[test] + fn test_with_init() { + let result = Arc::new(AtomicU32::new(0)); + let result_clone = Arc::clone(&result); + + let mut controller = CaptureLoopController::start_with_init( + "test-init-loop", + || Ok(42u32), // Init returns 42 + move |state| { + result_clone.store(*state, Ordering::SeqCst); + LoopAction::Stop + }, + ); + + controller.join(); + assert_eq!(result.load(Ordering::SeqCst), 42); + } + + #[test] + fn test_init_failure() { + let ran = Arc::new(AtomicBool::new(false)); + let ran_clone = Arc::clone(&ran); + + let mut controller = CaptureLoopController::start_with_init( + "test-fail-init", + || Err::<(), _>("Init failed".to_string()), + move |_: &mut ()| { + ran_clone.store(true, Ordering::SeqCst); + LoopAction::Stop + }, + ); + + controller.join(); + // Loop function should never run if init fails + assert!(!ran.load(Ordering::SeqCst)); + } + + #[test] + fn test_is_running() { + let controller = CaptureLoopController::start("test-running", || { + thread::sleep(Duration::from_millis(100)); + LoopAction::Continue + }); + + assert!(controller.is_running()); + + // Drop will stop it + drop(controller); + } +} diff --git a/src/backends/camera/mod.rs b/src/backends/camera/mod.rs index 15705399..b83595cd 100644 --- a/src/backends/camera/mod.rs +++ b/src/backends/camera/mod.rs @@ -29,13 +29,202 @@ //! └────────┘ //! ``` +// freedepth-specific modules (require both x86_64 and freedepth feature) +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub mod depth_controller; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub mod depth_native; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub mod v4l2_depth; + +// Kernel driver modules (work without freedepth) +pub mod format_converters; +pub mod frame_loop; pub mod manager; +pub mod motor_control; pub mod pipewire; pub mod types; pub mod v4l2_controls; +pub mod v4l2_depth_controls; +pub mod v4l2_kernel_depth; + +// freedepth exports +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub use depth_controller::{DepthController, is_depth_camera}; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub use depth_native::{ + DEPTH_PATH_PREFIX, NativeDepthBackend, can_use_native_backend, depth_device_index, + enumerate_depth_cameras, get_depth_formats, is_depth_native_device, rgb_to_rgba, +}; +// Kernel driver exports (always available) pub use manager::CameraBackendManager; +pub use motor_control::{MotorBackend, MotorController, TILT_MAX_DEGREES, TILT_MIN_DEGREES}; pub use types::*; +pub use v4l2_depth_controls::{ + DepthCapabilities, DepthExtrinsics, DepthIntrinsics, DepthSensorType, KernelRegistrationData, + REG_X_VAL_SCALE, V4l2DeviceInfo, has_depth_controls, query_depth_capabilities, + query_device_info, +}; +pub use v4l2_kernel_depth::{ + KERNEL_DEPTH_PREFIX, KERNEL_KINECT_PREFIX, KinectDevicePair, V4l2KernelDepthBackend, + find_kernel_kinect_pairs, has_kernel_kinect_devices, is_kernel_depth_device, + is_kernel_kinect_device, parse_kernel_kinect_path, +}; + +// Stubs when freedepth is disabled (either not x86_64 or feature disabled) +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn is_depth_camera(_device: &CameraDevice) -> bool { + // With kernel driver, depth cameras are detected via find_kernel_kinect_pairs + false +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn is_depth_native_device(_device_path: &str) -> bool { + false +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn enumerate_depth_cameras() -> Vec { + Vec::new() +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub const DEPTH_PATH_PREFIX: &str = "kinect:"; + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn get_depth_formats(_device: &CameraDevice) -> Vec { + Vec::new() +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn depth_device_index(_device_path: &str) -> Option { + None +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn can_use_native_backend() -> bool { + false +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub fn rgb_to_rgba(rgb: &[u8]) -> Vec { + // Convert RGB to RGBA by adding alpha channel + let pixels = rgb.len() / 3; + let mut rgba = Vec::with_capacity(pixels * 4); + for chunk in rgb.chunks_exact(3) { + rgba.extend_from_slice(chunk); + rgba.push(255); + } + rgba +} + +// Stub NativeDepthBackend when freedepth is disabled +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub struct NativeDepthBackend; + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +impl NativeDepthBackend { + pub fn new() -> Self { + Self + } +} + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +impl CameraBackend for NativeDepthBackend { + fn enumerate_cameras(&self) -> Vec { + Vec::new() + } + + fn get_formats(&self, _device: &CameraDevice, _video_mode: bool) -> Vec { + Vec::new() + } + + fn initialize(&mut self, _device: &CameraDevice, _format: &CameraFormat) -> BackendResult<()> { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn shutdown(&mut self) -> BackendResult<()> { + Ok(()) + } + + fn is_initialized(&self) -> bool { + false + } + + fn recover(&mut self) -> BackendResult<()> { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn switch_camera(&mut self, _device: &CameraDevice) -> BackendResult<()> { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn apply_format(&mut self, _format: &CameraFormat) -> BackendResult<()> { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn capture_photo(&self) -> BackendResult { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn start_recording(&mut self, _output_path: std::path::PathBuf) -> BackendResult<()> { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn stop_recording(&mut self) -> BackendResult { + Err(BackendError::NotAvailable( + "freedepth feature not enabled".to_string(), + )) + } + + fn is_recording(&self) -> bool { + false + } + + fn get_preview_receiver(&self) -> Option { + None + } + + fn backend_type(&self) -> CameraBackendType { + CameraBackendType::PipeWire + } + + fn is_available(&self) -> bool { + false + } + + fn current_device(&self) -> Option<&CameraDevice> { + None + } + + fn current_format(&self) -> Option<&CameraFormat> { + None + } +} + +// Stub DepthController when freedepth is disabled +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +pub struct DepthController; + +#[cfg(not(all(target_arch = "x86_64", feature = "freedepth")))] +impl DepthController { + pub fn new(_device_path: String) -> Result { + Err("freedepth feature not enabled".to_string()) + } +} use std::path::PathBuf; @@ -194,3 +383,193 @@ pub fn get_backend() -> Box { pub fn get_default_backend() -> CameraBackendType { CameraBackendType::PipeWire } + +// ============================================================================ +// Runtime Depth Backend Selection +// ============================================================================ + +/// Depth backend type - kernel driver or userspace (freedepth) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DepthBackendType { + /// Use the V4L2 kernel driver with depth controls + /// Preferred when available - no USB unbind/rebind needed + KernelDriver, + /// Use freedepth userspace library + /// Fallback when kernel driver doesn't support depth controls + #[cfg(target_arch = "x86_64")] + Freedepth, +} + +impl std::fmt::Display for DepthBackendType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::KernelDriver => write!(f, "Kernel Driver"), + #[cfg(target_arch = "x86_64")] + Self::Freedepth => write!(f, "freedepth"), + } + } +} + +/// Detect the best depth backend for a V4L2 device +/// +/// Checks if the device has V4L2 depth control support (kernel driver). +/// If so, returns KernelDriver; otherwise returns Freedepth on x86_64. +/// +/// # Arguments +/// * `v4l2_path` - Path to the V4L2 device (e.g., "/dev/video2") +/// +/// # Returns +/// * `Some(DepthBackendType)` - The recommended backend +/// * `None` - Device is not a depth camera +pub fn detect_depth_backend(v4l2_path: &str) -> Option { + use tracing::info; + + // First check if this device has kernel depth controls + if v4l2_depth_controls::has_depth_controls(v4l2_path) { + info!( + path = %v4l2_path, + backend = "KernelDriver", + "Detected depth camera with kernel driver support" + ); + return Some(DepthBackendType::KernelDriver); + } + + // On x86_64, fall back to freedepth + #[cfg(target_arch = "x86_64")] + { + // Check if this is a Kinect device (by driver name) + if let Ok(file) = std::fs::File::open(v4l2_path) { + use std::os::unix::io::AsRawFd; + let fd = file.as_raw_fd(); + + // Query driver name via VIDIOC_QUERYCAP + #[repr(C)] + struct V4l2Capability { + driver: [u8; 16], + card: [u8; 32], + bus_info: [u8; 32], + version: u32, + capabilities: u32, + device_caps: u32, + reserved: [u32; 3], + } + + let mut caps = V4l2Capability { + driver: [0; 16], + card: [0; 32], + bus_info: [0; 32], + version: 0, + capabilities: 0, + device_caps: 0, + reserved: [0; 3], + }; + + const VIDIOC_QUERYCAP: libc::c_ulong = 0x8056_5600; + + if unsafe { libc::ioctl(fd, VIDIOC_QUERYCAP, &mut caps as *mut _) } == 0 { + let driver = std::str::from_utf8(&caps.driver) + .unwrap_or("") + .trim_end_matches('\0'); + + if driver == "kinect" || driver == "gspca_kinect" { + info!( + path = %v4l2_path, + driver = %driver, + backend = "freedepth", + "Detected Kinect without kernel depth controls, using freedepth" + ); + return Some(DepthBackendType::Freedepth); + } + } + } + } + + None +} + +/// Check if the kernel depth driver is available for any device +pub fn has_kernel_depth_driver() -> bool { + for entry in std::fs::read_dir("/dev").into_iter().flatten() { + if let Ok(entry) = entry { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with("video") { + if v4l2_depth_controls::has_depth_controls(&path.to_string_lossy()) { + return true; + } + } + } + } + } + false +} + +/// Enumerate depth cameras with runtime backend selection +/// +/// This function prioritizes the kernel driver over freedepth. When the kernel +/// driver is present, freedepth is NOT used at all - complete isolation. +/// +/// Returns a list of cameras with their paths indicating the appropriate backend: +/// - `v4l2-kinect:/dev/videoX:/dev/videoY` for kernel driver (color:depth pair) +/// - `kinect:N` for freedepth (only when kernel driver not available) +pub fn enumerate_depth_cameras_with_backend() -> Vec<(CameraDevice, DepthBackendType)> { + use tracing::info; + + let mut cameras = Vec::new(); + + // Priority 1: Check for kernel driver device pairs + let kernel_pairs = find_kernel_kinect_pairs(); + if !kernel_pairs.is_empty() { + info!( + count = kernel_pairs.len(), + "Found Kinect devices with kernel driver - NOT using freedepth" + ); + + for pair in kernel_pairs { + let device = CameraDevice { + name: format!("{} (Kernel)", pair.card_name), + path: format!( + "{}{}:{}", + KERNEL_KINECT_PREFIX, pair.color_path, pair.depth_path + ), + metadata_path: Some(pair.depth_path.clone()), + device_info: Some(DeviceInfo { + card: pair.card_name.clone(), + driver: "kinect".to_string(), + path: pair.color_path.clone(), + real_path: pair.depth_path.clone(), + }), + }; + cameras.push((device, DepthBackendType::KernelDriver)); + } + + // Return early - do NOT fall back to freedepth when kernel driver is present + return cameras; + } + + // Priority 2: Fall back to freedepth on x86_64 (only if NO kernel devices found) + #[cfg(target_arch = "x86_64")] + { + info!("No kernel Kinect driver found - falling back to freedepth"); + for cam in enumerate_depth_cameras() { + cameras.push((cam, DepthBackendType::Freedepth)); + } + } + + info!( + count = cameras.len(), + backend = if cameras.is_empty() { + "none" + } else if cameras + .iter() + .any(|(_, b)| matches!(b, DepthBackendType::KernelDriver)) + { + "kernel" + } else { + "freedepth" + }, + "Enumerated depth cameras with backend selection" + ); + + cameras +} diff --git a/src/backends/camera/motor_control.rs b/src/backends/camera/motor_control.rs new file mode 100644 index 00000000..03849d55 --- /dev/null +++ b/src/backends/camera/motor_control.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Unified Motor Control Abstraction +//! +//! This module provides a unified interface for controlling the Kinect motor +//! that works with both the kernel V4L2 driver and the freedepth userspace library. +//! +//! When the kernel driver is active, motor control uses V4L2 controls +//! (V4L2_CID_TILT_ABSOLUTE, V4L2_CID_TILT_RESET). +//! +//! When using freedepth, motor control goes through the freedepth USB interface. + +use tracing::{debug, info, warn}; + +/// Tilt angle limits (in degrees) +pub const TILT_MIN_DEGREES: i8 = -27; +pub const TILT_MAX_DEGREES: i8 = 27; + +/// Motor control backend type +#[derive(Debug, Clone)] +pub enum MotorBackend { + /// V4L2 kernel driver tilt control + KernelV4L2 { + /// Path to the V4L2 device that has tilt controls + device_path: String, + }, + /// freedepth USB motor control + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Freedepth, + /// No motor control available + None, +} + +/// Unified motor controller +/// +/// Provides a consistent interface for motor control regardless of whether +/// the kernel driver or freedepth is being used. +pub struct MotorController { + backend: MotorBackend, +} + +impl MotorController { + /// Create a motor controller for a kernel V4L2 device + pub fn for_kernel_device(device_path: &str) -> Self { + info!( + device_path, + "Creating motor controller for kernel V4L2 device" + ); + Self { + backend: MotorBackend::KernelV4L2 { + device_path: device_path.to_string(), + }, + } + } + + /// Create a motor controller using freedepth + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + pub fn for_freedepth() -> Self { + info!("Creating motor controller for freedepth"); + Self { + backend: MotorBackend::Freedepth, + } + } + + /// Create a no-op motor controller (when no motor control is available) + pub fn none() -> Self { + Self { + backend: MotorBackend::None, + } + } + + /// Get the current backend type + pub fn backend(&self) -> &MotorBackend { + &self.backend + } + + /// Set tilt angle (-27 to +27 degrees) + pub fn set_tilt(&self, degrees: i8) -> Result<(), String> { + let degrees = degrees.clamp(TILT_MIN_DEGREES, TILT_MAX_DEGREES); + + match &self.backend { + MotorBackend::KernelV4L2 { device_path } => set_tilt_v4l2(device_path, degrees), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + MotorBackend::Freedepth => set_tilt_freedepth(degrees), + MotorBackend::None => { + warn!("Motor control not available"); + Err("Motor control not available".to_string()) + } + } + } + + /// Get current tilt angle + pub fn get_tilt(&self) -> Result { + match &self.backend { + MotorBackend::KernelV4L2 { device_path } => get_tilt_v4l2(device_path), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + MotorBackend::Freedepth => get_tilt_freedepth(), + MotorBackend::None => Err("Motor control not available".to_string()), + } + } + + /// Reset tilt to center position + pub fn reset_tilt(&self) -> Result<(), String> { + match &self.backend { + MotorBackend::KernelV4L2 { device_path } => reset_tilt_v4l2(device_path), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + MotorBackend::Freedepth => { + // freedepth doesn't have a dedicated reset, just set to 0 + set_tilt_freedepth(0) + } + MotorBackend::None => Err("Motor control not available".to_string()), + } + } +} + +// V4L2 implementation +use super::v4l2_controls; + +fn set_tilt_v4l2(device_path: &str, degrees: i8) -> Result<(), String> { + debug!(device_path, degrees, "Setting tilt via V4L2"); + + // V4L2_CID_TILT_ABSOLUTE expects the value in degrees for the Kinect driver + v4l2_controls::set_control( + device_path, + v4l2_controls::V4L2_CID_TILT_ABSOLUTE, + degrees as i32, + ) + .map_err(|e| format!("Failed to set tilt: {}", e)) +} + +fn get_tilt_v4l2(device_path: &str) -> Result { + debug!(device_path, "Getting tilt via V4L2"); + + v4l2_controls::get_control(device_path, v4l2_controls::V4L2_CID_TILT_ABSOLUTE) + .map(|v| v as i8) + .ok_or_else(|| "Failed to get tilt".to_string()) +} + +fn reset_tilt_v4l2(device_path: &str) -> Result<(), String> { + debug!(device_path, "Resetting tilt via V4L2"); + + // V4L2_CID_TILT_RESET is a button control - write 1 to trigger + v4l2_controls::set_control(device_path, v4l2_controls::V4L2_CID_TILT_RESET, 1) + .map_err(|e| format!("Failed to reset tilt: {}", e)) +} + +// ============================================================================= +// freedepth USB Device Management +// ============================================================================= + +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +mod freedepth_motor { + use std::sync::{Arc, Mutex}; + use tracing::{debug, info}; + + /// Global USB device for motor control + /// Set when native depth backend starts, cleared when it stops + static MOTOR_USB_DEVICE: std::sync::OnceLock>>>> = + std::sync::OnceLock::new(); + + fn get_motor_usb() -> &'static Mutex>>> { + MOTOR_USB_DEVICE.get_or_init(|| Mutex::new(None)) + } + + /// Set the USB device for motor control (called when native backend starts) + pub fn set_motor_usb_device(usb: Arc>) { + if let Ok(mut guard) = get_motor_usb().lock() { + *guard = Some(usb); + info!("Motor control USB device set"); + } + } + + /// Clear the USB device for motor control (called when native backend stops) + pub fn clear_motor_usb_device() { + if let Ok(mut guard) = get_motor_usb().lock() { + *guard = None; + info!("Motor control USB device cleared"); + } + } + + /// Execute a closure with the motor USB device + fn with_motor_usb(f: F) -> Result + where + F: FnOnce(&mut freedepth::UsbDevice) -> freedepth::Result, + { + let guard = get_motor_usb() + .lock() + .map_err(|e| format!("Lock error: {}", e))?; + let usb_arc = guard.as_ref().ok_or("Motor USB device not available")?; + let mut usb = usb_arc + .lock() + .map_err(|e| format!("USB lock error: {}", e))?; + f(&mut usb).map_err(|e| e.to_string()) + } + + pub fn set_tilt(degrees: i8) -> Result<(), String> { + debug!(degrees, "Setting tilt via freedepth"); + with_motor_usb(|usb| freedepth::Motor::new(usb).set_tilt(degrees)) + } + + pub fn get_tilt() -> Result { + debug!("Getting tilt via freedepth"); + with_motor_usb(|usb| freedepth::Motor::new(usb).get_tilt()) + } + + pub fn is_available() -> bool { + get_motor_usb() + .lock() + .map(|guard| guard.is_some()) + .unwrap_or(false) + } +} + +// Re-export for external use +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub use freedepth_motor::{clear_motor_usb_device, set_motor_usb_device}; + +/// Global function to set motor tilt via freedepth +/// +/// This is a convenience function for handlers that need quick motor access +/// without maintaining a MotorController instance. +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub fn set_motor_tilt(degrees: i8) -> Result<(), String> { + let degrees = degrees.clamp(TILT_MIN_DEGREES, TILT_MAX_DEGREES); + freedepth_motor::set_tilt(degrees) +} + +/// Global function to get motor tilt via freedepth +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub fn get_motor_tilt() -> Result { + freedepth_motor::get_tilt() +} + +/// Check if motor control is available via freedepth +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub fn is_motor_available() -> bool { + freedepth_motor::is_available() +} + +// freedepth implementation +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +fn set_tilt_freedepth(degrees: i8) -> Result<(), String> { + freedepth_motor::set_tilt(degrees) +} + +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +fn get_tilt_freedepth() -> Result { + freedepth_motor::get_tilt() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tilt_clamping() { + // Tilt values should be clamped to valid range + assert_eq!(30_i8.clamp(TILT_MIN_DEGREES, TILT_MAX_DEGREES), 27); + assert_eq!((-30_i8).clamp(TILT_MIN_DEGREES, TILT_MAX_DEGREES), -27); + assert_eq!(0_i8.clamp(TILT_MIN_DEGREES, TILT_MAX_DEGREES), 0); + } +} diff --git a/src/backends/camera/pipewire/enumeration.rs b/src/backends/camera/pipewire/enumeration.rs index c1d2b913..5ed9af91 100644 --- a/src/backends/camera/pipewire/enumeration.rs +++ b/src/backends/camera/pipewire/enumeration.rs @@ -5,7 +5,7 @@ //! This module provides camera discovery and format enumeration using PipeWire. //! PipeWire handles all camera access, format negotiation, and decoding internally. -use super::super::types::{CameraDevice, CameraFormat, DeviceInfo}; +use super::super::types::{CameraDevice, CameraFormat, DeviceInfo, SensorType}; use crate::constants::formats; use tracing::{debug, info, warn}; @@ -100,13 +100,18 @@ fn try_enumerate_with_pw_cli() -> Option> { current_object_path.as_deref(), ); - debug!(id = %id, serial = ?current_serial, name = %name, path = %path, "Found video camera"); - cameras.push(CameraDevice { - name: name.clone(), - path, - metadata_path: Some(id.clone()), // Store node ID in metadata_path for format enumeration - device_info, - }); + // Skip Kinect devices - they should use the native backend + if is_kinect_v4l2_device(&device_info) { + debug!(name = %name, "Skipping Kinect V4L2 device (use native backend instead)"); + } else { + debug!(id = %id, serial = ?current_serial, name = %name, path = %path, "Found video camera"); + cameras.push(CameraDevice { + name: name.clone(), + path, + metadata_path: Some(id.clone()), // Store node ID in metadata_path for format enumeration + device_info, + }); + } } } } @@ -185,13 +190,18 @@ fn try_enumerate_with_pw_cli() -> Option> { let device_info = build_device_info(current_nick.as_deref(), current_object_path.as_deref()); - debug!(id = %id, serial = ?current_serial, name = %name, path = %path, "Found video camera (last)"); - cameras.push(CameraDevice { - name: name.clone(), - path, - metadata_path: Some(id.clone()), // Store node ID in metadata_path for format enumeration - device_info, - }); + // Skip Kinect devices - they should use the native backend + if is_kinect_v4l2_device(&device_info) { + debug!(name = %name, "Skipping Kinect V4L2 device (use native backend instead)"); + } else { + debug!(id = %id, serial = ?current_serial, name = %name, path = %path, "Found video camera (last)"); + cameras.push(CameraDevice { + name: name.clone(), + path, + metadata_path: Some(id.clone()), // Store node ID in metadata_path for format enumeration + device_info, + }); + } } } } @@ -241,6 +251,21 @@ fn build_device_info(nick: Option<&str>, object_path: Option<&str>) -> Option) -> bool { + if let Some(info) = device_info { + // Check for Kinect V4L2 drivers + let driver = info.driver.to_lowercase(); + if driver == "gspca_kinect" || driver == "kinect" { + debug!(driver = %info.driver, path = %info.path, "Filtering out Kinect V4L2 device (use native backend)"); + return true; + } + } + false +} + /// Get V4L2 driver name using ioctl fn get_v4l2_driver(device_path: &str) -> Option { use std::os::unix::io::AsRawFd; @@ -377,6 +402,7 @@ fn get_fallback_formats() -> Vec { framerate: Some(fps), hardware_accelerated: true, pixel_format: "MJPG".to_string(), + sensor_type: SensorType::Rgb, }); } } @@ -467,7 +493,7 @@ fn try_enumerate_formats_from_node(node_id: &str) -> Option> { // Determine the pixel format string: // - For raw formats: use VideoFormat (YUY2, NV12, etc.) // - For compressed formats: use MediaSubtype (MJPG, H264, etc.) - let pixel_format = if subtype == "raw" { + let mut pixel_format = if subtype == "raw" { current_video_format .clone() .unwrap_or_else(|| "YUY2".to_string()) @@ -475,17 +501,36 @@ fn try_enumerate_formats_from_node(node_id: &str) -> Option> { subtype.to_uppercase() }; - // Create a format for each framerate - for &fps in ¤t_framerates { - formats.push(CameraFormat { - width: w, - height: h, - framerate: Some(fps), - hardware_accelerated: pixel_format == "MJPG", // MJPEG is hardware accelerated - pixel_format: pixel_format.clone(), - }); + // Detect Y10B depth sensor by resolution (640x488 or 1280x1024 with UNKNOWN format) + // The Kinect depth sensor reports VideoFormat:UNKNOWN for Y10B + if pixel_format == "UNKNOWN" && (w == 640 && h == 488 || w == 1280 && h == 1024) { + pixel_format = "Y10B".to_string(); + debug!(width = w, height = h, "Detected Y10B depth sensor format"); + } + + // Only add formats with supported pixel formats + if is_supported_pixel_format(&pixel_format) { + // Determine sensor type based on pixel format + let sensor_type = if pixel_format == "Y10B" { + SensorType::Depth + } else { + SensorType::Rgb + }; + + for &fps in ¤t_framerates { + formats.push(CameraFormat { + width: w, + height: h, + framerate: Some(fps), + hardware_accelerated: pixel_format == "MJPG", // MJPEG is hardware accelerated + pixel_format: pixel_format.clone(), + sensor_type, + }); + } + debug!(width = w, height = h, pixel_format = %pixel_format, ?sensor_type, framerates = current_framerates.len(), "Completed format group"); + } else { + debug!(width = w, height = h, pixel_format = %pixel_format, "Skipping unsupported format"); } - debug!(width = w, height = h, pixel_format = %pixel_format, framerates = current_framerates.len(), "Completed format group"); } current_width = None; current_height = None; @@ -499,7 +544,7 @@ fn try_enumerate_formats_from_node(node_id: &str) -> Option> { if !current_framerates.is_empty() { if let (Some(w), Some(h), Some(subtype)) = (current_width, current_height, ¤t_subtype) { - let pixel_format = if subtype == "raw" { + let mut pixel_format = if subtype == "raw" { current_video_format .clone() .unwrap_or_else(|| "YUY2".to_string()) @@ -507,14 +552,37 @@ fn try_enumerate_formats_from_node(node_id: &str) -> Option> { subtype.to_uppercase() }; - for &fps in ¤t_framerates { - formats.push(CameraFormat { - width: w, - height: h, - framerate: Some(fps), - hardware_accelerated: pixel_format == "MJPG", - pixel_format: pixel_format.clone(), - }); + // Detect Y10B depth sensor by resolution + if pixel_format == "UNKNOWN" && (w == 640 && h == 488 || w == 1280 && h == 1024) { + pixel_format = "Y10B".to_string(); + debug!( + width = w, + height = h, + "Detected Y10B depth sensor format (last)" + ); + } + + // Only add formats with supported pixel formats + if is_supported_pixel_format(&pixel_format) { + // Determine sensor type based on pixel format + let sensor_type = if pixel_format == "Y10B" { + SensorType::Depth + } else { + SensorType::Rgb + }; + + for &fps in ¤t_framerates { + formats.push(CameraFormat { + width: w, + height: h, + framerate: Some(fps), + hardware_accelerated: pixel_format == "MJPG", + pixel_format: pixel_format.clone(), + sensor_type, + }); + } + } else { + debug!(width = w, height = h, pixel_format = %pixel_format, "Skipping unsupported format (last)"); } } } @@ -526,6 +594,16 @@ fn try_enumerate_formats_from_node(node_id: &str) -> Option> { } } +/// Check if a pixel format is supported by the pipeline +/// +/// Filters out formats that the GStreamer pipeline cannot handle, +/// such as Y10B (10-bit packed grayscale) and raw Bayer patterns. +fn is_supported_pixel_format(pixel_format: &str) -> bool { + use crate::media::Codec; + let codec = Codec::from_fourcc(pixel_format); + codec != Codec::Unknown +} + /// Test if PipeWire is available and working pub fn is_pipewire_available() -> bool { if gstreamer::init().is_err() { diff --git a/src/backends/camera/pipewire/mod.rs b/src/backends/camera/pipewire/mod.rs index 1651798b..1117251c 100644 --- a/src/backends/camera/pipewire/mod.rs +++ b/src/backends/camera/pipewire/mod.rs @@ -6,6 +6,14 @@ //! //! This backend uses PipeWire for camera enumeration, format detection, and capture. //! It's the modern, recommended approach for Linux camera access. +//! +//! ## Depth Camera Device Handling +//! +//! Depth cameras are automatically routed to the native freedepth backend: +//! - Depth cameras are enumerated via freedepth (not V4L2/PipeWire) +//! - When a depth camera is selected, NativeDepthBackend is used +//! - This provides simultaneous RGB + depth streaming at 30fps +//! - V4L2 kernel driver is NOT used for depth cameras mod enumeration; mod pipeline; @@ -14,22 +22,52 @@ pub use enumeration::{enumerate_pipewire_cameras, get_pipewire_formats, is_pipew pub use pipeline::PipeWirePipeline; use super::CameraBackend; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use super::depth_controller::DepthController; use super::types::*; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use super::v4l2_depth::{DepthFrameReceiver, DepthFrameSender, V4l2DepthPipeline}; +use super::{NativeDepthBackend, is_depth_native_device}; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use super::{enumerate_depth_cameras, get_depth_formats}; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use crate::shaders::depth::{is_depth_colormap_enabled, is_depth_only_mode, unpack_y10b_gpu}; use std::path::PathBuf; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use std::sync::Arc; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +use tracing::warn; use tracing::{debug, info}; +use super::V4l2KernelDepthBackend; + +/// Active capture pipeline (GStreamer, V4L2 depth, or native depth camera) +enum ActivePipeline { + /// Standard GStreamer pipeline via PipeWire + GStreamer(pipeline::PipeWirePipeline), + /// Direct V4L2 depth capture for Y10B format (freedepth only) + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Depth(V4l2DepthPipeline), + /// Native depth camera backend (freedepth - bypasses V4L2 entirely) + DepthCamera(NativeDepthBackend), + /// V4L2 kernel depth backend (uses kernel driver with depth controls) + KernelDepth(V4l2KernelDepthBackend), +} + /// PipeWire backend implementation pub struct PipeWireBackend { /// Currently active camera device (if initialized) current_device: Option, /// Currently active format (if initialized) current_format: Option, - /// Active GStreamer pipeline for preview - pipeline: Option, + /// Active capture pipeline + active_pipeline: Option, /// Frame sender for preview stream frame_sender: Option, /// Frame receiver for preview stream (given to UI) frame_receiver: Option, + /// Depth frame processing task handle (for Y10B) + depth_task_handle: Option>, } impl PipeWireBackend { @@ -38,53 +76,289 @@ impl PipeWireBackend { Self { current_device: None, current_format: None, - pipeline: None, + active_pipeline: None, frame_sender: None, frame_receiver: None, + depth_task_handle: None, } } + /// Check if format is Y10B depth sensor + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + fn is_depth_format(format: &CameraFormat) -> bool { + format.pixel_format == "Y10B" + } + /// Internal method to create preview pipeline fn create_pipeline(&mut self) -> BackendResult<()> { + use super::is_kernel_depth_device; + let device = self .current_device - .as_ref() + .clone() .ok_or_else(|| BackendError::Other("No device set".to_string()))?; let format = self .current_format - .as_ref() + .clone() .ok_or_else(|| BackendError::Other("No format set".to_string()))?; + debug!( + device_path = %device.path, + is_kernel_depth = is_kernel_depth_device(&device.path), + is_freedepth = is_depth_native_device(&device.path), + "create_pipeline checking device type" + ); + + // Check if this is a kernel depth device - use V4L2 kernel backend + if is_kernel_depth_device(&device.path) { + info!(device = %device.name, "Using V4L2 kernel depth backend"); + return self.create_kernel_depth_pipeline(&device, &format); + } + + // Check if this is a freedepth depth camera device (only when no kernel driver) + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + if is_depth_native_device(&device.path) { + info!(device = %device.name, "Using native depth camera backend (freedepth)"); + return self.create_depth_camera_pipeline(&device, &format); + } + + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + { + if Self::is_depth_format(&format) { + // Y10B depth format - use V4L2 direct capture + return self.create_depth_pipeline(&device, &format); + } + } + // Standard format - use GStreamer via PipeWire + self.create_gstreamer_pipeline(&device, &format) + } + + /// Create kernel depth camera pipeline via V4L2 with depth controls + fn create_kernel_depth_pipeline( + &mut self, + device: &CameraDevice, + format: &CameraFormat, + ) -> BackendResult<()> { + use super::V4l2KernelDepthBackend; + + info!( + device = %device.name, + format = %format, + "Creating kernel depth camera pipeline" + ); + + // Create kernel depth backend + let mut kernel_backend = V4l2KernelDepthBackend::new(); + kernel_backend.initialize(device, format)?; + + // Use the KernelDepth pipeline variant + self.active_pipeline = Some(ActivePipeline::KernelDepth(kernel_backend)); + + info!("Kernel depth camera pipeline created successfully"); + Ok(()) + } + + /// Create native depth camera pipeline via freedepth + fn create_depth_camera_pipeline( + &mut self, + device: &CameraDevice, + format: &CameraFormat, + ) -> BackendResult<()> { + info!( + device = %device.name, + format = %format, + "Creating native depth camera pipeline via freedepth" + ); + + // Create native depth camera backend + let mut depth_backend = NativeDepthBackend::new(); + depth_backend.initialize(device, format)?; + + self.active_pipeline = Some(ActivePipeline::DepthCamera(depth_backend)); + + info!("Native depth camera pipeline created successfully"); + Ok(()) + } + + /// Create GStreamer pipeline for standard formats + fn create_gstreamer_pipeline( + &mut self, + device: &CameraDevice, + format: &CameraFormat, + ) -> BackendResult<()> { // Create frame channel let (sender, receiver) = cosmic::iced::futures::channel::mpsc::channel(100); // Create pipeline let pipeline = pipeline::PipeWirePipeline::new(device, format, sender.clone())?; - pipeline.start()?; - self.pipeline = Some(pipeline); + self.active_pipeline = Some(ActivePipeline::GStreamer(pipeline)); self.frame_sender = Some(sender); self.frame_receiver = Some(receiver); Ok(()) } + + /// Create V4L2 depth pipeline for Y10B format + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + fn create_depth_pipeline( + &mut self, + device: &CameraDevice, + format: &CameraFormat, + ) -> BackendResult<()> { + info!( + device = %device.name, + format = %format, + "Creating V4L2 depth pipeline for Y10B format" + ); + + // Create channels for depth frames and processed frames + let (depth_sender, mut depth_receiver): (DepthFrameSender, DepthFrameReceiver) = + cosmic::iced::futures::channel::mpsc::channel(10); + let (frame_sender, frame_receiver): (FrameSender, FrameReceiver) = + cosmic::iced::futures::channel::mpsc::channel(100); + + // Create V4L2 depth pipeline + let depth_pipeline = V4l2DepthPipeline::new(device, format, depth_sender)?; + + // Spawn task to process depth frames with GPU shader + let width = format.width; + let height = format.height; + let mut frame_sender_clone = frame_sender.clone(); + + let task_handle = tokio::spawn(async move { + use futures::StreamExt; + + info!("Depth frame processing task started"); + + while let Some(depth_frame) = depth_receiver.next().await { + // Process with GPU shader + // Check visualization modes from shared state + let use_colormap = is_depth_colormap_enabled(); + let depth_only = is_depth_only_mode(); + match unpack_y10b_gpu( + &depth_frame.raw_data, + width, + height, + use_colormap, + depth_only, + ) + .await + { + Ok(result) => { + // Apply depth camera calibration if available to convert raw depth to mm + let calibrated_depth = if DepthController::is_initialized() { + // Convert raw 10-bit values (shifted to 16-bit) to millimeters + DepthController::convert_depth_to_mm(&result.depth_u16) + .map(|mm_values| Arc::from(mm_values.into_boxed_slice())) + } else { + // No calibration available, use raw values + Some(Arc::from(result.depth_u16.into_boxed_slice())) + }; + + // Create CameraFrame with RGBA preview and depth data + let camera_frame = CameraFrame { + width: result.width, + height: result.height, + data: Arc::from(result.rgba_preview.into_boxed_slice()), + format: PixelFormat::Depth16, + stride: result.width * 4, + captured_at: depth_frame.captured_at, + depth_data: calibrated_depth, + depth_width: result.width, + depth_height: result.height, + video_timestamp: None, + }; + + // Send to preview channel + if frame_sender_clone.try_send(camera_frame).is_err() { + debug!("Preview channel full, dropping depth frame"); + } + } + Err(e) => { + warn!(error = %e, "Failed to unpack Y10B frame with GPU"); + } + } + } + + info!("Depth frame processing task ended"); + }); + + self.active_pipeline = Some(ActivePipeline::Depth(depth_pipeline)); + self.frame_sender = Some(frame_sender); + self.frame_receiver = Some(frame_receiver); + self.depth_task_handle = Some(task_handle); + + info!("V4L2 depth pipeline created successfully"); + Ok(()) + } } impl CameraBackend for PipeWireBackend { fn enumerate_cameras(&self) -> Vec { - info!("Using PipeWire backend for camera enumeration"); - - if let Some(cameras) = enumerate_pipewire_cameras() { - info!(count = cameras.len(), "PipeWire cameras enumerated"); - cameras + use super::{V4l2KernelDepthBackend, has_kernel_depth_driver}; + + info!("Enumerating cameras (PipeWire + depth cameras)"); + + let mut cameras = Vec::new(); + + // Check if kernel depth driver is available - if so, use it exclusively + // and skip freedepth entirely + if has_kernel_depth_driver() { + info!("Kernel depth driver detected - using V4L2 kernel backend for depth cameras"); + let kernel_backend = V4l2KernelDepthBackend::new(); + let kernel_cameras = kernel_backend.enumerate_cameras(); + if !kernel_cameras.is_empty() { + info!( + count = kernel_cameras.len(), + "Found depth cameras via kernel driver" + ); + cameras.extend(kernel_cameras); + } } else { - info!("PipeWire enumeration returned None"); - Vec::new() + // No kernel driver - use freedepth on x86_64 (if feature enabled) + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + { + let depth_cameras = enumerate_depth_cameras(); + if !depth_cameras.is_empty() { + info!( + count = depth_cameras.len(), + "Found depth cameras via freedepth (no kernel driver)" + ); + cameras.extend(depth_cameras); + } + } + } + + // Then enumerate other cameras via PipeWire (excludes depth camera V4L2 devices) + if let Some(pw_cameras) = enumerate_pipewire_cameras() { + info!(count = pw_cameras.len(), "Found cameras via PipeWire"); + cameras.extend(pw_cameras); } + + info!(total = cameras.len(), "Total cameras enumerated"); + cameras } fn get_formats(&self, device: &CameraDevice, _video_mode: bool) -> Vec { + use super::{V4l2KernelDepthBackend, is_kernel_depth_device}; + + // Check for kernel depth device first + if is_kernel_depth_device(&device.path) { + info!(device_path = %device.path, "Getting formats for kernel depth camera device"); + let kernel_backend = V4l2KernelDepthBackend::new(); + return kernel_backend.get_formats(device, _video_mode); + } + + // Use native formats for freedepth depth camera devices (only if no kernel driver) + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + if is_depth_native_device(&device.path) { + info!(device_path = %device.path, "Getting formats for freedepth depth camera device"); + return get_depth_formats(device); + } + + // Use PipeWire formats for other devices info!(device_path = %device.path, "Getting formats via PipeWire backend"); get_pipewire_formats(&device.path, device.metadata_path.as_deref()) } @@ -113,11 +387,26 @@ impl CameraBackend for PipeWireBackend { } fn shutdown(&mut self) -> BackendResult<()> { - info!("Shutting down PipeWire backend"); + info!("Shutting down backend"); + + // Cancel depth processing task if running + if let Some(handle) = self.depth_task_handle.take() { + handle.abort(); + } // Stop pipeline - if let Some(pipeline) = self.pipeline.take() { - pipeline.stop()?; + if let Some(pipeline) = self.active_pipeline.take() { + match pipeline { + ActivePipeline::GStreamer(p) => p.stop()?, + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + ActivePipeline::Depth(p) => p.stop()?, + ActivePipeline::DepthCamera(mut d) => { + d.shutdown()?; + } + ActivePipeline::KernelDepth(mut k) => { + k.shutdown()?; + } + } } // Clear state @@ -126,12 +415,12 @@ impl CameraBackend for PipeWireBackend { self.current_device = None; self.current_format = None; - info!("PipeWire backend shut down"); + info!("Backend shut down"); Ok(()) } fn is_initialized(&self) -> bool { - self.pipeline.is_some() && self.current_device.is_some() + self.active_pipeline.is_some() && self.current_device.is_some() } fn recover(&mut self) -> BackendResult<()> { @@ -187,43 +476,71 @@ impl CameraBackend for PipeWireBackend { } fn capture_photo(&self) -> BackendResult { - debug!("Capturing photo via PipeWire backend"); - - let pipeline = self - .pipeline - .as_ref() - .ok_or_else(|| BackendError::Other("Pipeline not initialized".to_string()))?; - - pipeline.capture_frame() + debug!("Capturing photo"); + + match &self.active_pipeline { + Some(ActivePipeline::GStreamer(pipeline)) => pipeline.capture_frame(), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Some(ActivePipeline::Depth(_)) => { + // For depth sensor, capture is handled via the frame stream + // The UI should use the last received frame + Err(BackendError::Other( + "Use frame stream for depth capture".to_string(), + )) + } + Some(ActivePipeline::DepthCamera(depth)) => { + // Capture from native depth camera backend + depth.capture_photo() + } + Some(ActivePipeline::KernelDepth(kernel)) => { + // Capture from kernel depth camera backend + kernel.capture_photo() + } + None => Err(BackendError::Other("Pipeline not initialized".to_string())), + } } fn start_recording(&mut self, output_path: PathBuf) -> BackendResult<()> { info!(path = %output_path.display(), "Starting recording"); - let pipeline = self - .pipeline - .as_mut() - .ok_or_else(|| BackendError::Other("Pipeline not initialized".to_string()))?; - - pipeline.start_recording(output_path) + match &mut self.active_pipeline { + Some(ActivePipeline::GStreamer(pipeline)) => pipeline.start_recording(output_path), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Some(ActivePipeline::Depth(_)) => Err(BackendError::Other( + "Video recording not supported for depth sensor".to_string(), + )), + Some(ActivePipeline::DepthCamera(_)) => Err(BackendError::Other( + "Video recording not yet implemented for native depth camera backend".to_string(), + )), + Some(ActivePipeline::KernelDepth(_)) => Err(BackendError::Other( + "Video recording not yet implemented for kernel depth camera backend".to_string(), + )), + None => Err(BackendError::Other("Pipeline not initialized".to_string())), + } } fn stop_recording(&mut self) -> BackendResult { info!("Stopping recording"); - let pipeline = self - .pipeline - .as_mut() - .ok_or_else(|| BackendError::Other("Pipeline not initialized".to_string()))?; - - pipeline.stop_recording() + match &mut self.active_pipeline { + Some(ActivePipeline::GStreamer(pipeline)) => pipeline.stop_recording(), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Some(ActivePipeline::Depth(_)) => Err(BackendError::NoRecordingInProgress), + Some(ActivePipeline::DepthCamera(_)) => Err(BackendError::NoRecordingInProgress), + Some(ActivePipeline::KernelDepth(_)) => Err(BackendError::NoRecordingInProgress), + None => Err(BackendError::Other("Pipeline not initialized".to_string())), + } } fn is_recording(&self) -> bool { - self.pipeline - .as_ref() - .map(|p| p.is_recording()) - .unwrap_or(false) + match &self.active_pipeline { + Some(ActivePipeline::GStreamer(pipeline)) => pipeline.is_recording(), + #[cfg(all(target_arch = "x86_64", feature = "freedepth"))] + Some(ActivePipeline::Depth(_)) => false, + Some(ActivePipeline::DepthCamera(_)) => false, + Some(ActivePipeline::KernelDepth(_)) => false, + None => false, + } } fn get_preview_receiver(&self) -> Option { diff --git a/src/backends/camera/pipewire/pipeline.rs b/src/backends/camera/pipewire/pipeline.rs index d82d7893..5215f4d1 100644 --- a/src/backends/camera/pipewire/pipeline.rs +++ b/src/backends/camera/pipewire/pipeline.rs @@ -265,6 +265,10 @@ impl PipeWirePipeline { format: PixelFormat::RGBA, // Pipeline outputs RGBA stride, captured_at: frame_start, // Use frame_start as capture timestamp + depth_data: None, // Regular RGB frames have no depth + depth_width: 0, + depth_height: 0, + video_timestamp: None, // PipeWire doesn't provide Kinect-style timestamps }; // Send frame to the app (non-blocking using try_send) diff --git a/src/backends/camera/types.rs b/src/backends/camera/types.rs index 24b6ef2b..2e92d77a 100644 --- a/src/backends/camera/types.rs +++ b/src/backends/camera/types.rs @@ -49,6 +49,28 @@ pub struct CameraDevice { pub device_info: Option, // V4L2 device information (card, driver, path, real_path) } +/// Type of sensor (distinguishes RGB camera from depth/IR sensors) +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SensorType { + /// Standard RGB camera (video/photo) + #[default] + Rgb, + /// Depth sensor (e.g., Kinect Y10B) + Depth, + /// Infrared sensor (e.g., Kinect IR) + Ir, +} + +impl std::fmt::Display for SensorType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SensorType::Rgb => write!(f, "Camera"), + SensorType::Depth => write!(f, "Depth"), + SensorType::Ir => write!(f, "IR"), + } + } +} + /// Camera format specification #[derive(Debug, Clone, PartialEq, Eq)] pub struct CameraFormat { @@ -57,6 +79,7 @@ pub struct CameraFormat { pub framerate: Option, // None for photo mode pub hardware_accelerated: bool, // True for MJPEG and raw formats with HW support pub pixel_format: String, // FourCC code (e.g., "MJPG", "H264", "YUYV") + pub sensor_type: SensorType, // RGB camera vs depth sensor } impl std::fmt::Display for CameraFormat { @@ -75,6 +98,9 @@ pub enum PixelFormat { /// RGBA - 32-bit with alpha (4 bytes per pixel) /// This is the native format used throughout the pipeline RGBA, + /// Depth16 - 16-bit grayscale depth data + /// Used for depth sensors like the Kinect Y10B format + Depth16, } /// A single frame from the camera @@ -82,10 +108,20 @@ pub enum PixelFormat { pub struct CameraFrame { pub width: u32, pub height: u32, - pub data: Arc<[u8]>, // Zero-copy frame data (RGBA format) - pub format: PixelFormat, // Pixel format of the data (always RGBA) + pub data: Arc<[u8]>, // Zero-copy frame data (RGBA format for preview) + pub format: PixelFormat, // Pixel format of the data (RGBA or Depth16) pub stride: u32, // Row stride (bytes per row, may include padding) pub captured_at: Instant, // Timestamp when frame was captured (for latency diagnostics) + /// Optional 16-bit depth data for depth sensor frames + /// Contains the full precision depth values when available + pub depth_data: Option>, + /// Depth dimensions (may differ from RGB width/height) + /// Only set when depth_data is Some + pub depth_width: u32, + pub depth_height: u32, + /// Video frame timestamp from hardware (for synchronizing depth/color at different frame rates) + /// Only used by Kinect native backend where depth (30fps) and color (10fps high-res) differ + pub video_timestamp: Option, } /// Frame receiver type for preview streams @@ -137,3 +173,36 @@ impl std::fmt::Display for BackendError { } impl std::error::Error for BackendError {} + +/// Processed video/color frame data ready for rendering +/// +/// Used by depth camera backends to provide color frame output. +/// Note: Pixel format varies by backend: +/// - NativeDepthBackend: RGB (3 bytes per pixel) +/// - V4l2KernelDepthBackend: RGBA (4 bytes per pixel) +#[derive(Debug, Clone)] +pub struct VideoFrameData { + /// Frame width + pub width: u32, + /// Frame height + pub height: u32, + /// Pixel data (format varies by backend) + pub data: Vec, + /// Frame timestamp + pub timestamp: u32, +} + +/// Processed depth frame data ready for 3D rendering +/// +/// Used by depth camera backends to provide depth frame output +#[derive(Debug, Clone)] +pub struct DepthFrameData { + /// Frame width + pub width: u32, + /// Frame height + pub height: u32, + /// Depth values in millimeters (u16 per pixel) + pub depth_mm: Vec, + /// Frame timestamp + pub timestamp: u32, +} diff --git a/src/backends/camera/v4l2_depth.rs b/src/backends/camera/v4l2_depth.rs new file mode 100644 index 00000000..a8b582a0 --- /dev/null +++ b/src/backends/camera/v4l2_depth.rs @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(all(target_arch = "x86_64", feature = "freedepth"))] + +//! Direct V4L2 depth sensor capture for Y10B format +//! +//! GStreamer doesn't support Y10B (10-bit packed grayscale) format, +//! so we use the v4l crate to capture raw bytes directly from the +//! Kinect depth sensor and process them with WGPU shaders. + +use super::types::*; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Instant; +use tracing::{debug, error, info, warn}; +use v4l::buffer::Type; +use v4l::io::traits::CaptureStream; +use v4l::prelude::*; +use v4l::video::Capture; + +// Import depth format constants from freedepth +use freedepth::y10b_packed_size; + +/// Raw depth frame from Y10B capture +#[derive(Debug, Clone)] +pub struct DepthFrame { + /// Frame width in pixels + pub width: u32, + /// Frame height in pixels + pub height: u32, + /// Raw Y10B packed data (4 pixels = 5 bytes) + pub raw_data: Arc<[u8]>, + /// Frame sequence number + pub sequence: u32, + /// Timestamp when frame was captured + pub captured_at: Instant, +} + +/// Depth frame sender type +pub type DepthFrameSender = cosmic::iced::futures::channel::mpsc::Sender; + +/// Depth frame receiver type +pub type DepthFrameReceiver = cosmic::iced::futures::channel::mpsc::Receiver; + +/// Y10B depth sensor pipeline using direct V4L2 capture +pub struct V4l2DepthPipeline { + device_path: String, + width: u32, + height: u32, + running: Arc, + thread_handle: Option>, +} + +impl V4l2DepthPipeline { + /// Create a new V4L2 depth capture pipeline + /// + /// # Arguments + /// * `device` - Camera device (must have a valid V4L2 path in device_info) + /// * `format` - Camera format (must be Y10B pixel format) + /// * `depth_sender` - Channel to send captured depth frames + pub fn new( + device: &CameraDevice, + format: &CameraFormat, + depth_sender: DepthFrameSender, + ) -> BackendResult { + // Get the V4L2 device path + let v4l2_path = device + .device_info + .as_ref() + .map(|info| info.real_path.clone()) + .or_else(|| { + // Try to extract from path if it's a v4l2: prefixed path + if device.path.starts_with("v4l2:") { + Some( + device + .path + .strip_prefix("v4l2:") + .unwrap_or(&device.path) + .to_string(), + ) + } else if device.path.starts_with("/dev/video") { + Some(device.path.clone()) + } else { + None + } + }) + .ok_or_else(|| { + BackendError::DeviceNotFound( + "No V4L2 device path available for depth sensor".to_string(), + ) + })?; + + info!( + device_path = %v4l2_path, + width = format.width, + height = format.height, + "Creating V4L2 depth capture pipeline for Y10B format" + ); + + let width = format.width; + let height = format.height; + let running = Arc::new(AtomicBool::new(true)); + let running_clone = running.clone(); + + // Spawn capture thread + let device_path_clone = v4l2_path.clone(); + let thread_handle = std::thread::spawn(move || { + if let Err(e) = capture_loop( + &device_path_clone, + width, + height, + depth_sender, + running_clone, + ) { + error!(error = %e, "Depth capture loop failed"); + } + }); + + Ok(Self { + device_path: v4l2_path, + width, + height, + running, + thread_handle: Some(thread_handle), + }) + } + + /// Stop the depth capture pipeline + pub fn stop(mut self) -> BackendResult<()> { + info!("Stopping V4L2 depth pipeline"); + self.running.store(false, Ordering::SeqCst); + + if let Some(handle) = self.thread_handle.take() { + match handle.join() { + Ok(_) => info!("Depth capture thread stopped"), + Err(_) => warn!("Depth capture thread panicked"), + } + } + + Ok(()) + } +} + +impl Drop for V4l2DepthPipeline { + fn drop(&mut self) { + info!("Dropping V4L2 depth pipeline"); + self.running.store(false, Ordering::SeqCst); + // Don't wait for thread in drop - it may already be finished + } +} + +/// Main capture loop running in a separate thread +fn capture_loop( + device_path: &str, + width: u32, + height: u32, + mut frame_sender: DepthFrameSender, + running: Arc, +) -> Result<(), Box> { + static FRAME_COUNTER: AtomicU64 = AtomicU64::new(0); + + info!( + device_path, + width, height, "Opening V4L2 device for Y10B capture" + ); + + // Open the device + let mut dev = Device::with_path(device_path) + .map_err(|e| format!("Failed to open V4L2 device {}: {}", device_path, e))?; + + // Query current format to see what the device supports + let current_format = dev + .format() + .map_err(|e| format!("Failed to query format: {}", e))?; + info!( + current_width = current_format.width, + current_height = current_format.height, + fourcc = ?current_format.fourcc, + "Current device format" + ); + + // Create Y10B FourCC: 'Y' '1' '0' 'B' = 0x42303159 + let y10b_fourcc = v4l::FourCC::new(b"Y10B"); + info!(?y10b_fourcc, "Creating Y10B format"); + + // Set format to Y10B with specified dimensions + let mut format = dev + .format() + .map_err(|e| format!("Failed to get format: {}", e))?; + format.width = width; + format.height = height; + format.fourcc = y10b_fourcc; + + // Try to set the format + match dev.set_format(&format) { + Ok(f) => { + info!( + width = f.width, + height = f.height, + fourcc = ?f.fourcc, + "Set V4L2 format" + ); + // Verify we got Y10B + if f.fourcc != y10b_fourcc { + warn!( + expected = ?y10b_fourcc, + got = ?f.fourcc, + "Device did not accept Y10B format, depth capture may not work correctly" + ); + } + } + Err(e) => { + warn!(error = %e, "Could not set format, using current device format"); + } + } + + // Calculate expected frame size for Y10B (10 bits per pixel, packed) + let expected_size = y10b_packed_size(width, height); + info!(expected_size, "Expected Y10B frame size"); + + // Create memory-mapped stream with 4 buffers + let mut stream = MmapStream::with_buffers(&mut dev, Type::VideoCapture, 4) + .map_err(|e| format!("Failed to create buffer stream: {}", e))?; + + info!("V4L2 depth capture stream started"); + + // Capture loop + while running.load(Ordering::SeqCst) { + let frame_start = Instant::now(); + + match stream.next() { + Ok((buf, meta)) => { + let frame_num = FRAME_COUNTER.fetch_add(1, Ordering::Relaxed); + + // Validate buffer size + if buf.len() != expected_size { + if frame_num % 30 == 0 { + warn!( + frame = frame_num, + got = buf.len(), + expected = expected_size, + "Unexpected buffer size" + ); + } + } + + // Create depth frame + let depth_frame = DepthFrame { + width, + height, + raw_data: Arc::from(buf), + sequence: meta.sequence, + captured_at: frame_start, + }; + + // Send frame (non-blocking) + match frame_sender.try_send(depth_frame) { + Ok(_) => { + if frame_num % 60 == 0 { + let elapsed = frame_start.elapsed(); + debug!( + frame = frame_num, + sequence = meta.sequence, + size = buf.len(), + elapsed_us = elapsed.as_micros(), + "Depth frame captured" + ); + } + } + Err(e) => { + if frame_num % 30 == 0 { + debug!(frame = frame_num, error = ?e, "Depth frame dropped (channel full)"); + } + } + } + } + Err(e) => { + warn!(error = %e, "Failed to capture depth frame"); + // Brief sleep before retry + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + } + + info!("V4L2 depth capture loop ended"); + Ok(()) +} + +/// Unpack Y10B data to 16-bit depth values +/// +/// This function is for CPU fallback - prefer GPU shader for real-time processing. +/// Uses freedepth's unpacking function and shifts to use full 16-bit range. +pub fn unpack_y10b_to_u16(raw_data: &[u8], width: u32, height: u32) -> Vec { + // Use freedepth's unpacking function + let raw_10bit = freedepth::unpack_10bit_ir(raw_data, width, height); + + // Shift left by 6 to use full 16-bit range (10-bit -> 16-bit) + raw_10bit.into_iter().map(|v| v << 6).collect() +} + +/// Convert 16-bit depth values to 8-bit grayscale for preview +pub fn depth_to_grayscale(depth_values: &[u16]) -> Vec { + depth_values + .iter() + .map(|&d| (d >> 8) as u8) // Take upper 8 bits + .collect() +} + +/// Convert 16-bit depth values to RGBA for display +pub fn depth_to_rgba(depth_values: &[u16]) -> Vec { + let mut rgba = Vec::with_capacity(depth_values.len() * 4); + for &depth in depth_values { + let gray = (depth >> 8) as u8; + rgba.push(gray); // R + rgba.push(gray); // G + rgba.push(gray); // B + rgba.push(255); // A + } + rgba +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_y10b_unpacking() { + // Test data: 4 pixels with known values + // P0 = 1023 (all 1s), P1 = 512, P2 = 256, P3 = 0 + // P0 = 0b11_1111_1111 = 1023 + // P1 = 0b10_0000_0000 = 512 + // P2 = 0b01_0000_0000 = 256 + // P3 = 0b00_0000_0000 = 0 + // + // Byte 0: P0[9:2] = 0b1111_1111 = 255 + // Byte 1: P0[1:0]|P1[9:4] = 0b11|10_0000 = 0b1110_0000 = 224 + // Byte 2: P1[3:0]|P2[9:6] = 0b0000|0100 = 0b0000_0100 = 4 + // Byte 3: P2[5:0]|P3[9:8] = 0b00_0000|00 = 0b0000_0000 = 0 + // Byte 4: P3[7:0] = 0b0000_0000 = 0 + let raw_data = vec![255u8, 224, 4, 0, 0]; + + let depth = unpack_y10b_to_u16(&raw_data, 2, 2); + + // Values are shifted left by 6 to use full 16-bit range + assert_eq!(depth[0], 1023 << 6); + assert_eq!(depth[1], 512 << 6); + assert_eq!(depth[2], 256 << 6); + assert_eq!(depth[3], 0); + } + + #[test] + fn test_depth_to_rgba() { + let depth = vec![0xFFFF, 0x8000, 0x0000]; + let rgba = depth_to_rgba(&depth); + + assert_eq!(rgba.len(), 12); + // First pixel (max depth) + assert_eq!(rgba[0], 255); // R + assert_eq!(rgba[1], 255); // G + assert_eq!(rgba[2], 255); // B + assert_eq!(rgba[3], 255); // A + // Second pixel (mid depth) + assert_eq!(rgba[4], 128); // R + // Third pixel (min depth) + assert_eq!(rgba[8], 0); // R + } +} diff --git a/src/backends/camera/v4l2_depth_controls.rs b/src/backends/camera/v4l2_depth_controls.rs new file mode 100644 index 00000000..8a275697 --- /dev/null +++ b/src/backends/camera/v4l2_depth_controls.rs @@ -0,0 +1,640 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! V4L2 Depth Camera Controls +//! +//! This module provides detection and access to depth cameras using the new +//! V4L2 depth control class extensions (V4L2_CTRL_CLASS_DEPTH). +//! +//! When the kernel driver supports these controls, we can use V4L2 directly +//! instead of the freedepth userspace library. + +use std::fs::File; +use std::os::unix::io::AsRawFd; +use tracing::{debug, info}; + +// V4L2 Depth Control Class and IDs +// These match the kernel header definitions from include/uapi/linux/v4l2-controls.h +const V4L2_CTRL_CLASS_DEPTH: u32 = 0x00a6_0000; +const V4L2_CID_DEPTH_CLASS_BASE: u32 = V4L2_CTRL_CLASS_DEPTH | 0x900; + +/// V4L2 Depth Control IDs +#[allow(dead_code)] +pub mod cid { + use super::V4L2_CID_DEPTH_CLASS_BASE; + + pub const DEPTH_SENSOR_TYPE: u32 = V4L2_CID_DEPTH_CLASS_BASE; + pub const DEPTH_UNITS: u32 = V4L2_CID_DEPTH_CLASS_BASE + 1; + pub const DEPTH_MIN_DISTANCE: u32 = V4L2_CID_DEPTH_CLASS_BASE + 2; + pub const DEPTH_MAX_DISTANCE: u32 = V4L2_CID_DEPTH_CLASS_BASE + 3; + pub const DEPTH_INTRINSICS: u32 = V4L2_CID_DEPTH_CLASS_BASE + 4; + pub const DEPTH_EXTRINSICS: u32 = V4L2_CID_DEPTH_CLASS_BASE + 5; + pub const DEPTH_CALIBRATION_VERSION: u32 = V4L2_CID_DEPTH_CLASS_BASE + 6; + pub const DEPTH_ILLUMINATOR_ENABLE: u32 = V4L2_CID_DEPTH_CLASS_BASE + 7; + pub const DEPTH_ILLUMINATOR_POWER: u32 = V4L2_CID_DEPTH_CLASS_BASE + 8; + pub const DEPTH_INVALID_VALUE: u32 = V4L2_CID_DEPTH_CLASS_BASE + 9; +} + +/// Depth sensor types (V4L2_DEPTH_SENSOR_*) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum DepthSensorType { + StructuredLight = 0, + TimeOfFlight = 1, + Stereo = 2, + ActiveStereo = 3, +} + +impl TryFrom for DepthSensorType { + type Error = (); + + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(Self::StructuredLight), + 1 => Ok(Self::TimeOfFlight), + 2 => Ok(Self::Stereo), + 3 => Ok(Self::ActiveStereo), + _ => Err(()), + } + } +} + +/// Camera intrinsic calibration parameters +/// Values are in Q16.16 fixed-point format (divide by 65536 for floating point) +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub struct DepthIntrinsics { + pub width: u32, + pub height: u32, + pub fx: i32, // Q16.16 focal length X + pub fy: i32, // Q16.16 focal length Y + pub cx: i32, // Q16.16 principal point X + pub cy: i32, // Q16.16 principal point Y + pub model: u32, + pub k1: i32, + pub k2: i32, + pub k3: i32, + pub p1: i32, + pub p2: i32, + pub reserved: [u32; 4], +} + +impl DepthIntrinsics { + /// Convert Q16.16 fixed-point values to floating point + pub fn fx_f32(&self) -> f32 { + self.fx as f32 / 65536.0 + } + + pub fn fy_f32(&self) -> f32 { + self.fy as f32 / 65536.0 + } + + pub fn cx_f32(&self) -> f32 { + self.cx as f32 / 65536.0 + } + + pub fn cy_f32(&self) -> f32 { + self.cy as f32 / 65536.0 + } +} + +/// Camera extrinsic calibration parameters (depth-to-RGB transform) +/// Rotation is Q2.30 fixed-point, translation is in micrometers +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub struct DepthExtrinsics { + pub rotation: [i32; 9], // 3x3 rotation matrix (Q2.30) + pub translation: [i32; 3], // Translation in micrometers + pub reserved: [u32; 4], +} + +impl DepthExtrinsics { + /// Get baseline (depth-to-RGB distance) in millimeters + pub fn baseline_mm(&self) -> f32 { + // Translation[0] is typically the X offset (baseline) in micrometers + self.translation[0] as f32 / 1000.0 + } +} + +/// Depth camera capabilities detected via V4L2 controls +#[derive(Debug, Clone)] +pub struct DepthCapabilities { + pub sensor_type: DepthSensorType, + pub units_um: u32, // Depth units in micrometers (1000 = mm) + pub min_distance_mm: u32, + pub max_distance_mm: u32, + pub invalid_value: u32, + pub intrinsics: Option, + pub extrinsics: Option, + pub has_illuminator: bool, +} + +// V4L2 ioctl definitions +const VIDIOC_QUERYCAP: libc::c_ulong = 0x8068_5600; // _IOR('V', 0, struct v4l2_capability) +const VIDIOC_QUERYCTRL: libc::c_ulong = 0xc044_5624; // _IOWR('V', 36, struct v4l2_queryctrl) - 68 bytes struct +const VIDIOC_G_CTRL: libc::c_ulong = 0xc008_561b; // _IOWR('V', 27, struct v4l2_control) +const VIDIOC_G_EXT_CTRLS: libc::c_ulong = 0xc040_5647; // _IOWR('V', 71, struct v4l2_ext_controls) + +/// V4L2 device capability structure for QUERYCAP +#[repr(C)] +struct V4l2Capability { + driver: [u8; 16], + card: [u8; 32], + bus_info: [u8; 32], + version: u32, + capabilities: u32, + device_caps: u32, + reserved: [u32; 3], +} + +/// Device information from V4L2 QUERYCAP +#[derive(Debug, Clone)] +pub struct V4l2DeviceInfo { + pub driver: String, + pub card: String, + pub bus_info: String, +} + +/// Query device information (driver, card name, bus_info) via QUERYCAP +pub fn query_device_info(device_path: &str) -> Option { + let file = File::open(device_path).ok()?; + let fd = file.as_raw_fd(); + + let mut caps = V4l2Capability { + driver: [0; 16], + card: [0; 32], + bus_info: [0; 32], + version: 0, + capabilities: 0, + device_caps: 0, + reserved: [0; 3], + }; + + let result = unsafe { libc::ioctl(fd, VIDIOC_QUERYCAP, &mut caps as *mut _) }; + + if result == 0 { + let driver = std::str::from_utf8(&caps.driver) + .unwrap_or("") + .trim_end_matches('\0') + .to_string(); + let card = std::str::from_utf8(&caps.card) + .unwrap_or("") + .trim_end_matches('\0') + .to_string(); + let bus_info = std::str::from_utf8(&caps.bus_info) + .unwrap_or("") + .trim_end_matches('\0') + .to_string(); + + Some(V4l2DeviceInfo { + driver, + card, + bus_info, + }) + } else { + None + } +} + +#[repr(C)] +struct V4l2Queryctrl { + id: u32, + type_: u32, + name: [u8; 32], + minimum: i32, + maximum: i32, + step: i32, + default_value: i32, + flags: u32, + reserved: [u32; 2], +} + +#[repr(C)] +struct V4l2Control { + id: u32, + value: i32, +} + +#[repr(C)] +struct V4l2ExtControl { + id: u32, + size: u32, + reserved2: [u32; 1], + value_or_ptr: u64, // Union: value (i32/i64) or pointer +} + +#[repr(C)] +struct V4l2ExtControls { + which: u32, + count: u32, + error_idx: u32, + request_fd: i32, + reserved: [u32; 1], + controls: *mut V4l2ExtControl, +} + +/// Check if a V4L2 device supports depth camera controls +/// +/// Returns true if the device has V4L2_CID_DEPTH_SENSOR_TYPE control, +/// indicating it's a depth camera with kernel driver support. +pub fn has_depth_controls(device_path: &str) -> bool { + let file = match File::open(device_path) { + Ok(f) => f, + Err(e) => { + debug!(path = %device_path, error = %e, "Failed to open device for depth control check"); + return false; + } + }; + + let fd = file.as_raw_fd(); + + // Query if DEPTH_SENSOR_TYPE control exists + let mut query = V4l2Queryctrl { + id: cid::DEPTH_SENSOR_TYPE, + type_: 0, + name: [0; 32], + minimum: 0, + maximum: 0, + step: 0, + default_value: 0, + flags: 0, + reserved: [0; 2], + }; + + let result = unsafe { libc::ioctl(fd, VIDIOC_QUERYCTRL, &mut query as *mut _) }; + + if result == 0 { + let name = std::str::from_utf8(&query.name) + .unwrap_or("unknown") + .trim_end_matches('\0'); + info!( + path = %device_path, + control_name = %name, + "Device has V4L2 depth controls (kernel driver)" + ); + true + } else { + debug!(path = %device_path, "Device does not have V4L2 depth controls"); + false + } +} + +/// Get a simple V4L2 control value +fn get_control(fd: i32, id: u32) -> Option { + let mut ctrl = V4l2Control { id, value: 0 }; + + let result = unsafe { libc::ioctl(fd, VIDIOC_G_CTRL, &mut ctrl as *mut _) }; + + if result == 0 { Some(ctrl.value) } else { None } +} + +/// Query depth camera capabilities via V4L2 controls +/// +/// This reads all available depth controls to build a complete +/// picture of the camera's capabilities. +pub fn query_depth_capabilities(device_path: &str) -> Option { + let file = File::open(device_path).ok()?; + let fd = file.as_raw_fd(); + + // Check for depth sensor type (required) + let sensor_type_val = get_control(fd, cid::DEPTH_SENSOR_TYPE)?; + let sensor_type = DepthSensorType::try_from(sensor_type_val as u32).ok()?; + + info!( + path = %device_path, + sensor_type = ?sensor_type, + "Querying depth camera capabilities" + ); + + // Get other standard controls + let units_um = get_control(fd, cid::DEPTH_UNITS).unwrap_or(1000) as u32; + let min_distance_mm = get_control(fd, cid::DEPTH_MIN_DISTANCE).unwrap_or(500) as u32; + let max_distance_mm = get_control(fd, cid::DEPTH_MAX_DISTANCE).unwrap_or(4000) as u32; + let invalid_value = get_control(fd, cid::DEPTH_INVALID_VALUE).unwrap_or(2047) as u32; + + // Check for illuminator control + let has_illuminator = { + let mut query = V4l2Queryctrl { + id: cid::DEPTH_ILLUMINATOR_ENABLE, + type_: 0, + name: [0; 32], + minimum: 0, + maximum: 0, + step: 0, + default_value: 0, + flags: 0, + reserved: [0; 2], + }; + let result = unsafe { libc::ioctl(fd, VIDIOC_QUERYCTRL, &mut query as *mut _) }; + result == 0 + }; + + // Try to get intrinsics (compound control - may not be available yet) + let intrinsics = get_intrinsics(fd); + let extrinsics = get_extrinsics(fd); + + Some(DepthCapabilities { + sensor_type, + units_um, + min_distance_mm, + max_distance_mm, + invalid_value, + intrinsics, + extrinsics, + has_illuminator, + }) +} + +/// Get depth intrinsics via extended control +fn get_intrinsics(fd: i32) -> Option { + let mut intrinsics = DepthIntrinsics::default(); + let size = std::mem::size_of::() as u32; + + let mut ext_ctrl = V4l2ExtControl { + id: cid::DEPTH_INTRINSICS, + size, + reserved2: [0], + value_or_ptr: &mut intrinsics as *mut _ as u64, + }; + + let mut ext_ctrls = V4l2ExtControls { + which: V4L2_CTRL_CLASS_DEPTH, + count: 1, + error_idx: 0, + request_fd: 0, + reserved: [0], + controls: &mut ext_ctrl, + }; + + let result = unsafe { libc::ioctl(fd, VIDIOC_G_EXT_CTRLS, &mut ext_ctrls as *mut _) }; + + if result == 0 { + debug!( + fx = intrinsics.fx_f32(), + fy = intrinsics.fy_f32(), + cx = intrinsics.cx_f32(), + cy = intrinsics.cy_f32(), + "Got depth intrinsics from kernel" + ); + Some(intrinsics) + } else { + debug!("Depth intrinsics control not available"); + None + } +} + +/// Get depth extrinsics via extended control +fn get_extrinsics(fd: i32) -> Option { + let mut extrinsics = DepthExtrinsics::default(); + let size = std::mem::size_of::() as u32; + + let mut ext_ctrl = V4l2ExtControl { + id: cid::DEPTH_EXTRINSICS, + size, + reserved2: [0], + value_or_ptr: &mut extrinsics as *mut _ as u64, + }; + + let mut ext_ctrls = V4l2ExtControls { + which: V4L2_CTRL_CLASS_DEPTH, + count: 1, + error_idx: 0, + request_fd: 0, + reserved: [0], + controls: &mut ext_ctrl, + }; + + let result = unsafe { libc::ioctl(fd, VIDIOC_G_EXT_CTRLS, &mut ext_ctrls as *mut _) }; + + if result == 0 { + debug!( + baseline_mm = extrinsics.baseline_mm(), + "Got depth extrinsics from kernel" + ); + Some(extrinsics) + } else { + debug!("Depth extrinsics control not available"); + None + } +} + +/// Enable or disable the depth illuminator (IR projector) +pub fn set_illuminator_enabled(device_path: &str, enabled: bool) -> Result<(), String> { + let file = File::open(device_path).map_err(|e| format!("Failed to open device: {}", e))?; + let fd = file.as_raw_fd(); + + let ctrl = V4l2Control { + id: cid::DEPTH_ILLUMINATOR_ENABLE, + value: if enabled { 1 } else { 0 }, + }; + + // Use VIDIOC_S_CTRL for setting + const VIDIOC_S_CTRL: libc::c_ulong = 0xc008_561c; + + let result = unsafe { libc::ioctl(fd, VIDIOC_S_CTRL, &ctrl as *const _) }; + + if result == 0 { + info!(enabled, "Set depth illuminator state"); + Ok(()) + } else { + Err(format!( + "Failed to set illuminator: {}", + std::io::Error::last_os_error() + )) + } +} + +/// Check if illuminator is currently enabled +pub fn is_illuminator_enabled(device_path: &str) -> Option { + let file = File::open(device_path).ok()?; + let fd = file.as_raw_fd(); + + get_control(fd, cid::DEPTH_ILLUMINATOR_ENABLE).map(|v| v != 0) +} + +// ============================================================================ +// Registration Data Conversion (Kernel Calibration -> Shader Format) +// ============================================================================ + +/// Depth image dimensions (standard Kinect) +const DEPTH_WIDTH: u32 = 640; +const DEPTH_HEIGHT: u32 = 480; +const DEPTH_MM_MAX: u32 = 10000; + +/// Fixed-point scale factor for x coordinates (matches freedepth) +pub const REG_X_VAL_SCALE: i32 = 256; + +/// Registration data for GPU shaders +/// Matches the format expected by point_cloud and mesh shaders +#[derive(Clone)] +pub struct KernelRegistrationData { + /// Registration table: 640*480 [x_scaled, y] pairs + /// x is scaled by REG_X_VAL_SCALE (256) + pub registration_table: Vec<[i32; 2]>, + /// Depth-to-RGB shift table: 10001 i32 values indexed by depth_mm + pub depth_to_rgb_shift: Vec, + /// Target offset (typically 0 for kernel driver) + pub target_offset: u32, +} + +impl KernelRegistrationData { + /// Create registration data from kernel intrinsics and extrinsics + /// + /// For the kernel driver, we use a simplified pinhole camera model: + /// - Registration table maps each depth pixel to RGB space + /// - Depth-to-RGB shift handles stereo baseline disparity + pub fn from_kernel_calibration( + intrinsics: &DepthIntrinsics, + extrinsics: Option<&DepthExtrinsics>, + ) -> Self { + let registration_table = build_registration_table_from_intrinsics(intrinsics); + let depth_to_rgb_shift = build_depth_to_rgb_shift_from_extrinsics(intrinsics, extrinsics); + + info!( + table_size = registration_table.len(), + shift_size = depth_to_rgb_shift.len(), + fx = intrinsics.fx_f32(), + fy = intrinsics.fy_f32(), + cx = intrinsics.cx_f32(), + cy = intrinsics.cy_f32(), + baseline_mm = extrinsics.map(|e| e.baseline_mm()).unwrap_or(0.0), + "Built registration data from kernel calibration" + ); + + Self { + registration_table, + depth_to_rgb_shift, + target_offset: 0, // Kernel driver doesn't use pad offset + } + } + + /// Convert to the shader RegistrationData format + pub fn to_shader_format(&self) -> crate::shaders::RegistrationData { + crate::shaders::RegistrationData { + registration_table: self.registration_table.clone(), + depth_to_rgb_shift: self.depth_to_rgb_shift.clone(), + target_offset: self.target_offset, + } + } +} + +/// Build registration table from intrinsics +/// +/// For a pinhole camera model, the registration table maps each depth pixel +/// to its corresponding RGB pixel position. Since depth and RGB cameras have +/// similar intrinsics, this is mostly an identity mapping with small corrections. +fn build_registration_table_from_intrinsics(intrinsics: &DepthIntrinsics) -> Vec<[i32; 2]> { + let mut table = Vec::with_capacity((DEPTH_WIDTH * DEPTH_HEIGHT) as usize); + + let fx = intrinsics.fx_f32(); + let fy = intrinsics.fy_f32(); + let cx = intrinsics.cx_f32(); + let cy = intrinsics.cy_f32(); + + // Default intrinsics if not calibrated + let fx = if fx > 0.0 { fx } else { 580.0 }; + let fy = if fy > 0.0 { fy } else { 580.0 }; + let cx = if cx > 0.0 { cx } else { 320.0 }; + let cy = if cy > 0.0 { cy } else { 240.0 }; + + for y in 0..DEPTH_HEIGHT { + for x in 0..DEPTH_WIDTH { + // For a simple pinhole model, depth and RGB pixels are roughly aligned + // The main offset is the stereo baseline (handled by depth_to_rgb_shift) + // + // Apply minor correction for principal point offset between cameras + // This is usually very small for Kinect + let depth_cx = DEPTH_WIDTH as f32 / 2.0; + let depth_cy = DEPTH_HEIGHT as f32 / 2.0; + + // Project from depth camera to RGB camera coordinate + // For aligned cameras, this is approximately identity with focal length scaling + let rgb_x = (x as f32 - depth_cx) * (fx / fx) + cx; + let rgb_y = (y as f32 - depth_cy) * (fy / fy) + cy; + + // Store scaled x and integer y (matching freedepth format) + // The x value is multiplied by REG_X_VAL_SCALE for fixed-point precision + let x_scaled = (rgb_x * REG_X_VAL_SCALE as f32) as i32; + let y_int = rgb_y as i32; + + table.push([x_scaled, y_int]); + } + } + + table +} + +/// Build depth-to-RGB shift table from extrinsics +/// +/// The shift table maps depth values (in mm) to horizontal pixel offsets +/// needed to align depth with RGB, accounting for stereo baseline. +/// +/// Formula: shift = baseline_mm * fx / depth_mm * REG_X_VAL_SCALE +fn build_depth_to_rgb_shift_from_extrinsics( + intrinsics: &DepthIntrinsics, + extrinsics: Option<&DepthExtrinsics>, +) -> Vec { + let mut table = vec![0i32; (DEPTH_MM_MAX + 1) as usize]; + + // Get baseline from extrinsics (or use default Kinect baseline of ~25mm) + let baseline_mm = extrinsics + .map(|e| e.baseline_mm().abs()) + .filter(|&b| b > 0.0) + .unwrap_or(25.0); + + // Get focal length + let fx = intrinsics.fx_f32(); + let fx = if fx > 0.0 { fx } else { 580.0 }; + + debug!( + baseline_mm, + fx, "Building depth-to-RGB shift table from kernel extrinsics" + ); + + for depth_mm in 1..=DEPTH_MM_MAX { + // Disparity formula: shift_pixels = baseline * focal_length / depth + // This gives the horizontal pixel offset due to stereo baseline + let shift_pixels = baseline_mm * fx / (depth_mm as f32); + + // Scale by REG_X_VAL_SCALE for fixed-point math + let shift_scaled = (shift_pixels * REG_X_VAL_SCALE as f32) as i32; + + table[depth_mm as usize] = shift_scaled; + } + + // Depth 0 has no shift + table[0] = 0; + + table +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_depth_sensor_type_conversion() { + assert_eq!( + DepthSensorType::try_from(0), + Ok(DepthSensorType::StructuredLight) + ); + assert_eq!( + DepthSensorType::try_from(1), + Ok(DepthSensorType::TimeOfFlight) + ); + assert!(DepthSensorType::try_from(99).is_err()); + } + + #[test] + fn test_intrinsics_conversion() { + let intrinsics = DepthIntrinsics { + fx: 580 * 65536, // 580 pixels in Q16.16 + fy: 580 * 65536, + cx: 320 * 65536, + cy: 240 * 65536, + ..Default::default() + }; + + assert!((intrinsics.fx_f32() - 580.0).abs() < 0.001); + assert!((intrinsics.cx_f32() - 320.0).abs() < 0.001); + } +} diff --git a/src/backends/camera/v4l2_kernel_depth.rs b/src/backends/camera/v4l2_kernel_depth.rs new file mode 100644 index 00000000..e627c11a --- /dev/null +++ b/src/backends/camera/v4l2_kernel_depth.rs @@ -0,0 +1,1104 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! V4L2 Kernel Depth Camera Backend +//! +//! This backend uses the V4L2 kernel driver with depth control extensions +//! for depth camera streaming. It's preferred over the freedepth userspace +//! library when the kernel driver is available. +//! +//! # Detection +//! +//! The kernel driver is detected by checking for V4L2_CID_DEPTH_SENSOR_TYPE +//! control on the device. If present, this backend is used; otherwise, +//! we fall back to freedepth. +//! +//! # Advantages over freedepth +//! +//! - No need to unbind/rebind kernel driver +//! - Better integration with V4L2 ecosystem +//! - Calibration data exposed via standard V4L2 controls +//! - Works with standard V4L2 tools (v4l2-ctl, etc.) + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Instant; + +use tracing::{debug, error, info, warn}; +use v4l::buffer::Type; +use v4l::io::mmap::Stream; +use v4l::io::traits::CaptureStream; +use v4l::prelude::*; +use v4l::video::Capture; +use v4l::{Format, FourCC}; + +use super::CameraBackend; +use super::format_converters::{ + self, DepthVisualizationOptions, grbg_to_rgba, unpack_y10b, uyvy_to_rgba, +}; +use super::types::*; +use super::v4l2_depth_controls::{ + self, DepthCapabilities, DepthIntrinsics, KernelRegistrationData, V4l2DeviceInfo, +}; + +/// Path prefix for kernel depth camera devices (single depth device) +pub const KERNEL_DEPTH_PREFIX: &str = "v4l2-depth:"; + +/// Path prefix for kernel Kinect paired devices (color:depth) +pub const KERNEL_KINECT_PREFIX: &str = "v4l2-kinect:"; + +/// A paired Kinect device (color + depth) from kernel driver +/// +/// The kernel Kinect driver creates two separate V4L2 video devices: +/// - Color camera (SGRBG8/UYVY formats) +/// - Depth camera (Y10B/Y16 formats, with depth controls) +/// +/// They share the same `bus_info` (e.g., "usb-1.11") which allows pairing. +#[derive(Debug, Clone)] +pub struct KinectDevicePair { + /// Path to color camera device (e.g., /dev/video4) + pub color_path: String, + /// Path to depth camera device (e.g., /dev/video5) + pub depth_path: String, + /// USB bus info shared by both devices (e.g., "usb-0000:00:14.0-11") + pub bus_info: String, + /// Card name (e.g., "Kinect") + pub card_name: String, +} + +/// Device type detected during scanning +#[derive(Debug, Clone)] +enum KinectDeviceType { + /// Color camera (has Bayer/UYVY formats, no depth controls) + Color, + /// Depth camera (has Y10B/Y16 formats and depth controls) + Depth, +} + +/// Candidate device found during scanning +#[derive(Debug)] +struct KinectCandidate { + path: String, + device_type: KinectDeviceType, + info: V4l2DeviceInfo, +} + +/// Find Kinect device pairs from the kernel driver +/// +/// Scans /dev/video* devices for the Kinect kernel driver signature: +/// - Devices with V4L2_CID_DEPTH_SENSOR_TYPE control are depth devices +/// - Devices with "kinect" driver and color formats are color devices +/// - Devices are paired by matching `bus_info` +/// +/// Returns a list of paired devices. Each pair represents one physical Kinect. +pub fn find_kernel_kinect_pairs() -> Vec { + use std::collections::HashMap; + use v4l::prelude::*; + + let mut candidates: Vec = Vec::new(); + + // Scan /dev/video* devices + let entries: Vec<_> = std::fs::read_dir("/dev") + .into_iter() + .flatten() + .flatten() + .filter(|e| { + e.path() + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n.starts_with("video")) + .unwrap_or(false) + }) + .collect(); + + for entry in entries { + let path = entry.path(); + let path_str = path.to_string_lossy().to_string(); + + // Get device info via QUERYCAP + let Some(info) = v4l2_depth_controls::query_device_info(&path_str) else { + continue; + }; + + // Check if this is a Kinect driver device + if info.driver != "kinect" { + continue; + } + + // Determine device type by checking video formats + // Note: Both devices may have depth controls, so we MUST check formats first + // - Depth camera: has Y10B/Y16 formats + // - Color camera: has GRBG/UYVY formats + let device_type = if let Ok(dev) = Device::with_path(&path) { + let formats: Vec<_> = dev.enum_formats().into_iter().flatten().collect(); + + // FourCC constants for comparison + let fourcc_y10b = FourCC::new(b"Y10B"); + let fourcc_y16 = FourCC::new(b"Y16 "); + let fourcc_grbg = FourCC::new(b"GRBG"); + let fourcc_uyvy = FourCC::new(b"UYVY"); + + // Check for depth formats (Y10B, Y16) + let has_depth_format = formats + .iter() + .any(|f| f.fourcc == fourcc_y10b || f.fourcc == fourcc_y16); + + // Check for color formats (Bayer GRBG or UYVY) + let has_color_format = formats + .iter() + .any(|f| f.fourcc == fourcc_grbg || f.fourcc == fourcc_uyvy); + + if has_depth_format { + KinectDeviceType::Depth + } else if has_color_format { + KinectDeviceType::Color + } else { + debug!(path = %path_str, "Unknown Kinect device type - no recognized formats"); + continue; + } + } else { + continue; + }; + + debug!( + path = %path_str, + device_type = ?device_type, + bus_info = %info.bus_info, + card = %info.card, + "Found Kinect kernel driver device" + ); + + candidates.push(KinectCandidate { + path: path_str, + device_type, + info, + }); + } + + // Group by bus_info to find pairs + let mut by_bus: HashMap> = HashMap::new(); + for candidate in candidates { + by_bus + .entry(candidate.info.bus_info.clone()) + .or_default() + .push(candidate); + } + + // Build pairs from grouped devices + let mut pairs = Vec::new(); + for (bus_info, devices) in by_bus { + let mut color_path: Option = None; + let mut depth_path: Option = None; + let mut card_name = String::from("Kinect"); + + for dev in devices { + match dev.device_type { + KinectDeviceType::Color => { + color_path = Some(dev.path); + card_name = dev.info.card.clone(); + } + KinectDeviceType::Depth => { + depth_path = Some(dev.path); + } + } + } + + // Only create a pair if we have both color and depth + if let (Some(color), Some(depth)) = (color_path, depth_path) { + info!( + color_path = %color, + depth_path = %depth, + bus_info = %bus_info, + "Found Kinect device pair" + ); + pairs.push(KinectDevicePair { + color_path: color, + depth_path: depth, + bus_info, + card_name, + }); + } + } + + pairs +} + +/// Check if there are any Kinect kernel driver devices available +pub fn has_kernel_kinect_devices() -> bool { + !find_kernel_kinect_pairs().is_empty() +} + +/// V4L2 Kernel Depth Backend +/// +/// Uses the V4L2 kernel driver with depth control extensions for +/// depth camera streaming. Supports dual-stream capture (color + depth) +/// when initialized with a device pair. +pub struct V4l2KernelDepthBackend { + /// Current device (combined representation) + device: Option, + /// Current format + format: Option, + /// Depth capabilities from V4L2 controls + capabilities: Option, + /// Registration data from kernel calibration (for depth-RGB alignment) + registration_data: Option, + /// Device pair (if using kernel driver with color + depth) + device_pair: Option, + /// Depth camera V4L2 path + depth_v4l2_path: Option, + /// Color camera V4L2 path + color_v4l2_path: Option, + /// Depth capture thread handle + depth_capture_thread: Option>, + /// Color capture thread handle + color_capture_thread: Option>, + /// Signal to stop capture + stop_signal: Arc, + /// Latest combined frame (color + depth visualization) + latest_frame: Arc>>, + /// Latest depth frame data + latest_depth_frame: Arc>>, + /// Latest color frame data + latest_color_frame: Arc>>, + /// Frame receiver for preview + frame_receiver: Option, + /// Frame sender for preview + frame_sender: Option, +} + +impl V4l2KernelDepthBackend { + /// Create a new V4L2 kernel depth backend + pub fn new() -> Self { + Self { + device: None, + format: None, + capabilities: None, + registration_data: None, + device_pair: None, + depth_v4l2_path: None, + color_v4l2_path: None, + depth_capture_thread: None, + color_capture_thread: None, + stop_signal: Arc::new(AtomicBool::new(false)), + latest_frame: Arc::new(Mutex::new(None)), + latest_depth_frame: Arc::new(Mutex::new(None)), + latest_color_frame: Arc::new(Mutex::new(None)), + frame_receiver: None, + frame_sender: None, + } + } + + /// Check if a V4L2 device has kernel depth driver support + pub fn has_kernel_depth_support(v4l2_path: &str) -> bool { + v4l2_depth_controls::has_depth_controls(v4l2_path) + } + + /// Get depth capabilities for a device + pub fn get_capabilities(&self) -> Option<&DepthCapabilities> { + self.capabilities.as_ref() + } + + /// Get depth intrinsics (camera calibration) + pub fn get_intrinsics(&self) -> Option<&DepthIntrinsics> { + self.capabilities.as_ref()?.intrinsics.as_ref() + } + + /// Get registration data (for depth-to-RGB alignment) + /// + /// Built from kernel intrinsics/extrinsics during initialization. + /// Returns None if calibration data was not available from the kernel. + pub fn get_registration_data(&self) -> Option<&KernelRegistrationData> { + self.registration_data.as_ref() + } + + /// Get registration data converted to shader format + pub fn get_shader_registration_data(&self) -> Option { + self.registration_data + .as_ref() + .map(|r| r.to_shader_format()) + } + + /// Get the device pair if available + pub fn get_device_pair(&self) -> Option<&KinectDevicePair> { + self.device_pair.as_ref() + } + + /// Get latest depth frame data + pub fn get_latest_depth(&self) -> Option { + self.latest_depth_frame.lock().ok()?.clone() + } + + /// Get latest color frame data + pub fn get_latest_color(&self) -> Option { + self.latest_color_frame.lock().ok()?.clone() + } + + /// Get the latest frame for preview (polling method) + pub fn get_frame(&self) -> Option { + self.latest_frame.lock().ok()?.clone() + } + + /// Initialize from a device pair (color + depth) + pub fn initialize_from_pair(&mut self, pair: &KinectDevicePair) -> BackendResult<()> { + info!( + color_path = %pair.color_path, + depth_path = %pair.depth_path, + "Initializing kernel depth backend from device pair" + ); + + // Query depth capabilities from depth device + let capabilities = v4l2_depth_controls::query_depth_capabilities(&pair.depth_path); + + // Build registration data from kernel calibration + let registration_data = if let Some(ref caps) = capabilities { + info!( + sensor_type = ?caps.sensor_type, + min_distance = caps.min_distance_mm, + max_distance = caps.max_distance_mm, + invalid_value = caps.invalid_value, + has_intrinsics = caps.intrinsics.is_some(), + has_extrinsics = caps.extrinsics.is_some(), + "Depth camera capabilities" + ); + + // Build registration from intrinsics/extrinsics if available + caps.intrinsics.as_ref().map(|intrinsics| { + KernelRegistrationData::from_kernel_calibration( + intrinsics, + caps.extrinsics.as_ref(), + ) + }) + } else { + None + }; + + // Create combined device representation + let device = CameraDevice { + name: format!("{} (Kernel)", pair.card_name), + path: format!( + "{}{}:{}", + KERNEL_KINECT_PREFIX, pair.color_path, pair.depth_path + ), + metadata_path: Some(pair.depth_path.clone()), + device_info: Some(DeviceInfo { + card: pair.card_name.clone(), + driver: "kinect".to_string(), + path: pair.color_path.clone(), + real_path: pair.depth_path.clone(), + }), + }; + + // Default format (640x480 depth) + let format = CameraFormat { + width: 640, + height: 480, + framerate: Some(30), + hardware_accelerated: true, + pixel_format: "Y10B".to_string(), + sensor_type: SensorType::Depth, + }; + + self.device = Some(device); + self.format = Some(format.clone()); + self.capabilities = capabilities; + self.registration_data = registration_data; + self.device_pair = Some(pair.clone()); + self.depth_v4l2_path = Some(pair.depth_path.clone()); + self.color_v4l2_path = Some(pair.color_path.clone()); + + // Start dual capture + self.start_dual_capture(pair, &format)?; + + info!("Kernel depth backend initialized with device pair"); + Ok(()) + } + + /// Start dual capture (color + depth streams) + fn start_dual_capture( + &mut self, + pair: &KinectDevicePair, + format: &CameraFormat, + ) -> BackendResult<()> { + let stop_signal = self.stop_signal.clone(); + let latest_frame = self.latest_frame.clone(); + let latest_depth = self.latest_depth_frame.clone(); + let latest_color = self.latest_color_frame.clone(); + + // Create frame channel + let (sender, receiver) = cosmic::iced::futures::channel::mpsc::channel(30); + self.frame_sender = Some(sender.clone()); + self.frame_receiver = Some(receiver); + + // Get capabilities for depth processing + let capabilities = self.capabilities.clone(); + + // Start depth capture thread + let depth_path = pair.depth_path.clone(); + let depth_width = format.width; + let depth_height = format.height; + let depth_stop = stop_signal.clone(); + let depth_latest = latest_depth.clone(); + let depth_frame_latest = latest_frame.clone(); + let depth_sender = sender.clone(); + let depth_caps = capabilities.clone(); + + let depth_handle = thread::spawn(move || { + if let Err(e) = depth_capture_loop( + &depth_path, + depth_width, + depth_height, + depth_stop, + depth_latest, + depth_frame_latest, + depth_sender, + depth_caps, + ) { + error!(error = %e, "Depth capture loop error"); + } + }); + + self.depth_capture_thread = Some(depth_handle); + + // Start color capture thread + let color_path = pair.color_path.clone(); + let color_stop = stop_signal.clone(); + let color_latest = latest_color.clone(); + + let color_handle = thread::spawn(move || { + if let Err(e) = color_capture_loop(&color_path, color_stop, color_latest) { + error!(error = %e, "Color capture loop error"); + } + }); + + self.color_capture_thread = Some(color_handle); + + info!("Dual capture started (color + depth)"); + Ok(()) + } + + /// Start single depth capture (legacy mode) + fn start_capture(&mut self, v4l2_path: &str, format: &CameraFormat) -> BackendResult<()> { + let path = v4l2_path.to_string(); + let width = format.width; + let height = format.height; + let stop_signal = self.stop_signal.clone(); + let latest_frame = self.latest_frame.clone(); + let latest_depth = self.latest_depth_frame.clone(); + + // Create frame channel + let (sender, receiver) = cosmic::iced::futures::channel::mpsc::channel(30); + self.frame_sender = Some(sender.clone()); + self.frame_receiver = Some(receiver); + + // Get capabilities for depth processing + let capabilities = self.capabilities.clone(); + + let handle = thread::spawn(move || { + if let Err(e) = depth_capture_loop( + &path, + width, + height, + stop_signal, + latest_depth, + latest_frame, + sender, + capabilities, + ) { + error!(error = %e, "Capture loop error"); + } + }); + + self.depth_capture_thread = Some(handle); + Ok(()) + } + + /// Stop all capture threads + fn stop_capture(&mut self) { + self.stop_signal.store(true, Ordering::SeqCst); + + if let Some(handle) = self.depth_capture_thread.take() { + let _ = handle.join(); + } + + if let Some(handle) = self.color_capture_thread.take() { + let _ = handle.join(); + } + + self.stop_signal.store(false, Ordering::SeqCst); + self.frame_sender = None; + self.frame_receiver = None; + } +} + +/// Depth capture loop running in a separate thread +fn depth_capture_loop( + path: &str, + width: u32, + height: u32, + stop_signal: Arc, + latest_depth: Arc>>, + latest_frame: Arc>>, + mut sender: FrameSender, + capabilities: Option, +) -> Result<(), String> { + info!(path, width, height, "Starting V4L2 kernel depth capture"); + + // Open V4L2 device + let dev = Device::with_path(path).map_err(|e| format!("Failed to open device: {}", e))?; + + // Set format - try Y10B first (11-bit packed), fall back to Y16 + let fourcc_y10b = FourCC::new(b"Y10B"); + let fourcc_y16 = FourCC::new(b"Y16 "); + + let format = Format::new(width, height, fourcc_y10b); + let actual_format = match dev.set_format(&format) { + Ok(f) => f, + Err(_) => { + // Try Y16 as fallback + let format = Format::new(width, height, fourcc_y16); + dev.set_format(&format) + .map_err(|e| format!("Failed to set format: {}", e))? + } + }; + + info!( + width = actual_format.width, + height = actual_format.height, + fourcc = ?actual_format.fourcc, + "V4L2 depth format configured" + ); + + // Create memory-mapped stream + let mut stream = Stream::with_buffers(&dev, Type::VideoCapture, 4) + .map_err(|e| format!("Failed to create stream: {}", e))?; + + // Get invalid depth value from capabilities + let invalid_value = capabilities + .as_ref() + .map(|c| c.invalid_value as u16) + .unwrap_or(2047); + + // Configure depth visualization (auto-range grayscale for kernel backend) + let viz_options = DepthVisualizationOptions { + grayscale: true, + invalid_value, + ..DepthVisualizationOptions::auto_range() + }; + + info!(invalid_value, "Depth capture loop started"); + + while !stop_signal.load(Ordering::SeqCst) { + // Capture frame + let (buf, meta) = match stream.next() { + Ok(frame) => frame, + Err(e) => { + warn!(error = %e, "Failed to capture depth frame"); + continue; + } + }; + + let captured_at = Instant::now(); + let timestamp = (meta.timestamp.sec as u32) + .wrapping_mul(1_000_000) + .wrapping_add(meta.timestamp.usec as u32); + + // Process depth data + let depth_data: Vec = if actual_format.fourcc == FourCC::new(b"Y16 ") { + // 16-bit depth - direct copy + buf.chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect() + } else if actual_format.fourcc == FourCC::new(b"Y10B") { + // 10-bit packed - unpack + unpack_y10b(buf, width, height) + } else { + warn!(fourcc = ?actual_format.fourcc, "Unsupported depth format"); + continue; + }; + + // Update latest depth frame data + if let Ok(mut guard) = latest_depth.lock() { + *guard = Some(DepthFrameData { + width, + height, + depth_mm: depth_data.clone(), + timestamp, + }); + } + + // Convert depth to RGBA visualization + let rgba = format_converters::depth_to_rgba(&depth_data, width, height, &viz_options); + + // Create camera frame for preview + let frame = CameraFrame { + width, + height, + data: Arc::from(rgba.into_boxed_slice()), + format: PixelFormat::Depth16, + stride: width * 4, + captured_at, + depth_data: Some(Arc::from(depth_data.into_boxed_slice())), + depth_width: width, + depth_height: height, + video_timestamp: Some(timestamp), + }; + + // Update latest frame + if let Ok(mut guard) = latest_frame.lock() { + *guard = Some(frame.clone()); + } + + // Send to channel + if sender.try_send(frame).is_err() { + debug!("Frame channel full, dropping frame"); + } + } + + info!("Depth capture loop stopped"); + Ok(()) +} + +/// Color capture loop running in a separate thread +fn color_capture_loop( + path: &str, + stop_signal: Arc, + latest_color: Arc>>, +) -> Result<(), String> { + info!(path, "Starting V4L2 kernel color capture"); + + // Open V4L2 device + let dev = Device::with_path(path).map_err(|e| format!("Failed to open device: {}", e))?; + + // Try to set UYVY format at 640x480 (preferred for color) + let fourcc_uyvy = FourCC::new(b"UYVY"); + let fourcc_grbg = FourCC::new(b"GRBG"); + + let format = Format::new(640, 480, fourcc_uyvy); + let actual_format = match dev.set_format(&format) { + Ok(f) => f, + Err(_) => { + // Try Bayer GRBG as fallback + let format = Format::new(640, 480, fourcc_grbg); + dev.set_format(&format) + .map_err(|e| format!("Failed to set color format: {}", e))? + } + }; + + let width = actual_format.width; + let height = actual_format.height; + + info!( + width, + height, + fourcc = ?actual_format.fourcc, + "V4L2 color format configured" + ); + + // Create memory-mapped stream + let mut stream = Stream::with_buffers(&dev, Type::VideoCapture, 4) + .map_err(|e| format!("Failed to create color stream: {}", e))?; + + info!("Color capture loop started"); + + while !stop_signal.load(Ordering::SeqCst) { + // Capture frame + let (buf, meta) = match stream.next() { + Ok(frame) => frame, + Err(e) => { + warn!(error = %e, "Failed to capture color frame"); + continue; + } + }; + + let timestamp = (meta.timestamp.sec as u32) + .wrapping_mul(1_000_000) + .wrapping_add(meta.timestamp.usec as u32); + + // Convert to RGBA based on format + let rgba = if actual_format.fourcc == FourCC::new(b"UYVY") { + uyvy_to_rgba(buf, width, height) + } else if actual_format.fourcc == FourCC::new(b"GRBG") { + grbg_to_rgba(buf, width, height) + } else { + warn!(fourcc = ?actual_format.fourcc, "Unsupported color format"); + continue; + }; + + // Update latest color frame + if let Ok(mut guard) = latest_color.lock() { + *guard = Some(VideoFrameData { + width, + height, + data: rgba, + timestamp, + }); + } + } + + info!("Color capture loop stopped"); + Ok(()) +} + +impl Default for V4l2KernelDepthBackend { + fn default() -> Self { + Self::new() + } +} + +impl CameraBackend for V4l2KernelDepthBackend { + fn enumerate_cameras(&self) -> Vec { + let mut cameras = Vec::new(); + + // First, look for paired Kinect devices (color + depth from kernel driver) + let pairs = find_kernel_kinect_pairs(); + for pair in pairs { + let device = CameraDevice { + name: format!("{} (Kernel)", pair.card_name), + path: format!( + "{}{}:{}", + KERNEL_KINECT_PREFIX, pair.color_path, pair.depth_path + ), + metadata_path: Some(pair.depth_path.clone()), + device_info: Some(DeviceInfo { + card: pair.card_name.clone(), + driver: "kinect".to_string(), + path: pair.color_path.clone(), + real_path: pair.depth_path.clone(), + }), + }; + + info!( + name = %device.name, + path = %device.path, + color = %pair.color_path, + depth = %pair.depth_path, + "Found kernel Kinect device pair" + ); + + cameras.push(device); + } + + // If we found paired devices, don't scan for individual depth devices + // (they're already included in pairs) + if !cameras.is_empty() { + return cameras; + } + + // Fallback: scan for individual depth devices (non-paired) + for entry in std::fs::read_dir("/dev").into_iter().flatten() { + if let Ok(entry) = entry { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with("video") { + let path_str = path.to_string_lossy(); + if Self::has_kernel_depth_support(&path_str) { + // Get device info + if let Ok(dev) = Device::with_path(&path) { + if let Ok(caps) = dev.query_caps() { + let device = CameraDevice { + name: caps.card.clone(), + path: format!("{}{}", KERNEL_DEPTH_PREFIX, path_str), + metadata_path: Some(path_str.to_string()), + device_info: Some(DeviceInfo { + card: caps.card.clone(), + driver: caps.driver.clone(), + path: path_str.to_string(), + real_path: path_str.to_string(), + }), + }; + + info!( + name = %device.name, + path = %device.path, + "Found kernel depth camera" + ); + + cameras.push(device); + } + } + } + } + } + } + } + + cameras + } + + fn get_formats(&self, device: &CameraDevice, _video_mode: bool) -> Vec { + let v4l2_path = device + .path + .strip_prefix(KERNEL_DEPTH_PREFIX) + .unwrap_or(&device.path); + + let dev = match Device::with_path(v4l2_path) { + Ok(d) => d, + Err(_) => return Vec::new(), + }; + + let mut formats = Vec::new(); + + // Query supported formats + if let Ok(format_iter) = dev.enum_formats() { + for fmt_desc in format_iter { + if let Ok(frame_sizes) = dev.enum_framesizes(fmt_desc.fourcc) { + for size in frame_sizes { + match size.size { + v4l::framesize::FrameSizeEnum::Discrete(discrete) => { + // Query framerates + if let Ok(intervals) = dev.enum_frameintervals( + fmt_desc.fourcc, + discrete.width, + discrete.height, + ) { + for interval in intervals { + let fps = match interval.interval { + v4l::frameinterval::FrameIntervalEnum::Discrete( + frac, + ) => { + if frac.numerator > 0 { + Some(frac.denominator / frac.numerator) + } else { + Some(30) + } + } + _ => Some(30), + }; + + formats.push(CameraFormat { + width: discrete.width, + height: discrete.height, + framerate: fps, + hardware_accelerated: true, + pixel_format: format!("{:?}", fmt_desc.fourcc), + sensor_type: SensorType::Depth, + }); + } + } + } + v4l::framesize::FrameSizeEnum::Stepwise(step) => { + // Add common resolutions + for (w, h) in [(640, 480), (320, 240)] { + if w >= step.min_width + && w <= step.max_width + && h >= step.min_height + && h <= step.max_height + { + formats.push(CameraFormat { + width: w, + height: h, + framerate: Some(30), + hardware_accelerated: true, + pixel_format: format!("{:?}", fmt_desc.fourcc), + sensor_type: SensorType::Depth, + }); + } + } + } + } + } + } + } + } + + formats + } + + fn initialize(&mut self, device: &CameraDevice, format: &CameraFormat) -> BackendResult<()> { + info!( + device = %device.name, + format = %format, + "Initializing V4L2 kernel depth backend" + ); + + // Check if this is a paired device (v4l2-kinect:color:depth format) + if let Some(paths) = device.path.strip_prefix(KERNEL_KINECT_PREFIX) { + // Parse color:depth paths + let parts: Vec<&str> = paths.split(':').collect(); + if parts.len() == 2 { + let pair = KinectDevicePair { + color_path: parts[0].to_string(), + depth_path: parts[1].to_string(), + bus_info: String::new(), // Not needed for initialization + card_name: device.name.clone(), + }; + return self.initialize_from_pair(&pair); + } + } + + // Single depth device path (v4l2-depth:/dev/videoX format) + let v4l2_path = device + .path + .strip_prefix(KERNEL_DEPTH_PREFIX) + .unwrap_or(&device.path) + .to_string(); + + // Query depth capabilities + let capabilities = v4l2_depth_controls::query_depth_capabilities(&v4l2_path); + + if let Some(ref caps) = capabilities { + info!( + sensor_type = ?caps.sensor_type, + min_distance = caps.min_distance_mm, + max_distance = caps.max_distance_mm, + invalid_value = caps.invalid_value, + "Depth camera capabilities" + ); + } + + self.device = Some(device.clone()); + self.format = Some(format.clone()); + self.capabilities = capabilities; + self.depth_v4l2_path = Some(v4l2_path.clone()); + + // Start single depth capture + self.start_capture(&v4l2_path, format)?; + + info!("V4L2 kernel depth backend initialized"); + Ok(()) + } + + fn shutdown(&mut self) -> BackendResult<()> { + info!("Shutting down V4L2 kernel depth backend"); + + self.stop_capture(); + + self.device = None; + self.format = None; + self.capabilities = None; + self.device_pair = None; + self.depth_v4l2_path = None; + self.color_v4l2_path = None; + + // Clear frame data + if let Ok(mut guard) = self.latest_depth_frame.lock() { + *guard = None; + } + if let Ok(mut guard) = self.latest_color_frame.lock() { + *guard = None; + } + + Ok(()) + } + + fn is_initialized(&self) -> bool { + self.device.is_some() && self.depth_capture_thread.is_some() + } + + fn recover(&mut self) -> BackendResult<()> { + let device = self + .device + .clone() + .ok_or_else(|| BackendError::Other("No device to recover".to_string()))?; + let format = self + .format + .clone() + .ok_or_else(|| BackendError::Other("No format to recover".to_string()))?; + + self.shutdown()?; + self.initialize(&device, &format) + } + + fn switch_camera(&mut self, device: &CameraDevice) -> BackendResult<()> { + let formats = self.get_formats(device, false); + let format = formats + .first() + .cloned() + .ok_or_else(|| BackendError::FormatNotSupported("No formats available".to_string()))?; + + self.initialize(device, &format) + } + + fn apply_format(&mut self, format: &CameraFormat) -> BackendResult<()> { + let device = self + .device + .clone() + .ok_or_else(|| BackendError::Other("No active device".to_string()))?; + + self.initialize(&device, format) + } + + fn capture_photo(&self) -> BackendResult { + self.latest_frame + .lock() + .ok() + .and_then(|guard| guard.clone()) + .ok_or_else(|| BackendError::Other("No frame available".to_string())) + } + + fn start_recording(&mut self, _output_path: PathBuf) -> BackendResult<()> { + Err(BackendError::Other( + "Recording not yet implemented for kernel depth backend".to_string(), + )) + } + + fn stop_recording(&mut self) -> BackendResult { + Err(BackendError::NoRecordingInProgress) + } + + fn is_recording(&self) -> bool { + false + } + + fn get_preview_receiver(&self) -> Option { + None // Use subscription model instead + } + + fn backend_type(&self) -> CameraBackendType { + CameraBackendType::PipeWire // Reuse existing type + } + + fn is_available(&self) -> bool { + // Check if any device has kernel depth support + for entry in std::fs::read_dir("/dev").into_iter().flatten() { + if let Ok(entry) = entry { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with("video") { + if Self::has_kernel_depth_support(&path.to_string_lossy()) { + return true; + } + } + } + } + } + false + } + + fn current_device(&self) -> Option<&CameraDevice> { + self.device.as_ref() + } + + fn current_format(&self) -> Option<&CameraFormat> { + self.format.as_ref() + } +} + +/// Check if a device path refers to a kernel depth camera +/// Recognizes both single depth devices (v4l2-depth:) and paired devices (v4l2-kinect:) +pub fn is_kernel_depth_device(path: &str) -> bool { + path.starts_with(KERNEL_DEPTH_PREFIX) || path.starts_with(KERNEL_KINECT_PREFIX) +} + +/// Check if a device path refers to a kernel kinect paired device +pub fn is_kernel_kinect_device(path: &str) -> bool { + path.starts_with(KERNEL_KINECT_PREFIX) +} + +/// Extract the V4L2 device path from a kernel depth path +pub fn kernel_depth_v4l2_path(path: &str) -> Option<&str> { + path.strip_prefix(KERNEL_DEPTH_PREFIX) +} + +/// Parse a kernel kinect device path to get color and depth paths +/// Format: v4l2-kinect:/dev/video4:/dev/video5 +pub fn parse_kernel_kinect_path(path: &str) -> Option<(String, String)> { + let paths = path.strip_prefix(KERNEL_KINECT_PREFIX)?; + let parts: Vec<&str> = paths.split(':').collect(); + if parts.len() == 2 { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + None + } +} diff --git a/src/backends/kinect/mod.rs b/src/backends/kinect/mod.rs new file mode 100644 index 00000000..f5600160 --- /dev/null +++ b/src/backends/kinect/mod.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(all(target_arch = "x86_64", feature = "freedepth"))] + +//! Kinect support via freedepth +//! +//! This module re-exports types from freedepth for Kinect sensor control. +//! Depth/video streaming is handled via V4L2 (kernel driver), while freedepth +//! handles the control features that V4L2 doesn't expose: +//! +//! - Motor/tilt control (-27° to +27°) +//! - LED control (off, green, red, yellow, blinking patterns) +//! - Accelerometer readout +//! - Device calibration data for accurate depth conversion +//! +//! ## Usage Pattern +//! +//! 1. Use freedepth to detect Kinect devices and fetch calibration +//! 2. Use V4L2 (`/dev/video*`) for depth/video streaming via GStreamer or v4l crate +//! 3. Use freedepth's `DepthToMm` converter to transform raw 11-bit depth to mm +//! 4. Use freedepth for motor/LED control during capture +//! +//! ## Example +//! +//! ```ignore +//! use camera::backends::kinect::{Context, Led}; +//! +//! // Initialize and fetch calibration +//! let ctx = Context::new()?; +//! let mut device = ctx.open_device(0)?; +//! device.fetch_calibration()?; +//! +//! // Get depth converter +//! let converter = device.depth_to_mm().unwrap(); +//! +//! // Start V4L2 streaming separately (via GStreamer pipeline) +//! // For each raw depth value from V4L2: +//! let raw_depth: u16 = 800; +//! let mm = converter.convert(raw_depth); +//! ``` + +// Re-export everything from freedepth +pub use freedepth::*; diff --git a/src/backends/mod.rs b/src/backends/mod.rs index 7f711bcc..e1fa8bce 100644 --- a/src/backends/mod.rs +++ b/src/backends/mod.rs @@ -38,4 +38,6 @@ pub mod audio; pub mod camera; +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub mod kinect; pub mod virtual_camera; diff --git a/src/backends/virtual_camera/file_source.rs b/src/backends/virtual_camera/file_source.rs index 7e161372..38dc25be 100644 --- a/src/backends/virtual_camera/file_source.rs +++ b/src/backends/virtual_camera/file_source.rs @@ -207,6 +207,10 @@ fn extract_frame_from_sample(sample: &gstreamer::Sample) -> BackendResult BackendResult { stride: width * 4, // RGBA = 4 bytes per pixel format: PixelFormat::RGBA, captured_at: Instant::now(), + depth_data: None, + depth_width: 0, + depth_height: 0, + video_timestamp: None, }) } @@ -514,6 +522,10 @@ impl VideoDecoder { stride: self.width * 4, format: PixelFormat::RGBA, captured_at: Instant::now(), + depth_data: None, + depth_width: 0, + depth_height: 0, + video_timestamp: None, }) } diff --git a/src/cli.rs b/src/cli.rs index aec3553e..e7aea102 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -423,6 +423,7 @@ pub fn process_burst_mode( exposure_time: None, iso: None, gain: None, + depth_data: None, }; let output_path = rt.block_on(async { @@ -503,6 +504,10 @@ fn load_dng_frame(path: &PathBuf) -> Result &'static str { match codec { + // Compressed Codec::MJPEG => "image/jpeg", Codec::H264 => "video/x-h264", Codec::H265 => "video/x-h265", + + // Packed YUV 4:2:2 Codec::YUYV => "video/x-raw,format=YUYV", Codec::UYVY => "video/x-raw,format=UYVY", Codec::YUY2 => "video/x-raw,format=YUY2", Codec::YVYU => "video/x-raw,format=YVYU", Codec::VYUY => "video/x-raw,format=VYUY", + + // Planar YUV 4:2:0 Codec::NV12 => "video/x-raw,format=NV12", Codec::NV21 => "video/x-raw,format=NV21", Codec::YV12 => "video/x-raw,format=YV12", Codec::I420 => "video/x-raw,format=I420", + + // RGB Codec::RGB24 => "video/x-raw,format=RGB", Codec::RGB32 => "video/x-raw,format=RGBA", Codec::BGR24 => "video/x-raw,format=BGR", Codec::BGR32 => "video/x-raw,format=BGRA", + + // Bayer patterns Codec::BayerGRBG => "video/x-bayer,format=grbg", Codec::BayerRGGB => "video/x-bayer,format=rggb", Codec::BayerBGGR => "video/x-bayer,format=bggr", Codec::BayerGBRG => "video/x-bayer,format=gbrg", - Codec::Y10B => "video/x-raw,format=GRAY10_LE32", - Codec::IR10 => "video/x-raw,format=GRAY10_LE32", + + // Depth/IR + Codec::Y10B => "video/x-raw,format=GRAY16_LE", // Y10B needs special handling + Codec::IR10 => "video/x-raw,format=GRAY16_LE", // IR10 is 10-bit packed like Y10B Codec::Y16 => "video/x-raw,format=GRAY16_LE", Codec::GREY => "video/x-raw,format=GRAY8", + Codec::Unknown => "video/x-raw", } } @@ -58,6 +70,11 @@ mod tests { assert_eq!(codec_to_gst_caps(&Codec::MJPEG), "image/jpeg"); assert_eq!(codec_to_gst_caps(&Codec::H264), "video/x-h264"); assert_eq!(codec_to_gst_caps(&Codec::YUYV), "video/x-raw,format=YUYV"); + assert_eq!(codec_to_gst_caps(&Codec::UYVY), "video/x-raw,format=UYVY"); + assert_eq!( + codec_to_gst_caps(&Codec::BayerGRBG), + "video/x-bayer,format=grbg" + ); } #[test] diff --git a/src/pipelines/mod.rs b/src/pipelines/mod.rs index 34123302..d3a4666d 100644 --- a/src/pipelines/mod.rs +++ b/src/pipelines/mod.rs @@ -35,6 +35,8 @@ //! //! - [`photo`]: Async photo capture with filters and JPEG encoding //! - [`video`]: Video recording with GStreamer and hardware acceleration +//! - [`scene`]: 3D scene capture with depth, color, point cloud, and mesh export pub mod photo; +pub mod scene; pub mod video; diff --git a/src/pipelines/photo/capture.rs b/src/pipelines/photo/capture.rs index c3f5d8af..a3212161 100644 --- a/src/pipelines/photo/capture.rs +++ b/src/pipelines/photo/capture.rs @@ -117,6 +117,10 @@ mod tests { format: PixelFormat::RGBA, stride: 1920 * 4, // RGBA stride captured_at: std::time::Instant::now(), + depth_data: None, + depth_width: 0, + depth_height: 0, + video_timestamp: None, }; let captured = PhotoCapture::capture_from_frame(frame).unwrap(); diff --git a/src/pipelines/photo/encoding.rs b/src/pipelines/photo/encoding.rs index a6290e59..fae8d95b 100644 --- a/src/pipelines/photo/encoding.rs +++ b/src/pipelines/photo/encoding.rs @@ -100,6 +100,20 @@ pub struct CameraMetadata { pub iso: Option, /// Gain value (camera-specific units) pub gain: Option, + /// Optional 16-bit depth data for depth sensors (e.g., Kinect) + /// When present, DNG encoder will include depth as a second IFD (SubImage) + pub depth_data: Option, +} + +/// Depth data information for DNG encoding +#[derive(Debug, Clone)] +pub struct DepthDataInfo { + /// Raw 16-bit depth values + pub values: Vec, + /// Width of depth image + pub width: u32, + /// Height of depth image + pub height: u32, } /// Photo encoder @@ -253,7 +267,96 @@ impl PhotoEncoder { Ok(buffer) } +} + +/// Encode 16-bit depth data as grayscale PNG (lossless) +/// +/// This is used for depth sensor data (e.g., Kinect Y10B format) to preserve +/// full 16-bit precision in a standard image format. +/// +/// # Arguments +/// * `depth_data` - 16-bit depth values (width * height values) +/// * `width` - Frame width in pixels +/// * `height` - Frame height in pixels +/// +/// # Returns +/// * `Ok(Vec)` - PNG encoded bytes +/// * `Err(String)` - Error message +pub fn encode_depth_png(depth_data: &[u16], width: u32, height: u32) -> Result, String> { + use image::ImageEncoder; + use image::codecs::png::PngEncoder; + + let expected_len = (width * height) as usize; + if depth_data.len() != expected_len { + return Err(format!( + "Depth data size mismatch: got {} values, expected {} for {}x{}", + depth_data.len(), + expected_len, + width, + height + )); + } + + // Convert u16 slice to bytes (little-endian) + let bytes: Vec = depth_data.iter().flat_map(|&v| v.to_le_bytes()).collect(); + + let mut buffer = Vec::new(); + let encoder = PngEncoder::new(&mut buffer); + + encoder + .write_image(&bytes, width, height, image::ExtendedColorType::L16) + .map_err(|e| format!("Depth PNG encoding failed: {}", e))?; + + debug!( + width, + height, + depth_values = depth_data.len(), + png_size = buffer.len(), + "Encoded depth data to 16-bit PNG" + ); + + Ok(buffer) +} +/// Save depth data to PNG file +/// +/// # Arguments +/// * `depth_data` - 16-bit depth values +/// * `width` - Frame width +/// * `height` - Frame height +/// * `output_dir` - Directory to save the file +/// +/// # Returns +/// * `Ok(PathBuf)` - Path to saved file +/// * `Err(String)` - Error message +pub async fn save_depth_png( + depth_data: Vec, + width: u32, + height: u32, + output_dir: PathBuf, +) -> Result { + // Generate filename with timestamp and "depth" prefix + let timestamp = chrono::Local::now().format("%Y%m%d_%H%M%S"); + let filename = format!("DEPTH_{}.png", timestamp); + let filepath = output_dir.join(&filename); + + info!(path = %filepath.display(), "Saving depth image"); + + let filepath_clone = filepath.clone(); + tokio::task::spawn_blocking(move || { + let png_data = encode_depth_png(&depth_data, width, height)?; + std::fs::write(&filepath_clone, &png_data) + .map_err(|e| format!("Failed to save depth image: {}", e))?; + Ok::<_, String>(()) + }) + .await + .map_err(|e| format!("Save task error: {}", e))??; + + info!(path = %filepath.display(), "Depth image saved successfully"); + Ok(filepath) +} + +impl PhotoEncoder { /// Encode image as DNG (Digital Negative raw format) /// /// Creates a simple linear DNG file with RGB data stored as strips. @@ -344,12 +447,12 @@ impl PhotoEncoder { ifd.insert(tiff_tags::Software, IfdValue::Ascii(software_with_gain)); } - // Create an Offsets implementation for the raw data - struct RgbOffsets { + // Create an Offsets implementation for data + struct DataOffsets { data: Vec, } - impl Offsets for RgbOffsets { + impl Offsets for DataOffsets { fn size(&self) -> u32 { self.data.len() as u32 } @@ -359,17 +462,68 @@ impl PhotoEncoder { } } - let offsets: Arc = Arc::new(RgbOffsets { data: raw_data }); + let offsets: Arc = Arc::new(DataOffsets { data: raw_data }); // Add strip data using Offsets ifd.insert(tiff_tags::StripOffsets, IfdValue::Offsets(offsets)); ifd.insert(tiff_tags::StripByteCounts, IfdValue::Long(raw_data_len)); + // Collect all IFDs + let mut ifds = vec![ifd]; + + // Add depth data as second IFD if present + if let Some(depth_info) = &camera_metadata.depth_data { + debug!( + depth_width = depth_info.width, + depth_height = depth_info.height, + depth_values = depth_info.values.len(), + "Adding depth data to DNG as second IFD" + ); + + let mut depth_ifd = Ifd::default(); + + // Required TIFF tags for 16-bit grayscale + depth_ifd.insert(tiff_tags::ImageWidth, IfdValue::Long(depth_info.width)); + depth_ifd.insert(tiff_tags::ImageLength, IfdValue::Long(depth_info.height)); + depth_ifd.insert(tiff_tags::BitsPerSample, IfdValue::Short(16)); // 16-bit + depth_ifd.insert(tiff_tags::Compression, IfdValue::Short(1)); // No compression + depth_ifd.insert(tiff_tags::PhotometricInterpretation, IfdValue::Short(1)); // BlackIsZero (grayscale) + depth_ifd.insert(tiff_tags::SamplesPerPixel, IfdValue::Short(1)); // Grayscale = 1 sample + depth_ifd.insert(tiff_tags::RowsPerStrip, IfdValue::Long(depth_info.height)); // One strip + depth_ifd.insert(tiff_tags::PlanarConfiguration, IfdValue::Short(1)); // Chunky + + // Mark as depth image (NewSubfileType = 4 for depth map, but we'll use ImageDescription) + // SubfileType 0 = full-resolution image + depth_ifd.insert(tiff_tags::NewSubfileType, IfdValue::Long(0)); + + // Add description to identify as depth data + depth_ifd.insert( + tiff_tags::ImageDescription, + IfdValue::Ascii("Depth Map (16-bit)".to_string()), + ); + + // Convert u16 depth values to bytes (little-endian for TIFF) + let depth_bytes: Vec = depth_info + .values + .iter() + .flat_map(|&v| v.to_le_bytes()) + .collect(); + let depth_data_len = depth_bytes.len() as u32; + + let depth_offsets: Arc = + Arc::new(DataOffsets { data: depth_bytes }); + + depth_ifd.insert(tiff_tags::StripOffsets, IfdValue::Offsets(depth_offsets)); + depth_ifd.insert(tiff_tags::StripByteCounts, IfdValue::Long(depth_data_len)); + + ifds.push(depth_ifd); + } + // Write the DNG file to a buffer let mut buffer = Vec::new(); let cursor = Cursor::new(&mut buffer); - DngWriter::write_dng(cursor, true, FileType::Dng, vec![ifd]) + DngWriter::write_dng(cursor, true, FileType::Dng, ifds) .map_err(|e| format!("DNG encoding failed: {:?}", e))?; Ok(buffer) diff --git a/src/pipelines/photo/mod.rs b/src/pipelines/photo/mod.rs index 5d220dfa..a69285e1 100644 --- a/src/pipelines/photo/mod.rs +++ b/src/pipelines/photo/mod.rs @@ -31,7 +31,7 @@ pub mod capture; pub mod encoding; pub mod processing; -pub use encoding::{CameraMetadata, EncodingFormat, EncodingQuality, PhotoEncoder}; +pub use encoding::{CameraMetadata, DepthDataInfo, EncodingFormat, EncodingQuality, PhotoEncoder}; pub use processing::{PostProcessingConfig, PostProcessor}; use crate::backends::camera::types::CameraFrame; @@ -76,29 +76,64 @@ impl PhotoPipeline { /// This runs the complete pipeline: /// 1. Captures frame from camera (already provided) /// 2. Post-processes the frame - /// 3. Encodes to target format + /// 3. Encodes to target format (DNG will include depth data if present) /// 4. Saves to disk + /// 5. If depth data is present AND format is not DNG, saves a separate depth PNG /// /// # Arguments - /// * `frame` - Raw camera frame (RGBA format) + /// * `frame` - Raw camera frame (RGBA format, may include depth_data) /// * `output_dir` - Directory to save the photo /// /// # Returns - /// * `Ok(PathBuf)` - Path to saved photo + /// * `Ok(PathBuf)` - Path to saved RGB photo (depth saved separately with DEPTH_ prefix for non-DNG) /// * `Err(String)` - Error message pub async fn capture_and_save( &self, frame: Arc, output_dir: PathBuf, ) -> Result { + use tracing::info; + + // Extract depth data before processing (clone the Arc if present) + let depth_data = frame.depth_data.clone(); + let width = frame.width; + let height = frame.height; + // Stage 1: Post-process (async, CPU-bound) let processed = self.post_processor.process(frame).await?; // Stage 2: Encode (async, CPU-bound) + // For DNG format, depth data will be included in the file itself let encoded = self.encoder.encode(processed).await?; - - // Stage 3: Save to disk (async, I/O-bound) - let output_path = self.encoder.save(encoded, output_dir).await?; + let is_dng = encoded.format == EncodingFormat::Dng; + + // Stage 3: Save RGB to disk (async, I/O-bound) + let output_path = self.encoder.save(encoded, output_dir.clone()).await?; + + // Stage 4: Save depth data if present (as 16-bit PNG) - only for non-DNG formats + // For DNG format, depth data is already embedded in the file via camera_metadata + if let Some(depth_arc) = depth_data { + if !is_dng { + info!( + width, + height, + depth_values = depth_arc.len(), + "Saving depth data as separate 16-bit PNG" + ); + match encoding::save_depth_png(depth_arc.to_vec(), width, height, output_dir).await + { + Ok(depth_path) => { + info!(path = %depth_path.display(), "Depth image saved"); + } + Err(e) => { + // Log but don't fail the whole capture + tracing::warn!(error = %e, "Failed to save depth image"); + } + } + } else { + info!("Depth data included in DNG file"); + } + } Ok(output_path) } @@ -106,9 +141,10 @@ impl PhotoPipeline { /// Capture and save with progress callback /// /// Same as `capture_and_save` but calls the provided callback at each stage. + /// Also saves depth data as a separate 16-bit PNG if present. /// /// # Arguments - /// * `frame` - Raw camera frame + /// * `frame` - Raw camera frame (may include depth_data) /// * `output_dir` - Directory to save the photo /// * `progress` - Callback for progress updates (0.0 - 1.0) pub async fn capture_and_save_with_progress( @@ -120,8 +156,15 @@ impl PhotoPipeline { where F: FnMut(f32) + Send, { + use tracing::info; + progress(0.0); + // Extract depth data before processing + let depth_data = frame.depth_data.clone(); + let width = frame.width; + let height = frame.height; + // Post-process let processed = self.post_processor.process(frame).await?; progress(0.33); @@ -130,8 +173,28 @@ impl PhotoPipeline { let encoded = self.encoder.encode(processed).await?; progress(0.66); - // Save - let output_path = self.encoder.save(encoded, output_dir).await?; + // Save RGB + let output_path = self.encoder.save(encoded, output_dir.clone()).await?; + progress(0.90); + + // Save depth data if present + if let Some(depth_arc) = depth_data { + info!( + width, + height, + depth_values = depth_arc.len(), + "Saving depth data as separate 16-bit PNG" + ); + match encoding::save_depth_png(depth_arc.to_vec(), width, height, output_dir).await { + Ok(depth_path) => { + info!(path = %depth_path.display(), "Depth image saved"); + } + Err(e) => { + tracing::warn!(error = %e, "Failed to save depth image"); + } + } + } + progress(1.0); Ok(output_path) diff --git a/src/pipelines/scene/gltf_export.rs b/src/pipelines/scene/gltf_export.rs new file mode 100644 index 00000000..6794434c --- /dev/null +++ b/src/pipelines/scene/gltf_export.rs @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(target_arch = "x86_64")] + +//! GLTF mesh export +//! +//! Exports depth + color data as a GLB (binary glTF) mesh with a texture. +//! Applies depth-to-RGB registration for correct UV mapping. +//! Origin is at the Kinect camera position. + +use super::{CameraIntrinsics, RegistrationData, SceneCaptureConfig}; +use crate::shaders::kinect_intrinsics as kinect; +use std::path::PathBuf; +use tracing::{debug, info}; + +/// Depth discontinuity threshold in meters (same as mesh shader) +const DEPTH_DISCONTINUITY_THRESHOLD: f32 = 0.1; + +/// Export mesh as GLB file with texture (applies registration via UV coordinates) +pub async fn export_mesh_gltf( + rgb_data: &[u8], + rgb_width: u32, + rgb_height: u32, + depth_data: &[u16], + depth_width: u32, + depth_height: u32, + output_path: &PathBuf, + config: &SceneCaptureConfig, +) -> Result<(), String> { + let rgb_data = rgb_data.to_vec(); + let depth_data = depth_data.to_vec(); + let output_path = output_path.clone(); + let config = config.clone(); + + tokio::task::spawn_blocking(move || { + export_gltf_sync( + &rgb_data, + rgb_width, + rgb_height, + &depth_data, + depth_width, + depth_height, + &output_path, + &config, + ) + }) + .await + .map_err(|e| format!("Task join error: {}", e))? +} + +fn export_gltf_sync( + rgb_data: &[u8], + rgb_width: u32, + rgb_height: u32, + depth_data: &[u16], + depth_width: u32, + depth_height: u32, + output_path: &PathBuf, + config: &SceneCaptureConfig, +) -> Result<(), String> { + // Generate mesh data (vertices, UVs, indices) using grid-based triangulation + let (vertices, uvs, indices) = generate_mesh_data( + rgb_width, + rgb_height, + depth_data, + depth_width, + depth_height, + &config.intrinsics, + config.depth_format, + config.mirror, + config.registration.as_ref(), + )?; + + if vertices.is_empty() || indices.is_empty() { + return Err("No valid mesh triangles generated".to_string()); + } + + info!( + vertex_count = vertices.len() / 3, + triangle_count = indices.len() / 3, + rgb_resolution = format!("{}x{}", rgb_width, rgb_height), + path = %output_path.display(), + "Exporting mesh with texture" + ); + + // Encode RGB data as JPEG for embedding in GLB + let texture_data = encode_texture_jpeg(rgb_data, rgb_width, rgb_height)?; + + // Build GLB file with texture + build_glb_file(&vertices, &uvs, &indices, &texture_data, output_path) +} + +/// Get registered RGB coordinates for a depth pixel as UV coordinates (0-1 range) +/// Returns the UV coordinates, or None if out of bounds +fn get_registered_uv_coords( + x: u32, + y: u32, + depth_mm: u32, + depth_width: u32, + rgb_width: u32, + rgb_height: u32, + registration: &RegistrationData, +) -> Option<(f32, f32)> { + let (rgb_x, rgb_y) = + registration.get_rgb_coords(x, y, depth_mm, depth_width, rgb_width, rgb_height)?; + + // Convert to UV coordinates (0-1 range) + // Note: V is flipped in glTF (0 at top, 1 at bottom) + let u = rgb_x as f32 / rgb_width as f32; + let v = rgb_y as f32 / rgb_height as f32; + + Some((u, v)) +} + +/// Generate mesh data using grid-based triangulation +/// Returns vertices (positions) and UV coordinates for texture mapping +#[allow(clippy::too_many_arguments)] +fn generate_mesh_data( + rgb_width: u32, + rgb_height: u32, + depth_data: &[u16], + depth_width: u32, + depth_height: u32, + intrinsics: &CameraIntrinsics, + depth_format: crate::shaders::DepthFormat, + mirror: bool, + registration: Option<&RegistrationData>, +) -> Result<(Vec, Vec, Vec), String> { + // Debug logging + if let Some(reg) = registration { + info!( + target_offset = reg.target_offset, + reg_scale_x = reg.reg_scale_x, + reg_scale_y = reg.reg_scale_y, + "GLB export using texture with registration" + ); + } else { + info!("GLB export: no registration data, using simple UV mapping"); + } + + // First pass: compute depth in meters and mm, plus registered UVs + let mut depth_meters: Vec = vec![-1.0; (depth_width * depth_height) as usize]; + let mut depth_mm_values: Vec = vec![0; (depth_width * depth_height) as usize]; + let mut vertex_uvs: Vec<[f32; 2]> = vec![[0.0, 0.0]; (depth_width * depth_height) as usize]; + + for y in 0..depth_height { + for x in 0..depth_width { + let idx = (y * depth_width + x) as usize; + let depth_raw = depth_data[idx]; + + let (depth_m, depth_mm) = match depth_format { + crate::shaders::DepthFormat::Millimeters => { + if depth_raw == 0 || depth_raw >= 10000 { + continue; + } + (depth_raw as f32 / 1000.0, depth_raw as u32) + } + crate::shaders::DepthFormat::Disparity16 => { + if depth_raw >= 65472 { + continue; + } + let raw = (depth_raw >> 6) as f32; + let denom = raw * kinect::DEPTH_COEFF_A + kinect::DEPTH_COEFF_B; + if denom <= 0.01 { + continue; + } + let dm = 1.0 / denom; + (dm, (dm * 1000.0) as u32) + } + }; + + if depth_m < intrinsics.min_depth || depth_m > intrinsics.max_depth { + continue; + } + + depth_meters[idx] = depth_m; + depth_mm_values[idx] = depth_mm; + + // Get registered UV coordinates + let uv = if let Some(reg) = registration { + if let Some((u, v)) = get_registered_uv_coords( + x, + y, + depth_mm, + depth_width, + rgb_width, + rgb_height, + reg, + ) { + [u, v] + } else { + // Registration out of bounds - skip this point + depth_meters[idx] = -1.0; + continue; + } + } else { + // No registration - use simple mapping + let u = x as f32 / depth_width as f32; + let v = y as f32 / depth_height as f32; + [u, v] + }; + + vertex_uvs[idx] = uv; + } + } + + // Second pass: generate vertices and triangles + let mut vertices: Vec = Vec::new(); + let mut uvs: Vec = Vec::new(); + let mut indices: Vec = Vec::new(); + let mut vertex_map: Vec = vec![-1; (depth_width * depth_height) as usize]; + + // Generate triangles for each 2x2 quad + for y in 1..depth_height { + for x in 1..depth_width { + let idx00 = ((y - 1) * depth_width + (x - 1)) as usize; + let idx10 = ((y - 1) * depth_width + x) as usize; + let idx01 = (y * depth_width + (x - 1)) as usize; + let idx11 = (y * depth_width + x) as usize; + + let d00 = depth_meters[idx00]; + let d10 = depth_meters[idx10]; + let d01 = depth_meters[idx01]; + let d11 = depth_meters[idx11]; + + // Skip if any depth is invalid + if d00 < 0.0 || d10 < 0.0 || d01 < 0.0 || d11 < 0.0 { + continue; + } + + // Check depth discontinuity + let max_diff = (d00 - d10) + .abs() + .max((d00 - d01).abs()) + .max((d11 - d10).abs()) + .max((d11 - d01).abs()); + + if max_diff > DEPTH_DISCONTINUITY_THRESHOLD { + continue; + } + + // Helper closure to get or create vertex + let mut get_or_create_vertex = |px: u32, py: u32| -> Option { + let idx = (py * depth_width + px) as usize; + let depth_m = depth_meters[idx]; + + if depth_m < 0.0 { + return None; + } + + if vertex_map[idx] >= 0 { + return Some(vertex_map[idx] as u32); + } + + // Unproject to 3D - origin is at camera + let mut vx = (px as f32 - intrinsics.cx) * depth_m / intrinsics.fx; + let vy = -((py as f32 - intrinsics.cy) * depth_m / intrinsics.fy); + let vz = depth_m; + + if mirror { + vx = -vx; + } + + let vertex_idx = (vertices.len() / 3) as u32; + vertices.push(vx); + vertices.push(vy); + vertices.push(vz); + + // Add UV coordinates + let uv = vertex_uvs[idx]; + uvs.push(uv[0]); + uvs.push(uv[1]); + + vertex_map[idx] = vertex_idx as i32; + Some(vertex_idx) + }; + + // Get vertex indices for quad corners + let v00 = get_or_create_vertex(x - 1, y - 1); + let v10 = get_or_create_vertex(x, y - 1); + let v01 = get_or_create_vertex(x - 1, y); + let v11 = get_or_create_vertex(x, y); + + if let (Some(i00), Some(i10), Some(i01), Some(i11)) = (v00, v10, v01, v11) { + // glTF uses counter-clockwise winding for front faces + // Triangle 1: (00, 01, 10) + indices.push(i00); + indices.push(i01); + indices.push(i10); + + // Triangle 2: (10, 01, 11) + indices.push(i10); + indices.push(i01); + indices.push(i11); + } + } + } + + Ok((vertices, uvs, indices)) +} + +/// Encode RGBA data as JPEG for embedding in GLB +fn encode_texture_jpeg(rgb_data: &[u8], width: u32, height: u32) -> Result, String> { + use image::{ImageBuffer, Rgba}; + + // Create image from RGBA data + let img: ImageBuffer, Vec> = + ImageBuffer::from_raw(width, height, rgb_data.to_vec()) + .ok_or("Failed to create image buffer")?; + + // Convert to RGB and encode as JPEG + let rgb_img = image::DynamicImage::ImageRgba8(img).into_rgb8(); + + let mut jpeg_data = Vec::new(); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_data, 92); + encoder + .encode_image(&rgb_img) + .map_err(|e| format!("Failed to encode JPEG: {}", e))?; + + info!( + jpeg_size = jpeg_data.len(), + original_size = rgb_data.len(), + compression_ratio = format!("{:.1}x", rgb_data.len() as f32 / jpeg_data.len() as f32), + "Encoded texture as JPEG" + ); + + Ok(jpeg_data) +} + +/// Build a GLB (binary glTF) file with texture +fn build_glb_file( + vertices: &[f32], + uvs: &[f32], + indices: &[u32], + texture_data: &[u8], + output_path: &PathBuf, +) -> Result<(), String> { + // Calculate buffer sizes + let vertices_bytes: Vec = vertices.iter().flat_map(|f| f.to_le_bytes()).collect(); + let uvs_bytes: Vec = uvs.iter().flat_map(|f| f.to_le_bytes()).collect(); + let indices_bytes: Vec = indices.iter().flat_map(|i| i.to_le_bytes()).collect(); + + // Buffer layout: vertices | uvs | indices | texture + let vertex_offset = 0usize; + let vertex_len = vertices_bytes.len(); + let uv_offset = vertex_len; + let uv_len = uvs_bytes.len(); + let index_offset = uv_offset + uv_len; + let index_len = indices_bytes.len(); + let texture_offset = index_offset + index_len; + let texture_len = texture_data.len(); + let total_buffer_len = texture_offset + texture_len; + + // Pad to 4-byte alignment + let padding = (4 - (total_buffer_len % 4)) % 4; + let padded_buffer_len = total_buffer_len + padding; + + // Calculate min/max for vertices + let mut min_pos = [f32::MAX; 3]; + let mut max_pos = [f32::MIN; 3]; + for chunk in vertices.chunks(3) { + min_pos[0] = min_pos[0].min(chunk[0]); + min_pos[1] = min_pos[1].min(chunk[1]); + min_pos[2] = min_pos[2].min(chunk[2]); + max_pos[0] = max_pos[0].max(chunk[0]); + max_pos[1] = max_pos[1].max(chunk[1]); + max_pos[2] = max_pos[2].max(chunk[2]); + } + + // Build glTF JSON with texture + let gltf_json = serde_json::json!({ + "asset": { + "generator": "COSMIC Camera", + "version": "2.0" + }, + "scene": 0, + "scenes": [{ + "nodes": [0] + }], + "nodes": [{ + "mesh": 0 + }], + "meshes": [{ + "primitives": [{ + "attributes": { + "POSITION": 0, + "TEXCOORD_0": 1 + }, + "indices": 2, + "material": 0, + "mode": 4 + }] + }], + "materials": [{ + "pbrMetallicRoughness": { + "baseColorTexture": { + "index": 0 + }, + "metallicFactor": 0.0, + "roughnessFactor": 1.0 + }, + "doubleSided": true + }], + "textures": [{ + "sampler": 0, + "source": 0 + }], + "samplers": [{ + "magFilter": 9729, // LINEAR + "minFilter": 9987, // LINEAR_MIPMAP_LINEAR + "wrapS": 33071, // CLAMP_TO_EDGE + "wrapT": 33071 // CLAMP_TO_EDGE + }], + "images": [{ + "bufferView": 3, + "mimeType": "image/jpeg" + }], + "accessors": [ + { + "bufferView": 0, + "byteOffset": 0, + "componentType": 5126, // FLOAT + "count": vertices.len() / 3, + "type": "VEC3", + "min": min_pos, + "max": max_pos + }, + { + "bufferView": 1, + "byteOffset": 0, + "componentType": 5126, // FLOAT + "count": uvs.len() / 2, + "type": "VEC2" + }, + { + "bufferView": 2, + "byteOffset": 0, + "componentType": 5125, // UNSIGNED_INT + "count": indices.len(), + "type": "SCALAR" + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": vertex_offset, + "byteLength": vertex_len, + "byteStride": 12, + "target": 34962 // ARRAY_BUFFER + }, + { + "buffer": 0, + "byteOffset": uv_offset, + "byteLength": uv_len, + "byteStride": 8, + "target": 34962 // ARRAY_BUFFER + }, + { + "buffer": 0, + "byteOffset": index_offset, + "byteLength": index_len, + "target": 34963 // ELEMENT_ARRAY_BUFFER + }, + { + "buffer": 0, + "byteOffset": texture_offset, + "byteLength": texture_len + // No target for images + } + ], + "buffers": [{ + "byteLength": padded_buffer_len + }] + }); + + // Serialize JSON + let json_string = serde_json::to_string(&gltf_json) + .map_err(|e| format!("Failed to serialize glTF: {}", e))?; + let json_bytes = json_string.as_bytes(); + + // Pad JSON to 4-byte alignment + let json_padding = (4 - (json_bytes.len() % 4)) % 4; + let padded_json_len = json_bytes.len() + json_padding; + + // Build GLB file + let total_length = 12 + 8 + padded_json_len + 8 + padded_buffer_len; + + let mut glb_data: Vec = Vec::with_capacity(total_length); + + // GLB Header + glb_data.extend_from_slice(b"glTF"); // Magic + glb_data.extend_from_slice(&2u32.to_le_bytes()); // Version + glb_data.extend_from_slice(&(total_length as u32).to_le_bytes()); // Length + + // JSON chunk + glb_data.extend_from_slice(&(padded_json_len as u32).to_le_bytes()); // Chunk length + glb_data.extend_from_slice(&0x4E4F534Au32.to_le_bytes()); // Chunk type "JSON" + glb_data.extend_from_slice(json_bytes); + glb_data.extend(std::iter::repeat_n(0x20u8, json_padding)); // Space padding + + // Binary chunk + glb_data.extend_from_slice(&(padded_buffer_len as u32).to_le_bytes()); // Chunk length + glb_data.extend_from_slice(&0x004E4942u32.to_le_bytes()); // Chunk type "BIN\0" + glb_data.extend_from_slice(&vertices_bytes); + glb_data.extend_from_slice(&uvs_bytes); + glb_data.extend_from_slice(&indices_bytes); + glb_data.extend_from_slice(texture_data); + glb_data.extend(std::iter::repeat_n(0u8, padding)); // Null padding + + // Write GLB file + std::fs::write(output_path, glb_data) + .map_err(|e| format!("Failed to write GLB file: {}", e))?; + + debug!(path = %output_path.display(), "GLB export complete"); + + Ok(()) +} diff --git a/src/pipelines/scene/laz_export.rs b/src/pipelines/scene/laz_export.rs new file mode 100644 index 00000000..0681ef0d --- /dev/null +++ b/src/pipelines/scene/laz_export.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(target_arch = "x86_64")] + +//! LAS point cloud export +//! +//! Exports depth + color data as an uncompressed LAS point cloud file. +//! Applies depth-to-RGB registration for correct color alignment. + +use super::{CameraIntrinsics, RegistrationData, SceneCaptureConfig}; +use crate::shaders::kinect_intrinsics as kinect; +use las::{Builder, Color, Point, Writer}; +use std::path::PathBuf; +use tracing::{debug, info}; + +/// Export point cloud as LAS file with color +pub async fn export_point_cloud_las( + rgb_data: &[u8], + rgb_width: u32, + rgb_height: u32, + depth_data: &[u16], + depth_width: u32, + depth_height: u32, + output_path: &PathBuf, + config: &SceneCaptureConfig, +) -> Result<(), String> { + let rgb_data = rgb_data.to_vec(); + let depth_data = depth_data.to_vec(); + let output_path = output_path.clone(); + let intrinsics = config.intrinsics; + let depth_format = config.depth_format; + let mirror = config.mirror; + let registration = config.registration.clone(); + + tokio::task::spawn_blocking(move || { + export_las_sync( + &rgb_data, + rgb_width, + rgb_height, + &depth_data, + depth_width, + depth_height, + &output_path, + &intrinsics, + depth_format, + mirror, + registration.as_ref(), + ) + }) + .await + .map_err(|e| format!("Task join error: {}", e))? +} + +/// Get registered RGB coordinates for a depth pixel +/// Applies high-res scaling for 1280x1024 RGB mode (registration tables are built for 640x480) +fn get_registered_rgb_coords( + x: u32, + y: u32, + depth_mm: u32, + depth_width: u32, + rgb_width: u32, + rgb_height: u32, + registration: &RegistrationData, +) -> Option<(u32, u32)> { + let (rgb_x, rgb_y) = + registration.get_rgb_coords(x, y, depth_mm, depth_width, rgb_width, rgb_height)?; + Some((rgb_x as u32, rgb_y as u32)) +} + +#[allow(clippy::too_many_arguments)] +fn export_las_sync( + rgb_data: &[u8], + rgb_width: u32, + rgb_height: u32, + depth_data: &[u16], + depth_width: u32, + depth_height: u32, + output_path: &PathBuf, + intrinsics: &CameraIntrinsics, + depth_format: crate::shaders::DepthFormat, + mirror: bool, + registration: Option<&RegistrationData>, +) -> Result<(), String> { + // Debug logging to compare with shader values + if let Some(reg) = registration { + let center_idx = 240 * 640 + 320; + info!( + target_offset = reg.target_offset, + table_len = reg.registration_table.len(), + shift_len = reg.depth_to_rgb_shift.len(), + center_reg_x = reg + .registration_table + .get(center_idx) + .map(|v| v[0]) + .unwrap_or(-1), + center_reg_y = reg + .registration_table + .get(center_idx) + .map(|v| v[1]) + .unwrap_or(-1), + shift_1000mm = reg.depth_to_rgb_shift.get(1000).copied().unwrap_or(-1), + "LAS export registration data" + ); + } else { + info!("LAS export: no registration data available"); + } + + // Collect valid 3D points with color + let mut points: Vec<(f64, f64, f64, u16, u16, u16)> = Vec::new(); + + for y in 0..depth_height { + for x in 0..depth_width { + let depth_idx = (y * depth_width + x) as usize; + let depth_raw = depth_data[depth_idx]; + + // Convert to meters based on depth format + let (depth_m, depth_mm) = match depth_format { + crate::shaders::DepthFormat::Millimeters => { + if depth_raw == 0 || depth_raw >= 10000 { + continue; // Invalid depth + } + (depth_raw as f32 / 1000.0, depth_raw as u32) + } + crate::shaders::DepthFormat::Disparity16 => { + if depth_raw >= 65472 { + continue; // Invalid depth marker + } + let raw = (depth_raw >> 6) as f32; + let denom = raw * kinect::DEPTH_COEFF_A + kinect::DEPTH_COEFF_B; + if denom <= 0.01 { + continue; + } + let dm = 1.0 / denom; + (dm, (dm * 1000.0) as u32) + } + }; + + // Check depth range + if depth_m < intrinsics.min_depth || depth_m > intrinsics.max_depth { + continue; + } + + // Get RGB color with registration (same as shader) + let (rgb_x, rgb_y) = if let Some(reg) = registration { + if let Some(coords) = get_registered_rgb_coords( + x, + y, + depth_mm, + depth_width, + rgb_width, + rgb_height, + reg, + ) { + coords + } else { + // Registration out of bounds - skip this point + continue; + } + } else { + // No registration - use simple mapping + let rx = if rgb_width != depth_width { + ((x as f32 * rgb_width as f32 / depth_width as f32) as u32).min(rgb_width - 1) + } else { + x + }; + let ry = if rgb_height != depth_height { + ((y as f32 * rgb_height as f32 / depth_height as f32) as u32) + .min(rgb_height - 1) + } else { + y + }; + (rx, ry) + }; + + let rgb_idx = ((rgb_y * rgb_width + rgb_x) * 4) as usize; + let r = rgb_data.get(rgb_idx).copied().unwrap_or(128) as u16 * 256; + let g = rgb_data.get(rgb_idx + 1).copied().unwrap_or(128) as u16 * 256; + let b = rgb_data.get(rgb_idx + 2).copied().unwrap_or(128) as u16 * 256; + + // Unproject to 3D + let mut px = ((x as f32 - intrinsics.cx) * depth_m / intrinsics.fx) as f64; + let py = -((y as f32 - intrinsics.cy) * depth_m / intrinsics.fy) as f64; + let pz = depth_m as f64; + + if mirror { + px = -px; + } + + points.push((px, py, pz, r, g, b)); + } + } + + if points.is_empty() { + return Err("No valid depth points to export".to_string()); + } + + info!( + point_count = points.len(), + path = %output_path.display(), + "Exporting point cloud" + ); + + // Calculate bounds for LAS header + let (min_x, max_x) = points + .iter() + .map(|p| p.0) + .fold((f64::MAX, f64::MIN), |(min, max), x| { + (min.min(x), max.max(x)) + }); + let (min_y, max_y) = points + .iter() + .map(|p| p.1) + .fold((f64::MAX, f64::MIN), |(min, max), y| { + (min.min(y), max.max(y)) + }); + let (min_z, max_z) = points + .iter() + .map(|p| p.2) + .fold((f64::MAX, f64::MIN), |(min, max), z| { + (min.min(z), max.max(z)) + }); + + // Build LAS header + let mut builder = Builder::from((1, 4)); // LAS 1.4 + builder.point_format.has_color = true; + builder.point_format.is_compressed = false; // Uncompressed LAS + + // Set transforms for coordinate precision + let scale = 0.001; // 1mm precision + builder.transforms = las::Vector { + x: las::Transform { + scale, + offset: (min_x + max_x) / 2.0, + }, + y: las::Transform { + scale, + offset: (min_y + max_y) / 2.0, + }, + z: las::Transform { + scale, + offset: (min_z + max_z) / 2.0, + }, + }; + + let header = builder + .into_header() + .map_err(|e| format!("Failed to build LAS header: {}", e))?; + + // Create writer + let mut writer = Writer::from_path(output_path, header) + .map_err(|e| format!("Failed to create LAS writer: {}", e))?; + + // Write points + for (px, py, pz, r, g, b) in points { + let mut point = Point::default(); + point.x = px; + point.y = py; + point.z = pz; + point.color = Some(Color::new(r, g, b)); + + writer + .write_point(point) + .map_err(|e| format!("Failed to write point: {}", e))?; + } + + writer + .close() + .map_err(|e| format!("Failed to close LAS file: {}", e))?; + + debug!( + path = %output_path.display(), + "LAS export complete" + ); + + Ok(()) +} diff --git a/src/pipelines/scene/mod.rs b/src/pipelines/scene/mod.rs new file mode 100644 index 00000000..3eceb863 --- /dev/null +++ b/src/pipelines/scene/mod.rs @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Scene capture pipeline +//! +//! Captures 3D scene data including: +//! - Raw depth image (grayscale) +//! - Raw color image +//! - Preview image (rendered 3D view) +//! - Point cloud with color (LAZ format) +//! - 3D mesh with texture (GLTF format) + +#[cfg(target_arch = "x86_64")] +mod gltf_export; +#[cfg(target_arch = "x86_64")] +mod laz_export; + +#[cfg(target_arch = "x86_64")] +pub use gltf_export::export_mesh_gltf; +#[cfg(target_arch = "x86_64")] +pub use laz_export::export_point_cloud_las; + +#[cfg(not(target_arch = "x86_64"))] +pub async fn export_mesh_gltf( + _rgb_data: &[u8], + _rgb_width: u32, + _rgb_height: u32, + _depth_data: &[u16], + _depth_width: u32, + _depth_height: u32, + _output_path: &std::path::PathBuf, + _config: &SceneCaptureConfig, +) -> Result<(), String> { + Err("freedepth feature not enabled".to_string()) +} + +#[cfg(not(target_arch = "x86_64"))] +pub async fn export_point_cloud_las( + _rgb_data: &[u8], + _rgb_width: u32, + _rgb_height: u32, + _depth_data: &[u16], + _depth_width: u32, + _depth_height: u32, + _output_path: &std::path::PathBuf, + _config: &SceneCaptureConfig, +) -> Result<(), String> { + Err("freedepth feature not enabled".to_string()) +} + +use crate::pipelines::photo::encoding::EncodingFormat; +use crate::shaders::kinect_intrinsics as kinect; +use image::{GrayImage, RgbImage, RgbaImage}; +use std::path::PathBuf; +use tracing::{debug, info}; + +/// Scene capture configuration +#[derive(Clone)] +pub struct SceneCaptureConfig { + /// Output format for images (JPEG, PNG, DNG) + pub image_format: EncodingFormat, + /// Camera intrinsics for 3D reconstruction + pub intrinsics: CameraIntrinsics, + /// Depth format (millimeters or disparity) + pub depth_format: crate::shaders::DepthFormat, + /// Whether to mirror the output + pub mirror: bool, + /// Registration data for depth-to-RGB alignment (optional) + pub registration: Option, +} + +/// Registration data for depth-to-RGB alignment +#[derive(Clone)] +pub struct RegistrationData { + /// Registration table: 640*480 [x_scaled, y] pairs + pub registration_table: Vec<[i32; 2]>, + /// Depth-to-RGB shift table: 10001 i32 values indexed by depth_mm + pub depth_to_rgb_shift: Vec, + /// Target offset from pad_info + pub target_offset: u32, + /// X scale factor for high-res RGB (1.0 for 640, 2.0 for 1280) + pub reg_scale_x: f32, + /// Y scale factor for high-res RGB (same as X to maintain aspect ratio) + pub reg_scale_y: f32, + /// Y offset for high-res RGB (typically 0 for top-aligned crop) + pub reg_y_offset: i32, +} + +impl RegistrationData { + /// Get registered RGB pixel coordinates for a depth pixel + /// + /// Applies the registration transform from depth space to RGB space, + /// accounting for high-res scaling if needed. + /// + /// Returns None if the coordinates are out of bounds or registration data is invalid. + pub fn get_rgb_coords( + &self, + x: u32, + y: u32, + depth_mm: u32, + depth_width: u32, + rgb_width: u32, + rgb_height: u32, + ) -> Option<(i32, i32)> { + let reg_idx = (y * depth_width + x) as usize; + if reg_idx >= self.registration_table.len() { + return None; + } + + let reg = self.registration_table[reg_idx]; + let clamped_depth_mm = depth_mm.min(10000) as usize; + + if clamped_depth_mm >= self.depth_to_rgb_shift.len() { + return None; + } + + let shift = self.depth_to_rgb_shift[clamped_depth_mm]; + + // Calculate RGB coordinates using registration formula from libfreenect + // Base coordinates are in 640x480 space + // reg_x_val_scale is always 256 (freedepth::REG_X_VAL_SCALE) + let rgb_x_scaled = reg[0] + shift; + let rgb_x_base = rgb_x_scaled / 256; + let rgb_y_base = reg[1] - self.target_offset as i32; + + // Scale to actual RGB resolution (for 1280x1024, scale by 2.0) + let rgb_x = (rgb_x_base as f32 * self.reg_scale_x) as i32; + let rgb_y = (rgb_y_base as f32 * self.reg_scale_y) as i32 + self.reg_y_offset; + + // Check bounds + if rgb_x < 0 || rgb_x >= rgb_width as i32 || rgb_y < 0 || rgb_y >= rgb_height as i32 { + return None; + } + + Some((rgb_x, rgb_y)) + } +} + +/// Camera intrinsics for depth-to-3D unprojection +#[derive(Clone, Copy)] +pub struct CameraIntrinsics { + pub fx: f32, + pub fy: f32, + pub cx: f32, + pub cy: f32, + pub min_depth: f32, + pub max_depth: f32, +} + +impl Default for CameraIntrinsics { + fn default() -> Self { + // Kinect defaults for 640x480 base resolution + Self { + fx: kinect::FX, + fy: kinect::FY, + cx: kinect::CX, + cy: kinect::CY, + min_depth: 0.4, + max_depth: 4.0, + } + } +} + +/// Result of scene capture +pub struct SceneCaptureResult { + pub scene_dir: PathBuf, + pub depth_path: PathBuf, + pub color_path: PathBuf, + pub preview_path: PathBuf, + pub pointcloud_path: PathBuf, + pub mesh_path: PathBuf, +} + +/// Capture and save a complete scene +/// +/// Creates a directory containing: +/// - depth.{format} - Raw depth as grayscale +/// - color.{format} - Raw color image +/// - preview.{format} - Rendered 3D preview +/// - pointcloud.las - Point cloud with color +/// - mesh.glb - 3D mesh with texture +pub async fn capture_scene( + rgb_data: &[u8], + rgb_width: u32, + rgb_height: u32, + depth_data: &[u16], + depth_width: u32, + depth_height: u32, + preview_data: Option<&[u8]>, + preview_width: u32, + preview_height: u32, + output_dir: PathBuf, + config: SceneCaptureConfig, +) -> Result { + // Create timestamped scene directory + let timestamp = chrono::Local::now().format("%Y%m%d_%H%M%S"); + let scene_dir = output_dir.join(format!("scene_{}", timestamp)); + tokio::fs::create_dir_all(&scene_dir) + .await + .map_err(|e| format!("Failed to create scene directory: {}", e))?; + + info!(scene_dir = %scene_dir.display(), "Creating scene capture"); + + let ext = config.image_format.extension(); + + // 1. Save depth image (grayscale) + let depth_path = scene_dir.join(format!("depth.{}", ext)); + save_depth_image(depth_data, depth_width, depth_height, &depth_path, &config).await?; + debug!(path = %depth_path.display(), "Saved depth image"); + + // 2. Save color image + let color_path = scene_dir.join(format!("color.{}", ext)); + save_color_image(rgb_data, rgb_width, rgb_height, &color_path, &config).await?; + debug!(path = %color_path.display(), "Saved color image"); + + // 3. Save preview image (rendered 3D view) + let preview_path = scene_dir.join(format!("preview.{}", ext)); + if let Some(preview) = preview_data { + save_preview_image( + preview, + preview_width, + preview_height, + &preview_path, + &config, + ) + .await?; + debug!(path = %preview_path.display(), "Saved preview image"); + } else { + // If no preview, copy the color image + tokio::fs::copy(&color_path, &preview_path) + .await + .map_err(|e| format!("Failed to copy preview: {}", e))?; + } + + // 4. Export point cloud as LAS + let pointcloud_path = scene_dir.join("pointcloud.las"); + export_point_cloud_las( + rgb_data, + rgb_width, + rgb_height, + depth_data, + depth_width, + depth_height, + &pointcloud_path, + &config, + ) + .await?; + debug!(path = %pointcloud_path.display(), "Saved point cloud"); + + // 5. Export mesh as GLB with vertex colors (registration applied directly) + let mesh_path = scene_dir.join("mesh.glb"); + export_mesh_gltf( + rgb_data, + rgb_width, + rgb_height, + depth_data, + depth_width, + depth_height, + &mesh_path, + &config, + ) + .await?; + debug!(path = %mesh_path.display(), "Saved mesh"); + + info!( + scene_dir = %scene_dir.display(), + "Scene capture complete" + ); + + Ok(SceneCaptureResult { + scene_dir: scene_dir.clone(), + depth_path, + color_path, + preview_path, + pointcloud_path, + mesh_path, + }) +} + +/// Save depth data as grayscale image +async fn save_depth_image( + depth_data: &[u16], + width: u32, + height: u32, + path: &PathBuf, + config: &SceneCaptureConfig, +) -> Result<(), String> { + let depth_data = depth_data.to_vec(); + let path = path.clone(); + let format = config.image_format; + + tokio::task::spawn_blocking(move || { + // Convert 16-bit depth to 8-bit grayscale for visualization + // Normalize to 0-255 range based on valid depth range + let mut gray_data = Vec::with_capacity((width * height) as usize); + + for &d in &depth_data { + let normalized = if d == 0 || d >= 10000 { + 0u8 // Invalid depth = black + } else { + // Normalize depth to 0-255 (closer = brighter) + let depth_m = d as f32 / 1000.0; + let normalized = 1.0 - (depth_m - 0.4) / (4.0 - 0.4); + (normalized.clamp(0.0, 1.0) * 255.0) as u8 + }; + gray_data.push(normalized); + } + + let gray_image = GrayImage::from_raw(width, height, gray_data) + .ok_or("Failed to create grayscale image")?; + + match format { + EncodingFormat::Jpeg => { + let rgb_image: RgbImage = image::DynamicImage::ImageLuma8(gray_image).into_rgb8(); + rgb_image + .save(&path) + .map_err(|e| format!("Failed to save depth JPEG: {}", e)) + } + EncodingFormat::Png => gray_image + .save(&path) + .map_err(|e| format!("Failed to save depth PNG: {}", e)), + EncodingFormat::Dng => { + // For DNG, save as 16-bit PNG instead (DNG is for RGB) + let path = path.with_extension("png"); + // Save raw 16-bit depth + let img = image::ImageBuffer::, Vec>::from_raw( + width, height, depth_data, + ) + .ok_or("Failed to create 16-bit depth image")?; + img.save(&path) + .map_err(|e| format!("Failed to save depth 16-bit PNG: {}", e)) + } + } + }) + .await + .map_err(|e| format!("Task join error: {}", e))? +} + +/// Save color image +async fn save_color_image( + rgb_data: &[u8], + width: u32, + height: u32, + path: &PathBuf, + config: &SceneCaptureConfig, +) -> Result<(), String> { + let rgb_data = rgb_data.to_vec(); + let path = path.clone(); + let format = config.image_format; + + tokio::task::spawn_blocking(move || { + // Convert RGBA to RGB + let rgb_only: Vec = rgb_data.chunks(4).flat_map(|c| &c[0..3]).copied().collect(); + + let rgb_image = RgbImage::from_raw(width, height, rgb_only) + .ok_or("Failed to create RGB image from color data")?; + + match format { + EncodingFormat::Jpeg => { + let mut buf = Vec::new(); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 92); + encoder + .encode_image(&rgb_image) + .map_err(|e| format!("Failed to encode JPEG: {}", e))?; + std::fs::write(&path, buf).map_err(|e| format!("Failed to write JPEG: {}", e)) + } + EncodingFormat::Png | EncodingFormat::Dng => { + // Use PNG for both PNG and DNG (DNG is complex for simple RGB) + rgb_image + .save(&path) + .map_err(|e| format!("Failed to save PNG: {}", e)) + } + } + }) + .await + .map_err(|e| format!("Task join error: {}", e))? +} + +/// Save preview image (rendered 3D view) +async fn save_preview_image( + preview_data: &[u8], + width: u32, + height: u32, + path: &PathBuf, + config: &SceneCaptureConfig, +) -> Result<(), String> { + let preview_data = preview_data.to_vec(); + let path = path.clone(); + let format = config.image_format; + + tokio::task::spawn_blocking(move || { + // Preview data is RGBA from the shader + let rgba_image = RgbaImage::from_raw(width, height, preview_data) + .ok_or("Failed to create RGBA image from preview data")?; + + // Convert to RGB + let rgb_image: RgbImage = image::DynamicImage::ImageRgba8(rgba_image).into_rgb8(); + + match format { + EncodingFormat::Jpeg => { + let mut buf = Vec::new(); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 92); + encoder + .encode_image(&rgb_image) + .map_err(|e| format!("Failed to encode preview JPEG: {}", e))?; + std::fs::write(&path, buf) + .map_err(|e| format!("Failed to write preview JPEG: {}", e)) + } + EncodingFormat::Png | EncodingFormat::Dng => rgb_image + .save(&path) + .map_err(|e| format!("Failed to save preview PNG: {}", e)), + } + }) + .await + .map_err(|e| format!("Task join error: {}", e))? +} diff --git a/src/pipelines/video/recorder.rs b/src/pipelines/video/recorder.rs index f2916a32..eb91a316 100644 --- a/src/pipelines/video/recorder.rs +++ b/src/pipelines/video/recorder.rs @@ -524,6 +524,10 @@ impl VideoRecorder { crate::backends::camera::types::PixelFormat::RGBA, stride, captured_at: std::time::Instant::now(), + depth_data: None, + depth_width: 0, + depth_height: 0, + video_timestamp: None, }; let _ = preview_sender.send(frame).await; diff --git a/src/shaders/common/geometry.wgsl b/src/shaders/common/geometry.wgsl new file mode 100644 index 00000000..301e9dd2 --- /dev/null +++ b/src/shaders/common/geometry.wgsl @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-3.0-only +// +// Shared 3D Geometry Utilities +// ============================ +// +// These functions are concatenated into point_cloud.wgsl and mesh.wgsl at build time. +// They require a `params` struct to be defined with the following fields: +// fx, fy, cx, cy: Camera intrinsics +// output_width, output_height: Render dimensions +// view_distance: Camera Z position +// fov: Field of view + +// Create rotation matrix from pitch and yaw +fn rotation_matrix(pitch: f32, yaw: f32) -> mat3x3 { + let cp = cos(pitch); + let sp = sin(pitch); + let cy = cos(yaw); + let sy = sin(yaw); + + // Combined rotation: first yaw (around Y), then pitch (around X) + return mat3x3( + vec3(cy, 0.0, sy), + vec3(sp * sy, cp, -sp * cy), + vec3(-cp * sy, sp, cp * cy) + ); +} + +// Project 2D pixel + depth to 3D point +// Note: Y is negated to convert from image coordinates (Y down) to 3D (Y up) +fn unproject(u: f32, v: f32, depth: f32) -> vec3 { + let x = (u - params.cx) * depth / params.fx; + let y = -(v - params.cy) * depth / params.fy; + let z = depth; + return vec3(x, y, z); +} + +// Apply perspective projection to get screen coordinates +fn project_to_screen(point: vec3) -> vec3 { + // Camera position: view_distance moves INTO the scene + let camera_z = params.view_distance; + let p = vec3(point.x, point.y, point.z - camera_z); + + // Perspective division + if (p.z <= 0.01) { + return vec3(-1.0, -1.0, -1.0); // Behind camera + } + + let aspect = f32(params.output_width) / f32(params.output_height); + let fov_factor = tan(params.fov * 0.5); + + let screen_x = p.x / (p.z * fov_factor * aspect); + let screen_y = p.y / (p.z * fov_factor); + + // Convert from [-1, 1] to pixel coordinates + let px = (screen_x * 0.5 + 0.5) * f32(params.output_width); + let py = (0.5 - screen_y * 0.5) * f32(params.output_height); + + return vec3(px, py, p.z); +} + +// Unpack RGBA from u32 +fn unpack_rgba(packed: u32) -> vec4 { + let r = f32((packed >> 0u) & 0xFFu) / 255.0; + let g = f32((packed >> 8u) & 0xFFu) / 255.0; + let b = f32((packed >> 16u) & 0xFFu) / 255.0; + let a = f32((packed >> 24u) & 0xFFu) / 255.0; + return vec4(r, g, b, a); +} + +// Kinect invalid depth marker (1023 in 10-bit, shifted to 16-bit = 65472) +const DEPTH_INVALID_16BIT: u32 = 65472u; diff --git a/src/shaders/common/mod.rs b/src/shaders/common/mod.rs new file mode 100644 index 00000000..6a332d48 --- /dev/null +++ b/src/shaders/common/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Shared shader utilities +//! +//! Common WGSL functions shared between point cloud and mesh rendering shaders. +//! These are concatenated with shader-specific code at compile time. + +mod params; + +pub use params::Render3DParams; + +/// Shared geometry functions for 3D rendering +/// +/// Includes: +/// - `rotation_matrix(pitch, yaw)` - Creates rotation matrix from Euler angles +/// - `unproject(u, v, depth, cx, cy, fx, fy)` - Projects 2D pixel to 3D point +/// - `project_to_screen(point, view_distance, fov, width, height)` - Perspective projection +/// - `unpack_rgba(packed)` - Unpacks RGBA from u32 +pub const GEOMETRY_FUNCTIONS: &str = include_str!("geometry.wgsl"); diff --git a/src/shaders/common/params.rs b/src/shaders/common/params.rs new file mode 100644 index 00000000..a7003c80 --- /dev/null +++ b/src/shaders/common/params.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Unified 3D rendering parameters +//! +//! Shared parameter struct for point cloud and mesh rendering. +//! These processors share 90% of their parameters, so we unify them. + +/// Unified 3D rendering parameters for point cloud and mesh shaders +/// +/// This struct is used by both PointCloudProcessor and MeshProcessor. +/// Each processor uses the relevant fields for its rendering mode. +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +pub struct Render3DParams { + // === Dimensions === + /// Depth input width + pub input_width: u32, + /// Depth input height + pub input_height: u32, + /// Output image width + pub output_width: u32, + /// Output image height + pub output_height: u32, + /// RGB input width (may differ from depth) + pub rgb_width: u32, + /// RGB input height + pub rgb_height: u32, + + // === Camera intrinsics === + /// Focal length X (pixels) + pub fx: f32, + /// Focal length Y (pixels) + pub fy: f32, + /// Principal point X (pixels) + pub cx: f32, + /// Principal point Y (pixels) + pub cy: f32, + + // === Depth format and conversion === + /// Depth format: 0 = millimeters, 1 = disparity (10-bit shifted to 16-bit) + pub depth_format: u32, + /// Depth conversion coefficient A (for disparity: 1/depth = raw * A + B) + pub depth_coeff_a: f32, + /// Depth conversion coefficient B + pub depth_coeff_b: f32, + /// Minimum valid depth (meters) + pub min_depth: f32, + /// Maximum valid depth (meters) + pub max_depth: f32, + + // === View transform === + /// Rotation around X axis (radians) + pub pitch: f32, + /// Rotation around Y axis (radians) + pub yaw: f32, + /// Field of view (radians) + pub fov: f32, + /// Camera distance from scene center + pub view_distance: f32, + + // === Registration parameters === + /// Whether to use registration lookup tables (1) or simple shift (0) + pub use_registration_tables: u32, + /// Y offset from pad_info for registration tables + pub target_offset: u32, + /// Fixed-point scale factor (typically 256) + pub reg_x_val_scale: i32, + /// Mirror horizontally (1 = yes, 0 = no) + pub mirror: u32, + /// X scale factor for high-res RGB (1.0 for 640, 2.0 for 1280) + pub reg_scale_x: f32, + /// Y scale factor for high-res RGB + pub reg_scale_y: f32, + /// Y offset for high-res (32 for 1280x1024, 0 for 640x480) + pub reg_y_offset: i32, + + // === Rendering mode-specific === + /// Point size (point cloud only, 0.0 for mesh) + pub point_size: f32, + /// Depth discontinuity threshold in meters (mesh only, 0.0 for point cloud) + pub depth_discontinuity_threshold: f32, + /// Color filter mode (0 = none, 1-14 = various filters) + pub filter_mode: u32, +} + +impl Default for Render3DParams { + fn default() -> Self { + Self { + input_width: 640, + input_height: 480, + output_width: 640, + output_height: 480, + rgb_width: 640, + rgb_height: 480, + // Default Kinect intrinsics + fx: 594.21, + fy: 591.04, + cx: 339.5, + cy: 242.7, + depth_format: 0, + depth_coeff_a: -0.0030711, + depth_coeff_b: 3.3309495, + min_depth: 0.4, + max_depth: 4.0, + pitch: 0.0, + yaw: 0.0, + fov: std::f32::consts::FRAC_PI_4, + view_distance: 2.0, + use_registration_tables: 0, + target_offset: 0, + reg_x_val_scale: 256, + mirror: 0, + reg_scale_x: 1.0, + reg_scale_y: 1.0, + reg_y_offset: 0, + point_size: 2.0, + depth_discontinuity_threshold: 0.1, + filter_mode: 0, + } + } +} + +impl Render3DParams { + /// Create params for point cloud rendering + pub fn for_point_cloud(point_size: f32) -> Self { + Self { + point_size, + depth_discontinuity_threshold: 0.0, + ..Default::default() + } + } + + /// Create params for mesh rendering + pub fn for_mesh(discontinuity_threshold: f32) -> Self { + Self { + point_size: 0.0, + depth_discontinuity_threshold: discontinuity_threshold, + ..Default::default() + } + } +} diff --git a/src/shaders/depth/constants.rs b/src/shaders/depth/constants.rs new file mode 100644 index 00000000..ef03bba0 --- /dev/null +++ b/src/shaders/depth/constants.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Depth sensor constants - Single source of truth +//! +//! All depth range, visualization, and sensor-specific constants live here. +//! These values are used across the depth processing pipeline. + +/// Kinect sensor depth range limits (millimeters) +/// Based on Xbox Kinect v1 sensor specifications +pub const DEPTH_MIN_MM: f32 = 400.0; +pub const DEPTH_MAX_MM: f32 = 4000.0; + +/// Integer versions for UI display +pub const DEPTH_MIN_MM_U16: u16 = 400; +pub const DEPTH_MAX_MM_U16: u16 = 4000; + +/// Invalid depth marker values +pub const DEPTH_INVALID_MM: u16 = 0; +/// Maximum valid depth value (values above this are considered invalid) +pub const DEPTH_MAX_VALID_MM: u16 = 8000; + +/// Number of quantization bands for depth colormap visualization +pub const DEPTH_COLORMAP_BANDS: f32 = 32.0; diff --git a/src/shaders/depth/mod.rs b/src/shaders/depth/mod.rs new file mode 100644 index 00000000..62abb916 --- /dev/null +++ b/src/shaders/depth/mod.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(all(target_arch = "x86_64", feature = "freedepth"))] + +//! GPU-accelerated depth processing for Y10B format +//! +//! This module provides GPU-based unpacking of Y10B depth sensor data (Kinect) +//! into viewable RGBA preview and lossless 16-bit depth values. + +mod constants; +mod processor; +mod visualization; + +pub use constants::*; +pub use visualization::{depth_mm_to_rgba, rgb_to_rgba}; + +/// Kinect camera intrinsics - re-exported from kinect_intrinsics module +pub use crate::shaders::kinect_intrinsics as kinect; + +pub use processor::{DepthProcessor, unpack_y10b_gpu}; + +/// Y10B shader source +pub const Y10B_UNPACK_SHADER: &str = include_str!("y10b_unpack.wgsl"); + +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Global depth visualization settings +/// These can be updated from the UI thread and read from the processing thread +static DEPTH_COLORMAP_ENABLED: AtomicBool = AtomicBool::new(false); +static DEPTH_ONLY_MODE: AtomicBool = AtomicBool::new(false); +static DEPTH_GRAYSCALE_MODE: AtomicBool = AtomicBool::new(false); + +/// Set whether the depth colormap should be enabled +pub fn set_depth_colormap_enabled(enabled: bool) { + DEPTH_COLORMAP_ENABLED.store(enabled, Ordering::Relaxed); +} + +/// Get whether the depth colormap is enabled +pub fn is_depth_colormap_enabled() -> bool { + DEPTH_COLORMAP_ENABLED.load(Ordering::Relaxed) +} + +/// Set whether depth-only mode is enabled (pure colormap without blending) +pub fn set_depth_only_mode(enabled: bool) { + DEPTH_ONLY_MODE.store(enabled, Ordering::Relaxed); +} + +/// Get whether depth-only mode is enabled +pub fn is_depth_only_mode() -> bool { + DEPTH_ONLY_MODE.load(Ordering::Relaxed) +} + +/// Set whether grayscale depth mode is enabled (grayscale instead of colormap) +pub fn set_depth_grayscale_mode(enabled: bool) { + DEPTH_GRAYSCALE_MODE.store(enabled, Ordering::Relaxed); +} + +/// Get whether grayscale depth mode is enabled +pub fn is_depth_grayscale_mode() -> bool { + DEPTH_GRAYSCALE_MODE.load(Ordering::Relaxed) +} diff --git a/src/shaders/depth/processor.rs b/src/shaders/depth/processor.rs new file mode 100644 index 00000000..01d8eef2 --- /dev/null +++ b/src/shaders/depth/processor.rs @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! GPU depth processor for Y10B format unpacking +//! +//! Uses WGPU compute shaders to efficiently unpack Y10B depth data from +//! the Kinect depth sensor into: +//! - RGBA preview images for display +//! - 16-bit depth values for lossless storage + +use crate::gpu::{self, wgpu}; +use crate::gpu_processor_singleton; +use crate::shaders::{CachedDimensions, compute_dispatch_size}; +use std::sync::Arc; +use tracing::{debug, info, warn}; + +// Import depth format constants from freedepth +use freedepth::{DEPTH_10BIT_NO_VALUE, y10b_packed_size}; + +/// Depth processing parameters +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +struct DepthParams { + width: u32, + height: u32, + min_depth: u32, + max_depth: u32, + use_colormap: u32, // 0 = grayscale, 1 = turbo colormap + depth_only: u32, // 0 = normal, 1 = depth-only mode (always use colormap) +} + +/// Result of depth unpacking containing both preview and depth data +pub struct DepthUnpackResult { + /// RGBA data for preview display (width * height * 4 bytes) + pub rgba_preview: Vec, + /// 16-bit depth values for lossless storage (width * height values) + pub depth_u16: Vec, + /// Frame width + pub width: u32, + /// Frame height + pub height: u32, +} + +/// GPU depth processor for Y10B format +pub struct DepthProcessor { + device: Arc, + queue: Arc, + pipeline: wgpu::ComputePipeline, + bind_group_layout: wgpu::BindGroupLayout, + uniform_buffer: wgpu::Buffer, + // Cached resources for current dimensions + cached_dims: CachedDimensions, + input_buffer: Option, + output_rgba_texture: Option, + output_depth_buffer: Option, + staging_rgba_buffer: Option, + staging_depth_buffer: Option, +} + +impl DepthProcessor { + /// Create a new GPU depth processor + pub async fn new() -> Result { + info!("Initializing GPU depth processor for Y10B format"); + + // Create device with low-priority queue + let (device, queue, gpu_info) = + gpu::create_low_priority_compute_device("depth_processor_gpu").await?; + + info!( + adapter_name = %gpu_info.adapter_name, + adapter_backend = ?gpu_info.backend, + low_priority = gpu_info.low_priority_enabled, + "GPU device created for depth processing" + ); + + // Create shader + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("y10b_unpack_shader"), + source: wgpu::ShaderSource::Wgsl(super::Y10B_UNPACK_SHADER.into()), + }); + + // Create bind group layout + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("depth_bind_group_layout"), + entries: &[ + // Input bytes buffer (read as u32) + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Output RGBA texture + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::StorageTexture { + access: wgpu::StorageTextureAccess::WriteOnly, + format: wgpu::TextureFormat::Rgba8Unorm, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + // Output depth buffer (u32 containing 16-bit values) + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: false }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Uniform parameters + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + + // Create pipeline layout + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("depth_pipeline_layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + // Create compute pipeline + let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("y10b_unpack_pipeline"), + layout: Some(&pipeline_layout), + module: &shader, + entry_point: Some("main"), + compilation_options: Default::default(), + cache: None, + }); + + // Create uniform buffer + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("depth_uniform_buffer"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + Ok(Self { + device, + queue, + pipeline, + bind_group_layout, + uniform_buffer, + cached_dims: CachedDimensions::default(), + input_buffer: None, + output_rgba_texture: None, + output_depth_buffer: None, + staging_rgba_buffer: None, + staging_depth_buffer: None, + }) + } + + /// Ensure resources are allocated for the given dimensions + fn ensure_resources(&mut self, width: u32, height: u32) { + if !self.cached_dims.needs_update(width, height) { + return; + } + + debug!(width, height, "Allocating depth processor resources"); + + let pixel_count = (width * height) as u64; + + // Y10B input size from freedepth, rounded up to u32 alignment + let y10b_size = y10b_packed_size(width, height) as u64; + let input_size = ((y10b_size + 3) / 4) * 4; // Round up to u32 alignment + + // RGBA output size + let rgba_size = pixel_count * 4; + + // Depth output size (u32 per pixel for alignment) + let depth_size = pixel_count * 4; + + // Create input buffer + self.input_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("depth_input_buffer"), + size: input_size, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + // Create output RGBA texture + self.output_rgba_texture = Some(self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("depth_rgba_texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + })); + + // Create output depth buffer + self.output_depth_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("depth_output_buffer"), + size: depth_size, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC, + mapped_at_creation: false, + })); + + // Create staging buffers for CPU readback + self.staging_rgba_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("depth_staging_rgba"), + size: rgba_size, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + + self.staging_depth_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("depth_staging_depth"), + size: depth_size, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + + self.cached_dims.update(width, height); + } + + /// Unpack Y10B data to RGBA preview and 16-bit depth + /// + /// # Arguments + /// * `y10b_data` - Raw Y10B packed bytes from depth sensor + /// * `width` - Frame width in pixels + /// * `height` - Frame height in pixels + /// * `min_depth` - Minimum depth value for visualization (default 0) + /// * `max_depth` - Maximum depth value for visualization (default 1023) + /// * `use_colormap` - Whether to apply turbo colormap (true) or grayscale (false) + /// * `depth_only` - Whether to force colormap mode (depth-only visualization) + pub async fn unpack( + &mut self, + y10b_data: &[u8], + width: u32, + height: u32, + min_depth: u32, + max_depth: u32, + use_colormap: bool, + depth_only: bool, + ) -> Result { + self.ensure_resources(width, height); + + let input_buffer = self + .input_buffer + .as_ref() + .ok_or("Input buffer not allocated")?; + let output_rgba_texture = self + .output_rgba_texture + .as_ref() + .ok_or("RGBA texture not allocated")?; + let output_depth_buffer = self + .output_depth_buffer + .as_ref() + .ok_or("Depth buffer not allocated")?; + let staging_rgba_buffer = self + .staging_rgba_buffer + .as_ref() + .ok_or("RGBA staging buffer not allocated")?; + let staging_depth_buffer = self + .staging_depth_buffer + .as_ref() + .ok_or("Depth staging buffer not allocated")?; + + // Calculate expected Y10B size for this resolution (from freedepth) + let expected_y10b_size = y10b_packed_size(width, height); + + // Truncate input data to expected size if larger (device may add row padding) + let actual_data = if y10b_data.len() > expected_y10b_size { + debug!( + got = y10b_data.len(), + expected = expected_y10b_size, + "Input data larger than expected, truncating" + ); + &y10b_data[..expected_y10b_size] + } else if y10b_data.len() < expected_y10b_size { + warn!( + got = y10b_data.len(), + expected = expected_y10b_size, + "Input data smaller than expected, depth unpacking may be incorrect" + ); + y10b_data + } else { + y10b_data + }; + + // Upload Y10B data to input buffer (pad to 4-byte alignment if needed) + let padded_data = if actual_data.len() % 4 != 0 { + let padding = 4 - (actual_data.len() % 4); + let mut padded = actual_data.to_vec(); + padded.extend(std::iter::repeat(0u8).take(padding)); + padded + } else { + actual_data.to_vec() + }; + self.queue.write_buffer(input_buffer, 0, &padded_data); + + // Update uniform buffer + let params = DepthParams { + width, + height, + min_depth, + max_depth, + use_colormap: if use_colormap { 1 } else { 0 }, + depth_only: if depth_only { 1 } else { 0 }, + }; + self.queue + .write_buffer(&self.uniform_buffer, 0, bytemuck::bytes_of(¶ms)); + + // Create bind group + let rgba_view = output_rgba_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("depth_bind_group"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: input_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&rgba_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: output_depth_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: self.uniform_buffer.as_entire_binding(), + }, + ], + }); + + // Create and submit command buffer + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("depth_encoder"), + }); + + { + let mut compute_pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("depth_compute_pass"), + timestamp_writes: None, + }); + + compute_pass.set_pipeline(&self.pipeline); + compute_pass.set_bind_group(0, Some(&bind_group), &[]); + + // Dispatch workgroups (16x16 threads per workgroup) + let workgroups_x = compute_dispatch_size(width, 16); + let workgroups_y = compute_dispatch_size(height, 16); + compute_pass.dispatch_workgroups(workgroups_x, workgroups_y, 1); + } + + // Copy RGBA texture to staging buffer + let pixel_count = (width * height) as u64; + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: output_rgba_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: staging_rgba_buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(width * 4), + rows_per_image: Some(height), + }, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + + // Copy depth buffer to staging + encoder.copy_buffer_to_buffer( + output_depth_buffer, + 0, + staging_depth_buffer, + 0, + pixel_count * 4, + ); + + self.queue.submit(std::iter::once(encoder.finish())); + + // Read back both buffers using shared helper + use crate::shaders::gpu_processor::read_buffer_async; + let rgba_preview = read_buffer_async(&self.device, &staging_rgba_buffer).await?; + let depth_data = read_buffer_async(&self.device, &staging_depth_buffer).await?; + + // Read depth data (stored as u32, extract lower 16 bits) + let depth_u32: &[u32] = bytemuck::cast_slice(&depth_data); + let depth_u16: Vec = depth_u32.iter().map(|&d| d as u16).collect(); + + Ok(DepthUnpackResult { + rgba_preview, + depth_u16, + width, + height, + }) + } +} + +// Use the shared singleton macro for GPU processor management +gpu_processor_singleton!(DepthProcessor, GPU_DEPTH_PROCESSOR, get_depth_processor); + +/// Unpack Y10B data using the shared GPU processor +/// +/// This is the main entry point for unpacking depth data. +/// +/// # Arguments +/// * `y10b_data` - Raw Y10B packed bytes from depth sensor +/// * `width` - Frame width in pixels +/// * `height` - Frame height in pixels +/// * `use_colormap` - Whether to apply turbo colormap (true) or grayscale (false) +/// * `depth_only` - Whether to force colormap mode (depth-only visualization) +pub async fn unpack_y10b_gpu( + y10b_data: &[u8], + width: u32, + height: u32, + use_colormap: bool, + depth_only: bool, +) -> Result { + let mut guard = get_depth_processor().await?; + let processor = guard + .as_mut() + .ok_or("GPU depth processor not initialized")?; + + // Use full 10-bit range for visualization (0 to max valid value) + processor + .unpack( + y10b_data, + width, + height, + 0, + DEPTH_10BIT_NO_VALUE as u32, + use_colormap, + depth_only, + ) + .await +} diff --git a/src/shaders/depth/visualization.rs b/src/shaders/depth/visualization.rs new file mode 100644 index 00000000..91174ec8 --- /dev/null +++ b/src/shaders/depth/visualization.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Depth visualization helpers +//! +//! Provides functions for converting depth data to viewable formats: +//! - Turbo colormap (blue=near, red=far) +//! - Grayscale (bright=near, dark=far) +//! - RGB to RGBA conversion + +use super::constants::{DEPTH_COLORMAP_BANDS, DEPTH_MAX_MM, DEPTH_MAX_VALID_MM, DEPTH_MIN_MM}; + +/// Turbo colormap: perceptually uniform rainbow (blue=near, red=far) +/// +/// Based on: https://ai.googleblog.com/2019/08/turbo-improved-rainbow-colormap-for.html +/// Simplified version with polynomial approximation. +#[inline] +fn turbo(t: f32) -> [u8; 4] { + let r = (0.13572138 + + t * (4.6153926 + t * (-42.66032 + t * (132.13108 + t * (-152.54825 + t * 59.28144))))) + .clamp(0.0, 1.0); + let g = (0.09140261 + + t * (2.19418 + t * (4.84296 + t * (-14.18503 + t * (4.27805 + t * 2.53377))))) + .clamp(0.0, 1.0); + let b = (0.1066733 + + t * (12.64194 + t * (-60.58204 + t * (109.99648 + t * (-82.52904 + t * 20.43388))))) + .clamp(0.0, 1.0); + [(r * 255.0) as u8, (g * 255.0) as u8, (b * 255.0) as u8, 255] +} + +/// Convert depth data (in millimeters) to RGBA visualization +/// +/// # Arguments +/// * `depth_mm` - Depth values in millimeters (0 = invalid) +/// * `width` - Image width +/// * `height` - Image height +/// * `quantize` - If true, quantize to bands for smoother visualization +/// * `grayscale` - If true, use grayscale instead of colormap +/// +/// # Returns +/// RGBA pixel data (4 bytes per pixel) +pub fn depth_mm_to_rgba( + depth_mm: &[u16], + width: u32, + height: u32, + quantize: bool, + grayscale: bool, +) -> Vec { + let pixel_count = (width * height) as usize; + let mut rgba = Vec::with_capacity(pixel_count * 4); + + for &depth in depth_mm.iter().take(pixel_count) { + if depth == 0 || depth > DEPTH_MAX_VALID_MM { + // Invalid depth - black + rgba.extend_from_slice(&[0, 0, 0, 255]); + } else { + // Normalize to 0.0-1.0 range (near=0.0, far=1.0) + let mut t = ((depth as f32) - DEPTH_MIN_MM) / (DEPTH_MAX_MM - DEPTH_MIN_MM); + t = t.clamp(0.0, 1.0); + + // Quantize to bands for smoother visualization + if quantize { + t = (t * DEPTH_COLORMAP_BANDS).floor() / DEPTH_COLORMAP_BANDS; + } + + if grayscale { + // Grayscale: near=bright, far=dark (invert t) + let gray = ((1.0 - t) * 255.0) as u8; + rgba.extend_from_slice(&[gray, gray, gray, 255]); + } else { + // Colormap: turbo (blue=near, red=far) + let color = turbo(t); + rgba.extend_from_slice(&color); + } + } + } + rgba +} + +/// Convert RGB pixel data to RGBA (add alpha channel) +/// +/// # Arguments +/// * `rgb` - RGB pixel data (3 bytes per pixel) +/// +/// # Returns +/// RGBA pixel data (4 bytes per pixel, alpha = 255) +pub fn rgb_to_rgba(rgb: &[u8]) -> Vec { + let pixel_count = rgb.len() / 3; + let mut rgba = Vec::with_capacity(pixel_count * 4); + for chunk in rgb.chunks_exact(3) { + rgba.push(chunk[0]); // R + rgba.push(chunk[1]); // G + rgba.push(chunk[2]); // B + rgba.push(255); // A + } + rgba +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rgb_to_rgba() { + let rgb = vec![255, 0, 0, 0, 255, 0, 0, 0, 255]; + let rgba = rgb_to_rgba(&rgb); + assert_eq!(rgba, vec![255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255]); + } + + #[test] + fn test_depth_invalid() { + let depth = vec![0u16; 4]; + let rgba = depth_mm_to_rgba(&depth, 2, 2, false, false); + // All invalid pixels should be black + for chunk in rgba.chunks(4) { + assert_eq!(chunk, &[0, 0, 0, 255]); + } + } + + #[test] + fn test_depth_grayscale() { + // Near depth should be bright, far depth should be dark + let depth = vec![400u16, 4000u16]; + let rgba = depth_mm_to_rgba(&depth, 2, 1, false, true); + // Near (400mm) should be very bright + assert!(rgba[0] > 200); + // Far (4000mm) should be very dark + assert!(rgba[4] < 50); + } +} diff --git a/src/shaders/depth/y10b_unpack.wgsl b/src/shaders/depth/y10b_unpack.wgsl new file mode 100644 index 00000000..7c4f4883 --- /dev/null +++ b/src/shaders/depth/y10b_unpack.wgsl @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-3.0-only +// +// Y10B unpacking compute shader for Kinect depth sensor +// +// Y10B Format: +// - 10-bit packed grayscale (depth data from Kinect depth sensor) +// - 4 pixels are packed into 5 bytes (40 bits for 4 x 10-bit values) +// +// Byte layout: +// Byte 0: P0[9:2] (bits 9-2 of pixel 0) +// Byte 1: P0[1:0] | P1[9:4] (bits 1-0 of pixel 0, bits 9-4 of pixel 1) +// Byte 2: P1[3:0] | P2[9:6] (bits 3-0 of pixel 1, bits 9-6 of pixel 2) +// Byte 3: P2[5:0] | P3[9:8] (bits 5-0 of pixel 2, bits 9-8 of pixel 3) +// Byte 4: P3[7:0] (bits 7-0 of pixel 3) +// +// Outputs: +// - RGBA texture for preview display (turbo colormap: blue=near, red=far) +// - 16-bit depth buffer for lossless storage + +struct DepthParams { + width: u32, + height: u32, + min_depth: u32, // Minimum depth value for visualization (usually 0) + max_depth: u32, // Maximum depth value for visualization (usually 1023 or 0x3FF) + use_colormap: u32, // 0 = grayscale, 1 = turbo colormap + depth_only: u32, // 0 = normal, 1 = depth-only mode (always use colormap) +} + +// Input: Raw Y10B packed bytes (read as u32 for efficient access) +@group(0) @binding(0) +var input_bytes: array; + +// Output: RGBA texture for preview display +@group(0) @binding(1) +var output_rgba: texture_storage_2d; + +// Output: 16-bit depth values (stored as u32 for alignment, only lower 16 bits used) +@group(0) @binding(2) +var output_depth: array; + +// Uniform parameters +@group(0) @binding(3) +var params: DepthParams; + +// Extract a single byte from the packed u32 input array +fn get_byte(byte_index: u32) -> u32 { + let word_index = byte_index / 4u; + let byte_offset = byte_index % 4u; + let word = input_bytes[word_index]; + return (word >> (byte_offset * 8u)) & 0xFFu; +} + +// Unpack a single 10-bit depth value from Y10B packed data +fn unpack_pixel(pixel_index: u32) -> u32 { + // Which group of 4 pixels this belongs to + let group_index = pixel_index / 4u; + // Position within the group (0-3) + let in_group_index = pixel_index % 4u; + // Starting byte for this group + let byte_offset = group_index * 5u; + + // Get the 5 bytes for this group + let b0 = get_byte(byte_offset); + let b1 = get_byte(byte_offset + 1u); + let b2 = get_byte(byte_offset + 2u); + let b3 = get_byte(byte_offset + 3u); + let b4 = get_byte(byte_offset + 4u); + + // Extract 10-bit value based on position within the group + var depth_10bit: u32; + if (in_group_index == 0u) { + // Pixel 0: bits from b0[7:0] (high 8) and b1[7:6] (low 2) + depth_10bit = (b0 << 2u) | (b1 >> 6u); + } else if (in_group_index == 1u) { + // Pixel 1: bits from b1[5:0] (high 6) and b2[7:4] (low 4) + depth_10bit = ((b1 & 0x3Fu) << 4u) | (b2 >> 4u); + } else if (in_group_index == 2u) { + // Pixel 2: bits from b2[3:0] (high 4) and b3[7:2] (low 6) + depth_10bit = ((b2 & 0x0Fu) << 6u) | (b3 >> 2u); + } else { + // Pixel 3: bits from b3[1:0] (high 2) and b4[7:0] (low 8) + depth_10bit = ((b3 & 0x03u) << 8u) | b4; + } + + return depth_10bit; +} + +// Convert 10-bit depth to normalized value for visualization +fn normalize_depth(depth_10bit: u32) -> f32 { + // Clamp to visualization range + let min_d = f32(params.min_depth); + let max_d = f32(params.max_depth); + let range = max_d - min_d; + + if (range <= 0.0) { + return 0.0; + } + + let depth_f = f32(depth_10bit); + let normalized = clamp((depth_f - min_d) / range, 0.0, 1.0); + + return normalized; +} + +// Turbo-inspired colormap for depth visualization +// Maps 0.0 (near/blue) to 1.0 (far/red) with perceptually uniform colors +// Based on Google's Turbo colormap for scientific visualization +fn turbo_colormap(t: f32) -> vec3 { + // Clamp input + let x = clamp(t, 0.0, 1.0); + + // Polynomial approximation of turbo colormap + // Red channel: starts low, peaks in middle-high + let r = clamp( + 0.13572138 + x * (4.61539260 + x * (-42.66032258 + x * (132.13108234 + x * (-152.94239396 + x * 59.28637943)))), + 0.0, 1.0 + ); + + // Green channel: peaks in middle + let g = clamp( + 0.09140261 + x * (2.19418839 + x * (4.84296658 + x * (-14.18503333 + x * (4.27729857 + x * 2.82956604)))), + 0.0, 1.0 + ); + + // Blue channel: starts high, decreases + let b = clamp( + 0.10667330 + x * (12.64194608 + x * (-60.58204836 + x * (110.36276771 + x * (-89.90310912 + x * 27.34824973)))), + 0.0, 1.0 + ); + + return vec3(r, g, b); +} + +// Special value indicating no depth data (saturated/invalid) +// This matches freedepth::DEPTH_10BIT_NO_VALUE (1023 = 0x3FF for 10-bit data) +const DEPTH_NO_VALUE: u32 = 1023u; + +@compute @workgroup_size(16, 16) +fn main(@builtin(global_invocation_id) global_id: vec3) { + let x = global_id.x; + let y = global_id.y; + + // Bounds check + if (x >= params.width || y >= params.height) { + return; + } + + // Calculate pixel index + let pixel_index = y * params.width + x; + + // Unpack the 10-bit depth value + let depth_10bit = unpack_pixel(pixel_index); + + // Store 16-bit depth value (shift left by 6 to use full 16-bit range) + // This preserves full precision: 10-bit -> 16-bit + output_depth[pixel_index] = depth_10bit << 6u; + + // Determine if colormap should be used + // depth_only mode always uses colormap, otherwise check use_colormap flag + let should_use_colormap = (params.depth_only == 1u) || (params.use_colormap == 1u); + + // Handle invalid depth (no data) + if (depth_10bit >= DEPTH_NO_VALUE) { + if (should_use_colormap) { + // Black for invalid pixels when using colormap + textureStore(output_rgba, vec2(i32(x), i32(y)), vec4(0.0, 0.0, 0.0, 1.0)); + } else { + // Dark gray for invalid pixels in grayscale mode + textureStore(output_rgba, vec2(i32(x), i32(y)), vec4(0.1, 0.1, 0.1, 1.0)); + } + return; + } + + // Convert to normalized value + var normalized = normalize_depth(depth_10bit); + + // Apply colormap or grayscale based on mode + var color: vec3; + if (should_use_colormap) { + // In depth_only mode, quantize to discrete bands for pure depth visualization + // This removes the fine texture from the IR pattern + if (params.depth_only == 1u) { + // Quantize to 32 depth bands for smooth color regions + let bands = 32.0; + normalized = floor(normalized * bands) / bands; + } + // Apply turbo colormap (near=blue, far=red) + color = turbo_colormap(normalized); + } else { + // Grayscale output (near=bright, far=dark) + // In Kinect raw data: high values = near, low values = far + color = vec3(normalized, normalized, normalized); + } + + // Write RGBA output + textureStore(output_rgba, vec2(i32(x), i32(y)), vec4(color, 1.0)); +} diff --git a/src/shaders/gpu_filter.rs b/src/shaders/gpu_filter.rs index ec84ed6e..36f6dbd9 100644 --- a/src/shaders/gpu_filter.rs +++ b/src/shaders/gpu_filter.rs @@ -7,6 +7,7 @@ use crate::app::FilterType; use crate::gpu::{self, wgpu}; +use crate::shaders::gpu_processor::CachedDimensions; use std::sync::Arc; use tracing::{debug, info, warn}; @@ -29,8 +30,7 @@ pub struct GpuFilterPipeline { sampler: wgpu::Sampler, uniform_buffer: wgpu::Buffer, // Cached resources for current dimensions - cached_width: u32, - cached_height: u32, + cached_dims: CachedDimensions, input_texture: Option, output_buffer: Option, staging_buffer: Option, @@ -156,8 +156,7 @@ impl GpuFilterPipeline { bind_group_layout, sampler, uniform_buffer, - cached_width: 0, - cached_height: 0, + cached_dims: CachedDimensions::default(), input_texture: None, output_buffer: None, staging_buffer: None, @@ -166,7 +165,7 @@ impl GpuFilterPipeline { /// Ensure resources are allocated for the given dimensions fn ensure_resources(&mut self, width: u32, height: u32) { - if self.cached_width == width && self.cached_height == height { + if !self.cached_dims.needs_update(width, height) { return; } @@ -206,8 +205,7 @@ impl GpuFilterPipeline { mapped_at_creation: false, })); - self.cached_width = width; - self.cached_height = height; + self.cached_dims.update(width, height); } /// Apply a filter to RGBA data @@ -327,25 +325,8 @@ impl GpuFilterPipeline { self.queue.submit(std::iter::once(encoder.finish())); // Map staging buffer and read back result - let buffer_slice = staging_buffer.slice(..); - let (sender, receiver) = futures::channel::oneshot::channel(); - buffer_slice.map_async(wgpu::MapMode::Read, move |result| { - let _ = sender.send(result); - }); - - let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); - - receiver - .await - .map_err(|_| "Failed to receive buffer mapping result")? - .map_err(|e| format!("Failed to map buffer: {:?}", e))?; - - // Read RGBA data directly from buffer - let data = buffer_slice.get_mapped_range(); - let output = data.to_vec(); - - drop(data); - staging_buffer.unmap(); + let output = + crate::shaders::gpu_processor::read_buffer_async(&self.device, &staging_buffer).await?; Ok(output) } diff --git a/src/shaders/gpu_processor.rs b/src/shaders/gpu_processor.rs new file mode 100644 index 00000000..9d67ec94 --- /dev/null +++ b/src/shaders/gpu_processor.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Shared GPU processor infrastructure +//! +//! Provides common functionality for all GPU compute processors: +//! - Singleton management (OnceLock>>) +//! - Device/queue creation with low-priority settings +//! - Buffer/texture allocation with dimension caching +//! - Async buffer readback utilities +//! +//! This module reduces code duplication across the depth, point_cloud, +//! mesh, and yuv_convert processors. + +use crate::gpu::wgpu; + +/// Cached resource dimensions - avoids reallocation when dimensions match +/// +/// Used by processors to track if buffers need to be recreated when +/// input/output dimensions change. +#[derive(Default, Clone, Copy, PartialEq, Debug)] +pub struct CachedDimensions { + pub width: u32, + pub height: u32, +} + +impl CachedDimensions { + /// Create new cached dimensions + pub fn new(width: u32, height: u32) -> Self { + Self { width, height } + } + + /// Check if dimensions have changed and need update + pub fn needs_update(&self, width: u32, height: u32) -> bool { + self.width != width || self.height != height + } + + /// Update cached dimensions + pub fn update(&mut self, width: u32, height: u32) { + self.width = width; + self.height = height; + } + + /// Check if dimensions are initialized (non-zero) + pub fn is_initialized(&self) -> bool { + self.width > 0 && self.height > 0 + } +} + +/// Helper for async buffer readback (map, poll, read, unmap) +/// +/// This is the common pattern used by all GPU processors to read data back +/// from GPU buffers to CPU memory. +/// +/// # Arguments +/// * `device` - The wgpu device for polling +/// * `buffer` - The buffer to read from (must be MAP_READ) +/// +/// # Returns +/// The buffer contents as a Vec +pub async fn read_buffer_async( + device: &wgpu::Device, + buffer: &wgpu::Buffer, +) -> Result, String> { + let slice = buffer.slice(..); + let (sender, receiver) = futures::channel::oneshot::channel(); + + slice.map_async(wgpu::MapMode::Read, move |result| { + let _ = sender.send(result); + }); + + let _ = device.poll(wgpu::PollType::wait_indefinitely()); + + receiver + .await + .map_err(|_| "Failed to receive buffer mapping".to_string())? + .map_err(|e| format!("Failed to map buffer: {:?}", e))?; + + let data = slice.get_mapped_range().to_vec(); + buffer.unmap(); + + Ok(data) +} + +/// Calculate compute shader dispatch size (workgroups needed) +/// +/// Given a dimension and workgroup size, returns the number of workgroups +/// needed to cover the entire dimension. +/// +/// # Arguments +/// * `dimension` - The dimension to cover (width or height) +/// * `workgroup_size` - The workgroup size (typically 16) +/// +/// # Returns +/// Number of workgroups needed +#[inline] +pub fn compute_dispatch_size(dimension: u32, workgroup_size: u32) -> u32 { + dimension.div_ceil(workgroup_size) +} + +/// Macro for generating singleton accessor functions +/// +/// This eliminates the ~20 lines of boilerplate per processor for +/// singleton management. Each processor needs: +/// - A static OnceLock>> +/// - A get_processor() function that lazily initializes +/// +/// # Example +/// ```ignore +/// gpu_processor_singleton!(DepthProcessor, GPU_DEPTH_PROCESSOR, get_depth_processor); +/// ``` +#[macro_export] +macro_rules! gpu_processor_singleton { + ($processor:ty, $static_name:ident, $get_fn:ident) => { + /// Cached GPU processor instance + static $static_name: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + + /// Get or create the shared GPU processor instance + pub async fn $get_fn() + -> Result>, String> { + let lock = $static_name.get_or_init(|| tokio::sync::Mutex::new(None)); + let mut guard = lock.lock().await; + + if guard.is_none() { + match <$processor>::new().await { + Ok(processor) => { + *guard = Some(processor); + } + Err(e) => { + tracing::warn!( + concat!("Failed to initialize GPU ", stringify!($processor), ": {}"), + e + ); + return Err(e); + } + } + } + + Ok(guard) + } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cached_dimensions() { + let mut dims = CachedDimensions::default(); + assert!(!dims.is_initialized()); + assert!(dims.needs_update(640, 480)); + + dims.update(640, 480); + assert!(dims.is_initialized()); + assert!(!dims.needs_update(640, 480)); + assert!(dims.needs_update(1280, 720)); + } + + #[test] + fn test_compute_dispatch_size() { + assert_eq!(compute_dispatch_size(640, 16), 40); + assert_eq!(compute_dispatch_size(641, 16), 41); + assert_eq!(compute_dispatch_size(16, 16), 1); + assert_eq!(compute_dispatch_size(1, 16), 1); + } +} diff --git a/src/shaders/gpu_utils.rs b/src/shaders/gpu_utils.rs new file mode 100644 index 00000000..79cc1010 --- /dev/null +++ b/src/shaders/gpu_utils.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Shared GPU utilities for depth processors (point cloud, mesh) +//! +//! Provides common functionality used by both point cloud and mesh GPU processors: +//! - Bind group layout creation +//! - Registration buffer management +//! - Workgroup dispatch calculations + +use crate::gpu::wgpu; +#[cfg(not(target_arch = "x86_64"))] +use crate::shaders::RegistrationData; +#[cfg(target_arch = "x86_64")] +use crate::shaders::point_cloud::RegistrationData; + +/// Create the standard bind group layout for depth processors. +/// +/// Both point cloud and mesh processors use identical bind group layouts with 7 bindings: +/// - 0: RGB input buffer (storage, read-only) +/// - 1: Depth input buffer (storage, read-only) +/// - 2: Output texture (storage texture, write-only) +/// - 3: Depth test buffer (storage, read-write, atomic) +/// - 4: Uniform parameters +/// - 5: Registration table buffer (storage, read-only) +/// - 6: Depth-to-RGB shift buffer (storage, read-only) +pub fn create_depth_processor_bind_group_layout( + device: &wgpu::Device, + label: &str, +) -> wgpu::BindGroupLayout { + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some(label), + entries: &[ + // RGB input buffer + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Depth input buffer + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Output texture + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::StorageTexture { + access: wgpu::StorageTextureAccess::WriteOnly, + format: wgpu::TextureFormat::Rgba8Unorm, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + // Depth test buffer (atomic) + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: false }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Uniform parameters + wgpu::BindGroupLayoutEntry { + binding: 4, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Registration table buffer (640*480 [x,y] pairs) + wgpu::BindGroupLayoutEntry { + binding: 5, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Depth-to-RGB shift buffer (10001 i32 values) + wgpu::BindGroupLayoutEntry { + binding: 6, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }) +} + +/// Registration buffers for GPU depth-to-RGB alignment +pub struct RegistrationBuffers { + /// Registration table buffer: 640*480 [x_scaled, y] pairs (interleaved i32) + pub table_buffer: wgpu::Buffer, + /// Depth-to-RGB shift buffer: 10001 i32 values indexed by depth_mm + pub shift_buffer: wgpu::Buffer, + /// Target offset from pad_info + pub target_offset: u32, +} + +/// Create registration buffers from registration data. +/// +/// The registration table is flattened from Vec<[i32; 2]> to Vec (interleaved x, y). +pub fn create_registration_buffers( + device: &wgpu::Device, + queue: &wgpu::Queue, + data: &RegistrationData, + label_prefix: &str, +) -> RegistrationBuffers { + // Flatten registration table: [[x1, y1], [x2, y2], ...] -> [x1, y1, x2, y2, ...] + let reg_table_data: Vec = data + .registration_table + .iter() + .flat_map(|&[x, y]| [x, y]) + .collect(); + + let table_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some(&format!("{}_registration_table_buffer", label_prefix)), + size: (reg_table_data.len() * std::mem::size_of::()) as u64, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + queue.write_buffer(&table_buffer, 0, bytemuck::cast_slice(®_table_data)); + + let shift_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some(&format!("{}_depth_to_rgb_shift_buffer", label_prefix)), + size: (data.depth_to_rgb_shift.len() * std::mem::size_of::()) as u64, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + queue.write_buffer( + &shift_buffer, + 0, + bytemuck::cast_slice(&data.depth_to_rgb_shift), + ); + + RegistrationBuffers { + table_buffer, + shift_buffer, + target_offset: data.target_offset, + } +} diff --git a/src/shaders/kinect_intrinsics.rs b/src/shaders/kinect_intrinsics.rs new file mode 100644 index 00000000..d7241dc7 --- /dev/null +++ b/src/shaders/kinect_intrinsics.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Kinect camera intrinsics and depth coefficients +//! +//! These constants are used across the depth processing pipeline for: +//! - Point cloud rendering (unprojection from 2D to 3D) +//! - Mesh generation +//! - Scene export (LAZ, GLTF) +//! +//! Reference resolution: 640x480 (medium resolution depth mode) +//! +//! These are separated from the freedepth-specific depth processing so they +//! can be used with both the kernel driver and freedepth backends. + +/// Focal length X (pixels) at 640x480 base resolution +pub const FX: f32 = 594.21; +/// Focal length Y (pixels) at 640x480 base resolution +pub const FY: f32 = 591.04; +/// Principal point X (pixels) at 640x480 base resolution +pub const CX: f32 = 339.5; +/// Principal point Y (pixels) at 640x480 base resolution +pub const CY: f32 = 242.7; + +/// Disparity-to-depth coefficient A +/// Used in formula: depth_m = 1.0 / (raw * DEPTH_COEFF_A + DEPTH_COEFF_B) +pub const DEPTH_COEFF_A: f32 = -0.0030711; +/// Disparity-to-depth coefficient B +/// Used in formula: depth_m = 1.0 / (raw * DEPTH_COEFF_A + DEPTH_COEFF_B) +pub const DEPTH_COEFF_B: f32 = 3.3309495; + +/// Base width for intrinsics calculation +pub const BASE_WIDTH: f32 = 640.0; +/// Base height for intrinsics calculation +pub const BASE_HEIGHT: f32 = 480.0; diff --git a/src/shaders/mesh/mesh_main.wgsl b/src/shaders/mesh/mesh_main.wgsl new file mode 100644 index 00000000..9855c16f --- /dev/null +++ b/src/shaders/mesh/mesh_main.wgsl @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-3.0-only +// +// Mesh rendering compute shader - Main entry points +// +// Requires geometry.wgsl to be prepended for shared functions: +// rotation_matrix, unproject, project_to_screen, unpack_rgba, DEPTH_INVALID_16BIT + +// Unified 3D rendering parameters (shared layout with point cloud shader) +struct Render3DParams { + // === Dimensions === + input_width: u32, // Depth input width + input_height: u32, // Depth input height + output_width: u32, // Output image width + output_height: u32, // Output image height + rgb_width: u32, // RGB input width (may differ from depth) + rgb_height: u32, // RGB input height + + // === Camera intrinsics === + fx: f32, // Focal length X (594.21 for Kinect) + fy: f32, // Focal length Y (591.04 for Kinect) + cx: f32, // Principal point X (339.5 for Kinect 640x480) + cy: f32, // Principal point Y (242.7 for Kinect 640x480) + + // === Depth format and conversion === + depth_format: u32, // 0 = millimeters, 1 = disparity (10-bit shifted to 16-bit) + depth_coeff_a: f32, // Disparity coefficient A (-0.0030711 for Kinect) + depth_coeff_b: f32, // Disparity coefficient B (3.3309495 for Kinect) + min_depth: f32, // Minimum valid depth in meters + max_depth: f32, // Maximum valid depth in meters + + // === View transform === + pitch: f32, // Rotation around X axis (radians) + yaw: f32, // Rotation around Y axis (radians) + fov: f32, // Field of view for perspective projection + view_distance: f32, // Camera distance from origin + + // === Registration parameters === + use_registration_tables: u32, // 1 = use lookup tables, 0 = use simple shift + target_offset: u32, // Y offset from pad_info + reg_x_val_scale: i32, // Fixed-point scale factor (256) + mirror: u32, // 1 = mirror horizontally, 0 = normal + reg_scale_x: f32, // X scale factor (1.0 for 640, 2.0 for 1280) + reg_scale_y: f32, // Y scale factor + reg_y_offset: i32, // Y offset (0 for top-aligned crop) + + // === Mode-specific parameters === + point_size: f32, // Point size in pixels (point cloud only) + depth_discontinuity_threshold: f32, // Mesh discontinuity threshold (mesh only, 0 for point cloud) + filter_mode: u32, // Color filter mode (0 = none, 1-14 = various filters) +} + +// Input: RGB data (RGBA format) +@group(0) @binding(0) +var input_rgb: array; + +// Input: Depth data (16-bit values stored in u32) +@group(0) @binding(1) +var input_depth: array; + +// Output: Rendered mesh image +@group(0) @binding(2) +var output_texture: texture_storage_2d; + +// Depth buffer for z-ordering (atomic min for nearest point) +@group(0) @binding(3) +var depth_buffer: array>; + +// Parameters +@group(0) @binding(4) +var params: Render3DParams; + +// Registration table: 640*480 [x_scaled, y] pairs for depth-RGB alignment +@group(0) @binding(5) +var registration_table: array>; + +// Depth-to-RGB shift table: 10001 i32 values indexed by depth in mm (0-10000) +@group(0) @binding(6) +var depth_to_rgb_shift: array; + +// Get depth in meters for a pixel (returns -1 for invalid) +fn get_depth_meters(x: u32, y: u32) -> f32 { + if (x >= params.input_width || y >= params.input_height) { + return -1.0; + } + + let pixel_idx = y * params.input_width + x; + let depth_u16 = input_depth[pixel_idx] & 0xFFFFu; + + var depth_m: f32; + + if (params.depth_format == 0u) { + if (depth_u16 == 0u || depth_u16 >= 10000u) { + return -1.0; + } + depth_m = f32(depth_u16) / 1000.0; + } else { + if (depth_u16 >= DEPTH_INVALID_16BIT) { + return -1.0; + } + let depth_raw = f32(depth_u16 >> 6u); + let denom = depth_raw * params.depth_coeff_a + params.depth_coeff_b; + if (denom <= 0.01) { + return -1.0; + } + depth_m = 1.0 / denom; + } + + if (depth_m < params.min_depth || depth_m > params.max_depth) { + return -1.0; + } + + return depth_m; +} + +// Get RGB color for a depth pixel +// Returns alpha = 0 if registration fails (coords out of bounds) +fn get_color(x: u32, y: u32, depth_m: f32) -> vec4 { + var rgb_x: u32; + var rgb_y: u32; + + if (params.use_registration_tables == 1u) { + let reg_idx = y * params.input_width + x; + let reg = registration_table[reg_idx]; + + let depth_mm = u32(depth_m * 1000.0); + let clamped_depth_mm = clamp(depth_mm, 0u, 10000u); + let shift = depth_to_rgb_shift[clamped_depth_mm]; + + // Calculate base RGB coordinates (in 640x480 space) + let rgb_x_scaled = reg.x + shift; + let rgb_x_base = rgb_x_scaled / params.reg_x_val_scale; + let rgb_y_base = reg.y - i32(params.target_offset); + + // Scale to actual RGB resolution if different from base 640x480 + let rgb_x_i = i32(f32(rgb_x_base) * params.reg_scale_x); + let rgb_y_i = i32(f32(rgb_y_base) * params.reg_scale_y) + params.reg_y_offset; + + // Skip pixels that fall outside the RGB image bounds + // Return alpha = 0 to signal invalid color + if (rgb_x_i < 0 || rgb_x_i >= i32(params.rgb_width) || + rgb_y_i < 0 || rgb_y_i >= i32(params.rgb_height)) { + return vec4(0.0, 0.0, 0.0, 0.0); + } + + rgb_x = u32(rgb_x_i); + rgb_y = u32(rgb_y_i); + } else { + var rgb_x_f = f32(x); + var rgb_y_f = f32(y); + + if (params.rgb_width != params.input_width || params.rgb_height != params.input_height) { + rgb_x_f = rgb_x_f * f32(params.rgb_width) / f32(params.input_width); + rgb_y_f = rgb_y_f * f32(params.rgb_height) / f32(params.input_height); + } + + rgb_x = u32(clamp(rgb_x_f, 0.0, f32(params.rgb_width - 1u))); + rgb_y = u32(clamp(rgb_y_f, 0.0, f32(params.rgb_height - 1u))); + } + + let rgb_idx = rgb_y * params.rgb_width + rgb_x; + return unpack_rgba(input_rgb[rgb_idx]); +} + +// Transform 3D point with rotation and mirroring +fn transform_point(point: vec3) -> vec3 { + var p = point; + + // Apply horizontal mirror if enabled + if (params.mirror == 1u) { + p.x = -p.x; + } + + // Apply rotation around scene center + let rotation_center = 1.5; + p.z -= rotation_center; + let rot = rotation_matrix(params.pitch, params.yaw); + p = rot * p; + p.z += rotation_center; + + return p; +} + +// Compute barycentric coordinates for a point relative to a triangle +fn barycentric(p: vec2, a: vec2, b: vec2, c: vec2) -> vec3 { + let v0 = c - a; + let v1 = b - a; + let v2 = p - a; + + let dot00 = dot(v0, v0); + let dot01 = dot(v0, v1); + let dot02 = dot(v0, v2); + let dot11 = dot(v1, v1); + let dot12 = dot(v1, v2); + + let inv_denom = 1.0 / (dot00 * dot11 - dot01 * dot01); + let u = (dot11 * dot02 - dot01 * dot12) * inv_denom; + let v = (dot00 * dot12 - dot01 * dot02) * inv_denom; + + return vec3(1.0 - u - v, v, u); +} + +// Convert depth to integer for atomic comparison +fn depth_to_int(depth: f32) -> u32 { + return u32((1.0 - depth / (params.max_depth * 2.0)) * 4294967295.0); +} + +// Rasterize a single triangle +fn rasterize_triangle( + s0: vec3, s1: vec3, s2: vec3, + c0: vec4, c1: vec4, c2: vec4, +) { + // Skip degenerate or behind-camera triangles + if (s0.z < 0.0 || s1.z < 0.0 || s2.z < 0.0) { + return; + } + + // Compute bounding box + let min_x = max(0, i32(floor(min(s0.x, min(s1.x, s2.x))))); + let max_x = min(i32(params.output_width) - 1, i32(ceil(max(s0.x, max(s1.x, s2.x))))); + let min_y = max(0, i32(floor(min(s0.y, min(s1.y, s2.y))))); + let max_y = min(i32(params.output_height) - 1, i32(ceil(max(s0.y, max(s1.y, s2.y))))); + + // Skip if bounding box is empty or off-screen + if (min_x > max_x || min_y > max_y) { + return; + } + + // Limit triangle size to avoid excessive iteration (max 64x64 pixels) + let max_size = 64; + if (max_x - min_x > max_size || max_y - min_y > max_size) { + return; + } + + // Rasterize using barycentric coordinates + for (var py = min_y; py <= max_y; py = py + 1) { + for (var px = min_x; px <= max_x; px = px + 1) { + let p = vec2(f32(px) + 0.5, f32(py) + 0.5); + + // Compute barycentric coordinates + let bary = barycentric(p, s0.xy, s1.xy, s2.xy); + + // Check if point is inside triangle (all barycentric coords >= 0) + if (bary.x >= 0.0 && bary.y >= 0.0 && bary.z >= 0.0) { + // Interpolate depth + let depth = bary.x * s0.z + bary.y * s1.z + bary.z * s2.z; + + // Interpolate color + let color = bary.x * c0 + bary.y * c1 + bary.z * c2; + + // Atomic depth test + let depth_int = depth_to_int(depth); + let idx = u32(py) * params.output_width + u32(px); + let old = atomicMax(&depth_buffer[idx], depth_int); + + if (depth_int >= old) { + // Apply color filter if enabled (filter_mode 1-12) + var final_color = color; + if (params.filter_mode > 0u && params.filter_mode <= 12u) { + let tex_coords = vec2( + f32(px) / f32(params.output_width), + f32(py) / f32(params.output_height) + ); + final_color = vec4(apply_filter(color.rgb, params.filter_mode, tex_coords), color.a); + } + textureStore(output_texture, vec2(px, py), final_color); + } + } + } + } +} + +@compute @workgroup_size(16, 16) +fn main(@builtin(global_invocation_id) global_id: vec3) { + let x = global_id.x; + let y = global_id.y; + + // Each thread processes the bottom-right corner of a 2x2 quad + // Skip first row/column (need top-left neighbor) + if (x == 0u || y == 0u || x >= params.input_width || y >= params.input_height) { + return; + } + + // Get depths of 2x2 quad: (x-1,y-1), (x,y-1), (x-1,y), (x,y) + let d00 = get_depth_meters(x - 1u, y - 1u); + let d10 = get_depth_meters(x, y - 1u); + let d01 = get_depth_meters(x - 1u, y); + let d11 = get_depth_meters(x, y); + + // Skip if any depth is invalid + if (d00 < 0.0 || d10 < 0.0 || d01 < 0.0 || d11 < 0.0) { + return; + } + + // Check depth discontinuity - don't connect across large depth jumps + let max_diff = max(max(abs(d00 - d10), abs(d00 - d01)), + max(abs(d11 - d10), abs(d11 - d01))); + if (max_diff > params.depth_discontinuity_threshold) { + return; + } + + // Unproject all 4 corners to 3D and transform + let p00 = transform_point(unproject(f32(x - 1u), f32(y - 1u), d00)); + let p10 = transform_point(unproject(f32(x), f32(y - 1u), d10)); + let p01 = transform_point(unproject(f32(x - 1u), f32(y), d01)); + let p11 = transform_point(unproject(f32(x), f32(y), d11)); + + // Get colors for each corner + let c00 = get_color(x - 1u, y - 1u, d00); + let c10 = get_color(x, y - 1u, d10); + let c01 = get_color(x - 1u, y, d01); + let c11 = get_color(x, y, d11); + + // Skip quad if any corner has invalid color (alpha = 0 from failed registration) + if (c00.a == 0.0 || c10.a == 0.0 || c01.a == 0.0 || c11.a == 0.0) { + return; + } + + // Project to screen + let s00 = project_to_screen(p00); + let s10 = project_to_screen(p10); + let s01 = project_to_screen(p01); + let s11 = project_to_screen(p11); + + // Rasterize two triangles for the quad + // Triangle 1: (00, 10, 01) + rasterize_triangle(s00, s10, s01, c00, c10, c01); + // Triangle 2: (10, 11, 01) + rasterize_triangle(s10, s11, s01, c10, c11, c01); +} + +// Clear pass: clear depth buffer and output texture +@compute @workgroup_size(16, 16) +fn clear_buffers(@builtin(global_invocation_id) global_id: vec3) { + let x = global_id.x; + let y = global_id.y; + + if (x >= params.output_width || y >= params.output_height) { + return; + } + + let idx = y * params.output_width + x; + atomicStore(&depth_buffer[idx], 0u); + + // Clear output to dark background + textureStore(output_texture, vec2(i32(x), i32(y)), vec4(0.1, 0.1, 0.1, 1.0)); +} diff --git a/src/shaders/mesh/mod.rs b/src/shaders/mesh/mod.rs new file mode 100644 index 00000000..ba07eb71 --- /dev/null +++ b/src/shaders/mesh/mod.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(target_arch = "x86_64")] + +//! GPU-accelerated mesh rendering for 3D depth visualization +//! +//! Uses grid-based triangulation with depth discontinuity handling to create +//! a triangulated mesh from depth camera data with RGB texture colors. + +mod processor; + +pub use processor::{MeshProcessor, MeshResult, render_mesh, set_mesh_registration_data}; + +use std::sync::OnceLock; + +/// Shared geometry functions (rotation_matrix, unproject, project_to_screen, unpack_rgba) +const GEOMETRY_WGSL: &str = include_str!("../common/geometry.wgsl"); + +/// Shared filter functions (luminance, hash, apply_filter) +const FILTERS_WGSL: &str = include_str!("../filters.wgsl"); + +/// Mesh shader main entry points +const MESH_MAIN_WGSL: &str = include_str!("mesh_main.wgsl"); + +/// Combined mesh shader (geometry + filters + main) +static MESH_SHADER_COMBINED: OnceLock = OnceLock::new(); + +/// Get the combined mesh shader source +pub fn mesh_shader() -> &'static str { + MESH_SHADER_COMBINED.get_or_init(|| { + format!( + "{}\n\n{}\n\n{}", + MESH_MAIN_WGSL, GEOMETRY_WGSL, FILTERS_WGSL + ) + }) +} diff --git a/src/shaders/mesh/processor.rs b/src/shaders/mesh/processor.rs new file mode 100644 index 00000000..859788b0 --- /dev/null +++ b/src/shaders/mesh/processor.rs @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! GPU mesh processor for 3D depth visualization +//! +//! Takes depth + RGB data and renders a triangulated 3D mesh using grid-based +//! triangulation with depth discontinuity handling. + +use crate::gpu::{self, wgpu}; +use crate::gpu_processor_singleton; +use crate::shaders::common::Render3DParams; +use crate::shaders::compute_dispatch_size; +use crate::shaders::gpu_utils; +use crate::shaders::kinect_intrinsics as kinect; +use crate::shaders::point_cloud::DepthFormat; +use std::sync::Arc; +use tracing::{debug, info, warn}; + +// Mesh uses Render3DParams from common::params + +/// Result of mesh rendering +pub struct MeshResult { + /// Rendered RGBA image + pub rgba: Vec, + /// Width of output image + pub width: u32, + /// Height of output image + pub height: u32, +} + +/// GPU mesh processor +pub struct MeshProcessor { + device: Arc, + queue: Arc, + render_pipeline: wgpu::ComputePipeline, + clear_pipeline: wgpu::ComputePipeline, + bind_group_layout: wgpu::BindGroupLayout, + uniform_buffer: wgpu::Buffer, + // Cached resources + cached_input_width: u32, + cached_input_height: u32, + cached_output_width: u32, + cached_output_height: u32, + cached_rgb_width: u32, + cached_rgb_height: u32, + rgb_buffer: Option, + depth_buffer: Option, + depth_test_buffer: Option, + output_texture: Option, + staging_buffer: Option, + // Registration data buffers (None = using dummy buffers) + registration_table_buffer: Option, + depth_to_rgb_shift_buffer: Option, + registration_target_offset: u32, + has_registration_data: bool, + // Dummy buffers for when no registration data is available + dummy_reg_buffer: wgpu::Buffer, + dummy_shift_buffer: wgpu::Buffer, +} + +impl MeshProcessor { + /// Create a new GPU mesh processor + pub async fn new() -> Result { + info!("Initializing GPU mesh processor"); + + let (device, queue, gpu_info) = gpu::create_low_priority_compute_device("mesh_gpu").await?; + + info!( + adapter_name = %gpu_info.adapter_name, + adapter_backend = ?gpu_info.backend, + low_priority = gpu_info.low_priority_enabled, + "GPU device created for mesh processing" + ); + + // Create shader module (using concatenated shared geometry + main shader) + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("mesh_shader"), + source: wgpu::ShaderSource::Wgsl(super::mesh_shader().into()), + }); + + // Create bind group layout (using shared depth processor layout) + let bind_group_layout = + gpu_utils::create_depth_processor_bind_group_layout(&device, "mesh_bind_group_layout"); + + // Create pipeline layout + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("mesh_pipeline_layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + // Create render pipeline + let render_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("mesh_render_pipeline"), + layout: Some(&pipeline_layout), + module: &shader, + entry_point: Some("main"), + compilation_options: Default::default(), + cache: None, + }); + + // Create clear pipeline + let clear_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("mesh_clear_pipeline"), + layout: Some(&pipeline_layout), + module: &shader, + entry_point: Some("clear_buffers"), + compilation_options: Default::default(), + cache: None, + }); + + // Create uniform buffer + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_uniform_buffer"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + // Create dummy buffers for when no registration data is available + let dummy_reg_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_dummy_reg_buffer"), + size: 8, + usage: wgpu::BufferUsages::STORAGE, + mapped_at_creation: false, + }); + let dummy_shift_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_dummy_shift_buffer"), + size: 8, + usage: wgpu::BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + Ok(Self { + device, + queue, + render_pipeline, + clear_pipeline, + bind_group_layout, + uniform_buffer, + cached_input_width: 0, + cached_input_height: 0, + cached_output_width: 0, + cached_output_height: 0, + cached_rgb_width: 0, + cached_rgb_height: 0, + rgb_buffer: None, + depth_buffer: None, + depth_test_buffer: None, + output_texture: None, + staging_buffer: None, + registration_table_buffer: None, + depth_to_rgb_shift_buffer: None, + registration_target_offset: 0, + has_registration_data: false, + dummy_reg_buffer, + dummy_shift_buffer, + }) + } + + /// Ensure resources are allocated for given dimensions + fn ensure_resources( + &mut self, + rgb_width: u32, + rgb_height: u32, + depth_width: u32, + depth_height: u32, + output_width: u32, + output_height: u32, + ) { + let rgb_changed = + self.cached_rgb_width != rgb_width || self.cached_rgb_height != rgb_height; + let depth_changed = + self.cached_input_width != depth_width || self.cached_input_height != depth_height; + let output_changed = + self.cached_output_width != output_width || self.cached_output_height != output_height; + + if !rgb_changed && !depth_changed && !output_changed { + return; + } + + let rgb_pixels = (rgb_width * rgb_height) as u64; + let depth_pixels = (depth_width * depth_height) as u64; + let output_pixels = (output_width * output_height) as u64; + + if rgb_changed { + debug!(rgb_width, rgb_height, "Allocating mesh RGB buffer"); + + self.rgb_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_rgb_buffer"), + size: rgb_pixels * 4, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + self.cached_rgb_width = rgb_width; + self.cached_rgb_height = rgb_height; + } + + if depth_changed { + debug!(depth_width, depth_height, "Allocating mesh depth buffer"); + + self.depth_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_depth_buffer"), + size: depth_pixels * 4, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + self.cached_input_width = depth_width; + self.cached_input_height = depth_height; + } + + if output_changed { + debug!( + output_width, + output_height, "Allocating mesh output resources" + ); + + self.output_texture = Some(self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("mesh_output_texture"), + size: wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + })); + + self.depth_test_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_depth_test_buffer"), + size: output_pixels * 4, + usage: wgpu::BufferUsages::STORAGE, + mapped_at_creation: false, + })); + + self.staging_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("mesh_staging_buffer"), + size: output_pixels * 4, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + + self.cached_output_width = output_width; + self.cached_output_height = output_height; + } + } + + /// Set registration data for depth-to-RGB alignment + pub fn set_registration_data(&mut self, data: &crate::shaders::point_cloud::RegistrationData) { + let buffers = + gpu_utils::create_registration_buffers(&self.device, &self.queue, data, "mesh"); + + self.registration_table_buffer = Some(buffers.table_buffer); + self.depth_to_rgb_shift_buffer = Some(buffers.shift_buffer); + self.registration_target_offset = buffers.target_offset; + self.has_registration_data = true; + + debug!("Mesh registration data uploaded to GPU"); + } + + /// Check if registration data has been set + pub fn has_registration_data(&self) -> bool { + self.has_registration_data + } + + /// Render mesh from depth + RGB data + #[allow(clippy::too_many_arguments)] + pub async fn render( + &mut self, + rgb_data: &[u8], + depth_data: &[u16], + rgb_width: u32, + rgb_height: u32, + depth_width: u32, + depth_height: u32, + output_width: u32, + output_height: u32, + pitch: f32, + yaw: f32, + zoom: f32, + depth_format: DepthFormat, + mirror: bool, + apply_rgb_registration: bool, + depth_discontinuity_threshold: f32, + filter_mode: u32, + ) -> Result { + self.ensure_resources( + rgb_width, + rgb_height, + depth_width, + depth_height, + output_width, + output_height, + ); + + let input_width = depth_width; + let input_height = depth_height; + + let rgb_buffer = self.rgb_buffer.as_ref().ok_or("RGB buffer not allocated")?; + let depth_buffer = self + .depth_buffer + .as_ref() + .ok_or("Depth buffer not allocated")?; + let depth_test_buffer = self + .depth_test_buffer + .as_ref() + .ok_or("Depth test buffer not allocated")?; + let output_texture = self + .output_texture + .as_ref() + .ok_or("Output texture not allocated")?; + let staging_buffer = self + .staging_buffer + .as_ref() + .ok_or("Staging buffer not allocated")?; + + // Validate data sizes + let expected_rgb_size = (rgb_width * rgb_height * 4) as usize; + let expected_depth_size = (depth_width * depth_height) as usize; + + if rgb_data.len() != expected_rgb_size { + warn!( + actual = rgb_data.len(), + expected = expected_rgb_size, + rgb_width, + rgb_height, + "RGB data size mismatch - this may cause rendering issues" + ); + } + + if depth_data.len() != expected_depth_size { + warn!( + actual = depth_data.len(), + expected = expected_depth_size, + depth_width, + depth_height, + "Depth data size mismatch - this may cause rendering issues" + ); + } + + // Upload RGB data + self.queue.write_buffer(rgb_buffer, 0, rgb_data); + + // Upload depth data (convert u16 to u32) + let depth_u32: Vec = depth_data.iter().map(|&d| d as u32).collect(); + self.queue + .write_buffer(depth_buffer, 0, bytemuck::cast_slice(&depth_u32)); + + // Calculate camera intrinsics scaled for input resolution + let scale_x = input_width as f32 / 640.0; + let scale_y = input_height as f32 / 480.0; + + // Calculate registration scale factors for high-res RGB + // Registration tables are built for 640x480 RGB. For 1280x1024: + // - The 640x480 comes from 1280x1024 cropped to 1280x960, then scaled by 0.5 + // - So to map back: scale by 2.0 (not 2.133), top-aligned (offset=0) + let reg_scale_x = rgb_width as f32 / 640.0; + let reg_scale_y = reg_scale_x; // Same as X to maintain aspect ratio + let reg_y_offset = 0i32; + + // Update uniform buffer with parameters + let params = Render3DParams { + input_width, + input_height, + output_width, + output_height, + rgb_width, + rgb_height, + fx: kinect::FX * scale_x, + fy: kinect::FY * scale_y, + cx: kinect::CX * scale_x, + cy: kinect::CY * scale_y, + depth_format: depth_format as u32, + depth_coeff_a: kinect::DEPTH_COEFF_A, + depth_coeff_b: kinect::DEPTH_COEFF_B, + min_depth: 0.4, + max_depth: 4.0, + pitch, + yaw, + fov: 1.0, + view_distance: zoom, + use_registration_tables: if self.has_registration_data && apply_rgb_registration { + 1 + } else { + 0 + }, + target_offset: self.registration_target_offset, + reg_x_val_scale: 256, + mirror: if mirror { 1 } else { 0 }, + reg_scale_x, + reg_scale_y, + reg_y_offset, + // Mode-specific parameters + point_size: 0.0, // Not used for mesh + depth_discontinuity_threshold, + filter_mode, + }; + self.queue + .write_buffer(&self.uniform_buffer, 0, bytemuck::bytes_of(¶ms)); + + // Create bind group + let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + // Get registration buffers or use pre-allocated dummy buffers + let (reg_table_buffer, shift_buffer) = if self.has_registration_data { + ( + self.registration_table_buffer.as_ref().unwrap(), + self.depth_to_rgb_shift_buffer.as_ref().unwrap(), + ) + } else { + (&self.dummy_reg_buffer, &self.dummy_shift_buffer) + }; + + let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("mesh_bind_group"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: rgb_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: depth_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&output_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: depth_test_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: self.uniform_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 5, + resource: reg_table_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 6, + resource: shift_buffer.as_entire_binding(), + }, + ], + }); + + // Create and submit command buffer + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("mesh_encoder"), + }); + + // Pass 1: Clear buffers and output + { + let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("mesh_clear_pass"), + timestamp_writes: None, + }); + pass.set_pipeline(&self.clear_pipeline); + pass.set_bind_group(0, Some(&bind_group), &[]); + let workgroups_x = compute_dispatch_size(output_width, 16); + let workgroups_y = compute_dispatch_size(output_height, 16); + pass.dispatch_workgroups(workgroups_x, workgroups_y, 1); + } + + // Pass 2: Render mesh + { + let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("mesh_render_pass"), + timestamp_writes: None, + }); + pass.set_pipeline(&self.render_pipeline); + pass.set_bind_group(0, Some(&bind_group), &[]); + let workgroups_x = compute_dispatch_size(input_width, 16); + let workgroups_y = compute_dispatch_size(input_height, 16); + pass.dispatch_workgroups(workgroups_x, workgroups_y, 1); + } + + // Copy output to staging buffer + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: output_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: staging_buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(output_width * 4), + rows_per_image: Some(output_height), + }, + }, + wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, + ); + + self.queue.submit(std::iter::once(encoder.finish())); + + // Map and read back using shared helper + let rgba = + crate::shaders::gpu_processor::read_buffer_async(&self.device, &staging_buffer).await?; + + Ok(MeshResult { + rgba, + width: output_width, + height: output_height, + }) + } +} + +// Use the shared singleton macro for GPU processor management +gpu_processor_singleton!(MeshProcessor, GPU_MESH_PROCESSOR, get_mesh_processor); + +/// Set registration data for depth-to-RGB alignment on the shared GPU mesh processor +pub async fn set_mesh_registration_data( + data: &crate::shaders::point_cloud::RegistrationData, +) -> Result<(), String> { + let mut guard = get_mesh_processor().await?; + let processor = guard.as_mut().ok_or("GPU mesh processor not initialized")?; + + processor.set_registration_data(data); + Ok(()) +} + +/// Render mesh using the shared GPU processor +#[allow(clippy::too_many_arguments)] +pub async fn render_mesh( + rgb_data: &[u8], + depth_data: &[u16], + rgb_width: u32, + rgb_height: u32, + depth_width: u32, + depth_height: u32, + output_width: u32, + output_height: u32, + pitch: f32, + yaw: f32, + zoom: f32, + depth_format: DepthFormat, + mirror: bool, + apply_rgb_registration: bool, + depth_discontinuity_threshold: f32, + filter_mode: u32, +) -> Result { + let mut guard = get_mesh_processor().await?; + let processor = guard.as_mut().ok_or("GPU mesh processor not initialized")?; + + processor + .render( + rgb_data, + depth_data, + rgb_width, + rgb_height, + depth_width, + depth_height, + output_width, + output_height, + pitch, + yaw, + zoom, + depth_format, + mirror, + apply_rgb_registration, + depth_discontinuity_threshold, + filter_mode, + ) + .await +} diff --git a/src/shaders/mod.rs b/src/shaders/mod.rs index 3cc9fa94..5955a003 100644 --- a/src/shaders/mod.rs +++ b/src/shaders/mod.rs @@ -6,9 +6,156 @@ //! //! All filters operate directly on RGBA textures for simplicity and efficiency. +pub mod common; +// Y10B depth processing requires freedepth for unpacking +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub mod depth; mod gpu_filter; +mod gpu_processor; +pub mod gpu_utils; +// Kinect intrinsics are shared across all backends (freedepth and kernel driver) +#[cfg(target_arch = "x86_64")] +pub mod kinect_intrinsics; +// Mesh and point cloud work with both freedepth and kernel driver +#[cfg(target_arch = "x86_64")] +pub mod mesh; +#[cfg(target_arch = "x86_64")] +pub mod point_cloud; +pub mod yuv_convert; +pub use gpu_processor::{CachedDimensions, compute_dispatch_size, read_buffer_async}; + +#[cfg(all(target_arch = "x86_64", feature = "freedepth"))] +pub use depth::{DepthProcessor, kinect, unpack_y10b_gpu}; pub use gpu_filter::{GpuFilterPipeline, apply_filter_gpu_rgba, get_gpu_filter_pipeline}; +#[cfg(target_arch = "x86_64")] +pub use mesh::{MeshProcessor, MeshResult, render_mesh, set_mesh_registration_data}; +#[cfg(target_arch = "x86_64")] +pub use point_cloud::{ + DepthFormat, PointCloudProcessor, PointCloudResult, RegistrationData, + get_point_cloud_registration_data, has_point_cloud_registration_data, render_point_cloud, + set_point_cloud_registration_data, +}; +pub use yuv_convert::{ + YuvConvertProcessor, YuvConvertResult, YuvFormat, convert_yuv_to_rgba_gpu, + convert_yuv_to_rgba_gpu_texture, +}; + +// Kinect intrinsics - re-export for non-x86_64 builds +#[cfg(not(target_arch = "x86_64"))] +pub use kinect_intrinsics as kinect; + +#[cfg(not(target_arch = "x86_64"))] +pub struct DepthProcessor; + +#[cfg(not(target_arch = "x86_64"))] +pub async fn unpack_y10b_gpu( + _y10b_data: &[u8], + _width: u32, + _height: u32, +) -> Result<(Vec, Vec), String> { + Err("freedepth feature not enabled".to_string()) +} + +#[cfg(not(target_arch = "x86_64"))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DepthFormat { + Y10B, + Z16, + Millimeters, + Disparity16, +} + +#[cfg(not(target_arch = "x86_64"))] +#[derive(Clone)] +pub struct RegistrationData { + pub registration_table: Vec<[i32; 2]>, + pub depth_to_rgb_shift: Vec, + pub target_offset: u32, +} + +#[cfg(not(target_arch = "x86_64"))] +pub async fn render_point_cloud( + _rgb_data: &[u8], + _depth_data: &[u16], + _rgb_width: u32, + _rgb_height: u32, + _depth_width: u32, + _depth_height: u32, + _output_width: u32, + _output_height: u32, + _pitch: f32, + _yaw: f32, + _zoom: f32, + _depth_format: DepthFormat, + _mirror: bool, + _apply_rgb_registration: bool, + _filter_mode: u32, +) -> Result { + Err("freedepth feature not enabled".to_string()) +} + +#[cfg(not(target_arch = "x86_64"))] +pub async fn set_point_cloud_registration_data(_data: &RegistrationData) -> Result<(), String> { + Err("freedepth feature not enabled".to_string()) +} + +#[cfg(not(target_arch = "x86_64"))] +pub fn has_point_cloud_registration_data() -> bool { + false +} + +#[cfg(not(target_arch = "x86_64"))] +pub async fn get_point_cloud_registration_data() -> Result, String> { + Ok(None) +} + +#[cfg(not(target_arch = "x86_64"))] +pub struct PointCloudProcessor; + +#[cfg(not(target_arch = "x86_64"))] +pub struct PointCloudResult { + pub width: u32, + pub height: u32, + pub rgba: Vec, +} + +#[cfg(not(target_arch = "x86_64"))] +pub async fn render_mesh( + _rgb_data: &[u8], + _depth_data: &[u16], + _rgb_width: u32, + _rgb_height: u32, + _depth_width: u32, + _depth_height: u32, + _output_width: u32, + _output_height: u32, + _pitch: f32, + _yaw: f32, + _zoom: f32, + _depth_format: DepthFormat, + _mirror: bool, + _apply_rgb_registration: bool, + _depth_discontinuity_threshold: f32, + _filter_mode: u32, +) -> Result { + Err("freedepth feature not enabled".to_string()) +} + +#[cfg(not(target_arch = "x86_64"))] +pub async fn set_mesh_registration_data(_data: &RegistrationData) -> Result<(), String> { + Err("freedepth feature not enabled".to_string()) +} + +#[cfg(not(target_arch = "x86_64"))] +pub struct MeshProcessor; + +#[cfg(not(target_arch = "x86_64"))] +pub struct MeshResult { + pub width: u32, + pub height: u32, + pub rgba: Vec, +} /// Shared filter functions (WGSL) /// Contains: luminance(), hash(), apply_filter() diff --git a/src/shaders/point_cloud/mod.rs b/src/shaders/point_cloud/mod.rs new file mode 100644 index 00000000..4f2f142c --- /dev/null +++ b/src/shaders/point_cloud/mod.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-3.0-only + +#![cfg(target_arch = "x86_64")] + +//! GPU-accelerated point cloud rendering for 3D depth visualization +//! +//! This module provides GPU-based point cloud rendering from depth + RGB data, +//! creating an interactive 3D view that can be rotated with mouse input. + +mod processor; + +pub use processor::{ + DepthFormat, PointCloudProcessor, PointCloudResult, RegistrationData, + get_point_cloud_registration_data, has_point_cloud_registration_data, render_point_cloud, + set_point_cloud_registration_data, +}; + +use std::sync::OnceLock; + +/// Shared geometry functions (rotation_matrix, unproject, project_to_screen, unpack_rgba) +const GEOMETRY_WGSL: &str = include_str!("../common/geometry.wgsl"); + +/// Shared filter functions (luminance, hash, apply_filter) +const FILTERS_WGSL: &str = include_str!("../filters.wgsl"); + +/// Point cloud shader main entry points +const POINT_CLOUD_MAIN_WGSL: &str = include_str!("point_cloud_main.wgsl"); + +/// Combined point cloud shader (geometry + filters + main) +static POINT_CLOUD_SHADER_COMBINED: OnceLock = OnceLock::new(); + +/// Get the combined point cloud shader source +pub fn point_cloud_shader() -> &'static str { + POINT_CLOUD_SHADER_COMBINED.get_or_init(|| { + format!( + "{}\n\n{}\n\n{}", + POINT_CLOUD_MAIN_WGSL, GEOMETRY_WGSL, FILTERS_WGSL + ) + }) +} diff --git a/src/shaders/point_cloud/point_cloud_main.wgsl b/src/shaders/point_cloud/point_cloud_main.wgsl new file mode 100644 index 00000000..99234518 --- /dev/null +++ b/src/shaders/point_cloud/point_cloud_main.wgsl @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-3.0-only +// +// Point cloud rendering compute shader - Main entry points +// +// Requires geometry.wgsl to be prepended for shared functions: +// rotation_matrix, unproject, project_to_screen, unpack_rgba, DEPTH_INVALID_16BIT + +// Unified 3D rendering parameters (shared layout with mesh shader) +struct Render3DParams { + // === Dimensions === + input_width: u32, // Depth input width + input_height: u32, // Depth input height + output_width: u32, // Output image width + output_height: u32, // Output image height + rgb_width: u32, // RGB input width (may differ from depth) + rgb_height: u32, // RGB input height + + // === Camera intrinsics === + fx: f32, // Focal length X (594.21 for Kinect) + fy: f32, // Focal length Y (591.04 for Kinect) + cx: f32, // Principal point X (339.5 for Kinect 640x480) + cy: f32, // Principal point Y (242.7 for Kinect 640x480) + + // === Depth format and conversion === + depth_format: u32, // 0 = millimeters, 1 = disparity (10-bit shifted to 16-bit) + depth_coeff_a: f32, // Disparity coefficient A (-0.0030711 for Kinect) + depth_coeff_b: f32, // Disparity coefficient B (3.3309495 for Kinect) + min_depth: f32, // Minimum valid depth in meters + max_depth: f32, // Maximum valid depth in meters + + // === View transform === + pitch: f32, // Rotation around X axis (radians) + yaw: f32, // Rotation around Y axis (radians) + fov: f32, // Field of view for perspective projection + view_distance: f32, // Camera distance from origin + + // === Registration parameters === + use_registration_tables: u32, // 1 = use lookup tables, 0 = use simple shift + target_offset: u32, // Y offset from pad_info + reg_x_val_scale: i32, // Fixed-point scale factor (256) + mirror: u32, // 1 = mirror horizontally, 0 = normal + reg_scale_x: f32, // X scale factor (1.0 for 640, 2.0 for 1280) + reg_scale_y: f32, // Y scale factor + reg_y_offset: i32, // Y offset (0 for top-aligned crop) + + // === Mode-specific parameters === + point_size: f32, // Point size in pixels (point cloud only) + depth_discontinuity_threshold: f32, // Mesh discontinuity threshold (mesh only, 0 for point cloud) + filter_mode: u32, // Color filter mode (0 = none, 1-14 = various filters) +} + +// Input: RGB data (RGBA format) +@group(0) @binding(0) +var input_rgb: array; + +// Input: Depth data (16-bit values stored in u32) +@group(0) @binding(1) +var input_depth: array; + +// Output: Rendered point cloud image +@group(0) @binding(2) +var output_texture: texture_storage_2d; + +// Depth buffer for z-ordering (atomic min for nearest point) +@group(0) @binding(3) +var depth_buffer: array>; + +// Parameters +@group(0) @binding(4) +var params: Render3DParams; + +// Registration table: 640*480 [x_scaled, y] pairs for depth-RGB alignment +// x_scaled is multiplied by REG_X_VAL_SCALE (256), y is integer +@group(0) @binding(5) +var registration_table: array>; + +// Depth-to-RGB shift table: 10001 i32 values indexed by depth in mm (0-10000) +// Values are scaled by REG_X_VAL_SCALE and represent horizontal pixel shift +@group(0) @binding(6) +var depth_to_rgb_shift: array; + +@compute @workgroup_size(16, 16) +fn main(@builtin(global_invocation_id) global_id: vec3) { + let x = global_id.x; + let y = global_id.y; + + // Bounds check for input + if (x >= params.input_width || y >= params.input_height) { + return; + } + + let pixel_idx = y * params.input_width + x; + + // Get depth value (u16 stored in u32) + let depth_u16 = input_depth[pixel_idx] & 0xFFFFu; + + var depth_m: f32; + + if (params.depth_format == 0u) { + // Format 0: Depth in millimeters (from native Kinect backend) + // Skip invalid depth (0 = invalid) + if (depth_u16 == 0u || depth_u16 >= 10000u) { + return; + } + // Convert millimeters to meters + depth_m = f32(depth_u16) / 1000.0; + } else { + // Format 1: 10-bit disparity shifted to 16-bit (from V4L2 Y10B) + // Skip invalid depth (1023 << 6 = 65472) + if (depth_u16 >= DEPTH_INVALID_16BIT) { + return; + } + // Convert back to 10-bit raw value + let depth_raw = f32(depth_u16 >> 6u); + // Convert raw disparity to meters using Kinect formula: + // depth_m = 1.0 / (raw * coeff_a + coeff_b) + let denom = depth_raw * params.depth_coeff_a + params.depth_coeff_b; + // Avoid division by zero or negative values + if (denom <= 0.01) { + return; + } + depth_m = 1.0 / denom; + } + + // Skip depth outside valid range + if (depth_m < params.min_depth || depth_m > params.max_depth) { + return; + } + + // Get RGB color - apply stereo registration for depth-RGB alignment + var rgb_x: u32; + var rgb_y: u32; + var color: vec4; + + if (params.use_registration_tables == 1u) { + // Use proper registration lookup tables from device calibration + let reg_idx = y * params.input_width + x; + let reg = registration_table[reg_idx]; + + // Convert depth to mm for shift table lookup + let depth_mm = u32(depth_m * 1000.0); + let clamped_depth_mm = clamp(depth_mm, 0u, 10000u); + let shift = depth_to_rgb_shift[clamped_depth_mm]; + + // Calculate RGB coordinates using registration formula from libfreenect: + // rgb_x = (registration_table[idx][0] + depth_to_rgb_shift[depth_mm]) / REG_X_VAL_SCALE + // rgb_y = registration_table[idx][1] - target_offset + // These coordinates are in 640x480 space (base registration resolution) + let rgb_x_scaled = reg.x + shift; + let rgb_x_base = rgb_x_scaled / params.reg_x_val_scale; + let rgb_y_base = reg.y - i32(params.target_offset); + + // Scale to actual RGB resolution if different from base 640x480 + // For 1280x1024: scale by 2x and add Y offset for aspect ratio difference + // The 640x480 RGB is from 1280x1024 cropped to 1280x960 then scaled by 0.5 + // So to go back: scale by 2, then add offset for the cropped rows + let rgb_x_i = i32(f32(rgb_x_base) * params.reg_scale_x); + let rgb_y_i = i32(f32(rgb_y_base) * params.reg_scale_y) + params.reg_y_offset; + + // Skip pixels that fall outside the RGB image bounds + // (registration can push bottom/edge rows beyond RGB dimensions) + if (rgb_x_i < 0 || rgb_x_i >= i32(params.rgb_width) || + rgb_y_i < 0 || rgb_y_i >= i32(params.rgb_height)) { + return; + } + + rgb_x = u32(rgb_x_i); + rgb_y = u32(rgb_y_i); + + let rgb_idx = rgb_y * params.rgb_width + rgb_x; + color = unpack_rgba(input_rgb[rgb_idx]); + } else { + // Fallback: Use simple identity mapping (no registration) + // This is used when registration data is not available + var rgb_x_f = f32(x); + var rgb_y_f = f32(y); + + // Scale to RGB resolution if different from depth + if (params.rgb_width != params.input_width || params.rgb_height != params.input_height) { + rgb_x_f = rgb_x_f * f32(params.rgb_width) / f32(params.input_width); + rgb_y_f = rgb_y_f * f32(params.rgb_height) / f32(params.input_height); + } + + // Clamp to valid RGB coordinates + rgb_x = u32(clamp(rgb_x_f, 0.0, f32(params.rgb_width - 1u))); + rgb_y = u32(clamp(rgb_y_f, 0.0, f32(params.rgb_height - 1u))); + + let rgb_idx = rgb_y * params.rgb_width + rgb_x; + color = unpack_rgba(input_rgb[rgb_idx]); + } + + // Unproject to 3D + var point_3d = unproject(f32(x), f32(y), depth_m); + + // Apply horizontal mirror if enabled + if (params.mirror == 1u) { + point_3d.x = -point_3d.x; + } + + // Apply rotation around scene center (typical viewing distance ~1.5m for Kinect) + // This allows rotating the view while keeping the scene centered + let rotation_center = 1.5; // Rotate around 1.5m depth + point_3d.z -= rotation_center; + let rot = rotation_matrix(params.pitch, params.yaw); + point_3d = rot * point_3d; + point_3d.z += rotation_center; + + // Project to screen + let screen = project_to_screen(point_3d); + + // Check if point is visible + if (screen.x < 0.0 || screen.x >= f32(params.output_width) || + screen.y < 0.0 || screen.y >= f32(params.output_height) || + screen.z < 0.0) { + return; + } + + let screen_x = u32(screen.x); + let screen_y = u32(screen.y); + let out_idx = screen_y * params.output_width + screen_x; + + // Convert depth to integer for atomic comparison (closer = smaller z = larger integer) + let depth_int = u32((1.0 - screen.z / (params.max_depth * 2.0)) * 4294967295.0); + + // Atomic depth test - only render if this point is closer + let old_depth = atomicMax(&depth_buffer[out_idx], depth_int); + + if (depth_int >= old_depth) { + // This point is closest (or equal), render it + // Apply color filter if enabled (filter_mode 1-12) + var final_color = color; + if (params.filter_mode > 0u && params.filter_mode <= 12u) { + let tex_coords = vec2( + f32(screen_x) / f32(params.output_width), + f32(screen_y) / f32(params.output_height) + ); + final_color = vec4(apply_filter(color.rgb, params.filter_mode, tex_coords), color.a); + } + textureStore(output_texture, vec2(i32(screen_x), i32(screen_y)), final_color); + } +} + +// Second pass: clear depth buffer and optionally fill holes +@compute @workgroup_size(16, 16) +fn clear_buffers(@builtin(global_invocation_id) global_id: vec3) { + let x = global_id.x; + let y = global_id.y; + + if (x >= params.output_width || y >= params.output_height) { + return; + } + + let idx = y * params.output_width + x; + atomicStore(&depth_buffer[idx], 0u); + + // Clear output to dark background + textureStore(output_texture, vec2(i32(x), i32(y)), vec4(0.1, 0.1, 0.1, 1.0)); +} diff --git a/src/shaders/point_cloud/processor.rs b/src/shaders/point_cloud/processor.rs new file mode 100644 index 00000000..ff39afe0 --- /dev/null +++ b/src/shaders/point_cloud/processor.rs @@ -0,0 +1,751 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! GPU point cloud processor for 3D depth visualization +//! +//! Takes depth + RGB data and renders a rotatable 3D point cloud. + +use crate::gpu::{self, wgpu}; +use crate::gpu_processor_singleton; +use crate::shaders::common::Render3DParams; +use crate::shaders::compute_dispatch_size; +use crate::shaders::gpu_utils; +use crate::shaders::kinect_intrinsics as kinect; +use std::sync::Arc; +use tracing::{debug, info, warn}; + +/// Depth data format for point cloud shader +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum DepthFormat { + /// Depth in millimeters (from native Kinect backend) + Millimeters = 0, + /// 10-bit disparity shifted to 16-bit (from V4L2 Y10B) + Disparity16 = 1, +} + +// Point cloud uses Render3DParams from common::params + +/// Result of point cloud rendering +pub struct PointCloudResult { + /// Rendered RGBA image + pub rgba: Vec, + /// Width of output image + pub width: u32, + /// Height of output image + pub height: u32, +} + +/// Registration data for depth-to-RGB alignment +#[derive(Clone)] +pub struct RegistrationData { + /// Registration table: 640*480 [x_scaled, y] pairs + pub registration_table: Vec<[i32; 2]>, + /// Depth-to-RGB shift table: 10001 i32 values indexed by depth_mm + pub depth_to_rgb_shift: Vec, + /// Target offset from pad_info + pub target_offset: u32, +} + +/// GPU point cloud processor +pub struct PointCloudProcessor { + device: Arc, + queue: Arc, + render_pipeline: wgpu::ComputePipeline, + clear_pipeline: wgpu::ComputePipeline, + bind_group_layout: wgpu::BindGroupLayout, + uniform_buffer: wgpu::Buffer, + // Cached resources - depth/input dimensions + cached_input_width: u32, + cached_input_height: u32, + cached_output_width: u32, + cached_output_height: u32, + // Cached resources - RGB dimensions (may differ from depth) + cached_rgb_width: u32, + cached_rgb_height: u32, + rgb_buffer: Option, + depth_buffer: Option, + depth_test_buffer: Option, + output_texture: Option, + staging_buffer: Option, + // Registration data buffers + registration_table_buffer: Option, + depth_to_rgb_shift_buffer: Option, + registration_target_offset: u32, + has_registration_data: bool, + // CPU copy of registration data for retrieval + registration_data_copy: Option, + // Pre-allocated dummy buffers for when registration data is not set + dummy_reg_buffer: wgpu::Buffer, + dummy_shift_buffer: wgpu::Buffer, +} + +impl PointCloudProcessor { + /// Create a new GPU point cloud processor + pub async fn new() -> Result { + info!("Initializing GPU point cloud processor"); + + let (device, queue, gpu_info) = + gpu::create_low_priority_compute_device("point_cloud_gpu").await?; + + info!( + adapter_name = %gpu_info.adapter_name, + adapter_backend = ?gpu_info.backend, + low_priority = gpu_info.low_priority_enabled, + "GPU device created for point cloud processing" + ); + + // Create shader module (using concatenated shared geometry + main shader) + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("point_cloud_shader"), + source: wgpu::ShaderSource::Wgsl(super::point_cloud_shader().into()), + }); + + // Create bind group layout (using shared depth processor layout) + let bind_group_layout = gpu_utils::create_depth_processor_bind_group_layout( + &device, + "point_cloud_bind_group_layout", + ); + + // Create pipeline layout + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("point_cloud_pipeline_layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + // Create render pipeline + let render_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("point_cloud_render_pipeline"), + layout: Some(&pipeline_layout), + module: &shader, + entry_point: Some("main"), + compilation_options: Default::default(), + cache: None, + }); + + // Create clear pipeline + let clear_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("point_cloud_clear_pipeline"), + layout: Some(&pipeline_layout), + module: &shader, + entry_point: Some("clear_buffers"), + compilation_options: Default::default(), + cache: None, + }); + + // Create uniform buffer + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_uniform_buffer"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + // Create dummy registration buffers for bind group compatibility + // These are used when registration data is not set + let dummy_reg_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_dummy_reg_buffer"), + size: 8, + usage: wgpu::BufferUsages::STORAGE, + mapped_at_creation: false, + }); + let dummy_shift_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_dummy_shift_buffer"), + size: 8, + usage: wgpu::BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + Ok(Self { + device, + queue, + render_pipeline, + clear_pipeline, + bind_group_layout, + uniform_buffer, + cached_input_width: 0, + cached_input_height: 0, + cached_output_width: 0, + cached_output_height: 0, + cached_rgb_width: 0, + cached_rgb_height: 0, + rgb_buffer: None, + depth_buffer: None, + depth_test_buffer: None, + output_texture: None, + staging_buffer: None, + registration_table_buffer: None, + depth_to_rgb_shift_buffer: None, + registration_target_offset: 0, + has_registration_data: false, + registration_data_copy: None, + dummy_reg_buffer, + dummy_shift_buffer, + }) + } + + /// Ensure resources are allocated for given dimensions + fn ensure_resources( + &mut self, + rgb_width: u32, + rgb_height: u32, + depth_width: u32, + depth_height: u32, + output_width: u32, + output_height: u32, + ) { + let rgb_changed = + self.cached_rgb_width != rgb_width || self.cached_rgb_height != rgb_height; + let depth_changed = + self.cached_input_width != depth_width || self.cached_input_height != depth_height; + let output_changed = + self.cached_output_width != output_width || self.cached_output_height != output_height; + + if !rgb_changed && !depth_changed && !output_changed { + return; + } + + let rgb_pixels = (rgb_width * rgb_height) as u64; + let depth_pixels = (depth_width * depth_height) as u64; + let output_pixels = (output_width * output_height) as u64; + + if rgb_changed { + debug!(rgb_width, rgb_height, "Allocating point cloud RGB buffer"); + + // RGB buffer (RGBA u32 per pixel) - sized for RGB resolution + self.rgb_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_rgb_buffer"), + size: rgb_pixels * 4, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + self.cached_rgb_width = rgb_width; + self.cached_rgb_height = rgb_height; + } + + if depth_changed { + debug!( + depth_width, + depth_height, "Allocating point cloud depth buffer" + ); + + // Depth buffer (u32 per pixel) - sized for depth resolution + self.depth_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_depth_buffer"), + size: depth_pixels * 4, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + self.cached_input_width = depth_width; + self.cached_input_height = depth_height; + } + + if output_changed { + debug!( + output_width, + output_height, "Allocating point cloud output resources" + ); + + // Output texture + self.output_texture = Some(self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("point_cloud_output_texture"), + size: wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + })); + + // Depth test buffer (atomic u32 per output pixel) + self.depth_test_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_depth_test_buffer"), + size: output_pixels * 4, + usage: wgpu::BufferUsages::STORAGE, + mapped_at_creation: false, + })); + + // Staging buffer for readback + self.staging_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("point_cloud_staging_buffer"), + size: output_pixels * 4, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + + self.cached_output_width = output_width; + self.cached_output_height = output_height; + } + } + + /// Set registration data for depth-to-RGB alignment. + /// + /// This should be called once after device initialization with calibration + /// data from the Kinect device. Once set, the shader will use proper + /// polynomial registration instead of simple hardcoded shifts. + pub fn set_registration_data(&mut self, data: &RegistrationData) { + // Log sample values for debugging + let center_idx = 240 * 640 + 320; + let corner_idx = 0; + info!( + table_len = data.registration_table.len(), + shift_len = data.depth_to_rgb_shift.len(), + target_offset = data.target_offset, + center_reg_x = data + .registration_table + .get(center_idx) + .map(|v| v[0]) + .unwrap_or(-1), + center_reg_y = data + .registration_table + .get(center_idx) + .map(|v| v[1]) + .unwrap_or(-1), + corner_reg_x = data + .registration_table + .get(corner_idx) + .map(|v| v[0]) + .unwrap_or(-1), + corner_reg_y = data + .registration_table + .get(corner_idx) + .map(|v| v[1]) + .unwrap_or(-1), + shift_500mm = data.depth_to_rgb_shift.get(500).copied().unwrap_or(-1), + shift_1000mm = data.depth_to_rgb_shift.get(1000).copied().unwrap_or(-1), + shift_2000mm = data.depth_to_rgb_shift.get(2000).copied().unwrap_or(-1), + "Setting registration data for point cloud processor" + ); + + // Create registration buffers using shared utility + let buffers = + gpu_utils::create_registration_buffers(&self.device, &self.queue, data, "point_cloud"); + + self.registration_table_buffer = Some(buffers.table_buffer); + self.depth_to_rgb_shift_buffer = Some(buffers.shift_buffer); + self.registration_target_offset = buffers.target_offset; + self.has_registration_data = true; + + // Store CPU copy for retrieval + self.registration_data_copy = Some(RegistrationData { + registration_table: data.registration_table.clone(), + depth_to_rgb_shift: data.depth_to_rgb_shift.clone(), + target_offset: data.target_offset, + }); + + debug!("Registration data uploaded to GPU"); + } + + /// Check if registration data has been set + pub fn has_registration_data(&self) -> bool { + self.has_registration_data + } + + /// Get a clone of the registration data if set + pub fn get_registration_data(&self) -> Option { + self.registration_data_copy.clone() + } + + /// Render point cloud from depth + RGB data + /// + /// # Arguments + /// * `rgb_data` - RGBA data (4 bytes per pixel) + /// * `depth_data` - 16-bit depth values + /// * `rgb_width` - Width of RGB data (may differ from depth) + /// * `rgb_height` - Height of RGB data (may differ from depth) + /// * `depth_width` - Width of depth data (used as input dimensions) + /// * `depth_height` - Height of depth data (used as input dimensions) + /// * `output_width` - Width of output image + /// * `output_height` - Height of output image + /// * `pitch` - Rotation around X axis (radians) + /// * `yaw` - Rotation around Y axis (radians) + /// * `zoom` - Zoom level (1.0 = default, <1.0 = closer, >1.0 = farther) + /// * `depth_format` - Format of depth data (millimeters or disparity) + /// * `mirror` - Whether to mirror the point cloud horizontally + /// * `apply_rgb_registration` - Whether to apply stereo registration (true for RGB camera, false for IR/depth) + /// * `filter_mode` - Color filter mode (0 = none, 1-12 = various filters) + pub async fn render( + &mut self, + rgb_data: &[u8], + depth_data: &[u16], + rgb_width: u32, + rgb_height: u32, + depth_width: u32, + depth_height: u32, + output_width: u32, + output_height: u32, + pitch: f32, + yaw: f32, + zoom: f32, + depth_format: DepthFormat, + mirror: bool, + apply_rgb_registration: bool, + filter_mode: u32, + ) -> Result { + // Ensure resources are allocated for the various dimensions + // - RGB buffer sized for RGB resolution + // - Depth buffer sized for depth resolution (640x480 for Kinect) + // - Output sized as requested + self.ensure_resources( + rgb_width, + rgb_height, + depth_width, + depth_height, + output_width, + output_height, + ); + + // Depth dimensions are used for shader iteration (input_width/input_height in shader) + let input_width = depth_width; + let input_height = depth_height; + + let rgb_buffer = self.rgb_buffer.as_ref().ok_or("RGB buffer not allocated")?; + let depth_buffer = self + .depth_buffer + .as_ref() + .ok_or("Depth buffer not allocated")?; + let depth_test_buffer = self + .depth_test_buffer + .as_ref() + .ok_or("Depth test buffer not allocated")?; + let output_texture = self + .output_texture + .as_ref() + .ok_or("Output texture not allocated")?; + let staging_buffer = self + .staging_buffer + .as_ref() + .ok_or("Staging buffer not allocated")?; + + // Validate data sizes + let expected_rgb_size = (rgb_width * rgb_height * 4) as usize; + let expected_depth_size = (depth_width * depth_height) as usize; + + if rgb_data.len() != expected_rgb_size { + warn!( + actual = rgb_data.len(), + expected = expected_rgb_size, + rgb_width, + rgb_height, + "RGB data size mismatch - this may cause rendering issues" + ); + } + + if depth_data.len() != expected_depth_size { + warn!( + actual = depth_data.len(), + expected = expected_depth_size, + depth_width, + depth_height, + "Depth data size mismatch - this may cause rendering issues" + ); + } + + // Upload RGB data + self.queue.write_buffer(rgb_buffer, 0, rgb_data); + + // Upload depth data (convert u16 to u32) + let depth_u32: Vec = depth_data.iter().map(|&d| d as u32).collect(); + self.queue + .write_buffer(depth_buffer, 0, bytemuck::cast_slice(&depth_u32)); + + // Calculate camera intrinsics scaled for input resolution + // Kinect defaults are for 640x480 + let scale_x = input_width as f32 / 640.0; + let scale_y = input_height as f32 / 480.0; + + // Calculate registration scale factors for high-res RGB + // Registration tables are built for 640x480 RGB. For 1280x1024: + // - The 640x480 comes from 1280x1024 cropped to 1280x960, then scaled by 0.5 + // - So to map back: scale by 2.0 (not 2.133), top-aligned (offset=0) + let reg_scale_x = rgb_width as f32 / 640.0; + // Use 4:3 aspect scaling (960 from 480), not full height scaling + let reg_scale_y = reg_scale_x; // Same as X to maintain aspect ratio (2.0 for 1280) + // Top-aligned: the 960 rows start at row 0, extra 64 rows are at bottom + let reg_y_offset = 0i32; + + debug!( + rgb_width, + rgb_height, + reg_scale_x, + reg_scale_y, + reg_y_offset, + "Registration scaling for RGB resolution" + ); + + // Update uniform buffer with parameters + let params = Render3DParams { + input_width, + input_height, + output_width, + output_height, + rgb_width, + rgb_height, + // Kinect camera intrinsics (scaled for depth resolution) + fx: kinect::FX * scale_x, + fy: kinect::FY * scale_y, + cx: kinect::CX * scale_x, + cy: kinect::CY * scale_y, + // Depth format + depth_format: depth_format as u32, + // Kinect depth conversion coefficients from libfreenect glpclview.c + // Formula: depth_m = 1.0 / (raw * coeff_a + coeff_b) + depth_coeff_a: kinect::DEPTH_COEFF_A, + depth_coeff_b: kinect::DEPTH_COEFF_B, + min_depth: 0.4, // 0.4m minimum (Kinect near limit) + max_depth: 4.0, // 4.0m maximum (Kinect far limit) + pitch, + yaw, + fov: 1.0, // ~57 degrees + view_distance: zoom, // Camera Z position: 0=sensor position, higher=into scene + // Registration parameters + // Only apply registration if we have tables AND the color source is from RGB camera + // (IR data is already aligned with depth, so no registration needed) + use_registration_tables: if self.has_registration_data && apply_rgb_registration { + 1 + } else { + 0 + }, + target_offset: self.registration_target_offset, + reg_x_val_scale: 256, // freedepth::REG_X_VAL_SCALE + mirror: if mirror { 1 } else { 0 }, + // High-res RGB scaling + reg_scale_x, + reg_scale_y, + reg_y_offset, + // Mode-specific parameters + point_size: 1.0, + depth_discontinuity_threshold: 0.0, // Not used for point cloud + filter_mode, + }; + self.queue + .write_buffer(&self.uniform_buffer, 0, bytemuck::bytes_of(¶ms)); + + // Create bind group + let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + // Get registration buffers or use pre-allocated dummies + let (reg_table_buffer, shift_buffer) = if self.has_registration_data { + ( + self.registration_table_buffer.as_ref().unwrap(), + self.depth_to_rgb_shift_buffer.as_ref().unwrap(), + ) + } else { + (&self.dummy_reg_buffer, &self.dummy_shift_buffer) + }; + + let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("point_cloud_bind_group"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: rgb_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: depth_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&output_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: depth_test_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: self.uniform_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 5, + resource: reg_table_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 6, + resource: shift_buffer.as_entire_binding(), + }, + ], + }); + + // Create and submit command buffer + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("point_cloud_encoder"), + }); + + // Pass 1: Clear buffers and output + { + let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("point_cloud_clear_pass"), + timestamp_writes: None, + }); + pass.set_pipeline(&self.clear_pipeline); + pass.set_bind_group(0, Some(&bind_group), &[]); + let workgroups_x = compute_dispatch_size(output_width, 16); + let workgroups_y = compute_dispatch_size(output_height, 16); + pass.dispatch_workgroups(workgroups_x, workgroups_y, 1); + } + + // Pass 2: Render point cloud + { + let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("point_cloud_render_pass"), + timestamp_writes: None, + }); + pass.set_pipeline(&self.render_pipeline); + pass.set_bind_group(0, Some(&bind_group), &[]); + let workgroups_x = compute_dispatch_size(input_width, 16); + let workgroups_y = compute_dispatch_size(input_height, 16); + pass.dispatch_workgroups(workgroups_x, workgroups_y, 1); + } + + // Copy output to staging buffer + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: output_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: staging_buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(output_width * 4), + rows_per_image: Some(output_height), + }, + }, + wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, + ); + + self.queue.submit(std::iter::once(encoder.finish())); + + // Map and read back using shared helper + let rgba = + crate::shaders::gpu_processor::read_buffer_async(&self.device, &staging_buffer).await?; + + Ok(PointCloudResult { + rgba, + width: output_width, + height: output_height, + }) + } +} + +// Use the shared singleton macro for GPU processor management +gpu_processor_singleton!( + PointCloudProcessor, + GPU_POINT_CLOUD_PROCESSOR, + get_point_cloud_processor +); + +/// Set registration data for depth-to-RGB alignment on the shared GPU processor. +/// +/// This should be called once after the Kinect device starts streaming, with +/// calibration data fetched from the device. This enables proper depth-to-RGB +/// alignment using device-specific polynomial registration tables. +pub async fn set_point_cloud_registration_data(data: &RegistrationData) -> Result<(), String> { + let mut guard = get_point_cloud_processor().await?; + let processor = guard + .as_mut() + .ok_or("GPU point cloud processor not initialized")?; + + processor.set_registration_data(data); + Ok(()) +} + +/// Check if the shared GPU processor has registration data set. +pub async fn has_point_cloud_registration_data() -> Result { + let guard = get_point_cloud_processor().await?; + let processor = guard + .as_ref() + .ok_or("GPU point cloud processor not initialized")?; + + Ok(processor.has_registration_data()) +} + +/// Get the registration data from the shared GPU processor if set. +/// +/// This is useful for scene capture to use the same registration data as the preview shader. +pub async fn get_point_cloud_registration_data() -> Result, String> { + let guard = get_point_cloud_processor().await?; + let processor = guard + .as_ref() + .ok_or("GPU point cloud processor not initialized")?; + + Ok(processor.get_registration_data()) +} + +/// Render point cloud using the shared GPU processor +/// +/// # Arguments +/// * `rgb_data` - RGBA data (4 bytes per pixel) +/// * `depth_data` - 16-bit depth values +/// * `rgb_width` - Width of RGB data +/// * `rgb_height` - Height of RGB data +/// * `depth_width` - Width of depth data +/// * `depth_height` - Height of depth data +/// * `output_width` - Width of output image +/// * `output_height` - Height of output image +/// * `pitch` - Rotation around X axis (radians) +/// * `yaw` - Rotation around Y axis (radians) +/// * `zoom` - Zoom level (1.0 = default, <1.0 = closer, >1.0 = farther) +/// * `depth_format` - Format of depth data (millimeters or disparity) +/// * `mirror` - Whether to mirror the point cloud horizontally +/// * `apply_rgb_registration` - Whether to apply stereo registration (true for RGB camera, false for IR/depth) +/// * `filter_mode` - Color filter mode (0 = none, 1-12 = various filters) +pub async fn render_point_cloud( + rgb_data: &[u8], + depth_data: &[u16], + rgb_width: u32, + rgb_height: u32, + depth_width: u32, + depth_height: u32, + output_width: u32, + output_height: u32, + pitch: f32, + yaw: f32, + zoom: f32, + depth_format: DepthFormat, + mirror: bool, + apply_rgb_registration: bool, + filter_mode: u32, +) -> Result { + let mut guard = get_point_cloud_processor().await?; + let processor = guard + .as_mut() + .ok_or("GPU point cloud processor not initialized")?; + + processor + .render( + rgb_data, + depth_data, + rgb_width, + rgb_height, + depth_width, + depth_height, + output_width, + output_height, + pitch, + yaw, + zoom, + depth_format, + mirror, + apply_rgb_registration, + filter_mode, + ) + .await +} diff --git a/src/shaders/yuv_convert/mod.rs b/src/shaders/yuv_convert/mod.rs new file mode 100644 index 00000000..84f66cc3 --- /dev/null +++ b/src/shaders/yuv_convert/mod.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! GPU-accelerated YUV to RGBA conversion +//! +//! This module provides compute shader-based conversion of YUV 4:2:2 formats +//! to RGBA, keeping the output on GPU for efficient display or further processing. + +mod processor; + +pub use processor::{ + YuvConvertProcessor, YuvConvertResult, YuvFormat, convert_yuv_to_rgba_gpu, + convert_yuv_to_rgba_gpu_texture, +}; diff --git a/src/shaders/yuv_convert/processor.rs b/src/shaders/yuv_convert/processor.rs new file mode 100644 index 00000000..f575ea25 --- /dev/null +++ b/src/shaders/yuv_convert/processor.rs @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! GPU-accelerated YUV to RGBA conversion +//! +//! This module provides compute shader-based conversion of YUV 4:2:2 formats +//! (YUYV and UYVY) to RGBA. The output stays on GPU for efficient display +//! or further processing without CPU round-trips. + +use crate::gpu::{self, wgpu}; +use crate::shaders::compute_dispatch_size; +use crate::shaders::gpu_processor::CachedDimensions; +use std::sync::Arc; +use tracing::{debug, info}; + +/// YUV format variants +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum YuvFormat { + /// YUYV format: Y0, U, Y1, V + Yuyv = 0, + /// UYVY format: U, Y0, V, Y1 (used by Kinect) + Uyvy = 1, +} + +/// Uniform buffer for shader parameters +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +struct YuvParams { + width: u32, + height: u32, + format: u32, + _pad: u32, +} + +/// Result of YUV to RGBA conversion +pub struct YuvConvertResult { + /// Width of output image + pub width: u32, + /// Height of output image + pub height: u32, + /// RGBA data (4 bytes per pixel) - only populated if read back to CPU + pub rgba: Option>, + /// GPU texture handle for zero-copy display + pub texture: Option>, +} + +/// GPU processor for YUV to RGBA conversion +pub struct YuvConvertProcessor { + device: Arc, + queue: Arc, + pipeline: wgpu::ComputePipeline, + bind_group_layout: wgpu::BindGroupLayout, + // Cached resources for reuse + cached_dims: CachedDimensions, + uniform_buffer: Option, + input_buffer: Option, + output_texture: Option>, + staging_buffer: Option, +} + +impl YuvConvertProcessor { + /// Create a new YUV converter with GPU acceleration + pub async fn new() -> Result { + // Create low-priority GPU device for compute operations + let (device, queue, info) = gpu::create_low_priority_compute_device("YUV Convert").await?; + + info!( + adapter_name = %info.adapter_name, + low_priority = info.low_priority_enabled, + "GPU device created for YUV conversion" + ); + + // Load shader + let shader_source = include_str!("yuv_to_rgba.wgsl"); + let shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("YUV to RGBA Shader"), + source: wgpu::ShaderSource::Wgsl(shader_source.into()), + }); + + // Create bind group layout + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("YUV Convert Bind Group Layout"), + entries: &[ + // Params uniform + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Input YUV buffer + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + // Output RGBA texture + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::StorageTexture { + access: wgpu::StorageTextureAccess::WriteOnly, + format: wgpu::TextureFormat::Rgba8Unorm, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + ], + }); + + // Create pipeline layout + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("YUV Convert Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + // Create compute pipeline + let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("YUV to RGBA Pipeline"), + layout: Some(&pipeline_layout), + module: &shader_module, + entry_point: Some("main"), + compilation_options: Default::default(), + cache: None, + }); + + Ok(Self { + device, + queue, + pipeline, + bind_group_layout, + cached_dims: CachedDimensions::default(), + uniform_buffer: None, + input_buffer: None, + output_texture: None, + staging_buffer: None, + }) + } + + /// Ensure resources are allocated for the given dimensions + fn ensure_resources(&mut self, width: u32, height: u32) { + if !self.cached_dims.needs_update(width, height) { + return; + } + + debug!(width, height, "Allocating YUV convert resources"); + + // Create uniform buffer + self.uniform_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("YUV Params Buffer"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + // Input buffer: YUV 4:2:2 = 2 bytes per pixel + let input_size = (width * height * 2) as u64; + self.input_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("YUV Input Buffer"), + size: input_size, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + })); + + // Output texture + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("RGBA Output Texture"), + size: wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::STORAGE_BINDING + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + self.output_texture = Some(Arc::new(texture)); + + // Staging buffer for reading back to CPU (optional) + let output_size = (width * height * 4) as u64; + // Align to 256 bytes for COPY_DST + let aligned_size = (output_size + 255) & !255; + self.staging_buffer = Some(self.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("RGBA Staging Buffer"), + size: aligned_size, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + })); + + self.cached_dims.update(width, height); + } + + /// Convert YUV data to RGBA on GPU + /// + /// # Arguments + /// * `yuv_data` - Raw YUV 4:2:2 data (2 bytes per pixel) + /// * `width` - Image width in pixels + /// * `height` - Image height in pixels + /// * `format` - YUV format (YUYV or UYVY) + /// * `read_back` - If true, read RGBA data back to CPU; if false, keep on GPU only + /// + /// # Returns + /// Result containing the converted RGBA data and/or GPU texture handle + pub async fn convert( + &mut self, + yuv_data: &[u8], + width: u32, + height: u32, + format: YuvFormat, + read_back: bool, + ) -> Result { + // Validate input size + let expected_size = (width * height * 2) as usize; + if yuv_data.len() < expected_size { + return Err(format!( + "YUV data too small: {} bytes, expected {}", + yuv_data.len(), + expected_size + )); + } + + // Ensure resources are allocated + self.ensure_resources(width, height); + + let uniform_buffer = self.uniform_buffer.as_ref().unwrap(); + let input_buffer = self.input_buffer.as_ref().unwrap(); + let output_texture = self.output_texture.as_ref().unwrap(); + + // Update uniform buffer + let params = YuvParams { + width, + height, + format: format as u32, + _pad: 0, + }; + self.queue + .write_buffer(uniform_buffer, 0, bytemuck::bytes_of(¶ms)); + + // Upload YUV data + self.queue + .write_buffer(input_buffer, 0, &yuv_data[..expected_size]); + + // Create texture view + let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + // Create bind group + let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("YUV Convert Bind Group"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: input_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&output_view), + }, + ], + }); + + // Create command encoder + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("YUV Convert Encoder"), + }); + + // Dispatch compute shader + // Each workgroup processes 16x16 macro-pixels (32x16 actual pixels) + let workgroups_x = compute_dispatch_size(width / 2, 16); + let workgroups_y = compute_dispatch_size(height, 16); + + { + let mut compute_pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some("YUV to RGBA Pass"), + timestamp_writes: None, + }); + compute_pass.set_pipeline(&self.pipeline); + compute_pass.set_bind_group(0, &bind_group, &[]); + compute_pass.dispatch_workgroups(workgroups_x, workgroups_y, 1); + } + + // Optionally copy to staging buffer for CPU readback + if read_back { + let staging_buffer = self.staging_buffer.as_ref().unwrap(); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: output_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: staging_buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(width * 4), + rows_per_image: Some(height), + }, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + } + + // Submit commands + self.queue.submit(std::iter::once(encoder.finish())); + + // Read back if requested + let rgba = if read_back { + let staging_buffer = self.staging_buffer.as_ref().unwrap(); + let rgba_data = + crate::shaders::gpu_processor::read_buffer_async(&self.device, staging_buffer) + .await?; + Some(rgba_data) + } else { + None + }; + + Ok(YuvConvertResult { + width, + height, + rgba, + texture: Some(Arc::clone(output_texture)), + }) + } + + /// Get the GPU device for sharing with other GPU operations + pub fn device(&self) -> &Arc { + &self.device + } + + /// Get the GPU queue for sharing with other GPU operations + pub fn queue(&self) -> &Arc { + &self.queue + } + + /// Get the current output texture (if any) + pub fn output_texture(&self) -> Option<&Arc> { + self.output_texture.as_ref() + } +} + +// Global shared processor instance using the standard macro +crate::gpu_processor_singleton!(YuvConvertProcessor, GPU_YUV_PROCESSOR, get_yuv_processor); + +/// Convert YUV to RGBA using GPU acceleration +/// +/// This is the main public API for GPU-accelerated YUV conversion. +/// +/// # Arguments +/// * `yuv_data` - Raw YUV 4:2:2 data (2 bytes per pixel) +/// * `width` - Image width in pixels +/// * `height` - Image height in pixels +/// * `format` - YUV format (YUYV or UYVY) +/// +/// # Returns +/// RGBA data (4 bytes per pixel) as a Vec +pub async fn convert_yuv_to_rgba_gpu( + yuv_data: &[u8], + width: u32, + height: u32, + format: YuvFormat, +) -> Result, String> { + let mut guard = get_yuv_processor().await?; + let processor = guard.as_mut().ok_or("YUV GPU processor not available")?; + + let result = processor + .convert(yuv_data, width, height, format, true) + .await?; + result + .rgba + .ok_or_else(|| "No RGBA data returned".to_string()) +} + +/// Convert YUV to RGBA on GPU without CPU readback +/// +/// Returns the GPU texture handle for zero-copy display or further GPU processing. +/// +/// # Arguments +/// * `yuv_data` - Raw YUV 4:2:2 data (2 bytes per pixel) +/// * `width` - Image width in pixels +/// * `height` - Image height in pixels +/// * `format` - YUV format (YUYV or UYVY) +/// +/// # Returns +/// YuvConvertResult with GPU texture handle +pub async fn convert_yuv_to_rgba_gpu_texture( + yuv_data: &[u8], + width: u32, + height: u32, + format: YuvFormat, +) -> Result { + let mut guard = get_yuv_processor().await?; + let processor = guard.as_mut().ok_or("YUV GPU processor not available")?; + + processor + .convert(yuv_data, width, height, format, false) + .await +} diff --git a/src/shaders/yuv_convert/yuv_to_rgba.wgsl b/src/shaders/yuv_convert/yuv_to_rgba.wgsl new file mode 100644 index 00000000..49a8e4fd --- /dev/null +++ b/src/shaders/yuv_convert/yuv_to_rgba.wgsl @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-3.0-only +// +// YUV 4:2:2 to RGBA compute shader +// +// Converts YUYV or UYVY packed YUV data to RGBA on the GPU. +// The output texture stays on GPU for further processing or display. +// +// YUV 4:2:2 formats pack 2 pixels into 4 bytes: +// - YUYV: Y0, U, Y1, V (used by Kinect) +// - UYVY: U, Y0, V, Y1 + +// Conversion parameters +struct Params { + // Image dimensions + width: u32, + height: u32, + // Format: 0 = YUYV, 1 = UYVY + format: u32, + // Padding for alignment + _pad: u32, +} + +@group(0) @binding(0) var params: Params; +// Input: packed YUV data (2 bytes per pixel = width * height * 2 bytes total) +// Stored as u32 array (4 bytes = 2 pixels worth of YUV data) +@group(0) @binding(1) var input_yuv: array; +// Output: RGBA texture +@group(0) @binding(2) var output_rgba: texture_storage_2d; + +// ITU-R BT.601 YUV to RGB conversion +// R = 1.164*(Y-16) + 1.596*(V-128) +// G = 1.164*(Y-16) - 0.813*(V-128) - 0.391*(U-128) +// B = 1.164*(Y-16) + 2.018*(U-128) +fn yuv_to_rgb(y: i32, u: i32, v: i32) -> vec3 { + // Scale factors (multiplied by 256 for integer math, then divided) + let c = (y - 16) * 298; + let d = u - 128; + let e = v - 128; + + let r = (c + 409 * e + 128) >> 8; + let g = (c - 100 * d - 208 * e + 128) >> 8; + let b = (c + 516 * d + 128) >> 8; + + return vec3( + clamp(f32(r) / 255.0, 0.0, 1.0), + clamp(f32(g) / 255.0, 0.0, 1.0), + clamp(f32(b) / 255.0, 0.0, 1.0) + ); +} + +@compute @workgroup_size(16, 16, 1) +fn main(@builtin(global_invocation_id) global_id: vec3) { + let x = global_id.x; + let y = global_id.y; + + // Each thread processes 2 horizontal pixels (one YUYV/UYVY macro-pixel) + let pixel_x = x * 2u; + + // Bounds check + if (pixel_x >= params.width || y >= params.height) { + return; + } + + // Calculate input index + // Each u32 contains 4 bytes = one macro-pixel (2 pixels worth) + let input_idx = y * (params.width / 2u) + x; + + if (input_idx >= arrayLength(&input_yuv)) { + return; + } + + // Read 4 bytes as u32 + let packed = input_yuv[input_idx]; + + // Extract bytes based on format + let byte0 = i32((packed >> 0u) & 0xFFu); + let byte1 = i32((packed >> 8u) & 0xFFu); + let byte2 = i32((packed >> 16u) & 0xFFu); + let byte3 = i32((packed >> 24u) & 0xFFu); + + var y0: i32; + var u: i32; + var y1: i32; + var v: i32; + + if (params.format == 0u) { + // YUYV format: Y0, U, Y1, V + y0 = byte0; + u = byte1; + y1 = byte2; + v = byte3; + } else { + // UYVY format: U, Y0, V, Y1 + u = byte0; + y0 = byte1; + v = byte2; + y1 = byte3; + } + + // Convert both pixels + let rgb0 = yuv_to_rgb(y0, u, v); + let rgb1 = yuv_to_rgb(y1, u, v); + + // Write to output texture + textureStore(output_rgba, vec2(i32(pixel_x), i32(y)), vec4(rgb0, 1.0)); + + // Only write second pixel if within bounds + if (pixel_x + 1u < params.width) { + textureStore(output_rgba, vec2(i32(pixel_x) + 1, i32(y)), vec4(rgb1, 1.0)); + } +}