feat(simg4ox): support multithreaded Geant4 runs

README.md

@@ -69,6 +69,11 @@ docker run --rm --gpus all ghcr.io/bnlnpps/simphony simg4ox -g tests/geom/raindr
 apptainer exec --nv docker://ghcr.io/bnlnpps/simphony simg4ox -g /workspaces/simphony/tests/geom/raindrop.gdml -m /workspaces/simphony/tests/run.mac
 ```
 
+These commands use `simg4ox`'s default serial Geant4 run manager. The published
+images include multithreaded Geant4; use `tests/run_mt.mac` and add
+`--threads N` to run the same torch photons on `N` Geant4 CPU workers. Opticks
+GPU launches remain serialized because the GPU event context is process-wide.
+
 ### Install with Spack
 
 Simphony is also available through the BNLNPPS Spack repository:

docs/assets/simg4ox-event-processing.typ

@@ -0,0 +1,162 @@
+// Regenerate from the repository root with:
+//   typst compile --format svg docs/assets/simg4ox-event-processing.typ \
+//     docs/assets/simg4ox-event-processing.svg
+
+#let ink = rgb("#17324d")
+#let muted = rgb("#5d6975")
+#let panel-fill = rgb("#f8fafc")
+#let panel-stroke = rgb("#cbd5df")
+#let cpu-fill = rgb("#dceeff")
+#let cpu-stroke = rgb("#3978a8")
+#let gpu-fill = rgb("#eee3ff")
+#let gpu-stroke = rgb("#7651a8")
+#let wait-fill = rgb("#f0f2f4")
+#let wait-stroke = rgb("#8a929b")
+#let output-fill = rgb("#e3f5e8")
+#let output-stroke = rgb("#3d8b59")
+
+#set page(width: 1050pt, height: auto, margin: 22pt, fill: white)
+#set text(font: "DejaVu Sans", size: 10pt, fill: ink)
+#set par(leading: 0.7em)
+
+#let timeline-columns = (
+  90pt,
+  1fr, 1fr, 1fr, 1fr, 1fr, 1fr,
+  1fr, 1fr, 1fr, 1fr, 1fr, 1fr,
+)
+
+#let lane-label(body) = align(
+  right + horizon,
+  text(size: 8.5pt, weight: "bold", fill: ink, body),
+)
+
+#let segment(fill-color, border-color, body) = box(
+  width: 100%,
+  height: 29pt,
+  fill: fill-color,
+  stroke: 0.8pt + border-color,
+  radius: 4pt,
+  inset: (x: 3pt, y: 4pt),
+  align(center + horizon, text(size: 8pt, weight: "bold", fill: ink, body)),
+)
+
+#let panel(title, subtitle, body) = block(
+  width: 100%,
+  fill: panel-fill,
+  stroke: 0.8pt + panel-stroke,
+  radius: 7pt,
+  inset: 12pt,
+)[
+  #text(size: 12pt, weight: "bold")[#title]
+  #h(8pt)
+  #text(size: 8.5pt, fill: muted)[#subtitle]
+  #v(8pt)
+  #body
+]
+
+#align(center)[
+  #text(size: 17pt, weight: "bold")[Current `simg4ox` event processing]
+  #v(2pt)
+  #text(size: 9pt, fill: muted)[E0–E2 denote complete Geant4 events, each potentially containing many G4 tracks. The GPU event context is process-wide in both modes.]
+]
+
+#v(12pt)
+
+#panel(
+  [Serial (`--threads 1`)],
+  [Each Geant4 event and its GPU photon transport finish before the next event starts.],
+  grid(
+    columns: timeline-columns,
+    column-gutter: 2pt,
+    row-gutter: 5pt,
+    align: horizon,
+
+    [],
+    grid.cell(colspan: 12)[#align(right)[#text(size: 7.5pt, fill: muted)[time →]]],
+
+    [#lane-label[Geant4 CPU]],
+    grid.cell(colspan: 2)[#segment(cpu-fill, cpu-stroke)[process event E0]],
+    grid.cell(colspan: 2)[],
+    grid.cell(colspan: 2)[#segment(cpu-fill, cpu-stroke)[process event E1]],
+    grid.cell(colspan: 2)[],
+    grid.cell(colspan: 2)[#segment(cpu-fill, cpu-stroke)[process event E2]],
+    grid.cell(colspan: 2)[],
+
+    [#lane-label[Shared GPU]],
+    grid.cell(colspan: 2)[],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E0 photons]],
+    grid.cell(colspan: 2)[],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E1 photons]],
+    grid.cell(colspan: 2)[],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E2 photons]],
+  ),
+)
+
+#v(10pt)
+
+#panel(
+  [Multithreaded (`--threads 3`)],
+  [Geant4 events are processed concurrently; their photons enter GPU transport strictly in event-ID order.],
+  grid(
+    columns: timeline-columns,
+    column-gutter: 2pt,
+    row-gutter: 5pt,
+    align: horizon,
+
+    [],
+    grid.cell(colspan: 12)[#align(right)[#text(size: 7.5pt, fill: muted)[time →]]],
+
+    [#lane-label[Worker 0]],
+    grid.cell(colspan: 3)[#segment(cpu-fill, cpu-stroke)[process event E0]],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E0 photons]],
+    grid.cell(colspan: 7)[],
+
+    [#lane-label[Worker 1]],
+    grid.cell(colspan: 2)[#segment(cpu-fill, cpu-stroke)[process event E1]],
+    grid.cell(colspan: 3)[#segment(wait-fill, wait-stroke)[wait for E0]],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E1 photons]],
+    grid.cell(colspan: 5)[],
+
+    [#lane-label[Worker 2]],
+    grid.cell(colspan: 4)[#segment(cpu-fill, cpu-stroke)[process event E2]],
+    grid.cell(colspan: 3)[#segment(wait-fill, wait-stroke)[wait for E0–E1]],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E2 photons]],
+    grid.cell(colspan: 3)[],
+
+    [#lane-label[Shared GPU]],
+    grid.cell(colspan: 3)[],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E0 photons]],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E1 photons]],
+    grid.cell(colspan: 2)[#segment(gpu-fill, gpu-stroke)[transport E2 photons]],
+    grid.cell(colspan: 3)[],
+  ),
+)
+
+#v(10pt)
+
+#align(center)[
+  #box(width: 11pt, height: 11pt, fill: cpu-fill, stroke: 0.8pt + cpu-stroke, radius: 2pt)
+  #h(4pt) Geant4 event processing
+  #h(18pt)
+  #box(width: 11pt, height: 11pt, fill: wait-fill, stroke: 0.8pt + wait-stroke, radius: 2pt)
+  #h(4pt) Waiting for event-ID turn
+  #h(18pt)
+  #box(width: 11pt, height: 11pt, fill: gpu-fill, stroke: 0.8pt + gpu-stroke, radius: 2pt)
+  #h(4pt) Serialized GPU photon transport
+  #h(18pt)
+  #box(width: 11pt, height: 11pt, fill: output-fill, stroke: 0.8pt + output-stroke, radius: 2pt)
+  #h(4pt) Run-end merge and save
+]
+
+#v(8pt)
+
+#align(center)[
+  #box(
+    fill: output-fill,
+    stroke: 0.8pt + output-stroke,
+    radius: 4pt,
+    inset: (x: 10pt, y: 5pt),
+  )[
+    After all events: merge hits by event ID → `s_hits.npy` + `g_hits.npy`
+  ]
+]

docs/getting-started.md

@@ -126,8 +126,14 @@ only the target you are changing, and run a relevant test:
 ctest --test-dir build -N
 cmake --build build --target simg4ox
 ctest --test-dir build -R raindrop
+ctest --test-dir build -R '^Integration\.simg4ox_multithread$'
 ```
 
+The `simg4ox_multithread` integration test runs five optical-photon events
+with two Geant4 CPU workers and validates the merged CPU/GPU hit arrays. See
+[the simg4ox example](../examples/README.md#example-4-simg4ox-g4--gpu-validation)
+for serial and MT command lines and the supplied `tests/run_mt.mac` macro.
+
 The full suite includes GPU-backed tests. You can build without a GPU, but
 running those tests requires a compatible NVIDIA driver and GPU access from the
 container.
@@ -142,6 +148,10 @@ If you prefer to manage the toolchain yourself, install:
 - CMake 3.22+
 - Python 3.10+
 
+Build Geant4 with multithreading enabled to use `simg4ox --threads N` with
+`N>1`. The project container images already use a multithreaded Geant4 build;
+`--threads 1` remains available for serial validation.
+
 With those dependencies available, clone, build, and test the project:
 
 ```shell
@@ -182,13 +192,19 @@ To try the latest published release and confirm that GPU access works:
 docker run --rm --gpus all ghcr.io/bnlnpps/simphony simg4ox -g tests/geom/raindrop.gdml -m tests/run.mac
 ```
 
+This uses the default serial run manager. To exercise Geant4 MT in the
+container, use `tests/run_mt.mac` and append `--threads N`.
+
 To test an image built from your current checkout instead:
 
 ```shell
 docker build -t simphony:develop .
 docker run --rm --gpus all simphony:develop simg4ox -g tests/geom/raindrop.gdml -m tests/run.mac
 ```
 
+As with the published image, select `tests/run_mt.mac` and pass `--threads N`
+for an MT run.
+
 For day-to-day development, the Dev Container is more convenient because source
 edits are available immediately without rebuilding the image.
 
@@ -200,6 +216,9 @@ On systems that provide Apptainer, run the same published release with:
 apptainer exec --nv docker://ghcr.io/bnlnpps/simphony simg4ox -g /workspaces/simphony/tests/geom/raindrop.gdml -m /workspaces/simphony/tests/run.mac
 ```
 
+For MT, use the absolute macro path
+`/workspaces/simphony/tests/run_mt.mac` and add `--threads N`.
+
 Use `singularity` in place of `apptainer` on systems that provide the older
 command name.
 

docs/inputs-outputs.md

@@ -57,6 +57,28 @@ tracking is also run for validation.
 | `numphoton` | Number of photons to generate |
 | `wavelength` | Photon wavelength (nm) |
 
+### `simg4ox` execution model
+
+`simg4ox` gives the same generated torch photons to Geant4 CPU tracking and
+Simphony GPU tracking.
+
+Select the Geant4 run manager with `--threads N`. `N=1`, the default, uses the
+serial run manager. `N>1` requires a multithreaded Geant4 build and creates `N`
+CPU workers. This choice must be made before the macro is read, so
+`/run/numberOfThreads` is not the thread-selection interface for `simg4ox`.
+Use a macro such as `tests/run_mt.mac` for run initialization and event count:
+
+```bash
+simg4ox -g tests/geom/opticks_raindrop.gdml -c dev \
+    -m tests/run_mt.mac -s 42 --threads 4
+```
+
+In MT mode, actions and sensitive detectors are worker-local. Completed events
+share one process-wide Opticks GPU context, so GPU launches are serialized in
+event-ID order while Geant4 CPU tracking remains multithreaded. Run-wide CPU
+and GPU hits are merged in event-ID order. Full CPU-side Opticks photon-history
+recording is available only with the serial run manager.
+
 ## Defining primary particles
 
 For charged-particle examples, the user or developer defines the primary
@@ -150,6 +172,13 @@ without the final `A000` or `B000` event-index subdirectory. The app-level
 `Config` interface does not expose a separate save-mode field; prefer
 `Config::output_dir` for controlling where event folders are written.
 
+`simg4ox` also writes application-level run aggregates directly under
+`Config::output_dir`: `s_hits.npy` contains Simphony GPU hits and `g_hits.npy`
+contains Geant4 sensitive-detector hits. Both have shape `(H, 4, 4)`, dtype
+`float32`, and the `sphoton` layout described below. They are written in both
+serial and MT modes. In MT mode, CPU-side history folders are not produced, but
+`g_hits.npy` is still written from the merged Geant4 hit collections.
+
 ### File schemas
 
 The table assumes the default full photon representation and unmerged hits:

docs/performance-and-debugging.md

@@ -26,6 +26,20 @@ docker run --rm -t -v /tmp/out:/tmp/out simphony:release \
     run-performance -g tests/geom/opticks_raindrop.gdml -o /tmp/out/release
 ```
 
+### Interpreting `simg4ox` MT timings
+
+`simg4ox --threads N` parallelizes Geant4 CPU tracking of its configured torch
+photons. It does not run multiple Opticks launches concurrently: the current
+process-wide GPU event context requires launches to be serialized in event-ID
+order. Consequently, an end-to-end `simg4ox` wall-clock measurement includes
+parallel CPU work plus serialized GPU work and should not be interpreted as a
+pure GPU speed-up measurement.
+
+![Serial and multithreaded simg4ox event-processing timelines](assets/simg4ox-event-processing.svg)
+
+The supplied `tests/run_mt.mac` contains only five low-statistics events and is
+an integration check, not a benchmark.
+
 ## Debug analysis with `optiphy/ana/photon_history_summary.py`
 
 The script analyzes GPU optical photon simulation output to debug where

examples/README.md

@@ -7,7 +7,7 @@ Simphony provides several examples demonstrating GPU-accelerated optical photon
 | `simphox` | Optical photons (torch) | None | External project build and CPU/GPU photon generation smoke test |
 | `GPUCerenkov` | Cerenkov only | Simple nested boxes (raindrop) | Basic Cerenkov testing |
 | `GPURaytrace` | Cerenkov + Scintillation | 8x8 CsI crystal + SiPM array | Realistic detector simulation |
-| `simg4ox` | Optical photons (torch) | Any GDML | G4 + GPU side-by-side validation |
+| `simg4ox` | Optical photons (torch) | Any GDML | Serial/MT G4 + GPU side-by-side validation |
 | `GPUPhotonSourceMinimal` | Optical photons (torch) | Any GDML | GPU-only test |
 | `GPUPhotonFileSource` | Optical photons (text file) | Any GDML | GPU-only, user-defined photons from file |
 | WLS test | Wavelength shifting | WLS sphere + detector shell | Validate GPU WLS physics |
@@ -26,7 +26,7 @@ Geant4 optical-photon tracking is also run for validation.
 | Photon input from text file | No | No | No | No | Yes |
 | G4 optical photon tracking | Yes | Yes | Yes | No | No |
 | GPU simulation (Simphony) | Yes | Yes | Yes | Yes | Yes |
-| Multi-threaded | Yes | Yes | No | No | No |
+| Geant4 CPU multithreading | Yes | Yes | Yes | No | No |
 
 `GPUCerenkov` and `GPURaytrace` collect gensteps from charged-particle
 interactions and pass them to Simphony for GPU photon generation and tracing.
@@ -35,6 +35,10 @@ a torch configuration. `simg4ox` runs both G4 and GPU tracking for
 validation, while `GPUPhotonSourceMinimal` keeps only the GPU path.
 `GPUPhotonFileSource` reads user-defined photons from a text file.
 
+For `simg4ox`, multithreading applies to Geant4 CPU tracking. Its process-wide
+Opticks event context is protected by serializing GPU launches in event-ID
+order.
+
 ### Example 1: simphox (External build smoke test)
 
 Assuming Simphony is properly installed on the system, compile and run this
@@ -109,13 +113,14 @@ grep -c "CreationProcessID=1" opticks_hits_output.txt  # Scintillation
 
 ### Example 4: simg4ox (G4 + GPU Validation)
 
-`simg4ox` generates optical photons from a configurable torch source and runs
-both Geant4 and Simphony GPU simulation in parallel on the same input photons. This
+`simg4ox` generates optical photons from a configurable torch source and
+tracks the same input with Geant4 on the CPU and Simphony on the GPU. This
 enables direct comparison of hit counts and positions between the two engines.
 
-Both engines detect photons using the same mechanism: border surface physics. On the G4
-side the `SteppingAction` records a hit when `G4OpBoundaryProcess` reports Detection at
-the optical surface, matching how Simphony detects photons on the GPU.
+Both engines detect photons using border-surface physics. On the Geant4 side,
+the optical boundary process invokes the worker-local `PhotonSD` at configured
+sensitive surfaces; the detector records and terminates the photon. Simphony
+selects GPU hits from the corresponding optical-boundary result.
 
 | Argument | Description | Default |
 |----------|-------------|---------|
@@ -124,12 +129,32 @@ the optical surface, matching how Simphony detects photons on the GPU.
 | `-m, --macro` | Path to G4 macro | `run.mac` |
 | `-i, --interactive` | Open interactive viewer | off |
 | `-s, --seed` | Fixed random seed | Geant4 default |
+| `-t, --threads` | Geant4 CPU threads; `1` selects the serial run manager | `1` |
+
+The same executable supports serial and multi-threaded Geant4 runs:
 
 ```bash
-simg4ox -g tests/geom/opticks_raindrop.gdml -c dev -m run.mac -s 42
+# Serial Geant4 (default)
+simg4ox -g tests/geom/opticks_raindrop.gdml -c dev -m tests/run_5evt.mac -s 42
+
+# Geant4 MT with four CPU workers
+simg4ox -g tests/geom/opticks_raindrop.gdml -c dev -m tests/run_mt.mac -s 42 --threads 4
 ```
 
+The thread count is a command-line option because the executable must select
+the serial or MT run-manager type before Geant4 reads a macro. The example
+`tests/run_mt.mac` contains the remaining run initialization and event-count
+commands.
+
+In MT mode, Geant4 actions and sensitive detectors are worker-local and the
+run-wide results are merged in event-ID order. Opticks currently exposes one
+process-wide GPU event context, so GPU launches are serialized in that same
+order while Geant4 CPU tracking remains multi-threaded. Full CPU-side Opticks
+photon-history recording remains available in serial mode; both modes write
+the run-wide hit arrays below.
+
 **Output:**
+
 - `s_hits.npy` — Simphony GPU hits
 - `g_hits.npy` — Geant4 hits
 

src/CMakeLists.txt

@@ -14,7 +14,7 @@ target_include_directories(consgeo PRIVATE
 )
 target_link_libraries(consgeo simphony_argparse)
 
-# simg4ox runs Geant4 and OptiX simulations
+# simg4ox validates torch photons with serial/MT Geant4 and serialized OptiX launches
 add_executable(simg4ox simg4ox.cpp g4app.h)
 
 target_link_libraries(simg4ox SysRap simphony_g4_deps simphony_argparse)