diff --git a/.github/workflows/lingbot-demo-runtime.yml b/.github/workflows/lingbot-demo-runtime.yml new file mode 100644 index 000000000..98b575f6e --- /dev/null +++ b/.github/workflows/lingbot-demo-runtime.yml @@ -0,0 +1,334 @@ +name: LingBot Demo Runtime + +on: + push: + branches: + - main + - "pull-request/[0-9]+" + paths: + - ".github/workflows/lingbot-demo-runtime.yml" + - "pyproject.toml" + - "uv.lock" + - "flashdreams/pyproject.toml" + - "flashdreams/flashdreams/core/**" + - "flashdreams/flashdreams/infra/**" + - "flashdreams/flashdreams/runtime/**" + - "flashdreams/flashdreams/serving/**" + - "flashdreams/flashdreams/recipes/taehv/**" + - "flashdreams/flashdreams/recipes/wan/**" + - "flashdreams/tests/test_webrtc_*.py" + - "integrations/lingbot/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + demo-runtime: + name: null and MP4 + runs-on: linux-amd64-gpu-rtxpro6000-latest-2 + timeout-minutes: 180 + defaults: + run: + shell: bash + container: + image: nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 + options: --gpus all + env: + UV_PROJECT_ENVIRONMENT: /tmp/flashdreams-venv + UV_LINK_MODE: copy + UV_PYTHON: "3.12" + MAX_JOBS: 8 + HF_HOME: /tmp/huggingface + FLASHDREAMS_CACHE_DIR: /tmp/flashdreams-cache + # Streaming avoids the old duplicate merged safetensors cache. CI uses + # the generic reserve so the model-specific 200 GiB first-run budget does + # not reject runners that have enough room for the streamed shards. + FLASHDREAMS_MIN_CACHE_FREE_GB: "20" + ARTIFACT_DIR: artifacts/lingbot_demo_runtime + PRESET_ID: lingbot-world-v2-14b-causal-fast-taehv-window15-sink3 + BLOCKS: "5" + FPS: "16" + WIDTH: "640" + HEIGHT: "352" + EXPECTED_WIDTH: "640" + EXPECTED_HEIGHT: "352" + MIN_DURATION_SECONDS: "3" + MAX_DURATION_SECONDS: "5" + steps: + - name: Detect GPU architecture + id: gpu-arch + run: | + nvidia-smi + compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '[:space:]') + arch=$(echo "${compute_cap}" | tr -d '.') + echo "arch=${arch}" >> "$GITHUB_OUTPUT" + echo "Detected GPU compute capability: ${compute_cap} -> sm_${arch}" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ + python3 python3-dev python3-venv \ + ffmpeg \ + gcc g++ ninja-build \ + libnccl-dev \ + curl git ca-certificates jq unzip + rm -rf /var/lib/apt/lists/* + + - name: Setup proxy cache + uses: nv-gha-runners/setup-proxy-cache@main + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-suffix: "lingbot-demo-runtime-sm${{ steps.gpu-arch.outputs.arch }}" + prune-cache: false + + - name: Install dependencies + env: + NVTE_CUDA_ARCHS: ${{ steps.gpu-arch.outputs.arch }} + run: | + uv venv --clear + uv sync --locked --package flashdreams-lingbot --no-dev + + - name: Verify GPU availability + run: nvidia-smi + + - name: Run LingBot demo modes + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + set -uo pipefail + + log_dir="${ARTIFACT_DIR}/logs" + output_dir="${ARTIFACT_DIR}/outputs" + summary="${ARTIFACT_DIR}/summary.md" + status_file="${ARTIFACT_DIR}/command-status.env" + mkdir -p "${log_dir}" "${output_dir}" + : > "${status_file}" + + ldemo() { + uv run --no-sync --package flashdreams-lingbot lingbot-demo "$@" + } + + run_demo() { + local name="$1" + shift + local log="${log_dir}/${name}.log" + + { + printf '$' + printf ' %q' "$@" + printf '\n\n' + "$@" + } 2>&1 | tee "${log}" + + local rc="${PIPESTATUS[0]}" + echo "${name}=${rc}" >> "${status_file}" + echo "${name} exit code: ${rc}" | tee -a "${summary}" + return 0 + } + + { + echo "# LingBot Demo Runtime CI" + echo + echo "| Mode | Expected blocks | Output |" + echo "| --- | ---: | --- |" + echo "| null | ${BLOCKS} | none |" + echo "| MP4 | ${BLOCKS} | lingbot-demo-replay.mp4 |" + echo + echo "## Command Status" + } > "${summary}" + + run_demo null \ + ldemo replay \ + --device cuda:0 \ + --preset-id "${PRESET_ID}" \ + --example-idx 0 \ + --total-blocks "${BLOCKS}" \ + --fps "${FPS}" \ + --pixel-height "${HEIGHT}" \ + --pixel-width "${WIDTH}" \ + --output-mode null + + run_demo mp4 \ + ldemo replay \ + --device cuda:0 \ + --preset-id "${PRESET_ID}" \ + --example-idx 0 \ + --total-blocks "${BLOCKS}" \ + --fps "${FPS}" \ + --pixel-height "${HEIGHT}" \ + --pixel-width "${WIDTH}" \ + --output "${output_dir}/lingbot-demo-replay.mp4" + + - name: Validate LingBot demo artifacts + run: | + set -euo pipefail + + log_dir="${ARTIFACT_DIR}/logs" + output_dir="${ARTIFACT_DIR}/outputs" + probe_dir="${ARTIFACT_DIR}/ffprobe" + summary="${ARTIFACT_DIR}/summary.md" + status_file="${ARTIFACT_DIR}/command-status.env" + mkdir -p "${probe_dir}" + + status_of() { + awk -F= -v name="$1" '$1 == name { print $2 }' "${status_file}" + } + + assert_exit_zero() { + local name="$1" + local rc + rc="$(status_of "${name}")" + if [ "${rc}" != "0" ]; then + echo "${name} command failed with exit code ${rc}" >&2 + exit 1 + fi + } + + assert_clean_log() { + local log="$1" + if grep -En "ERROR|Traceback|Exception|status=failed|Run failed|failed run" "${log}"; then + echo "failure marker found in ${log}" >&2 + exit 1 + fi + } + + assert_log_contains() { + local log="$1" + local pattern="$2" + local label="$3" + if ! grep -Eq "${pattern}" "${log}"; then + echo "expected ${label} in ${log}" >&2 + exit 1 + fi + } + + assert_log_not_contains() { + local log="$1" + local pattern="$2" + local label="$3" + if grep -Eq "${pattern}" "${log}"; then + echo "unexpected ${label} in ${log}" >&2 + exit 1 + fi + } + + validate_mp4() { + local mode="$1" + local mp4="$2" + local metadata="${probe_dir}/${mode}.json" + + if [ ! -s "${mp4}" ]; then + echo "expected non-empty MP4 at ${mp4}" >&2 + exit 1 + fi + + ffprobe \ + -v error \ + -select_streams v:0 \ + -show_entries stream=width,height,r_frame_rate,avg_frame_rate,nb_frames,duration:format=duration \ + -of json \ + "${mp4}" > "${metadata}" + + local stream_count width height duration + stream_count="$(jq '.streams | length' "${metadata}")" + width="$(jq -r '.streams[0].width // ""' "${metadata}")" + height="$(jq -r '.streams[0].height // ""' "${metadata}")" + duration="$(jq -r '.streams[0].duration // .format.duration // "0"' "${metadata}")" + + if [ "${stream_count}" -lt 1 ]; then + echo "ffprobe found no video stream in ${mp4}" >&2 + exit 1 + fi + + if [ "${width}" != "${EXPECTED_WIDTH}" ] || [ "${height}" != "${EXPECTED_HEIGHT}" ]; then + echo "unexpected ${mode} resolution ${width}x${height}; expected ${EXPECTED_WIDTH}x${EXPECTED_HEIGHT}" >&2 + exit 1 + fi + + awk \ + -v duration="${duration}" \ + -v min_duration="${MIN_DURATION_SECONDS}" \ + -v max_duration="${MAX_DURATION_SECONDS}" \ + 'BEGIN { + if ((duration + 0) < min_duration || (duration + 0) > max_duration) { + exit 1 + } + }' || { + echo "unexpected ${mode} duration ${duration}s; expected ${MIN_DURATION_SECONDS}-${MAX_DURATION_SECONDS}s" >&2 + exit 1 + } + } + + null_log="${log_dir}/null.log" + mp4_log="${log_dir}/mp4.log" + + assert_exit_zero null + assert_exit_zero mp4 + + assert_clean_log "${null_log}" + assert_clean_log "${mp4_log}" + + assert_log_contains "${null_log}" "Streaming sharded safetensors checkpoint" "null streaming checkpoint load" + assert_log_contains "${null_log}" "Finished streaming .* safetensors shard" "null streamed checkpoint completion" + assert_log_contains "${null_log}" "AR 4 encode" "null final AR block" + assert_log_contains "${null_log}" "Lingbot runtime step 4 frames=" "null final replay step" + assert_log_not_contains "${null_log}" "Loading merged sharded checkpoint from cache|Saved merged sharded checkpoint" "merged safetensors cache usage" + + assert_log_contains "${mp4_log}" "Streaming sharded safetensors checkpoint" "MP4 streaming checkpoint load" + assert_log_contains "${mp4_log}" "Finished streaming .* safetensors shard" "MP4 streamed checkpoint completion" + assert_log_contains "${mp4_log}" "AR 4 encode" "MP4 final AR block" + assert_log_contains "${mp4_log}" "Lingbot runtime step 4 frames=" "MP4 final replay step" + assert_log_not_contains "${mp4_log}" "Loading merged sharded checkpoint from cache|Saved merged sharded checkpoint" "merged safetensors cache usage" + + validate_mp4 mp4 "${output_dir}/lingbot-demo-replay.mp4" + + { + echo + echo "## Validation" + echo + echo "- Null and MP4 commands exited zero." + echo "- Logs contained expected streaming-checkpoint and final AR-step markers." + echo "- Logs did not contain merged-safetensors cache markers." + echo "- MP4 output was non-empty and passed ffprobe stream checks." + } >> "${summary}" + + - name: Trim uv cache for upload + if: always() + run: | + cache_dir="${UV_CACHE_DIR:-/github/home/.cache/uv}" + echo "=== Cache size before trim ===" + du -sh "${cache_dir}" || true + du -sh "${cache_dir}"/*/ 2>/dev/null || true + + rm -rf "${cache_dir}/wheels-v6" + rm -rf "${cache_dir}/archive-v0" + + find "${cache_dir}/git-v0/checkouts" \ + \( -name "build" -o -name "*.egg-info" -o -name "__pycache__" \) \ + -type d -exec rm -rf {} + 2>/dev/null || true + + rm -rf "${cache_dir}/sdists-v9/editable" + + echo "" + echo "=== Cache size after trim ===" + du -sh "${cache_dir}" || true + du -sh "${cache_dir}"/*/ 2>/dev/null || true + echo "" + echo "=== Cached built wheels (sdists-v9) ===" + find "${cache_dir}/sdists-v9" -name "*.whl" -exec ls -lh {} \; 2>/dev/null || true + + - name: Upload demo runtime artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: lingbot-demo-runtime + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: ignore diff --git a/.github/workflows/omnidreams-demo-runtime.yml b/.github/workflows/omnidreams-demo-runtime.yml new file mode 100644 index 000000000..1f494fa83 --- /dev/null +++ b/.github/workflows/omnidreams-demo-runtime.yml @@ -0,0 +1,328 @@ +name: OmniDreams Demo Runtime + +on: + push: + branches: + - main + - "pull-request/[0-9]+" + paths: + - ".github/workflows/omnidreams-demo-runtime.yml" + - "pyproject.toml" + - "uv.lock" + - "flashdreams/pyproject.toml" + - "flashdreams/flashdreams/core/**" + - "flashdreams/flashdreams/infra/**" + - "flashdreams/flashdreams/runtime/**" + - "flashdreams/flashdreams/serving/**" + - "flashdreams/flashdreams/recipes/taehv/**" + - "flashdreams/flashdreams/recipes/wan/**" + - "integrations/omnidreams/**" + - "integrations/lingbot/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + demo-runtime: + name: null, precomputed MP4, and Ludus MP4 + runs-on: linux-amd64-gpu-rtxpro6000-latest-2 + timeout-minutes: 180 + defaults: + run: + shell: bash + container: + image: nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 + options: --gpus all + env: + UV_PROJECT_ENVIRONMENT: /tmp/flashdreams-venv + UV_LINK_MODE: copy + UV_PYTHON: "3.10" + MAX_JOBS: 8 + ARTIFACT_DIR: artifacts/omnidreams_demo_runtime + NULL_BLOCKS: "10" + PRECOMPUTED_BLOCKS: "75" + LUDUS_BLOCKS: "76" + FPS: "30" + EXAMPLE_DATA_UUID: 239560dc-33d1-11ef-9720-00044bcbccac + LUDUS_SCENE_UUID: 0d404ff7-2b66-498c-b047-1ed8cded60d4 + LUDUS_TRACE: integrations/omnidreams/omnidreams/demo/traces/ludus_forward_sweep_60s.json + EXPECTED_WIDTH: "1280" + EXPECTED_HEIGHT: "704" + MIN_DURATION_SECONDS: "18" + MAX_DURATION_SECONDS: "22" + steps: + - name: Detect GPU architecture + id: gpu-arch + run: | + nvidia-smi + compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '[:space:]') + arch=$(echo "${compute_cap}" | tr -d '.') + echo "arch=${arch}" >> "$GITHUB_OUTPUT" + echo "Detected GPU compute capability: ${compute_cap} -> sm_${arch}" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ + python3 python3-dev python3-venv \ + ffmpeg \ + gcc g++ ninja-build \ + libnccl-dev \ + curl git ca-certificates jq unzip + rm -rf /var/lib/apt/lists/* + + - name: Setup proxy cache + uses: nv-gha-runners/setup-proxy-cache@main + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-suffix: "omnidreams-demo-runtime-sm${{ steps.gpu-arch.outputs.arch }}" + prune-cache: false + + - name: Install dependencies + env: + NVTE_CUDA_ARCHS: ${{ steps.gpu-arch.outputs.arch }} + run: | + uv venv --clear + uv sync --locked --extra dev + + - name: Verify GPU availability + run: nvidia-smi + + - name: Run OmniDreams demo modes + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + set -uo pipefail + + log_dir="${ARTIFACT_DIR}/logs" + output_dir="${ARTIFACT_DIR}/outputs" + summary="${ARTIFACT_DIR}/summary.md" + status_file="${ARTIFACT_DIR}/command-status.env" + mkdir -p "${log_dir}" "${output_dir}" + : > "${status_file}" + + odemo() { + uv run --no-sync --package flashdreams-omnidreams omnidreams-demo "$@" + } + + run_demo() { + local name="$1" + shift + local log="${log_dir}/${name}.log" + + { + printf '$' + printf ' %q' "$@" + printf '\n\n' + "$@" + } 2>&1 | tee "${log}" + + local rc="${PIPESTATUS[0]}" + echo "${name}=${rc}" >> "${status_file}" + echo "${name} exit code: ${rc}" | tee -a "${summary}" + return 0 + } + + { + echo "# OmniDreams Demo Runtime CI" + echo + echo "| Mode | Expected blocks | Output |" + echo "| --- | ---: | --- |" + echo "| null | ${NULL_BLOCKS} | none |" + echo "| precomputed MP4 | ${PRECOMPUTED_BLOCKS} | omnidreams-demo-precomputed-20s.mp4 |" + echo "| Ludus MP4 | ${LUDUS_BLOCKS} | omnidreams-demo-ludus-20s.mp4 |" + echo + echo "## Command Status" + } > "${summary}" + + run_demo null \ + odemo replay \ + --output-mode null \ + --device cuda:0 \ + --total-blocks "${NULL_BLOCKS}" + + run_demo precomputed-mp4 \ + odemo replay \ + --device cuda:0 \ + --example-data \ + --example-data-uuid "${EXAMPLE_DATA_UUID}" \ + --total-blocks "${PRECOMPUTED_BLOCKS}" \ + --fps "${FPS}" \ + --output "${output_dir}/omnidreams-demo-precomputed-20s.mp4" + + run_demo ludus-mp4 \ + odemo replay \ + --conditioning-mode ludus-scene-driving \ + --keyboard-trace "${LUDUS_TRACE}" \ + --device cuda:0 \ + --scene-uuid "${LUDUS_SCENE_UUID}" \ + --seed 42 \ + --total-blocks "${LUDUS_BLOCKS}" \ + --output "${output_dir}/omnidreams-demo-ludus-20s.mp4" + + - name: Validate OmniDreams demo artifacts + run: | + set -euo pipefail + + log_dir="${ARTIFACT_DIR}/logs" + output_dir="${ARTIFACT_DIR}/outputs" + probe_dir="${ARTIFACT_DIR}/ffprobe" + summary="${ARTIFACT_DIR}/summary.md" + status_file="${ARTIFACT_DIR}/command-status.env" + mkdir -p "${probe_dir}" + + status_of() { + awk -F= -v name="$1" '$1 == name { print $2 }' "${status_file}" + } + + assert_exit_zero() { + local name="$1" + local rc + rc="$(status_of "${name}")" + if [ "${rc}" != "0" ]; then + echo "${name} command failed with exit code ${rc}" >&2 + exit 1 + fi + } + + assert_clean_log() { + local log="$1" + if grep -En "ERROR|Traceback|Exception|status=failed|Run failed|failed run" "${log}"; then + echo "failure marker found in ${log}" >&2 + exit 1 + fi + } + + assert_log_contains() { + local log="$1" + local pattern="$2" + local label="$3" + if ! grep -Eq "${pattern}" "${log}"; then + echo "expected ${label} in ${log}" >&2 + exit 1 + fi + } + + validate_mp4() { + local mode="$1" + local mp4="$2" + local metadata="${probe_dir}/${mode}.json" + + if [ ! -s "${mp4}" ]; then + echo "expected non-empty MP4 at ${mp4}" >&2 + exit 1 + fi + + ffprobe \ + -v error \ + -select_streams v:0 \ + -show_entries stream=width,height,r_frame_rate,avg_frame_rate,nb_frames,duration:format=duration \ + -of json \ + "${mp4}" > "${metadata}" + + local stream_count width height duration + stream_count="$(jq '.streams | length' "${metadata}")" + width="$(jq -r '.streams[0].width // ""' "${metadata}")" + height="$(jq -r '.streams[0].height // ""' "${metadata}")" + duration="$(jq -r '.streams[0].duration // .format.duration // "0"' "${metadata}")" + + if [ "${stream_count}" -lt 1 ]; then + echo "ffprobe found no video stream in ${mp4}" >&2 + exit 1 + fi + + if [ "${width}" != "${EXPECTED_WIDTH}" ] || [ "${height}" != "${EXPECTED_HEIGHT}" ]; then + echo "unexpected ${mode} resolution ${width}x${height}; expected ${EXPECTED_WIDTH}x${EXPECTED_HEIGHT}" >&2 + exit 1 + fi + + awk \ + -v duration="${duration}" \ + -v min_duration="${MIN_DURATION_SECONDS}" \ + -v max_duration="${MAX_DURATION_SECONDS}" \ + 'BEGIN { + if ((duration + 0) < min_duration || (duration + 0) > max_duration) { + exit 1 + } + }' || { + echo "unexpected ${mode} duration ${duration}s; expected ${MIN_DURATION_SECONDS}-${MAX_DURATION_SECONDS}s" >&2 + exit 1 + } + } + + null_log="${log_dir}/null.log" + precomputed_log="${log_dir}/precomputed-mp4.log" + ludus_log="${log_dir}/ludus-mp4.log" + + assert_exit_zero null + assert_exit_zero precomputed-mp4 + assert_exit_zero ludus-mp4 + + assert_clean_log "${null_log}" + assert_clean_log "${precomputed_log}" + assert_clean_log "${ludus_log}" + + assert_log_contains "${null_log}" "AR 9 encode" "null final AR block" + assert_log_contains "${null_log}" "OmniDreams demo replay step 9 frames=" "null final replay step" + assert_log_contains "${null_log}" "Loaded OmniDreams demo HDMaps shape=.*views=1" "null precomputed HDMaps" + + assert_log_contains "${precomputed_log}" "AR 74 encode" "precomputed final AR block" + assert_log_contains "${precomputed_log}" "OmniDreams demo replay step 74 frames=" "precomputed final replay step" + assert_log_contains "${precomputed_log}" "Loaded OmniDreams demo HDMaps shape=.*views=1" "precomputed HDMaps" + + assert_log_contains "${ludus_log}" "AR 75 encode" "Ludus final AR block" + assert_log_contains "${ludus_log}" "OmniDreams demo replay step 75 frames=" "Ludus final replay step" + assert_log_contains "${ludus_log}" "ludus_backend=cuda" "Ludus CUDA backend" + assert_log_contains "${ludus_log}" "trace_events=[1-9][0-9]*" "nonzero Ludus trace events" + + validate_mp4 precomputed-mp4 "${output_dir}/omnidreams-demo-precomputed-20s.mp4" + validate_mp4 ludus-mp4 "${output_dir}/omnidreams-demo-ludus-20s.mp4" + + { + echo + echo "## Validation" + echo + echo "- All commands exited zero." + echo "- Logs contained expected final AR blocks and provider markers." + echo "- MP4 outputs were non-empty and passed ffprobe stream checks." + } >> "${summary}" + + - name: Trim uv cache for upload + if: always() + run: | + cache_dir="${UV_CACHE_DIR:-/github/home/.cache/uv}" + echo "=== Cache size before trim ===" + du -sh "${cache_dir}" || true + du -sh "${cache_dir}"/*/ 2>/dev/null || true + + rm -rf "${cache_dir}/wheels-v6" + rm -rf "${cache_dir}/archive-v0" + + find "${cache_dir}/git-v0/checkouts" \ + \( -name "build" -o -name "*.egg-info" -o -name "__pycache__" \) \ + -type d -exec rm -rf {} + 2>/dev/null || true + + rm -rf "${cache_dir}/sdists-v9/editable" + + echo "" + echo "=== Cache size after trim ===" + du -sh "${cache_dir}" || true + du -sh "${cache_dir}"/*/ 2>/dev/null || true + echo "" + echo "=== Cached built wheels (sdists-v9) ===" + find "${cache_dir}/sdists-v9" -name "*.whl" -exec ls -lh {} \; 2>/dev/null || true + + - name: Upload demo runtime artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: omnidreams-demo-runtime + path: ${{ env.ARTIFACT_DIR }} + if-no-files-found: ignore diff --git a/configs/omnidreams_demo_replay_benchmarks.json b/configs/omnidreams_demo_replay_benchmarks.json new file mode 100644 index 000000000..3ac6e4019 --- /dev/null +++ b/configs/omnidreams_demo_replay_benchmarks.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "description": "Manual one-minute local benchmark scenarios for comparing the legacy Omnidreams single-view runner against the experimental shared demo replay path. The runner writes the legacy stacked HDMap/RGB canvas while the shared demo writes generated RGB output, so use the report for manual MP4 comparison rather than automatic pixel quality scoring.", + "scenarios": [ + { + "id": "omnidreams-sv-runner-baseline", + "name": "Omnidreams single-view runner baseline", + "description": "Runs the stable legacy Omnidreams single-view runner with the bundled example data for the same one-minute block count used by the shipped Omnidreams baseline.", + "report_group": { + "id": "omnidreams-demo", + "name": "Omnidreams Demo Comparison" + }, + "tags": [ + "manual", + "gpu", + "real-demo", + "omnidreams", + "i2v", + "replay", + "baseline" + ], + "env": { + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" + }, + "command": [ + "uv", + "run", + "--project", + "integrations/omnidreams", + "flashdreams-run", + "omnidreams-sv-2steps-chunk2-loc6-lightvae-lighttae", + "--example-data", + "True", + "--example-data-uuid", + "239560dc-33d1-11ef-9720-00044bcbccac", + "--total-blocks", + "226" + ], + "warmup_steps": 1, + "quality_baseline_compare": false, + "timeout_s": 7200 + }, + { + "id": "omnidreams-sv-demo-replay", + "name": "Omnidreams shared demo replay", + "description": "Runs the experimental shared demo API replay path with the same stable non-perf preset, bundled example data, and one-minute block count as the legacy runner.", + "report_group": { + "id": "omnidreams-demo", + "name": "Omnidreams Demo Comparison" + }, + "tags": [ + "manual", + "gpu", + "real-demo", + "omnidreams", + "i2v", + "replay", + "shared-demo" + ], + "env": { + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" + }, + "command": [ + "uv", + "run", + "--project", + "integrations/omnidreams", + "omnidreams-demo", + "replay", + "--preset-id", + "omnidreams-sv-2steps-chunk2-loc6-lightvae-lighttae", + "--example-data", + "--example-data-uuid", + "239560dc-33d1-11ef-9720-00044bcbccac", + "--total-blocks", + "226", + "--output", + "{output_dir}/omnidreams-sv-demo-replay.mp4" + ], + "output_dir_arg": null, + "warmup_steps": 1, + "quality_baseline_compare": false, + "timeout_s": 7200 + } + ] +} diff --git a/docs/inference_runtime_api_design.md b/docs/inference_runtime_api_design.md new file mode 100644 index 000000000..260b24b24 --- /dev/null +++ b/docs/inference_runtime_api_design.md @@ -0,0 +1,786 @@ + + +# FlashDreams Inference Runtime API Design Proposal + +Date: July 30, 2026 + +## Summary + +This proposal defines a standard inference runtime API for FlashDreams +integrations. The goal is to make world-model integrations easier to build, +benchmark, and run without forcing every model into the same input shape or +optimization stack. + +The proposed API separates the pieces that are currently mixed together in +integration-specific runner code: + +- `InferenceConfig`: how the model and inference stack should run; +- `UserInputs`: controls or events from an app, replay trace, or benchmark; +- `InferenceInput`: prompts, frames, videos, trajectories, maps, scene data, and + other values required by a specific model; +- input mapping: model/application-specific conversion from user-facing inputs + into model-facing inputs; +- runtime/session execution: model setup, warmup, per-rollout state, and + stepping; +- output targets: WebRTC, native display, MP4, benchmark artifacts, or headless + runs; +- metrics/profiling: timings, memory, traces, NVTX ranges, and benchmark + outputs. + +Current T2/T3 implementation notes are in +`docs/inference_runtime_inputs_implementation.md`. + +The supported-model input inventory used to revisit T2/T3 is in +`docs/inference_runtime_supported_inputs_inventory.md`. + +The API should standardize the envelope and lifecycle. It should not pretend +that all world models have the same inputs, that all models use the same +optimization stack, or that a raw checkpoint can fully describe how to run the +model. + +## Current Implementation Plan + +Implementation should happen on an experimental integration branch. PRs for this +work should target that branch until the API shape and migrated demo paths are +working well enough to merge to `main` together. + +The experimental branch can temporarily break or simplify command-line options +while the demos are being moved to the new API. The required outcome for this +branch is that the OmniDreams demo runs through the new shared demo/runtime path, +and that benchmark and manual WebRTC checks can confirm it is at least broadly +healthy before the branch is merged back to `main`. + +Initial scope: + +- define the minimal runtime API envelope; +- migrate OmniDreams to use it through a shared demo-level API; +- support selectable output modes such as MP4, JPEG/MJPEG stream, WebRTC, and + headless/null where appropriate; +- use or update benchmark tooling to verify the migrated OmniDreams demo; +- defer additional model migrations, hosted execution, full autotune, and polished + metrics until the first branch proves the API shape. + +## Task Tracker + +| ID | Status | Workstream | Can run in parallel? | Depends on | Done when | +| --- | --- | --- | --- | --- | --- | +| T0 | Complete | Create experimental branch and contribution rules. | No, this starts the work. | None. | Branch exists, PR target is agreed, and main merge criteria are written down. | +| T1 | Complete | Minimal API envelope and naming. | Partly. | T0. | `InferenceConfig`, `UserInputs`, `InferenceInput`, runtime/session, output target, and mapping boundaries are defined well enough for demos to use. | +| T2 | Complete | Event-based `UserInputs`. | Yes, after T1 direction is agreed. | T1. | User inputs are primarily timestamped events; replay traces and derived snapshots are supported where needed. | +| T3 | Complete | `CanonicalInputs`, `InferenceInput`, schemas, and mapping boundary. | Yes, after T1 direction is agreed. | T1. | Models can declare required global/per-step inputs, and mappings can convert canonical inputs into inference inputs. | +| T4 | Complete | `ModelRunner`, `InferenceRuntime`, and `InferenceSession` skeleton. | Partly. | T1. | A minimal standard loop can initialize a runtime, run at least one sequential session, and close cleanly. | +| T5 | Planned | Output mode selection. | Yes, after the result/output shape is agreed. | T1, T4. | A run can choose output behavior such as MP4, JPEG/MJPEG stream, WebRTC, benchmark artifact, or headless/null without changing model code. | +| T6 | Partially complete | LingBot migration and live model-input cleanup. | Yes. | T2, T3, T4. | Scene/prompt/first-frame model inputs flow through `InferenceInput.global_conditioning` or a typed model-input object, static runtime settings remain in runtime config, LingBot uses the shared WebRTC hook shape, and any retained per-model WebRTC/demo wrappers are deliberate compatibility shims. | +| T7 | Partially complete | OmniDreams migration. | Yes, once T2-T4 have a usable skeleton. | T2, T3, T4. | OmniDreams replay and WebRTC run through the shared demo API path; remaining work is output/stat integration, legacy demo retirement, and cleanup. | +| T8 | Partially complete | Benchmark/smoke verification for OmniDreams. | Preparation can run early; final gate is late. | T5, T7. | Existing or updated benchmark tooling can run the migrated OmniDreams demo and produce enough evidence that it still works. | +| T9 | Planned | Metrics and profiling normalization for the branch. | Yes, but final integration is late. | T4, T5, T8. | Basic canonical metrics are emitted for migrated demos; deeper metrics can remain follow-up work. | +| T10 | Planned | CLI compatibility, legacy retirement, and migration cleanup. | Yes, after demo migrations start. | T5, T7, T8. | Required demo commands are restored or replaced, old interactive-drive and old OmniDreams demo/server paths are removed or reduced to compatibility shims, code used only by retired demos is removed, and user-facing docs/notes match the branch behavior. | +| T11 | Planned | Stabilize and merge experimental branch to `main`. | No, final integration step. | T5, T7-T10. | OmniDreams passes agreed smoke/benchmark checks, review feedback is addressed, and the branch can merge as one API transition. | + +Current LingBot migration status: + +- LingBot has partial runtime/session plumbing, but live WebRTC still retains + the segment-based `generate_chunk(segments, frame_times)` entry point while + the model-input boundary is cleaned up. +- Scene, prompt, and first-frame style model inputs should move through + `InferenceInput.global_conditioning` or a typed object carried there instead + of being hidden inside runtime config. +- Runtime config should keep static execution settings: pipeline config, + device, resolution/FPS, warmup, encoder options, movement speeds, and + cache/layout options. +- Browser-only session options, such as OmniDreams postprocess preset + selection, should remain pending session input unless they become true model + conditioning. +- LingBot still delegates parts of the WebRTC path through + `lingbot.webrtc.server.create_app()` and `LingbotWebRTCSessionManager`. + Follow-up work should move it to the same hook shape as OmniDreams: + `WebRTCManagerOptions`, `WebRTCAppExtension`, and shared route/resource + helpers. +- Full realtime `UserInputs` / `InputMapping` integration can be deferred + until after the live model-input handling is explicit. +- Later cleanup should remove or reduce old per-model WebRTC server wrappers + and obsolete demo code once both demos are fully on the shared path. + +Current OmniDreams migration status: + +- The shared `flashdreams.runtime.demo` API and OmniDreams demo adapter exist. +- OmniDreams MP4 replay runs through the shared replay runner and MP4 output + target. +- The one-minute benchmark comparison can run the legacy replay path and the new + shared demo replay path side by side. +- OmniDreams WebRTC runs through `serve_flashdreams_demo(...)` and the shared + WebRTC manager path while still using the existing OmniDreams runtime and + packaged browser app. +- The migration is not complete until the new output target/stat artifact work + lands, the new OmniDreams path is updated to use it, the old interactive-drive + and old OmniDreams demo/server paths are removed or reduced to deliberate + compatibility shims, code used only by retired demos is deleted, and the + experimental demo/runtime/input code is cleaned up. + +Suggested parallel split: + +- one person owns T4 and keeps it aligned with the completed T1 envelope, + because the standard loop is now the critical path; +- one person owns T2/T3, because event inputs, schemas, and mapping need to + stay coherent; +- one person owns T5/T8/T9, because outputs, benchmarks, and metrics are tightly + related; +- one person owns T6's LingBot follow-up: live model inputs, shared WebRTC hook + migration, and cleanup of retained per-model wrappers; +- one person should track branch health, CLI compatibility, and merge readiness. + +## Architecture + +```text +Optional discovery for CLI, benchmark, hosted, or installed-package flows: + Model/preset registry + -> adapter/preset/default setup/scenario metadata + -> contributes defaults to the app-supplied run setup + +Main runtime flow: +App / integration / benchmark / transport + chooses how the run is driven and where output goes + supplies run setup: + InferenceConfig + UserInputs + InferenceInput + output/metrics options + | + v +ModelRunner / standard loop + orchestrates validation, lifecycle, stepping, output, and metrics + uses input mapping to: + validate that user/app inputs can drive the model + build global and per-step InferenceInput during the run + | + v +InferenceRuntime + reusable heavyweight lifecycle: distributed init, model load, compile, warmup + load once; create sessions sequentially unless the backend supports concurrency + | + v +InferenceSession + one rollout/stream: global conditioning, cache/state, current step, reset + keeps per-run state from leaking across prompts, clients, or benchmark repeats + | + v +Model implementation / inference pipeline + hot path: encode -> model step -> decode -> cache/finalize + | + v +Output target + WebRTC | native window | MP4 | benchmark | headless/null + | + v +Metrics / artifacts / logs / reports / traces +``` + +## Example Sequential Session Flow + +The runtime/session split is primarily about reusing expensive model setup while +keeping each rollout's state isolated. The default mental model should be +sequential sessions, not required concurrent sessions. + +```text +ModelRunner / standard loop + | + v +Create InferenceRuntime from InferenceConfig + load checkpoint/model + initialize distributed/backend state + compile/capture/warm up if configured + | + v +Start InferenceSession A + global conditioning: prompt/frame/scene/etc. + per-session state: cache, current step, reset state + step 0 -> step 1 -> ... -> done + outputs -> Output target + metrics -> Metrics recorder + close session A + | + v +Start InferenceSession B + new global conditioning or replay scenario + independent cache/state + step 0 -> step 1 -> ... -> done + outputs -> Output target + metrics -> Metrics recorder + close session B + | + v +Close InferenceRuntime + release model/backend resources +``` + +For v0, an `InferenceRuntime` may support only one active session at a time. +Concurrent sessions should be treated as an optional backend/model capability, +not a baseline API requirement. + +`StreamInferencePipeline` should remain an important local implementation path +for models that already use it, but it should not be treated as the only +possible model boundary. A session may call `StreamInferencePipeline`, another +local model implementation, a Dynamo-like backend, or a hosted service. + +## System Components + +| Component | Role | Boundary | +| --- | --- | --- | +| Model/preset registry | Lists what can run: model/preset slugs, scenarios, capabilities, resource hints, and supported output modes. | Must remain cheap to query and must not load checkpoints. | +| App / integration / benchmark / transport | Owns the user-facing mode: CLI, native integration, WebRTC, benchmark, hosted request, or replay. | Supplies run setup, user inputs, model inputs, and output target selection. | +| User input library | Normalizes live or replayed controls into FlashDreams-supported user input events/windows. | Shared primitives for keyboard, reset, prompt/image selection, traces, and future scalar controls. | +| Input mapping | Converts user/app inputs plus global conditioning into the model-specific inputs needed by the session. | A model adapter may provide a default mapping; runtimes, applications, benchmarks, and replay tools may override it without changing the model step. | +| ModelRunner / standard loop | Orchestrates one run from setup through runtime initialization, stepping, output, metrics, and teardown. | Shared orchestration layer used by CLIs, benchmarks, MP4 runs, and simple realtime flows. | +| InferenceRuntime | Owns heavyweight lifecycle: distributed init, model construction, checkpoint loading, compile/capture, warmup, hosted-service connection, and teardown. | Long-lived reusable runtime created from `InferenceConfig`; lets FlashDreams load/warm once and create sessions sequentially unless the backend supports concurrency. | +| InferenceSession | Owns one rollout or stream: global conditioning, cache state, current step, reset behavior, step requirements, and step execution. | Per-rollout interface consumed by the standard loop; keeps state isolated across prompts, browser clients, replay scenarios, or benchmark repeats. | +| Model implementation / inference pipeline | Implements encode, model step, decode, cache updates, and model-specific optimizations. | FlashDreams wraps this boundary; it should not replace every model implementation. | +| Output target | Consumes generated outputs and handles presentation or persistence. | Separate from model execution so the same session can feed WebRTC, MP4, benchmark, or headless output. | +| Metrics, artifacts, and profiling | Records timings, memory, quality data, logs, reports, traces, and optional NVTX ranges. | Shared observation layer for local runs, benchmarks, CI smoke, and hosted runs. | + +## API Layers + +FlashDreams should expose layered APIs rather than a single all-or-nothing +interface: + +```text +High-level runtime API + run setup -> standard loop -> output targets -> metrics/artifacts + +Adapter/runtime API + model adapter -> InferenceRuntime -> InferenceSession + +Low-level inference API + StreamInferencePipeline -> encoders/decoders -> cache/perf/profiling helpers +``` + +| Layer | Intended user | Provides | +| --- | --- | --- | +| High-level runtime API | Users who want FlashDreams to own the run loop. | Run setup, input mapping, runtime/session lifecycle, output targets, metrics, profiling, and benchmark artifacts. | +| Adapter/runtime API | Model owners who want their model to plug into the standard loop. | Model adapter, input requirements, runtime/session implementation, and model-specific mapping or validation. | +| Low-level inference API | Users who want to own their own loop while reusing FlashDreams building blocks. | `StreamInferencePipeline`, encoders, decoders, cache helpers, profiling tools, and optimization utilities. | + +These layers should remain compatible. The new runtime API sits above the +existing lower-level pieces; it does not replace them. + +## Goals + +- Make FlashDreams easier to use for new world-model integrations. +- Keep model-specific input semantics explicit instead of hiding them in runner + code. +- Avoid a single monolithic inference stack; different models should be able to + validate and use different optimization features. +- Separate model execution from presentation and persistence. +- Support both live input and deterministic replay through the same + runtime/session boundary. +- Make metrics, benchmark artifacts, and profiling first-class without forcing + profiling overhead into normal runs. +- Preserve room for local single-GPU, local distributed, Dynamo-like, and hosted + execution. + +## Non-Goals + +- Do not infer arbitrary model semantics from a raw checkpoint. +- Do not require every model to use the same encoder, decoder, scheduler, + control representation, transport, or optimization set. +- Do not make WebRTC or native display part of the model API. +- Do not make autotuning part of normal inference startup. +- Do not require users to use the high-level standard loop when they only need + lower-level inference building blocks. +- Do not require every existing integration to migrate in one large change. + +## API Placement + +The new API should sit above the existing `flashdreams.infra` layer. Existing +pipelines, encoders, decoders, runner configs, realtime input helpers, WebRTC +code, and quality/benchmark utilities should be reused where possible. + +The exact package layout and class definitions can be decided during +implementation. This document should define responsibilities and boundaries, not +the final Python shape. + +## InferenceConfig + +`InferenceConfig` describes how to run the model/runtime. It should cover: + +- model or preset identity; +- checkpoint or model asset selection; +- execution backend, such as local single GPU, local multi-GPU, Dynamo-like, or + hosted/external execution; +- device placement, precision, and resource hints; +- optimization choices such as compile, CUDA graph capture, attention backend, + cache policy, overlap, prefetch, and native extensions; +- runtime-affecting profiling or tracing options. + +It should not contain prompts, keyboard state, browser settings, MP4 paths, +benchmark output directories, or other app/output settings. Those belong in the +run setup around `InferenceConfig`. + +Existing `StreamInferencePipelineConfig` and `InstantiateConfig` style configs +can remain valid model references behind this layer. The model adapter should +validate which execution and optimization choices are supported. Unsupported +choices should fail clearly or be explicitly handled only when the user selected +an automatic mode. + +## UserInputs + +`UserInputs` describes user-facing controls produced by a live UI, browser, +native app, replay trace, synthetic benchmark driver, or no-op source. + +User inputs should primarily be represented as timestamped events. This gives +live apps, replay traces, and benchmarks the same basic shape, and lets +FlashDreams route, drain, or window those events when a model session asks for +the next chunk of inputs. Resampling and interpolation should remain +input-specific mapping or helper behavior, because controls such as rotations, +poses, or controller state may need semantics that generic runtime code cannot +infer safely. + +Initial supported user input types should stay close to what FlashDreams already +uses: + +- keyboard keydown/keyup events; +- reset requests; +- prompt or image selection/update events; +- future scalar controls such as throttle, brake, steer, or camera axes once an + integration needs them. + +Snapshot-style inputs, such as current key state, can still be supported when +useful. They should be treated as a derived or compatibility form rather than +the primary user-input abstraction. + +User inputs are not model inputs. A keyboard event does not have one universal +meaning. One model may map it to pose segments, another to steering commands, +and another may ignore it. + +## CanonicalInputs And InferenceInput + +Inputs move through three layers: + +```text +UserInputs -> CanonicalInputs -> InferenceInput + raw canonicalized encoded +``` + +Raw device events for live control are canonicalized into device-independent +modalities before application or mapping logic consumes them, so adding a +keyboard, gamepad, or wheel is a converter registration rather than an +application change. Global conditioning is application-owned and reaches +`InferenceInput` directly; it does not pass through live device canonicalization. +`InferenceInput` is what an `InferenceSession` actually receives. + +`CanonicalInputs` describes device-independent live control for one requested +input window. `InferenceInput` describes the data the model or inference +pipeline actually requires, split into two conditioning slots: + +- global conditioning: values that condition the whole rollout; +- per-step conditioning: values needed for one generated chunk or frame window. + +Examples of global conditioning include prompt, negative prompt, conditioning +frame, input video, scene id, HD map asset, camera calibration, initial camera +pose, seed, or model-specific fields. + +Global conditioning establishes session-global model state when a session +starts or resets. During an active rollout, a non-empty global-conditioning +payload passed to `InferenceSession.step()` asks the session to update that +state when the model supports it. Reset remains a separate explicit session +method. + +Examples of per-step conditioning include frame timestamps, pose segments, +camera trajectory chunks, rendered HD map frames, conditioning video windows, +control tensors, event markers, or model-specific fields. + +Inference input payloads should use semantic names, not only modality names. For +example, a first frame and an HD map frame should be distinct inputs even if +both are image-like values. + +Model input names, input modalities, and schema metadata should be open-ended. +Supported integrations such as SANA-WM, LingBot, Omnidreams, and future +external adapters may need different semantic fields. Adding a new model should +usually mean adding adapter-owned schema declarations and mappings, not changing +a central FlashDreams enum. + +Consumption cadence is a separate hint from input scope. A field may be +provided through global conditioning because it is session-global state, while +the adapter consumes or slices it during every step. That can be recorded as +`frequency_consumed` metadata without changing whether the field belongs in +`global_conditioning_fields` or `step_fields`. + +For interactive runs, most `InferenceInput` values will be app-owned global +conditioning plus per-step inputs produced by input mapping. For MP4 generation +and benchmarking, the API should also support fixed per-step model inputs so +runs can be deterministic. + +## Schemas + +The API should support lightweight `UserInputSchema`, `CanonicalInputSchema`, +and `InferenceInputSchema` +metadata. + +These schemas are not meant to be a rich type system or a replacement for +model-specific validation. They should be just enough to answer: + +- what can this app, transport, trace, or benchmark source provide? +- what does this model require before session start and at each step? +- can this event source drive this model with the selected mapping? + +The purpose is to fail early before expensive model initialization, produce +clearer errors, make fixed scenarios easier to validate, and avoid ambiguous +dict payloads where keys only describe modality. + +Schema objects may carry open-ended metadata for query-time hints such as +coordinate frame, units, rough shape summary, accepted file suffixes, schema +URI, model family, or source/transport details. Metadata should help humans and +adapter selection code, but compatibility should still be based on the declared +event capabilities, semantic model fields, input modalities, and schema phases. +Consumption-cadence hints are descriptive and adapter-owned. + +For simple CLI text-to-video or image-to-video runs, `UserInputSchema` can be +trivial or omitted because there may be no live controls. `InferenceInputSchema` is +more important because each supported model still needs to declare the +model-facing values it expects. + +## Model Requirements + +A raw checkpoint should not be treated as self-describing. It may imply tensor +shapes or architecture details, but it usually does not fully define: + +- required semantic inputs; +- initial versus per-step inputs; +- units for timestamps, poses, or calibration values; +- how user controls become model controls; +- preprocessing, encoder, decoder, mask, prompt, or cache rules. + +Therefore, a FlashDreams-supported model should have an adapter or integration +layer that declares its model input requirements, declares any user inputs it can +map by default, and prepares inputs for the underlying model implementation. + +Users running an existing FlashDreams-supported model should not need to write +that adapter. Developers bringing a new world model to FlashDreams should expect +to provide one. + +## External Model Usage + +Users should be able to run their own models without adding those models to the +FlashDreams repository. The flow depends on which API layer they use: + +```text +High-level runtime API + user supplies or installs model adapter + FlashDreams owns standard loop, outputs, metrics, benchmarks + +Adapter/runtime API + model owner implements adapter/runtime/session + adapter can be passed directly or registered by an installed package + +Low-level inference API + user owns loop and lifecycle + user reuses pipeline, encoder/decoder, cache, profiling, or optimization tools +``` + +| Flow | Registry needed? | Who provides model-specific code? | Result | +| --- | --- | --- | --- | +| Direct Python | No. | User or model owner passes an adapter/setup directly. | FlashDreams can run the standard loop without the model living in the repo. | +| Installed package | Yes, for discovery. | External or internal package registers adapters/presets. | CLIs, benchmarks, and hosted schedulers can discover the model cheaply. | +| Low-level only | No. | User owns the loop and calls lower-level FlashDreams pieces directly. | Useful when the user wants optimizations or pipeline helpers but not the standard loop. | + +The model adapter is a role/boundary, not necessarily a concrete class. It is +the model-specific code that declares input requirements, validates supported +configs, creates the runtime/session, and connects FlashDreams to the actual +model implementation. + +The registry should not be treated as a central FlashDreams-owned catalog of all +possible models. It is a discovery mechanism for installed adapters. Built-in +public integrations, internal GitLab-only integrations, and third-party packages +can all participate through the same mechanism. + +FlashDreams should not claim to run an arbitrary checkpoint with no adapter +unless the checkpoint already matches a supported generic adapter. + +## Input Mapping + +Input mapping is required whenever `UserInputs` need to become per-step +`InferenceInput`. In the T1 envelope this boundary is represented by a separate +`InputMapping` protocol. A model adapter may provide the default mapper because +it knows how its supported user controls affect model-facing inputs. Applications, +benchmarks, replay tools, or hosted runtimes may replace that mapper when they +need a different wire surface or aggregation policy. + +The selected mapping may be a single mapper or a composed set of mappers, so one +run can combine separate prompt, first-frame, and live-control mappings instead +of routing everything through one object. + +There are two separate moments to keep clear: + +- before runtime initialization, FlashDreams should select the mapping or mapper + set and check obvious compatibility between the app event source and the + model; +- during the standard loop, the runtime or runner passes app-owned global + `InferenceInput` through the selected mapping before session start, then + queues and timestamps user events, canonicalizes the session-requested window, + and uses the selected mapping to build per-step `InferenceInput`. + +This keeps the Reactor-style contract intact: the model-side integration can +declare user inputs, declare model inputs, and provide a default mapping, while +the runtime owns transport, event validation, timestamping, input queue/window +selection, output delivery, and optional overrides. + +`StepRequest` and `StepResult` are per-step runtime messages, not declarative +schemas. `InferenceSession.next_step_request()` returns a `StepRequest` to say +which step is next, which user-input time window to map, and whether this step +has any narrower `InferenceInputSchema` than the session default. The runner or +application then builds an `InferenceInput` and calls `InferenceSession.step()`, +which returns a `StepResult` carrying the generated output, output timing, +metrics, and step metadata. + +Examples: + +- T2V mapping validates a prompt and creates no per-step control inputs. +- I2V mapping validates a prompt plus first frame and creates no live controls. +- A keyboard-driven integration maps key events or event windows into pose + segments or steering controls. +- OmniDreams-like integrations may map driving commands into camera poses, HD + map frames, and dynamic actor state. +- Benchmark mapping can read fixed event traces and produce identical step + inputs each run. + +The compatibility check should be treated as early validation, not a guarantee +that the run will succeed. It can catch obvious mismatches, but the model +adapter/runtime still owns deep tensor validation and model semantics. + +## Runtime And Standard Loop + +The standard loop should be shared by CLI generation, headless playback, MP4 +generation, benchmarks, and simple realtime applications. + +The current v0 production loop is `flashdreams.runtime.run_inference_session()`. +It is intentionally narrow: one adapter, one config, one canonicalizer/source, +one selected mapping, one initial input, one output target, one metrics +recorder, and one synchronous sequential session. + +A run should: + +1. Discover the model or preset without loading checkpoints. +2. Resolve inference config, user inputs, model inputs, output target, metrics, + profiling, and optional scenario setup. +3. Validate that the event source and mapping can drive the selected model. +4. Initialize the runtime. +5. Start a session from global conditioning inputs. +6. For each step, ask the session what it needs, gather live or fixed inputs, + build step model inputs, run the session step, route outputs, and record + metrics. +7. Finalize output artifacts, metrics, logs, reports, and traces. + +Realtime transports may need an async variant, backpressure, and explicit flow +control, but the conceptual boundary should remain the same: event/input source, +input mapping, session, output target, metrics. + +The session should expose what it needs for the next step rather than requiring +the app or output layer to guess. This matters because AR step 0 can differ from +steady-state steps, and encoder/decoder temporal compression can produce +different input and output frame windows. + +Input and output timing should share a session timeline even when raw capture +rates and presentation rates differ. A session can request a user-input window +for mapping, then return an output window or equivalent metadata so an output +target can present the generated chunk at the intended cadence. + +## Output Targets + +Output handling should be separate from model execution. The model session +returns generated outputs and metadata; the output target decides what to do +with them. + +Expected output targets include: + +- WebRTC streaming; +- native window display; +- MJPEG or lightweight remote preview; +- MP4 writing; +- benchmark artifact writing; +- headless playback; +- null output for pure throughput measurements. + +Display and transport can still affect measured performance through copies, +encoding, queueing, backpressure, and presentation timing. Those costs should be +measured as output-target or end-to-end metrics instead of being mixed into core +model-stage timings. + +## Fixed Inputs, Benchmarks + +The API should support fixed runs as a first-class case. This is needed for MP4 +generation, benchmarks, regression testing, and autotune. + +Two replay levels should be supported: + +- user-event replay: records timestamped key events, prompt or image + selection/update events, reset events, and timing, then runs normal input + mapping; +- model-input replay: records or defines already-mapped per-step model inputs + for stricter model-level regression tests. + +User-event replay tests more of the application stack. Model-input replay is +better for isolating model runtime performance and reproducibility. + +## Metrics And Profiling + +Metrics should have a small canonical baseline plus optional extras. + +The baseline should cover: + +- lifecycle timing: startup, load, warmup, first-step latency; +- model-stage timing: encode, model step, decode, finalize/cache update; +- memory: allocated, reserved, peak, and per-rank where applicable; +- throughput: frames per second, chunks per second, real-time factor. + +Realtime runs may add input-to-present latency, jitter, missed deadlines, queue +depth, dropped frames, WebRTC stats, encoder bitrate, and client stats. +Benchmark runs may add quality metrics, logs, MP4/image previews, and reports. + +Persisted timing metrics should use seconds as the canonical unit because +seconds compose cleanly across Python timers, traces, and long-running +durations. Reports and UIs can display milliseconds for short latencies. + +Profiling should be optional and controlled separately from normal metrics. +NVTX ranges should be supported for Nsight profiling, but profiling should not +be required for normal inference or benchmark runs. + +## Autotune + +Autotune should be a separate harness that evaluates candidate +`InferenceConfig` variants against fixed scenarios. It should not be part of +normal startup. + +Autotune may search over compile, CUDA graph capture, attention backend, +precision, cache policy, overlap, prefetch, native extensions, and chunk size +when the model supports those knobs. + +Results are only valid for a specific model, checkpoint, hardware, driver, +FlashDreams commit, and scenario. First-run compile/capture cost should be +separated from steady-state metrics. Agent assistance could help propose search +spaces or summarize results, but the measured selection process should be +deterministic code. + +## Distributed And Hosted Execution + +The API should leave room for local single-GPU, local multi-GPU, Dynamo-like +execution, and hosted execution such as a Reactor-style platform. + +At this stage, the proposal should not define Reactor- or Dynamo-specific +contracts in detail. It should preserve the right boundary: execution backend +selection belongs in `InferenceConfig`, while backend-specific scheduling, +authentication, asset access, output streaming, artifact handling, and failure +behavior belong behind the runtime/backend implementation. + +The practical order should be local first, then local distributed, then +hosted/distributed backends once concrete backend owners can validate the +requirements. + +## Existing Code And Migration + +The new API should reuse existing code instead of replacing everything: + +- keep `flashdreams.infra.pipeline` as the common local encode/model/decode + implementation path; +- keep existing encoder and decoder contracts and reuse temporal size helpers; +- keep existing runner configs and CLI compatibility during migration; +- reuse `KeyboardResampler` and realtime input helpers behind the new input + boundary; +- treat WebRTC as a transport/output adapter and bridge it gradually; +- reuse existing quality and benchmark utilities where applicable; +- keep internal-only integrations registered only in the GitLab/internal + workspace. + +The task tracker near the start of this document is the source of truth for the +first implementation branch. The first milestone is intentionally narrower than +the full design: prove the API with OmniDreams, add shared output/stat artifact +selection, retire the old OmniDreams demo paths, clean up the experimental +runtime/demo code, and collect enough benchmark/smoke evidence to merge the +experimental branch back to `main` safely. LingBot should be handled in a +separate follow-up plan. + +## Design Risks + +- `InferenceConfig` could become too broad if prompts, controls, output paths, + browser settings, and benchmark settings are added to it. Keep it focused on + model/runtime execution. +- Dict-like model inputs are flexible but can fail late. Keep dict payloads for + flexibility, but require lightweight schemas and adapter validation for + supported models. +- Schemas could become too heavy. Keep them minimal and role-oriented. +- User inputs are not model inputs. Keep input mapping explicit and + model/application-owned. +- Per-frame, per-chunk, and AR-step clocks are easy to confuse. The session + should expose step requirements instead of making app code guess. +- Output separation is necessary but not free. Measure output and transport + costs separately from core model timings. +- Hosted/distributed execution is still under-specified. Keep the API boundary + open until backend owners validate concrete requirements. +- Existing WebRTC behavior is nontrivial. Bridge it gradually to avoid + regressions. +- Public/internal boundaries must remain clean. Internal adapters, slugs, and + scenarios should not leak into the public repo. + +## Decisions Made In T1 + +Task T1 settles the initial package and naming envelope without committing to a +registry, standard loop, concrete output modes, or model migrations: + +- The experimental API lives under `flashdreams.runtime`. +- The model-specific integration boundary is named `ModelAdapter`. +- Heavyweight lifecycle is split into `InferenceRuntime` and + `InferenceSession`. +- Step data carriers are named `StepRequest` and `StepResult`. They are runtime + messages around one call to `InferenceSession.step()`, not schema + declarations; a session returns `None` from `next_step_request()` when the + rollout is complete. +- Raw inputs use `UserInputs`, canonicalized inputs use `CanonicalInputs`, and + model-facing inputs use `InferenceInput`. + Both remain lightweight payload envelopes with shallow read-only mappings. +- `UserInputSchema`, `CanonicalInputSchema`, and `InferenceInputSchema` stay + intentionally small: they + declare supported event types and required named fields for early validation, + not a full type system. +- Input mapping is represented by a separate `InputMapping` protocol. Model + adapters may provide a default mapping; runtimes and applications may override + it while preserving the `CanonicalInputs` to `InferenceInput` boundary. Simple + fixed-input runs can use `IdentityInputMapping`. +- Output handling is represented by `OutputTarget`; `NullOutputTarget` is the + initial headless implementation. +- Metrics collection is represented by `MetricsRecorder`; timing samples use + seconds as the canonical unit. +- The minimum v0 user input shape is timestamped `UserInputEvent` records plus + optional snapshot data. Concrete event-type catalogs are left to T2 and demo + migrations. + +## Remaining Decisions + +- What direct-Python API should let users pass an external adapter without + registering it? +- What package registration mechanism should third-party and internal adapters + use for CLI discovery and benchmarks? +- Which model should migrate after OmniDreams settles the shared demo API shape? +- What metrics are required for every benchmark run? +- What metadata must be discoverable without loading checkpoints? +- What requirements do Dynamo/Reactor-style backends need before we commit to + hosted execution details? + +The document currently uses "integration" for model-specific packages and app +entrypoints. If the team prefers "model" as the public term, that can be changed +later without changing the architecture. + +## Recommendation + +Proceed with the proposed split: + +- `InferenceConfig` for model/runtime execution; +- `UserInputs` for app-facing controls and replay traces; +- `CanonicalInputs` for device-independent application-facing inputs; +- `InferenceInput` for model-facing global and per-step conditioning; +- input mapping for model/application-specific conversion; +- runtime/session boundaries for lifecycle and stepping; +- output targets for display, streaming, files, and benchmarks; +- shared metrics and optional profiling. + +The main constraint is that arbitrary world-model inputs cannot be standardized +away. FlashDreams can provide the shared envelope, loop, metrics, replay, and +output tools, but each supported model still needs an adapter that declares and +validates its own input contract. diff --git a/docs/inference_runtime_inputs_implementation.md b/docs/inference_runtime_inputs_implementation.md new file mode 100644 index 000000000..75460cced --- /dev/null +++ b/docs/inference_runtime_inputs_implementation.md @@ -0,0 +1,289 @@ + + +# Inference Runtime Inputs Implementation Notes + +This note documents the input layers of the experimental runtime API: what +exists, how the pieces fit together, what the compatibility query answers, and +what is intentionally still outside this layer. + +Implementation lives in `flashdreams.runtime`: + +- `flashdreams/flashdreams/runtime/inputs.py` — the input types and schemas +- `flashdreams/flashdreams/runtime/canonical.py` — raw device to canonical + modality conversion +- `flashdreams/flashdreams/runtime/mapping.py` — canonical to encoded mapping + and compatibility +- `flashdreams/tests/test_runtime_canonical.py` +- `flashdreams/tests/test_runtime_input_mapping.py` +- `flashdreams/tests/test_inference_runtime_api.py` — the T1 envelope tests +- `flashdreams/tests/test_runtime_runner.py` — the production standard loop + tests that exercise all three input layers with runtime/session cleanup + +The supported-model input inventory that informed this work is in +`docs/inference_runtime_supported_inputs_inventory.md`. + +## The Three Layers + +```text +UserInputs ──InputCanonicalizer──▶ CanonicalInputs ──InputMapping──▶ InferenceInput + raw canonicalized encoded +(device events) (device-independent) (what the session gets) +``` + +| Layer | Type | Owner | Example | +| --- | --- | --- | --- | +| raw | `UserInputs` / `UserInputEvent` | transport, replay loader, benchmark driver | `key_down {"key": "w"}`, wheel axis reading | +| canonicalized | `CanonicalInputs` | device converters registered on `InputCanonicalizer` | `driver_command {throttle, brake, steer, ...}` | +| encoded | `InferenceInput` | the selected `InputMapping` | whatever the model's session consumes | + +Applications and mappings consume `CanonicalInputs`. They never read raw device +events: `InputMapping.map_step_inputs` takes `canonical_inputs`, not +`user_inputs`, so this is enforced by the signature rather than by convention. +Adding a keyboard, gamepad, or wheel is an `InputCanonicalizer.register` call +that touches no application, mapping, or model code. + +This path covers **live user control only**. Global conditioning is +application-owned data and reaches `InferenceInput` directly, without passing +through canonicalization or a device converter. Session start/reset establishes +that global conditioning. During an active rollout, a non-empty +`global_conditioning` payload passed to `step()` requests an update of the +session-global state when the model supports it. + +## Conditioning Slots + +The encoded layer splits model-facing inputs into two slots: + +- **global conditioning** — session-global model state: prompt, conditioning + frame, scene. +- **per-step conditioning** — needed to generate the next chunk or frame: + steering, HD map frames, camera trajectory. + +`InputPhase` is `Literal["global_conditioning", "step"]`. The phase names the +`InferenceInput` slot the caller provides. + +`InputField.frequency_consumed` is independent query metadata. It says how the +adapter consumes a field internally, such as `once` or `per_step`; it does not +decide whether the caller provides the field through `global_conditioning` or +`step`. + +## Global Conditioning Is Session-Global State + +`InferenceInput.global_conditioning` carries session-scoped inputs. A runtime +passes those values to `InferenceRuntime.start_session()` or to +`InferenceSession.reset()` when the backend supports resetting a rollout. +During an active rollout, passing a non-empty `global_conditioning` payload to +`InferenceSession.step()` asks the session to update that session-global state. +The model/session owns whether that update is supported. + +```python +from flashdreams.runtime import InferenceInput, InferenceInputSchema, InputField + +schema = InferenceInputSchema( + global_conditioning_fields=( + InputField(name="prompt"), + InputField(name="scene_id"), + ) +) +schema.require_global_conditioning( + InferenceInput(global_conditioning={"prompt": "drive", "scene_id": "town_02"}) +) + +step_with_prompt_update = InferenceInput( + global_conditioning={"prompt": "heavy rain"}, + step={"steering": 0.0}, +) +``` + +Per-step conditioning is different: those values are supplied through +`InferenceInput.step` for each generated chunk or frame. Converters still emit +every window, because live control is level-triggered: a key held across a step +emits no events but still means full throttle. + +## Raw Inputs + +`UserInputEvent` carries `timestamp_s`, `event_type`, `payload`, `source`, and +`source_event_id`. `UserInputs` holds an ordered batch plus a `snapshot` and +`metadata`, and slices to a half-open `TimeWindow`: + +```python +from flashdreams.runtime import TimeWindow, UserInputEvent, UserInputs + +inputs = UserInputs( + events=( + UserInputEvent(timestamp_s=0.0, event_type="prompt_set", + payload={"prompt": "drive forward"}), + UserInputEvent(timestamp_s=0.5, event_type="key_down", payload={"key": "w"}), + ) +) +step_window = inputs.window(TimeWindow(start_s=0.0, end_s=1.0)) +``` + +`UserInputSchema` describes what a transport, replay trace, or benchmark driver +can provide. `event_types` declares only that an event type exists; +`UserInputCapability` additionally pins the payload fields it carries, so a +converter can require `key_down` events that actually have a `key`. A bare +`event_types` entry still satisfies any consumer needing no specific payload +fields, so schemas written before capabilities existed keep working. + +## Canonical Modalities + +A `CanonicalModality` is a device-independent input: a name and the payload +fields it guarantees. Converters implement `DeviceConverter`, declaring +what raw capabilities they consume and which modality they produce. + +```python +from flashdreams.runtime import ( + DRIVER_COMMAND, InputCanonicalizer, KeyboardToDriverCommand, TimeWindow, +) + +canonicalizer = InputCanonicalizer([KeyboardToDriverCommand()]) +canonicalizer.register(WheelToDriverCommand()) # a wheel is one call + +canonical = canonicalizer.canonicalize( + user_inputs, window=TimeWindow(start_s=0.0, end_s=1.0), source_schema=browser +) +canonical.values["driver_command"]["throttle"] +``` + +`DRIVER_COMMAND` is the one shipped modality. `KeyboardToDriverCommand` reuses +`KeyboardState`/`normalize_key` from `flashdreams.serving.realtime.input` and +mirrors the semantics the Omnidreams interactive-drive keyboard backend already +has. Its key bindings are data (`DEFAULT_DRIVING_BINDINGS`), and the set of +tracked keys is derived from them, so a rebound layout cannot leave an action +unreachable. + +`ScriptedModality` is the mock/replay converter. It consumes no raw +capabilities, so a benchmark or test can author a scenario at the canonical +level without knowing any device vocabulary: + +```python +canonicalizer = InputCanonicalizer([ + ScriptedModality(modality=DRIVER_COMMAND, timeline=[(0.0, full_throttle)]), +]) +canonicalizer.canonicalize( + UserInputs(), window=step_window, source_schema=UserInputSchema() +) +``` + +Application code is identical between a real run and a scripted one. + +Converters are stateful, so feed windows in session order and call +`InputCanonicalizer.reset()` at a rollout boundary. Replaying the same window +sequence reproduces the same `CanonicalInputs`. + +When several devices produce the same modality, the highest-priority one that +returned a value wins; `CanonicalInputs.metadata["canonical_sources"]` records +which device supplied each. Every feedable converter still sees each window, so +a preempted device's state stays current and unplugging the higher-priority +device does not resume from stale state. + +## Mapping And Compatibility + +`InputMapping` is the canonical-to-encoded boundary. `InputMappingSchema` is its +declarative surface: `consumes` names canonical modalities; +`produces_global_conditioning` and `produces_step` name the `InferenceInput` +fields it can build. + +`InputMapping.validate()` raises, which fails a run late and cannot say *which* +optional model input a source would enable or *which* missing modality makes a +required one unreachable. `check_mapping_compatibility` answers those before +expensive runtime initialization: + +```python +from flashdreams.runtime import check_mapping_set_compatibility + +compatibility = check_mapping_set_compatibility( + canonical_schema=canonicalizer.canonical_schema(browser), + inference_input_schema=adapter.inference_input_schema, + mapping_schemas=(prompt_mapping, frame_mapping, steering_mapping), +) +if not compatibility.can_drive: + compatibility.raise_if_incompatible() +``` + +`MappingCompatibility` reports `missing_modalities`, +`missing_required_model_fields`, `satisfied_required_model_fields`, +`available_optional_model_fields`, and `unavailable_mapping_schemas`. + +Compatibility is evaluated per mapping rather than over a flattened bag, so each +mapping keeps its own consumes/produces link. A mapping the source cannot feed +is dropped and reported, costing only the inputs it produced. So a dropped +mapping that fed only optional fields degrades the run instead of vetoing it, +and those fields are correctly absent from `available_optional_model_fields`; a +dropped mapping that was the only producer of a required field still blocks. + +Because a mapping consumes modalities rather than raw events, one mapping +written against `driver_command` works for a keyboard, a wheel, or any device +registered later, with no change to the mapping or the model schema. + +`undeclared_inference_inputs()` reports payload keys a mapping produced but did +not declare, which keeps hand-written schemas honest as the code drifts. + +`StepRequest` and `StepResult` sit around a single `InferenceSession.step()` +call. They are not schema declarations. A session returns `StepRequest` from +`next_step_request()` to name the next step, optionally provide a narrower +`InferenceInputSchema`, and request a `TimeWindow` of user inputs. The runner +then builds `InferenceInput` and calls `step()`, which returns a `StepResult` +for the output target and metrics recorder. + +## What This Does Not Validate + +The schemas intentionally avoid becoming a rich type system. These remain the +responsibility of the model adapter, runtime, session, or mapping: + +- tensor shape and dtype, image decode details; +- camera coordinate systems, pose and timestamp units; +- prompt-embedding mechanics; +- whether a model can actually apply a requested global-conditioning update; +- enforcing consumption-cadence metadata; +- deep validation of scene, HD map, or actor-state data. + +The layer answers "can this source plausibly drive this model through this +mapping?" It does not replace model-owned validation. + +## Open Questions + +Tracked against the runtime API discussion, not yet settled: + +- **Alternative valid input combinations.** `InferenceInputSchema` has one flat + required set, so "accepts `{prompt}` OR `{prompt, conditioning_frame}`" cannot + be expressed. `MappingCompatibility.missing_required_model_fields` assumes a + single required set too. +- **`step()` returning a future**, for models with a dependency on their own + output. `InferenceSession.step()` is currently synchronous. +- **`Input System` ownership.** The diagrams show it pulling events, so the + Application owns an input system. `InputCanonicalizer` is currently a pure + function over a supplied window and owns no source. Whether it needs to grow + one depends on the loop-ownership decision. Mock input and key binding are + handled (`ScriptedModality`, `DEFAULT_DRIVING_BINDINGS`). + +## Owned Elsewhere + +Named here only so the boundary is explicit; these are not gaps in the input +layer: + +- **`FrameStream`**, which the architecture diagrams place between + `InferenceSession` and `Output Target`. The code writes `StepResult` straight + to `OutputTarget.write()`. Output shape is T5. +- **Declared output modalities**, so an output target or quality-eval can state + what it requires and be matched the way inputs now are. T5/T8. +- **Full `Application` ownership**, the class that has-a input system, input + map, global conditioning, session, and output target. T4 now provides the + narrow synchronous runner; richer application ownership remains outside T4. +- **Loop ownership** — whether the application or the runtime/session drives the + main event loop, and whether inputs are queued and batched. + +## Validation + +```bash +.venv/bin/pytest flashdreams/tests/test_runtime_canonical.py \ + flashdreams/tests/test_runtime_input_mapping.py \ + flashdreams/tests/test_inference_runtime_api.py \ + flashdreams/tests/test_runtime_runner.py -q +.venv/bin/ty check flashdreams/flashdreams/runtime +``` + +At the time of writing these pass: 87 tests, and `ty` is clean. diff --git a/docs/inference_runtime_serving_architecture_improvements.md b/docs/inference_runtime_serving_architecture_improvements.md new file mode 100644 index 000000000..9d71e5d6a --- /dev/null +++ b/docs/inference_runtime_serving_architecture_improvements.md @@ -0,0 +1,417 @@ +# Inference runtime and serving architecture improvements + +## Status + +Proposed. This document records the follow-up work needed to make the runtime, +output, WebRTC, and local-window architecture match the intended component +boundaries. It is an implementation checklist, not a compatibility promise. + +## Goal + +Use one model-session implementation and one generated-video result boundary +for runner CLI, WebRTC, and local-window execution: + +```mermaid +flowchart LR + INPUTS["CLI, WebRTC, and local input adapters"] --> WORKER["Model runtime worker"] + WORKER --> SESSION["Model sessionpipeline, cache, AR state"] + SESSION --> STREAM["VideoOutputStream"] + STREAM --> RESULT["StepResult"] + RESULT --> MP4["MP4 collector"] + RESULT --> WEBRTC["WebRTC encoder"] + RESULT --> LOCAL["Local presenter"] +``` + +The model integration owns conditioning, pipeline/cache state, and generation. +Shared runtime code owns orchestration contracts. Output consumers own only +their transport or presentation behavior. + +## Non-goals + +- Do not change `StreamInferencePipeline.initialize_cache`, `generate`, or + `finalize`. +- Do not move Lingbot- or OmniDreams-specific conditioning into shared + `flashdreams` code. +- Do not force model output tensors to CPU before a consumer requires host + memory. +- Do not combine WebRTC encoding, MP4 writing, and local presentation into one + output class. +- Do not add a video-specific wrapper around `StepResult`. + +## Current problems + +### Parallel model runtimes + +The generic runtime API uses `InferenceRuntime` and `InferenceSession`, while +WebRTC uses a separate `WebRTCGenerationRuntime`. Lingbot and OmniDreams each +implement replay and WebRTC generation separately, and OmniDreams local-window +execution adds a third session implementation. + +This duplicates pipeline construction, cache lifecycle, AR indexing, +`generate`/`finalize`, reset behavior, and output packaging. + +### Duplicate result metadata + +The earlier design wrapped a video-specific result in `StepResult`, while both +carried equivalent step index, frame count, and metrics fields. Consumers need +one layout-aware `StepResult` boundary instead. + +### Mixed `VideoOutputStream` responsibilities + +`VideoOutputStream` currently performs post-processing, collection, statistics +collection, result construction, CUDA synchronization, MP4 conversion, and +writing. It also exposes both `process` and `make_step_result`, leaving callers +to choose between a tensor and a result object. + +### WebRTC discards the result abstraction + +The WebRTC manager receives `StepResult` but passes only +`result.video_chunk` to encoders. Encoder implementations then infer tensor +layout from rank and shape instead of consuming the declared layout. + +### Hidden WebRTC extension points + +The shared demo builder discovers undeclared adapter methods with `getattr`, +including runtime-config, session-manager, and app factories. Model-specific +manager subclasses also provide result metadata and session-reset behavior. +The effective server interface is therefore wider than the declared protocol. + +### Model behavior in the shared browser client + +The shared browser module owns peer connection, video, metrics, and data-channel +logic, but it also hardcodes driving controls and post-process REST behavior. +The model `adapter.js` contract is implicit and differs greatly between +integrations. + +### Hard-coded output launch routing + +`serving/output_targets.py` identifies integrations from runner-name prefixes +and launches different server families for Lingbot and OmniDreams. Adding an +integration or output mode requires editing shared routing code. + +## Target ownership + +### Shared runtime and infrastructure + +- Runtime/session protocols and orchestration. +- A thread-affine runtime worker for asynchronous serving. +- `StepResult` and layout-aware video conversion. +- Stateful output post-processing through `VideoOutputStream`. +- Generic output targets, WebRTC manager, encoders, and app construction. + +### Model integrations + +- Pipeline/config selection and checkpoint behavior. +- Model-specific global conditioning and per-step input mapping. +- One session core containing pipeline/cache/AR state. +- Model-specific session-input validation and optional browser routes/assets. +- Model-specific metadata placed on the generated result. + +### Output consumers + +- MP4: collect results and persist artifacts/statistics. +- WebRTC: encode and enqueue results, then report delivery metrics. +- Local window: convert results to lazy frames and present them. + +## Improvement workstreams + +### 1. Canonical step result + +Use one layout-aware `StepResult` as the direct boundary for generated video. +Do not introduce a separate video result type or nested result envelope. + +Target properties: + +- One step/chunk index. +- One frame count, derived from or validated against the declared layout. +- A required tensor layout. +- One metrics mapping. +- Optional output time window and model-specific metadata. +- No implicit CPU transfer. + +#### TODO + +- [x] Decide the final field names and update the runtime protocol. +- [x] Require `layout` on every video `StepResult`. +- [x] Derive or validate `num_frames` exactly once during construction. +- [x] Keep all step metrics in `StepResult.metrics`. +- [x] Move video output-window information onto the canonical result. +- [x] Change video sessions and output targets to pass `StepResult` directly. +- [x] Remove duplicate unwrap/type-check code from MP4 and runner output + targets. +- [x] Add CPU tests for layout validation, frame counts, metadata, and metrics. + +Acceptance criteria: + +- A generated video step crosses every model/output boundary as exactly one + layout-aware `StepResult`. +- No step index, frame count, or metrics mapping is duplicated in a second + envelope. + +### 2. Single `VideoOutputStream` operation + +Make the stream the only raw-tensor-to-generated-result stage: + +```python +result = output_stream.process( + video_chunk, + autoregressive_index=step_index, + metrics=metrics, +) +``` + +`process` should return `StepResult`. There should be no separate +`make_step_result` call. + +#### TODO + +- [x] Change `VideoOutputStream.process` to return `StepResult`. +- [x] Remove `VideoOutputStream.make_step_result`. +- [x] Keep streaming post-processing and result construction in the stream. +- [x] Move MP4 collection and writing into `Mp4VideoOutputTarget`. +- [x] Move runner statistics persistence into the runner/MP4 target. +- [x] Remove transport-specific CUDA synchronization from the stream. +- [x] Define how `finish` reports a buffered post-processor tail without + introducing a second result type. +- [x] Verify that a disabled postprocessor preserves tensor identity and device. +- [x] Verify that stateful postprocessors are reset between sessions. + +Acceptance criteria: + +- Every generated chunk makes one output-stream call. +- Post-processing occurs at most once per chunk. +- The stream does not know about WebRTC, local-window presentation, or MP4 + files. + +### 3. Result-aware WebRTC delivery + +The WebRTC manager and encoders should consume the complete generated result. + +#### TODO + +- [x] Change `VideoEncoder.deliver_chunk` to accept `StepResult`. +- [x] Pass the result directly from the session manager to the encoder. +- [x] Make software frame conversion use `result.layout`. +- [x] Make NVENC conversion use `result.layout` instead of tensor-rank + heuristics. +- [x] Move model-specific `chunk_done` fields into `result.metadata`. +- [x] Keep transport measurements such as enqueue time, queue depth, and + control latency in the WebRTC manager. +- [x] Test `tchw` and `bvtchw` delivery through both software and NVENC fakes. +- [x] Test that no host copy occurs before the software path requests one. + +Acceptance criteria: + +- The manager never unwraps `result.video_chunk` merely to cross the encoder + boundary. +- Encoder behavior is driven by the declared layout, not guessed shape. + +### 4. One model session core per integration + +Extract one synchronous model-session core for each integration. The core owns +pipeline/cache/AR state and returns `StepResult`. Input adapters prepare +the model-specific inputs for replay, WebRTC, or local use. + +#### Lingbot TODO + +- [x] Extract shared cache initialization, AR indexing, generation, finalize, + reset, and close logic from the replay and WebRTC sessions. +- [x] Reuse the core from the runner/replay path. +- [x] Map WebRTC keyboard actions and text events into the same per-step input + boundary. +- [x] Reuse the core from the WebRTC path. +- [x] Delete the duplicate Lingbot generation implementation. +- [x] Add parity tests comparing replay and live mappings for equivalent camera + inputs. + +#### OmniDreams TODO + +- [x] Extract shared pipeline/wrapper state, cache/finalization state, AR index, + post-processing, reset, and close logic. +- [x] Reuse the model-session boundary from replay and WebRTC. +- [x] Adapt interactive-drive trajectories to the same session-step input. +- [x] Carry `StepResult` to the local presentation boundary and use + `lazy_rgb_frames()` for presentation. +- [x] Preserve delayed-finalization behavior required by interactive drive. +- [x] Delete duplicate OmniDreams generation implementations after parity is + established. +- [x] Test RGB, debug-HDMap, post-process on/off, and scene-reset behavior. + +Acceptance criteria: + +- Each integration contains one implementation of cache initialization, + `generate`, `finalize`, reset, and AR-index advancement. +- Output mode changes input and presentation adapters, not model execution. + +### 5. Thread-affine runtime worker + +All asynchronous serving lifecycle calls must execute on one owned worker +thread so CUDA, Triton, and CUDA-graph state remain thread-affine. + +#### TODO + +- [x] Add a shared single-thread runtime worker under `flashdreams.runtime`. +- [x] Route runtime initialization, session creation/reset, step, and close + through that worker. +- [x] Set the CUDA device when the worker thread starts. +- [x] Keep distributed rank coordination inside model-owned operations. +- [x] Remove per-call `asyncio.to_thread` use from integration runtimes. +- [x] Make cancellation stop awaiting a call without abandoning runtime + cleanup. +- [x] Add CPU tests for call ordering, exception propagation, and shutdown. +- [x] Add a GPU regression test that runs enough chunks to exercise Triton and + CUDA-graph reuse on one thread. + +Acceptance criteria: + +- `initialize -> reset -> step* -> close` executes on the same OS thread for a + serving runtime. +- No integration independently invents its own thread-dispatch mechanism. + +### 6. Generic WebRTC session manager + +The shared manager should own only peer lifecycle, control-event timing, input +sampling, generation scheduling, encoding, and delivery. + +#### TODO + +- [x] Drive the canonical `StepRequest -> StepResult` runtime boundary instead + of a WebRTC-only `generate_chunk` method. +- [x] Use `StepRequest` metadata to determine the next input window and + frame count. +- [x] Replace model-specific reset hooks with mapped session inputs. +- [x] Replace `_model_name` with runtime/adapter identity. +- [x] Replace `_chunk_done_extra` with `StepResult.metadata`. +- [x] Replace integration-specific runtime-error tuples with shared runtime + errors. +- [x] Delete no-op manager wrappers. +- [x] Remove integration-specific manager subclasses; integration factories + configure the shared manager's control keys and generation-error policy. +- [x] Move model-specific HTTP input and preview behavior to app controllers. +- [x] Cover session negotiation, reset, reconnect, error, and warmup behavior in + shared CPU tests. + +Acceptance criteria: + +- Lingbot and OmniDreams use the same concrete manager unless a real transport + capability differs. +- The manager has no imports from integration packages. + +### 7. Explicit WebRTC app and browser adapter contracts + +Replace dynamic optional methods with explicit extension surfaces. + +#### Server TODO + +- [x] Declare a typed WebRTC demo-adapter protocol. +- [x] Replace `getattr` discovery of runtime-config, manager, and app factories. +- [x] Always construct the shared aiohttp/WebRTC app in shared code. +- [x] Let integrations provide model web resources and optional route + registration, not a complete replacement app factory. +- [x] Provide one generic session-input route that delegates parsing/validation + to the model adapter where practical. +- [x] Keep offer, health, static assets, preload, and shutdown routes shared. + +#### Browser TODO + +- [x] Document the `adapter.js` interface with a JSDoc typedef or equivalent. +- [x] Keep peer connection, video, heartbeat, metrics, and common control + rendering in the shared client. +- [x] Make control groups declarative instead of hardcoded as universal WSAD + controls. +- [x] Move model-specific session forms and control-message handling into the + model adapter. +- [x] Represent optional post-processing as an explicit capability. +- [x] Add shared adapter-contract tests for Lingbot and OmniDreams. + +Acceptance criteria: + +- The browser loads one shared client and one small model adapter. +- A model can add UI/session behavior without copying connection or playback + logic. +- The shared demo builder has no undeclared adapter calls. + +### 8. Capability-driven output launch + +Output discovery should come from registered model/demo adapters instead of +runner-name prefix checks. + +#### TODO + +- [x] Let adapters declare supported input and output modes. +- [x] Resolve `cli`, `webrtc`, and `local-window` through adapter capabilities. +- [x] Remove `_is_lingbot_runner` and `_is_omnidreams_runner` branches from + shared output routing. +- [x] Launch Lingbot and OmniDreams WebRTC through the same shared demo entry + point. +- [x] Keep local-window manifest selection inside the OmniDreams integration. +- [x] Add registry tests proving a new adapter can add an output without editing + shared routing code. + +Acceptance criteria: + +- Adding a model integration does not require a model-name branch under + `flashdreams/flashdreams`. +- All WebRTC-capable integrations use the same shared server construction. + +## Suggested pull-request sequence + +Keep each change behavior-preserving and independently testable: + +1. **Result contract:** canonicalize `StepResult` and remove duplicated + video fields from the outer result path. +2. **Output consumption:** simplify `VideoOutputStream`; make WebRTC encoders, + MP4, and local presentation consume the result directly. +3. **Runtime worker:** add thread-affine execution and migrate existing WebRTC + lifecycle calls without changing model behavior. +4. **Lingbot session:** unify replay/runner and WebRTC generation. +5. **OmniDreams session:** unify replay, WebRTC, and local-window generation. +6. **WebRTC manager:** remove model-specific manager hooks and wrappers. +7. **App/UI boundary:** formalize server and browser adapter contracts. +8. **Launch routing:** replace model-name branches with adapter capabilities. + +Do not combine the model-session migrations with the browser redesign. Keeping +those changes separate makes output parity and UI regressions easier to locate. + +## Verification checklist + +### Static and CPU checks + +- [x] `uv run --locked --group lint ty check` +- [x] `uv run --locked --group lint pre-commit run --all-files` +- [x] Runtime/result/output unit tests. +- [x] WebRTC manager, message, encoder, and server unit tests. +- [x] Lingbot and OmniDreams demo API CPU tests. +- [x] Local-window adapter and frame-conversion CPU tests. +- [x] Every new pytest test has exactly one CI marker. + +### GPU checks + +- [ ] Lingbot runner replay produces the expected chunk count and MP4. +- [ ] Lingbot WebRTC runs multiple chunks, resets, and reconnects. +- [ ] OmniDreams replay produces the expected chunk count and MP4. +- [ ] OmniDreams WebRTC runs multiple chunks with post-processing off and on. +- [ ] OmniDreams local window renders multiple chunks and resets scenes. +- [ ] Software and NVENC WebRTC delivery both work. +- [x] Compiled and CUDA-graph configurations run beyond capture/replay startup. +- [ ] Multi-GPU rank coordination still advances every AR step in order. + +### Parity checks + +- [x] Equivalent replay and live per-step inputs reach the same model session + shape and layout. +- [ ] Post-processing is applied once, with matching output across consumers. +- [ ] Frame count, step index, metrics, and metadata agree across CLI, WebRTC, + and local-window paths. +- [ ] No output consumer introduces an unexpected device transfer. + +## Definition of done + +- One model session implementation exists per integration. +- One `VideoOutputStream` call creates each generated `StepResult`. +- CLI, WebRTC, and local-window consumers accept that result directly. +- All WebRTC runtime lifecycle operations are thread-affine. +- Shared runtime and serving code contain no Lingbot/OmniDreams branches. +- The shared browser client owns connection/playback behavior; model adapters + own only model-specific UI and session behavior. +- CPU CI, lint/type checks, and targeted GPU serving tests pass. diff --git a/docs/inference_runtime_supported_inputs_inventory.md b/docs/inference_runtime_supported_inputs_inventory.md new file mode 100644 index 000000000..f82f3192b --- /dev/null +++ b/docs/inference_runtime_supported_inputs_inventory.md @@ -0,0 +1,354 @@ + + +# Supported Model Input Inventory + +This note inventories the inputs used by the currently supported FlashDreams +runners and interactive runtimes, plus the SANA-WM input surface on `main`, then +records the T2/T3 API implications. It is intentionally about input contracts, +not tensor shape validation or model quality. + +## Inventory + +WAN 2.1 T2V, Self-Forcing WAN 2.1 T2V, Causal-Forcing T2V, +FastVideo Causal WAN 2.2 T2V, and Cosmos Predict2 T2V: + +- Source/app inputs: prompt text or prompt text file, pixel height/width, and + fps or block count depending on runner. +- Model-facing global conditioning: prompt text plus latent/output height and width + derived from run config. +- Model-facing per-step inputs: no live controls; AR loop steps with fixed + session state. + +WAN 2.1 I2V, Causal-Forcing I2V, and Cosmos Predict2 I2V: + +- Source/app inputs: prompt text or prompt file, first-frame image path or URL, + and pixel height/width. +- Model-facing global conditioning: prompt text and decoded first-frame tensor. +- Model-facing per-step inputs: no live controls. + +FlashVSR: + +- Source/app inputs: input video path or URL, chunk size, crop region, sparse + ratio, and optional output FPS. +- Model-facing global conditioning: no explicit prompt at runner time; the prompt + tensor is configured in the pipeline. Input video dimensions affect + per-video runtime/pipeline setup. +- Model-facing per-step inputs: video chunks passed to + `pipeline.generate(input=clip)`. + +LingBot CLI: + +- Source/app inputs: prompt or prompt path, first-frame image path, pose path, + intrinsics path, total blocks, dimensions, and fps. +- Model-facing global conditioning: prompt text and first-frame tensor. +- Model-facing per-step inputs: `CamCtrlInput` with intrinsics, camera poses, + and world scale. + +LingBot WebRTC: + +- Source/app inputs: session prompt, uploaded/remote/default first-frame image, + keyboard events, reset requests, text-event catalog, and trigger events. +- Model-facing global conditioning: prompt text, first-frame tensor, base text + embeddings, precomputed text-event embeddings, base intrinsics, and world + scale. +- Model-facing per-step inputs: keyboard event windows become pose segments + and camera trajectories. Text-event triggers can replace rollout text + embeddings when the model supports it. + +HY-WorldPlay WAN I2V: + +- Source/app inputs: prompt or prompt path, first-frame image path or example + image, pose string or pose JSON, memory-selection settings, dimensions, fps, + and seed. +- Model-facing global conditioning: prompt text and first-frame tensor for + session setup. +- Model-facing per-step inputs: pose data is bound for the rollout as action + labels, view matrices, intrinsics, and memory-selection state before AR steps. + +Omnidreams CLI: + +- Source/app inputs: shared prompt or per-camera prompts, HDMap video paths, + first-frame image/video paths, camera names, example-data UUID, and optional + embedding save/load paths. +- Model-facing global conditioning: prompt list, first-frame tensor, view names; or + precomputed text/image/negative-text embeddings. +- Model-facing per-step inputs: HDMap video chunks passed per AR step. + +Omnidreams WebRTC: + +- Source/app inputs: scene directory or scene UUID, scene variant, camera name, + prompt/first-frame assets resolved from the scene, keyboard events, reset + requests, and optional postprocess preset. +- Model-facing global conditioning: scene data, renderer, first-frame tensor, prompt, + camera calibration/extrinsics, initial ego pose, and initial timestamp. +- Model-facing per-step inputs: keyboard event windows become ego poses, + camera poses per view, and frame timestamps. The wrapper renders HDMap + conditioning internally for each step. + +Omnidreams interactive drive: + +- Source/app inputs: scene bundle, keyboard events or wheel/controller samples, + view-mode/reset/scene-exit controls, and vehicle/chunk config. +- Model-facing global conditioning: scene bundle, selected camera, prompt, initial + RGB frame, initial rig pose, and initial timestamp. +- Model-facing per-step inputs: `DriverCommand` samples become trajectory + chunks, rendered frames, and world-model conditioning. + +Template recipe: + +- Source/app inputs: synthetic runner config: batch size, height, width, context + tokens, AR steps, and seed. +- Model-facing global conditioning: synthetic transformer context, optional negative + context, height, and width. +- Model-facing per-step inputs: optional synthetic control tensor. + +WAN 2.2 TI2V pipeline config: + +- Source/app inputs: downstream runners use this rather than a standalone runner + in this tree. +- Model-facing global conditioning: prompt text and first-frame image for + TI2V-style session setup. +- Model-facing per-step inputs: downstream runners decide controls; + HY-WorldPlay currently binds action/camera state around it. + +SANA-WM bidirectional and streaming on `main`: + +- Source/app inputs: first-frame image path, prompt or prompt path, optional + negative prompt, camera trajectory path or action DSL, optional intrinsics + path or derived intrinsics, frame count, fps, Stage-1 sampling knobs, seed, + precision/refiner options, and streaming chunk/block settings. +- Model-facing global conditioning: decoder context such as prompt, fps, + `save_stage1`, refiner seed, sink size, and streaming refiner window/block + parameters. +- Model-facing per-step inputs: bidirectional passes one + `SanaWMI2VConditioningRequest` into the single generation step. Streaming + passes one `SanaWMStreamingI2VConditioningRequest` repeatedly; the + conditioning encoder caches rollout-wide prompt, first-frame, camera, latent + shape, and chunk-boundary state, then slices per AR chunk. +- Model-facing semantic fields include prompt, negative prompt, first frame, + camera-to-world trajectory, intrinsics vec4 sequence, frame count, fps, + sampling parameters, seed, and streaming chunking parameters. + +## API Implications + +The inventory changes the T2/T3 shape in five concrete ways. + +First, a selected mapping is often a composition. A LingBot-like run needs prompt +mapping, first-frame mapping, and keyboard-to-camera mapping. Omnidreams may add +scene selection, camera selection, and HDMap mapping. The implementation should +support checking a set of mapping schemas as one compatibility surface, while +still allowing a single mapping object when that is simpler. + +Second, `InferenceInputSchema` needs explicit global-conditioning and per-step +schema slots. `global_conditioning_fields` describe the session-global state +carried through `InferenceInput.global_conditioning`. Start/reset establishes +that state; a non-empty global-conditioning payload in a step context asks the +session to update it when the model supports that. `step_fields` arrive through +`InferenceInput.step` for one generated chunk or frame window. + +This distinction matters for rollout-wide values such as full camera +trajectories, action labels, intrinsics sequences, and memory-selection config. +Those can be supplied in the global-conditioning slot, even if the adapter later +slices them internally while executing steps. If the caller must supply a fresh +value for every generated chunk, that value belongs in `step_fields`. + +`frequency_consumed` is a separate optional hint for how the adapter uses a +field internally, such as `once` or `per_step`. It does not decide where the +caller provides the value. A field can live in `global_conditioning_fields` and +still have `frequency_consumed="per_step"` when the adapter slices or reads +rollout-wide state during step execution. + +Third, `name` is the semantic model input role, while `input_modality` is only +a coarse value-kind hint. For example, `prompt` and `negative_prompt` are +different semantic names even though both usually have `input_modality="text"`. +The semantic input name is the main contract; source details such as path, URL, +bytes, decoded tensor layout, accepted suffixes, or file schema belong in +adapter validation or `metadata`. + +Fourth, schema objects need open-ended metadata for future adapters. This lets a +SANA-WM-like adapter advertise that `camera_trajectory_c2w` uses an +`[F,4,4]` OpenCV camera-to-world sequence, or lets another model advertise a +schema URI, units, coordinate frame, accepted file suffixes, cardinality hints, +or adapter notes. Metadata should remain query information and should not become +the compatibility type system. + +Fifth, `UserInputSchema` describes raw source capabilities, `CanonicalModality` +describes what an application consumes, and mapping schemas describe derived +model-facing semantics. A browser may provide `key_down`, `key_up`, +`prompt_set`, and `initial_frame_set` events. Those become canonical modalities +such as `driver_command` or `conditioning_prompt`; whether they can then drive +`steering`, `camera_trajectory`, or text embeddings depends on the +selected mapping and model schema. + +## Implemented T2/T3 Shape + +The implementation that came out of this inventory is: + +1. Keep `UserInputEvent` and `UserInputs` as the raw event API, sliced by a + half-open `TimeWindow`. Static session-start values remain timestamp-zero + events. +2. Keep `UserInputSchema` lightweight and source-facing. `event_types` declares + that an event type exists; `UserInputCapability` additionally pins the + payload fields it carries. +3. Add a canonical layer between raw and encoded. `CanonicalModality` names a + device-independent input and its payload fields; `InputCanonicalizer` + registers per-device converters and produces `CanonicalInputs`. Applications + and mappings consume canonical inputs and never read raw device events. +4. Split `InferenceInput` (formerly `ModelInputs`) into `global_conditioning` + and `step`. Global conditioning is session-global state; `step` is the + payload for one generated chunk or frame window. +5. Keep `InputField.input_modality`, `frequency_consumed`, and `metadata` as + lightweight query hints, while leaving tensor shape and model-specific + validation to adapters and sessions. `InputField.name` remains the semantic + payload key. +6. Keep `InputMappingSchema` as the canonical-to-encoded boundary, with + mapping-set compatibility helpers for composed mappings. +7. Keep input names, input modalities, and metadata open-ended. + Adding a new model should usually mean adding adapter-owned schema + declarations and mappings, not changing the core input dataclasses. +8. Leave deep validation to model adapters, sessions, and mappings. The schema + layer catches obvious source/mapping/model mismatches before expensive + runtime initialization; it does not validate every tensor and coordinate + convention. + +See `docs/inference_runtime_inputs_implementation.md` for the resulting API. + +## Extensibility Contract + +The inventory above is not a vocabulary freeze. The core API does not contain a +closed enum of allowed input names. New adapters can introduce semantic field +names that match the model boundary they own. + +Use these conventions when adding future model schemas: + +- Prefer semantic names over modality names, such as `camera_trajectory_c2w` + instead of `array`, or `hdmap_frames` instead of `image`. +- Use `input_modality` for a coarse value-kind hint, such as `text`, `image`, + `embedding`, `c2w_sequence`, or `intrinsics_vec4_sequence`. +- Use `metadata` for representation details such as paths, decoded tensor + layout, units, coordinate frame, shape summary, accepted suffixes, schema URI, + model family, value ranges, or cardinality. +- Use `frequency_consumed` for adapter-consumption cadence, such as `once` or + `per_step`; keep it independent from whether the field is declared under + `global_conditioning_fields` or `step_fields`. +- Keep deep validation in the adapter/mapping. The lightweight schemas answer + whether the selected source and mapping can plausibly drive the model before + expensive initialization. + +## Representative Schema Sketches + +These are not migration work for T4+, but they show that the current primitives +can describe the supported input surfaces. All use +`flashdreams.runtime.InferenceInputSchema` and `InputField`. + +```python +lingbot_model = InferenceInputSchema( + description="lingbot-world", + global_conditioning_fields=( + InputField(name="prompt", input_modality="text", frequency_consumed="once"), + InputField( + name="global_conditioning_frame", + input_modality="image", + frequency_consumed="once", + ), + InputField( + name="text_embeddings", + required=False, + input_modality="embedding", + frequency_consumed="once", + ), + ), + step_fields=( + InputField(name="camera_trajectory", frequency_consumed="per_step"), + ), +) +``` + +```python +omnidreams_model = InferenceInputSchema( + description="omnidreams", + global_conditioning_fields=( + InputField(name="prompts", input_modality="text", frequency_consumed="once"), + InputField( + name="global_conditioning_frames", + input_modality="image", + frequency_consumed="once", + ), + InputField(name="view_names", frequency_consumed="once"), + InputField( + name="text_embeddings", + required=False, + input_modality="embedding", + frequency_consumed="once", + ), + InputField( + name="image_embeddings", + required=False, + input_modality="embedding", + frequency_consumed="once", + ), + ), + step_fields=( + InputField(name="hdmap_frames", frequency_consumed="per_step"), + ), +) +``` + +```python +hy_worldplay_model = InferenceInputSchema( + description="hy-worldplay", + global_conditioning_fields=( + InputField(name="prompt", input_modality="text", frequency_consumed="once"), + InputField( + name="global_conditioning_frame", + input_modality="image", + frequency_consumed="once", + ), + InputField(name="action_labels", frequency_consumed="per_step"), + InputField(name="camera_viewmats", frequency_consumed="per_step"), + InputField(name="camera_intrinsics", frequency_consumed="per_step"), + InputField(name="memory_config", frequency_consumed="per_step"), + ), +) +``` + +```python +sana_wm_model = InferenceInputSchema( + description="sana-wm", + global_conditioning_fields=( + InputField(name="prompt", input_modality="text", frequency_consumed="once"), + InputField( + name="negative_prompt", + required=False, + input_modality="text", + frequency_consumed="once", + ), + InputField( + name="global_conditioning_frame", + input_modality="image", + frequency_consumed="once", + ), + InputField( + name="camera_trajectory_c2w", + input_modality="c2w_sequence", + frequency_consumed="per_step", + metadata={"shape": "[F,4,4]", "coordinates": "opencv_c2w"}, + ), + InputField( + name="camera_intrinsics_vec4", + required=False, + input_modality="intrinsics_vec4_sequence", + frequency_consumed="per_step", + metadata={"shape": "[F,4]"}, + ), + ), +) +``` + +SANA-WM's `stage1_sampling` and `streaming_chunking` are deliberately absent +above. They describe how to run the model rather than what conditions it, so +they belong in `InferenceConfig`, not in an input schema. Flagged here because +the runner currently threads them alongside the conditioning inputs. diff --git a/docs/source/developer_guides/local_benchmarks.rst b/docs/source/developer_guides/local_benchmarks.rst index 687d95aa2..62842ba32 100644 --- a/docs/source/developer_guides/local_benchmarks.rst +++ b/docs/source/developer_guides/local_benchmarks.rst @@ -132,6 +132,27 @@ input stream is shorter than the requested duration. ``interactive-drive`` is left out of this shipped MP4 suite for now because its public CLI is a live presenter rather than a file-writing runner. +Omnidreams Shared Demo Comparison +--------------------------------- + +``configs/omnidreams_demo_replay_benchmarks.json`` contains a one-minute manual +comparison between the legacy Omnidreams single-view runner and the experimental +shared demo replay path: + +.. code-block:: bash + + uv run flashdreams-benchmark \ + --scenario-file configs/omnidreams_demo_replay_benchmarks.json \ + --scenario omnidreams-sv-runner-baseline \ + --scenario omnidreams-sv-demo-replay \ + --output-dir artifacts/benchmarks/omnidreams-demo-replay-compare + +Use the generated report's MP4 links for side-by-side manual review. The legacy +runner writes the stacked HDMap/RGB canvas while the shared demo writes generated +RGB output, so this comparison intentionally disables automatic baseline quality +scoring until those output layouts are aligned. Both scenarios use ``226`` +blocks, matching the shipped Omnidreams one-minute baseline. + Quality Hooks ------------- diff --git a/flashdreams/flashdreams/core/checkpoint/load.py b/flashdreams/flashdreams/core/checkpoint/load.py index 4d11f7642..4432e1be5 100644 --- a/flashdreams/flashdreams/core/checkpoint/load.py +++ b/flashdreams/flashdreams/core/checkpoint/load.py @@ -20,14 +20,16 @@ import io import json import os +import time from collections.abc import Callable, Mapping -from concurrent.futures import ProcessPoolExecutor +from concurrent.futures import ThreadPoolExecutor from typing import Literal, overload from urllib.parse import unquote, urlparse import torch from huggingface_hub import hf_hub_download, try_to_load_from_cache from loguru import logger +from safetensors import safe_open from safetensors.torch import load as load_safetensors from safetensors.torch import load_file as load_safetensors_file from safetensors.torch import save_file as save_safetensors @@ -239,7 +241,7 @@ def _safetensors_device(map_location: str | torch.device) -> str: def _hf_hub_download_shard_task( args: tuple[str, str, str | None, str], ) -> tuple[str, str]: - """Picklable worker: download one shard; used by ProcessPoolExecutor.""" + """Download or resolve one Hugging Face shard.""" repo_id, shard_file, subfolder, revision = args settings: dict[str, object] = { "repo": repo_id, @@ -275,7 +277,7 @@ def _parallel_hf_hub_download_shards( subfolder: str | None, revision: str, ) -> dict[str, str]: - """Download unique shard files in parallel processes; returns shard -> local path.""" + """Download unique shard files in parallel workers; returns shard -> local path.""" if not shard_files: return {} if len(shard_files) == 1: @@ -297,10 +299,10 @@ def _parallel_hf_hub_download_shards( work = [(repo_id, s, subfolder, revision) for s in shard_files] logger.info( f"Downloading {len(shard_files)} Hugging Face safetensors shards " - f"with up to {max_workers} parallel processes" + f"with up to {max_workers} parallel workers" ) shard_to_path: dict[str, str] = {} - with ProcessPoolExecutor(max_workers=max_workers) as pool: + with ThreadPoolExecutor(max_workers=max_workers) as pool: for shard_file, path in pool.map(_hf_hub_download_shard_task, work): shard_to_path[shard_file] = path return shard_to_path @@ -737,12 +739,334 @@ def _load_checkpoint_from_local( ) -> dict[str, torch.Tensor]: """Load checkpoint from local filesystem.""" if ext == ".safetensors": - with open(path, "rb") as f: - return load_safetensors(f.read()) + return load_safetensors_file(path, device=_safetensors_device(map_location)) else: return torch.load(path, map_location=map_location, weights_only=False) +def _copy_checkpoint_tensor(destination: torch.Tensor, source: torch.Tensor) -> int: + """Copy one checkpoint tensor into ``destination`` with bounded staging.""" + checkpoint_bytes = source.numel() * source.element_size() + if destination.device.type != "cpu": + staged = source.to(dtype=destination.dtype) + if staged.data_ptr() == source.data_ptr(): + staged = staged.clone() + destination.copy_(staged) + if destination.device.type == "cuda": + # Keep the CPU staging buffer alive until CUDA has consumed it. + torch.cuda.synchronize(destination.device) + del staged + return checkpoint_bytes + + destination.copy_(source.to(device=destination.device, dtype=destination.dtype)) + return checkpoint_bytes + + +def _stream_safetensors_into_model( + model: torch.nn.Module, + path: str, +) -> torch.nn.Module: + """Copy a safetensors checkpoint into a model with bounded host residency.""" + model_state = model.state_dict() + + with safe_open(path, framework="pt", device="cpu") as source: + checkpoint_keys = set(source.keys()) + model_keys = set(model_state) + missing = sorted(model_keys - checkpoint_keys) + unexpected = sorted(checkpoint_keys - model_keys) + if missing or unexpected: + details = [] + if missing: + details.append(f"Missing key(s): {', '.join(missing[:20])}") + if unexpected: + details.append(f"Unexpected key(s): {', '.join(unexpected[:20])}") + raise RuntimeError( + f"Checkpoint does not match {type(model).__name__}: " + + "; ".join(details) + ) + + for name, destination in model_state.items(): + source_shape = tuple(source.get_slice(name).get_shape()) + if source_shape != tuple(destination.shape): + raise RuntimeError( + f"Checkpoint tensor {name!r} has shape {source_shape}, " + f"expected {tuple(destination.shape)}" + ) + + with torch.no_grad(): + for name, destination in model_state.items(): + tensor = source.get_tensor(name) + try: + _copy_checkpoint_tensor(destination, tensor) + finally: + del tensor + + return model + + +def _stream_sharded_safetensors_into_model( + model: torch.nn.Module, + *, + weight_map: Mapping[str, str], + resolve_shard_path: Callable[[str], str], +) -> torch.nn.Module: + """Copy a sharded safetensors checkpoint into a model one shard at a time.""" + model_state = model.state_dict() + checkpoint_keys = set(weight_map) + model_keys = set(model_state) + missing = sorted(model_keys - checkpoint_keys) + unexpected = sorted(checkpoint_keys - model_keys) + if missing or unexpected: + details = [] + if missing: + details.append(f"Missing key(s): {', '.join(missing[:20])}") + if unexpected: + details.append(f"Unexpected key(s): {', '.join(unexpected[:20])}") + raise RuntimeError( + f"Checkpoint does not match {type(model).__name__}: " + "; ".join(details) + ) + + keys_by_shard: dict[str, list[str]] = {} + for tensor_name, shard_file in weight_map.items(): + keys_by_shard.setdefault(shard_file, []).append(tensor_name) + + shard_files = sorted(keys_by_shard) + destination_devices = sorted( + {str(tensor.device) for tensor in model_state.values()} + ) + destination_dtypes = sorted({str(tensor.dtype) for tensor in model_state.values()}) + logger.info( + "Streaming sharded safetensors into {}: {} shard(s), {} tensor(s), " + "destination devices={}, dtypes={}", + type(model).__name__, + len(shard_files), + len(weight_map), + destination_devices, + destination_dtypes, + ) + + for shard_index, shard_file in enumerate(shard_files, start=1): + shard_path = resolve_shard_path(shard_file) + tensor_names = keys_by_shard[shard_file] + shard_size_gib = os.path.getsize(shard_path) / 1024**3 + started = time.perf_counter() + logger.info( + "Validating safetensors shard {}/{}: {} tensors, {:.2f} GiB, {}", + shard_index, + len(shard_files), + len(tensor_names), + shard_size_gib, + shard_file, + ) + with safe_open(shard_path, framework="pt", device="cpu") as source: + shard_keys = set(source.keys()) + for name in tensor_names: + if name not in shard_keys: + raise KeyError( + f"Key {name!r} missing from shard {shard_file!r} " + f"(path {shard_path!r})" + ) + source_shape = tuple(source.get_slice(name).get_shape()) + destination = model_state[name] + if source_shape != tuple(destination.shape): + raise RuntimeError( + f"Checkpoint tensor {name!r} has shape {source_shape}, " + f"expected {tuple(destination.shape)}" + ) + logger.info( + "Validated safetensors shard {}/{} in {:.1f}s: {}", + shard_index, + len(shard_files), + time.perf_counter() - started, + shard_file, + ) + + total_copied_bytes = 0 + total_started = time.perf_counter() + with torch.no_grad(): + for shard_index, shard_file in enumerate(shard_files, start=1): + shard_path = resolve_shard_path(shard_file) + tensor_names = keys_by_shard[shard_file] + shard_copied_bytes = 0 + started = time.perf_counter() + logger.info( + "Streaming safetensors shard {}/{} into model: {} tensors, {}", + shard_index, + len(shard_files), + len(tensor_names), + shard_file, + ) + with safe_open(shard_path, framework="pt", device="cpu") as source: + for name in tensor_names: + tensor = source.get_tensor(name) + try: + tensor_bytes = _copy_checkpoint_tensor( + model_state[name], tensor + ) + shard_copied_bytes += tensor_bytes + total_copied_bytes += tensor_bytes + finally: + del tensor + elapsed = time.perf_counter() - started + throughput = ( + shard_copied_bytes / 1024**3 / elapsed if elapsed > 0 else float("inf") + ) + logger.info( + "Streamed safetensors shard {}/{} in {:.1f}s: {:.2f} GiB copied " + "({:.2f} GiB/s), {}", + shard_index, + len(shard_files), + elapsed, + shard_copied_bytes / 1024**3, + throughput, + shard_file, + ) + + elapsed = time.perf_counter() - total_started + throughput = total_copied_bytes / 1024**3 / elapsed if elapsed > 0 else float("inf") + logger.info( + "Finished streaming {} safetensors shard(s) in {:.1f}s: {:.2f} GiB copied " + "({:.2f} GiB/s)", + len(shard_files), + elapsed, + total_copied_bytes / 1024**3, + throughput, + ) + + return model + + +def _stream_sharded_safetensors_index_into_model( + checkpoint_path: str, + *, + model: torch.nn.Module, + checkpoint_min_free_gb: float | None, +) -> torch.nn.Module | None: + """Stream a safetensors index checkpoint into ``model`` without merging.""" + if checkpoint_path.startswith("s3://"): + return None + + if _is_huggingface_checkpoint_url(checkpoint_path): + repo_id, index_filename, subfolder, revision = ( + _parse_huggingface_checkpoint_url(checkpoint_path) + ) + logger.info( + f"Streaming sharded safetensors checkpoint from Hugging Face: " + f"{checkpoint_path}" + ) + settings: dict[str, object] = { + "repo": repo_id, + "filename": index_filename, + "revision": revision, + } + _preflight_checkpoint_cache_requirement( + label="Hugging Face sharded checkpoint cache", + min_free_gb=checkpoint_min_free_gb, + settings=settings, + ) + min_bytes = _preflight_hf_cache( + label="Hugging Face checkpoint index cache", + settings=settings, + ) + try: + index_local = hf_hub_download( + repo_id=repo_id, + filename=index_filename, + subfolder=subfolder, + revision=revision, + ) + except Exception as exc: + _raise_hf_cache_disk_error( + exc, + label="Hugging Face checkpoint index cache", + required_bytes=min_bytes, + settings=settings, + ) + raise + with open(index_local) as f: + index = json.load(f) + weight_map = index.get("weight_map") + if not isinstance(weight_map, dict) or not weight_map: + raise ValueError( + f"Invalid or empty weight_map in safetensors index: {index_local}" + ) + + unique_shards = sorted(set(weight_map.values())) + shard_to_path = _parallel_hf_hub_download_shards( + repo_id=repo_id, + shard_files=unique_shards, + subfolder=subfolder, + revision=revision, + ) + + def resolve_shard_path(shard_file: str) -> str: + return shard_to_path[shard_file] + + return _stream_sharded_safetensors_into_model( + model, + weight_map=weight_map, + resolve_shard_path=resolve_shard_path, + ) + + if not os.path.isfile(checkpoint_path): + raise FileNotFoundError( + f"Sharded safetensors index not found: {checkpoint_path}" + ) + logger.info( + f"Streaming sharded safetensors checkpoint from local index: {checkpoint_path}" + ) + with open(checkpoint_path) as f: + index = json.load(f) + weight_map = index.get("weight_map") + if not isinstance(weight_map, dict) or not weight_map: + raise ValueError( + f"Invalid or empty weight_map in safetensors index: {checkpoint_path}" + ) + base_dir = os.path.dirname(os.path.abspath(checkpoint_path)) + + def resolve_shard_path(shard_file: str) -> str: + return os.path.join(base_dir, shard_file) + + return _stream_sharded_safetensors_into_model( + model, + weight_map=weight_map, + resolve_shard_path=resolve_shard_path, + ) + + +def _resolve_streamable_safetensors_path( + checkpoint_path: str, + *, + local_cache_dir: str, + checkpoint_min_free_gb: float | None, +) -> str | None: + """Resolve a locally available safetensors file for streaming model loads.""" + if _is_sharded_safetensors_index_checkpoint(checkpoint_path): + if checkpoint_path.startswith("s3://"): + return None + cache_path = _sharded_safetensors_merge_cache_path( + checkpoint_path, local_cache_dir + ) + if os.path.exists(cache_path): + logger.info(f"Streaming merged sharded checkpoint from cache: {cache_path}") + return cache_path + return None + + if _get_checkpoint_extension(checkpoint_path) != ".safetensors": + return None + if _is_huggingface_checkpoint_url(checkpoint_path): + return _download_checkpoint_from_huggingface_url( + checkpoint_path, + checkpoint_min_free_gb=checkpoint_min_free_gb, + ) + if checkpoint_path.startswith("s3://"): + cache_path = os.path.join( + local_cache_dir, checkpoint_path.removeprefix("s3://") + ) + return cache_path if os.path.exists(cache_path) else None + return checkpoint_path + + def _load_checkpoint_from_s3( s3_path: str, ext: str, @@ -837,8 +1161,9 @@ def load_checkpoint( Args: checkpoint_path: ``s3://`` URI, local path, or HF URL. Single-file or DCP directory. - model: Model to load weights into. Required for DCP. Optional for - single-file: when provided, ``load_state_dict`` is called. + model: Model to load weights into. Required for DCP. Cached + safetensors are streamed into a provided model; other single-file + formats use ``load_state_dict``. checkpoint_type: ``"auto"``, ``"single"``, or ``"distributed"``. local_cache_dir: Directory for caches. credential_path: S3 credentials path. @@ -873,6 +1198,25 @@ def load_checkpoint( checkpoint_type = "distributed" if checkpoint_type == "single": + if model is not None: + if _is_sharded_safetensors_index_checkpoint(checkpoint_path): + streamed_model = _stream_sharded_safetensors_index_into_model( + checkpoint_path, + model=model, + checkpoint_min_free_gb=checkpoint_min_free_gb, + ) + if streamed_model is not None: + logger.info(f"Streamed checkpoint into model: {checkpoint_path}") + return streamed_model + stream_path = _resolve_streamable_safetensors_path( + checkpoint_path, + local_cache_dir=local_cache_dir, + checkpoint_min_free_gb=checkpoint_min_free_gb, + ) + if stream_path is not None: + _stream_safetensors_into_model(model, stream_path) + logger.info(f"Streamed checkpoint into model: {checkpoint_path}") + return model state_dict = load_single_checkpoint( checkpoint_path=checkpoint_path, local_cache_dir=local_cache_dir, diff --git a/flashdreams/flashdreams/infra/postprocess/__init__.py b/flashdreams/flashdreams/infra/postprocess/__init__.py index c01c6ec5e..ded5fe12c 100644 --- a/flashdreams/flashdreams/infra/postprocess/__init__.py +++ b/flashdreams/flashdreams/infra/postprocess/__init__.py @@ -37,12 +37,14 @@ VideoPostprocessStepStats, VideoPostprocessStream, create_runner_postprocess_stream, + create_video_postprocess_stream, ) __all__ = [ "VideoPostprocessStream", "VideoPostprocessStepStats", "create_runner_postprocess_stream", + "create_video_postprocess_stream", "VideoChunk", "VideoPostProcessor", "VideoPostProcessorConfig", diff --git a/flashdreams/flashdreams/infra/postprocess/stream.py b/flashdreams/flashdreams/infra/postprocess/stream.py index 2dd90adf0..ca1eed49d 100644 --- a/flashdreams/flashdreams/infra/postprocess/stream.py +++ b/flashdreams/flashdreams/infra/postprocess/stream.py @@ -212,15 +212,17 @@ def _prepare(self, output: Tensor) -> None: self._prepared = True -def create_runner_postprocess_stream( - config: RunnerConfigT, +def create_video_postprocess_stream( *, + postprocess: VideoPostprocessChainConfig, + output_layout: VideoTensorLayout, + fps: float | None, + per_view: bool, world_size: int, is_rank_zero: bool = True, - fps: float | None = None, + profile: bool = False, ) -> VideoPostprocessStream | None: - """Create a runner post-processing stream, or ``None`` when skipped.""" - postprocess = getattr(config, "postprocess") + """Create a post-processing stream for one generated video rollout.""" if not postprocess.is_enabled(): return None postprocess.validate_execution(world_size=world_size) @@ -230,7 +232,27 @@ def create_runner_postprocess_stream( and not postprocess.requires_all_ranks(world_size=world_size) ): return None + return VideoPostprocessStream( + postprocess=postprocess, + output_layout=output_layout, + fps=fps, + per_view=per_view, + world_size=world_size, + profile=profile, + ) + +def create_runner_postprocess_stream( + config: RunnerConfigT, + *, + world_size: int, + is_rank_zero: bool = True, + fps: float | None = None, +) -> VideoPostprocessStream | None: + """Create a runner post-processing stream, or ``None`` when skipped.""" + postprocess = getattr(config, "postprocess") + if not postprocess.is_enabled(): + return None output_layout = getattr(config, "postprocess_output_layout") if output_layout is None: raise ValueError( @@ -242,12 +264,13 @@ def create_runner_postprocess_stream( if configured_fps is None: configured_fps = getattr(config, "fps", getattr(config, "output_fps", None)) - return VideoPostprocessStream( + return create_video_postprocess_stream( postprocess=postprocess, output_layout=output_layout, fps=configured_fps, per_view=getattr(config, "postprocess_per_view"), world_size=world_size, + is_rank_zero=is_rank_zero, profile=bool( getattr(getattr(config, "pipeline", None), "enable_sync_and_profile", False) ), diff --git a/flashdreams/flashdreams/infra/results.py b/flashdreams/flashdreams/infra/results.py new file mode 100644 index 000000000..d6d69488e --- /dev/null +++ b/flashdreams/flashdreams/infra/results.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generated inference result contracts shared by runtimes and consumers.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Any + +from torch import Tensor + +from flashdreams.infra.postprocess import VideoTensorLayout +from flashdreams.infra.time import TimeWindow + +if TYPE_CHECKING: + from flashdreams.infra.video_output import LazyRGBFrame + + +@dataclass(frozen=True, kw_only=True, slots=True) +class StepResult: + """Generated output and metadata returned by one inference step. + + Video results use :meth:`from_video_chunk`, which records a required tensor + layout and derives the frame count once. Non-video results may use the + regular constructor without a layout. + """ + + __hash__ = None + + step_index: int + output: Any = None + frame_count: int = 0 + layout: VideoTensorLayout | None = None + output_window: TimeWindow | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + metrics: Mapping[str, float | int] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.step_index < 0: + raise ValueError("StepResult.step_index must be >= 0.") + if self.frame_count < 0: + raise ValueError("StepResult.frame_count must be >= 0.") + if self.layout is not None: + from flashdreams.infra.video_output import infer_video_num_frames + + video_chunk = self.video_chunk + derived_frame_count = infer_video_num_frames( + video_chunk, + layout=self.layout, + ) + if self.frame_count not in (0, derived_frame_count): + raise ValueError( + "StepResult.frame_count does not match the declared video " + f"layout: expected {derived_frame_count}, got {self.frame_count}." + ) + object.__setattr__(self, "frame_count", derived_frame_count) + object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata))) + object.__setattr__(self, "metrics", MappingProxyType(dict(self.metrics))) + + @classmethod + def from_video_chunk( + cls, + *, + step_index: int, + video_chunk: Tensor, + layout: VideoTensorLayout, + output_window: TimeWindow | None = None, + metadata: Mapping[str, Any] | None = None, + metrics: Mapping[str, float | int] | None = None, + ) -> StepResult: + """Build one layout-aware generated-video result.""" + return cls( + step_index=step_index, + output=video_chunk, + layout=layout, + output_window=output_window, + metadata=dict(metadata or {}), + metrics=dict(metrics or {}), + ) + + @property + def video_chunk(self) -> Tensor: + """Return the video tensor or fail if this is not a video result.""" + if self.layout is None: + raise ValueError("StepResult.layout is required for video output.") + if not isinstance(self.output, Tensor): + raise TypeError( + "A video StepResult requires a torch.Tensor output, " + f"got {type(self.output).__name__}." + ) + return self.output + + def lazy_rgb_frames( + self, + *, + batch_index: int = 0, + view_index: int = 0, + record_cuda_event: bool = True, + ) -> list[LazyRGBFrame]: + """Expose this video result as lazy per-frame RGB handles.""" + from flashdreams.infra.video_output import lazy_rgb_frames_from_video_tensor + + return lazy_rgb_frames_from_video_tensor( + self.video_chunk, + layout=self._video_layout(), + batch_index=batch_index, + view_index=view_index, + record_cuda_event=record_cuda_event, + ) + + def video_hwc_uint8( + self, + *, + batch_index: int = 0, + view_index: int = 0, + ) -> Tensor: + """Return this video result as uint8 ``[T,H,W,C]`` on its device.""" + from flashdreams.infra.video_output import video_tensor_to_hwc_uint8 + + return video_tensor_to_hwc_uint8( + self.video_chunk, + layout=self._video_layout(), + batch_index=batch_index, + view_index=view_index, + ) + + def _video_layout(self) -> VideoTensorLayout: + if self.layout is None: + raise ValueError("StepResult.layout is required for video output.") + return self.layout + + +__all__ = ["StepResult"] diff --git a/flashdreams/flashdreams/infra/runner.py b/flashdreams/flashdreams/infra/runner.py index 2ef2fc296..060adbd31 100644 --- a/flashdreams/flashdreams/infra/runner.py +++ b/flashdreams/flashdreams/infra/runner.py @@ -39,7 +39,7 @@ VideoTensorLayout, create_runner_postprocess_stream, ) -from flashdreams.infra.video_output import RunnerVideoOutputStream +from flashdreams.infra.video_output import VideoOutputStream def _is_torchrun_env() -> bool: @@ -63,6 +63,9 @@ class RunnerConfig(InstantiateConfig): per-runner ``--help`` (it's metadata, not a knob); a non-empty value is enforced for in-tree runners by the registry test.""" + output_adapter: Annotated[str | None, tyro.conf.Suppress] = None + """Optional ``module:attribute`` implementing non-CLI output capabilities.""" + pipeline: StreamInferencePipelineConfig """Wrapped pipeline config; the runner instantiates and drives it.""" @@ -183,19 +186,16 @@ def create_video_output_stream( self, *, fps: float | None = None, - move_to_cpu: bool = True, - ) -> RunnerVideoOutputStream: - """Create the standard runner video output stream for one rollout.""" + ) -> VideoOutputStream: + """Create the standard post-processing stream for one rollout.""" layout = self.config.postprocess_output_layout if layout is None: raise ValueError( "Runner video output collection requires an output layout." ) - return RunnerVideoOutputStream( + return VideoOutputStream( postprocess_stream=self.create_postprocess_stream(fps=fps), output_layout=layout, - collect_output=self.is_rank_zero, - move_to_cpu=move_to_cpu, ) @abstractmethod diff --git a/flashdreams/flashdreams/infra/time.py b/flashdreams/flashdreams/infra/time.py new file mode 100644 index 000000000..65629e2ec --- /dev/null +++ b/flashdreams/flashdreams/infra/time.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared time-domain value objects.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeWindow: + """Half-open time window in seconds since session start.""" + + start_s: float + end_s: float + + def __post_init__(self) -> None: + if not math.isfinite(self.start_s) or not math.isfinite(self.end_s): + raise ValueError("TimeWindow bounds must be finite seconds.") + if self.start_s < 0 or self.end_s < 0: + raise ValueError("TimeWindow bounds must be non-negative.") + if self.end_s < self.start_s: + raise ValueError("TimeWindow.end_s must be >= start_s.") + + def contains(self, timestamp_s: float) -> bool: + """Return whether ``timestamp_s`` falls within this half-open window.""" + return self.start_s <= timestamp_s < self.end_s + + +__all__ = ["TimeWindow"] diff --git a/flashdreams/flashdreams/infra/video_output.py b/flashdreams/flashdreams/infra/video_output.py index 930bcf572..199c36b5c 100644 --- a/flashdreams/flashdreams/infra/video_output.py +++ b/flashdreams/flashdreams/infra/video_output.py @@ -18,14 +18,17 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal, TypeAlias, cast import torch from torch import Tensor from flashdreams.infra.acceleration.frame_prefetch import LazyCudaFrame from flashdreams.infra.postprocess import VideoPostprocessStream, VideoTensorLayout +from flashdreams.infra.results import StepResult +from flashdreams.infra.time import TimeWindow + +WritableVideoTensorLayout: TypeAlias = Literal["thwc", "tchw", "btchw", "bcthw"] def video_layout_time_dim(layout: VideoTensorLayout) -> int: @@ -41,6 +44,19 @@ def video_layout_time_dim(layout: VideoTensorLayout) -> int: def infer_video_num_frames(tensor: Tensor, *, layout: VideoTensorLayout) -> int: """Infer a video chunk's frame count from its declared layout.""" + expected_ndim = { + "tchw": 4, + "btchw": 5, + "bcthw": 5, + "bvtchw": 6, + }.get(layout) + if expected_ndim is None: + raise ValueError(f"unsupported video layout: {layout!r}") + if tensor.ndim != expected_ndim: + raise ValueError( + f"layout={layout!r} expects a {expected_ndim}D tensor, " + f"got shape {tuple(tensor.shape)}." + ) return int(tensor.shape[video_layout_time_dim(layout)]) @@ -132,152 +148,120 @@ def lazy_rgb_frames_from_video_tensor( ] -@dataclass(slots=True) -class VideoStepResult: - """One generated video chunk plus per-step metadata. - - The field names intentionally match the pre-existing WebRTC result shape - so serving runtimes and output helpers share layout-aware chunk metadata. - """ - - chunk_index: int - num_frames: int - video_chunk: Tensor - stats: dict[str, float] | None = None - layout: VideoTensorLayout | None = None - metadata: dict[str, Any] = field(default_factory=dict) - - @classmethod - def from_video_chunk( - cls, - *, - chunk_index: int, - video_chunk: Tensor, - layout: VideoTensorLayout, - stats: dict[str, float] | None = None, - metadata: Mapping[str, Any] | None = None, - ) -> VideoStepResult: - """Build a result and infer ``num_frames`` from ``layout``.""" - return cls( - chunk_index=chunk_index, - num_frames=infer_video_num_frames(video_chunk, layout=layout), - video_chunk=video_chunk, - stats=stats, - layout=layout, - metadata=dict(metadata or {}), - ) - - def lazy_rgb_frames( - self, - *, - batch_index: int = 0, - view_index: int = 0, - record_cuda_event: bool = True, - ) -> list[LazyRGBFrame]: - """Expose this chunk as lazy per-frame RGB handles.""" - if self.layout is None: - raise ValueError("VideoStepResult.layout is required for frame extraction") - return lazy_rgb_frames_from_video_tensor( - self.video_chunk, - layout=self.layout, - batch_index=batch_index, - view_index=view_index, - record_cuda_event=record_cuda_event, - ) - - def video_hwc_uint8( - self, - *, - batch_index: int = 0, - view_index: int = 0, - ) -> Tensor: - """Return this chunk as a uint8 ``[T,H,W,C]`` tensor on its source device.""" - if self.layout is None: - raise ValueError("VideoStepResult.layout is required for frame extraction") - return video_tensor_to_hwc_uint8( - self.video_chunk, - layout=self.layout, - batch_index=batch_index, - view_index=view_index, - ) - - -class RunnerVideoOutputStream: - """Post-process, collect, and summarize runner video chunks.""" +class VideoOutputStream: + """Turn generated tensors into post-processed step results.""" def __init__( self, *, postprocess_stream: VideoPostprocessStream | None, output_layout: VideoTensorLayout, - collect_output: bool = True, - move_to_cpu: bool = True, - empty_message: str = "runner emitted no video frames", ) -> None: self.postprocess_stream = postprocess_stream self.output_layout = output_layout - self._time_dim = video_layout_time_dim(output_layout) - self._collect_output = collect_output - self.move_to_cpu = move_to_cpu - self.empty_message = empty_message - self._chunks: list[Tensor] = [] self._closed = False - self.stats_history: list[dict[str, object]] = [] - - @property - def collect_output(self) -> bool: - """Return whether this stream collects chunks for rank-zero writing.""" - return self._collect_output + self._last_step_index: int | None = None def process( self, video_chunk: Tensor, *, autoregressive_index: int, - stats: dict[str, float] | None = None, - stats_extra: Mapping[str, object] | None = None, - ) -> None: - """Process one generated chunk and collect it when this rank writes output.""" + metrics: Mapping[str, float | int] | None = None, + metadata: Mapping[str, Any] | None = None, + output_window: TimeWindow | None = None, + ) -> StepResult: + """Post-process one generated chunk into the shared result boundary.""" if self._closed: raise RuntimeError("cannot process video after finish()") processed = video_chunk + result_metadata = dict(metadata or {}) if self.postprocess_stream is not None: processed = self.postprocess_stream.process( video_chunk, autoregressive_index=autoregressive_index, ) - self._append_if_nonempty(processed) - if self.collect_output and stats is not None: - if self.postprocess_stream is None: - combined_stats: dict[str, object] = dict(stats) - else: - combined_stats = self.postprocess_stream.add_process_stats(stats) - entry: dict[str, object] = { - "autoregressive_index": autoregressive_index, - **combined_stats, - } - if stats_extra is not None: - entry.update(stats_extra) - self.stats_history.append(entry) + postprocess_stats = self.postprocess_stream.last_process_stats + if postprocess_stats is not None: + result_metadata["postprocess"] = postprocess_stats.as_dict() + self._last_step_index = autoregressive_index + return StepResult.from_video_chunk( + step_index=autoregressive_index, + video_chunk=processed.detach(), + layout=self.output_layout, + output_window=output_window, + metrics=metrics, + metadata=result_metadata, + ) - def finish(self) -> Tensor | None: - """Flush post-processing and return the collected rank-zero video.""" + def finish(self) -> StepResult | None: + """Close the stream and return a post-processing tail, when present.""" if self._closed: return None self._closed = True - if self.postprocess_stream is not None: - flushed = self.postprocess_stream.finish() - if flushed is not None: - self._append_if_nonempty(flushed) - return self._collected_output() + if self.postprocess_stream is None: + return None + flushed = self.postprocess_stream.finish() + if flushed is None: + return None + if self._last_step_index is None: + raise RuntimeError("post-processing emitted a tail before any video step") + return StepResult.from_video_chunk( + step_index=self._last_step_index, + video_chunk=flushed.detach(), + layout=self.output_layout, + metadata={"postprocess_tail": True}, + ) + + +class VideoResultCollector: + """Collect video results for persistence or composed presentation.""" - def _append_if_nonempty(self, output: Tensor) -> None: - if not self.collect_output or output.shape[self._time_dim] == 0: + def __init__( + self, + *, + output_layout: VideoTensorLayout, + enabled: bool = True, + move_to_cpu: bool = True, + empty_message: str = "runner emitted no video frames", + ) -> None: + self.output_layout = output_layout + self.enabled = enabled + self.move_to_cpu = move_to_cpu + self.empty_message = empty_message + self._time_dim = video_layout_time_dim(output_layout) + self._chunks: list[Tensor] = [] + self.stats_history: list[dict[str, object]] = [] + + def add(self, result: StepResult) -> None: + """Collect one video result and its serializable statistics.""" + if result.layout != self.output_layout: + raise ValueError( + f"collector expected layout {self.output_layout!r}, " + f"got {result.layout!r}." + ) + if not self.enabled: return - self._chunks.append(output.cpu() if self.move_to_cpu else output) + if result.frame_count > 0: + chunk = result.video_chunk + self._chunks.append(chunk.cpu() if self.move_to_cpu else chunk) + entry: dict[str, object] = { + "step_index": result.step_index, + "frames": result.frame_count, + **result.metrics, + } + if result.output_window is not None: + entry["output_start_s"] = result.output_window.start_s + entry["output_end_s"] = result.output_window.end_s + if "postprocess" in result.metadata: + entry["postprocess"] = result.metadata["postprocess"] + if result.metadata.get("postprocess_tail"): + entry["postprocess_tail"] = True + self.stats_history.append(entry) - def _collected_output(self) -> Tensor | None: - if not self.collect_output: + def finish(self) -> Tensor | None: + """Concatenate and return all collected video chunks.""" + if not self.enabled: return None if not self._chunks: raise ValueError(self.empty_message) @@ -288,12 +272,43 @@ def _collected_output(self) -> Tensor | None: return output +def prepare_video_for_mp4( + video: Tensor, + *, + layout: VideoTensorLayout | str, +) -> tuple[Tensor, WritableVideoTensorLayout]: + """Convert a stream output into a layout accepted by runner MP4 I/O.""" + if layout in {"thwc", "tchw", "btchw", "bcthw"}: + return video, cast(WritableVideoTensorLayout, layout) + if layout == "bvtchw": + if video.ndim != 6: + raise ValueError( + "layout='bvtchw' expects a 6D [B,V,T,C,H,W] tensor, " + f"got {tuple(video.shape)}." + ) + if video.shape[0] != 1: + raise ValueError( + "layout='bvtchw' MP4 writing expects a single batch element, " + f"got {tuple(video.shape)}." + ) + _, views, frames, channels, height, width = video.shape + canvas = ( + video[0] + .permute(1, 3, 0, 4, 2) + .contiguous() + .reshape(frames, height, views * width, channels) + ) + return canvas, "thwc" + raise ValueError(f"unsupported video layout for MP4: {layout!r}") + + __all__ = [ "LazyRGBFrame", - "RunnerVideoOutputStream", - "VideoStepResult", + "VideoOutputStream", + "VideoResultCollector", "infer_video_num_frames", "lazy_rgb_frames_from_video_tensor", + "prepare_video_for_mp4", "video_layout_time_dim", "video_tensor_to_hwc_uint8", ] diff --git a/flashdreams/flashdreams/recipes/wan/transformer/wan21.py b/flashdreams/flashdreams/recipes/wan/transformer/wan21.py index c1ae0ab04..0022f1fc2 100644 --- a/flashdreams/flashdreams/recipes/wan/transformer/wan21.py +++ b/flashdreams/flashdreams/recipes/wan/transformer/wan21.py @@ -157,6 +157,16 @@ class Wan21TransformerConfig(TransformerConfig): """Pre-load state-dict remap (e.g. Self-Forcing's ``generator_ema.model.…`` layout).""" + stream_checkpoint: bool = False + """Load cached safetensors directly into the model with bounded host residency.""" + + init_device: str | None = None + """Optional device used for initial network parameter allocation. + + Large streaming-checkpoint models can set this to the final runtime device + so the module is not first materialized as fp32 CPU tensors. + """ + batch_shape: tuple[int, ...] = (1,) """Batch dims of the latent (excluding the L, D dims).""" @@ -273,19 +283,29 @@ def __init__(self, config: Wan21TransformerConfig) -> None: self._output_height: int | None = None self._output_width: int | None = None - self.network = config.network.setup() - self.network = self.network.to(dtype=config.dtype) + self.network = self._setup_network(config) self.network.eval() self.network.set_context_parallel_group(cp_group=self._cp_group) if config.checkpoint_path is not None: - state_dict = load_checkpoint( - config.checkpoint_path, - checkpoint_min_free_gb=config.checkpoint_min_free_gb, - ) - if config.state_dict_transform is not None: - state_dict = config.state_dict_transform(state_dict) - self.network.load_state_dict(state_dict) + if config.stream_checkpoint: + if config.state_dict_transform is not None: + raise ValueError( + "stream_checkpoint does not support state_dict_transform" + ) + load_checkpoint( + config.checkpoint_path, + model=self.network, + checkpoint_min_free_gb=config.checkpoint_min_free_gb, + ) + else: + state_dict = load_checkpoint( + config.checkpoint_path, + checkpoint_min_free_gb=config.checkpoint_min_free_gb, + ) + if config.state_dict_transform is not None: + state_dict = config.state_dict_transform(state_dict) + self.network.load_state_dict(state_dict) self.network.update_parameters_after_loading_checkpoint() if config.compile_network: @@ -308,6 +328,23 @@ def __init__(self, config: Wan21TransformerConfig) -> None: self._cuda_graph_dispatch.uncond_call or self.network ) + @staticmethod + def _setup_network(config: Wan21TransformerConfig) -> WanDiTNetwork: + init_device = ( + None if config.init_device is None else torch.device(config.init_device) + ) + if init_device is None: + return config.network.setup().to(dtype=config.dtype) + + previous_dtype = torch.get_default_dtype() + try: + torch.set_default_dtype(config.dtype) + with torch.device(init_device): + network = config.network.setup() + finally: + torch.set_default_dtype(previous_dtype) + return network.to(device=init_device, dtype=config.dtype) + @property def latent_shape(self) -> tuple[int, ...]: """Per-rank post-patchify latent shape ``[*batch_shape, L/cp, D]``. diff --git a/flashdreams/flashdreams/runtime/__init__.py b/flashdreams/flashdreams/runtime/__init__.py new file mode 100644 index 000000000..3280e6862 --- /dev/null +++ b/flashdreams/flashdreams/runtime/__init__.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Experimental inference runtime API envelope. + +This package defines the small v0 boundary above ``flashdreams.infra``. It is +intentionally additive while integrations migrate onto it. +""" + +from flashdreams.runtime.canonical import ( + DEFAULT_DRIVING_BINDINGS, + DRIVER_COMMAND, + DeviceConverter, + DeviceConverterSchema, + InputCanonicalizer, + KeyboardToDriverCommand, + ScriptedModality, +) +from flashdreams.runtime.config import ExecutionBackend, InferenceConfig, Precision +from flashdreams.runtime.inputs import ( + INPUT_PHASES, + CanonicalInputs, + CanonicalInputSchema, + CanonicalModality, + InferenceInput, + InferenceInputSchema, + InputField, + InputPhase, + TimeWindow, + UserInputCapability, + UserInputEvent, + UserInputs, + UserInputSchema, + validate_phase, +) +from flashdreams.runtime.interfaces import ( + InferenceRuntime, + InferenceSession, + ModelAdapter, +) +from flashdreams.runtime.keyboard import ( + DEFAULT_SUPPORTED_KEYS, + DRIVING_SUPPORTED_KEYS, + KEY_ALIASES, + WSAD_SUPPORTED_KEYS, + ImageRequest, + KeyboardState, + PromptRequest, + ResetRequest, + SparseInputSnapshot, + normalize_key, +) +from flashdreams.runtime.mapping import ( + DeclaresMappingSchema, + IdentityInputMapping, + InputMapping, + InputMappingSchema, + MappingCompatibility, + check_mapping_compatibility, + check_mapping_set_compatibility, + combine_mapping_schemas, + undeclared_inference_inputs, +) +from flashdreams.runtime.metrics import ( + InMemoryMetricsRecorder, + MetricsRecorder, + MetricsSnapshot, + NullMetricsRecorder, + RuntimeMetricSample, +) +from flashdreams.runtime.output import NullOutputTarget, OutputArtifact, OutputTarget +from flashdreams.runtime.runner import run_inference_session +from flashdreams.runtime.types import ( + StepRequest, + StepRequirements, + StepResult, + step_requirements_from_request, +) +from flashdreams.runtime.video_output import Mp4VideoOutputTarget +from flashdreams.runtime.worker import ModelExecutionWorker, ThreadAffineRuntimeWorker + +__all__ = [ + "CanonicalInputs", + "CanonicalInputSchema", + "CanonicalModality", + "check_mapping_compatibility", + "check_mapping_set_compatibility", + "combine_mapping_schemas", + "DeclaresMappingSchema", + "DEFAULT_DRIVING_BINDINGS", + "DEFAULT_SUPPORTED_KEYS", + "DeviceConverter", + "DeviceConverterSchema", + "DRIVING_SUPPORTED_KEYS", + "DRIVER_COMMAND", + "ExecutionBackend", + "IdentityInputMapping", + "InferenceConfig", + "InferenceInput", + "InferenceInputSchema", + "InferenceRuntime", + "InferenceSession", + "InMemoryMetricsRecorder", + "INPUT_PHASES", + "InputCanonicalizer", + "InputField", + "InputMapping", + "InputMappingSchema", + "InputPhase", + "ImageRequest", + "KEY_ALIASES", + "KeyboardState", + "KeyboardToDriverCommand", + "MappingCompatibility", + "MetricsRecorder", + "MetricsSnapshot", + "ModelAdapter", + "ModelExecutionWorker", + "Mp4VideoOutputTarget", + "NullMetricsRecorder", + "NullOutputTarget", + "OutputArtifact", + "OutputTarget", + "Precision", + "PromptRequest", + "ResetRequest", + "RuntimeMetricSample", + "ScriptedModality", + "SparseInputSnapshot", + "StepRequest", + "StepRequirements", + "StepResult", + "TimeWindow", + "ThreadAffineRuntimeWorker", + "run_inference_session", + "step_requirements_from_request", + "undeclared_inference_inputs", + "UserInputCapability", + "UserInputEvent", + "UserInputs", + "UserInputSchema", + "WSAD_SUPPORTED_KEYS", + "normalize_key", + "validate_phase", +] diff --git a/flashdreams/flashdreams/runtime/_utils.py b/flashdreams/flashdreams/runtime/_utils.py new file mode 100644 index 000000000..d8016c6b7 --- /dev/null +++ b/flashdreams/flashdreams/runtime/_utils.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small helpers shared by the experimental runtime API.""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import TypeVar + +ValueT = TypeVar("ValueT") + + +def freeze_mapping(value: Mapping[str, ValueT]) -> Mapping[str, ValueT]: + """Return a read-only shallow copy of ``value``.""" + return MappingProxyType(dict(value)) diff --git a/flashdreams/flashdreams/runtime/canonical.py b/flashdreams/flashdreams/runtime/canonical.py new file mode 100644 index 000000000..bf3960380 --- /dev/null +++ b/flashdreams/flashdreams/runtime/canonical.py @@ -0,0 +1,387 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Raw device input to canonical modality conversion. + +This is the ``raw input -> canonicalized input`` leg. Applications consume +:class:`~flashdreams.runtime.inputs.CanonicalInputs`; they never read raw device +events. Adding a keyboard, gamepad, or force-feedback wheel is therefore a +:meth:`InputCanonicalizer.register` call that touches no application, mapping, +or model code. + +Converters are stateful, because HID input is edge-triggered while per-step +conditioning is level-triggered: a key held across a step emits no events yet +still means full throttle. Feed windows in session order and call +:meth:`InputCanonicalizer.reset` at a rollout boundary; replaying the same +window sequence then reproduces the same canonical inputs. + +This layer covers live user control only. Global conditioning such as a prompt +or conditioning frame is application-owned and reaches ``InferenceInput`` +directly, without passing through canonicalization. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, Protocol, runtime_checkable + +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.inputs import ( + CanonicalInputs, + CanonicalInputSchema, + CanonicalModality, + TimeWindow, + UserInputCapability, + UserInputs, + UserInputSchema, +) +from flashdreams.runtime.keyboard import KeyboardState, normalize_key + +DriverBindings = Mapping[str, frozenset[str]] + +DEFAULT_DRIVING_BINDINGS: DriverBindings = MappingProxyType( + { + "throttle": frozenset({"w", "up"}), + "brake": frozenset({"s", "down"}), + "steer_left": frozenset({"a", "left"}), + "steer_right": frozenset({"d", "right"}), + "stop": frozenset({"space"}), + "reverse": frozenset(), + } +) +"""Default key bindings for :class:`KeyboardToDriverCommand`. + +Bindings are data so a layout can be rebound without editing the converter, and +so the set of tracked keys is derived from them rather than declared twice. +""" + +_DRIVER_ACTIONS = frozenset(DEFAULT_DRIVING_BINDINGS) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DeviceConverterSchema: + """Metadata for one device-to-canonical-modality converter.""" + + name: str + produces: CanonicalModality + consumes: tuple[UserInputCapability, ...] = () + device_kind: str | None = None + priority: int = 0 + metadata: Mapping[str, Any] = field( + default_factory=dict, + compare=False, + hash=False, + ) + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("DeviceConverterSchema.name must be non-empty.") + if not isinstance(self.produces, CanonicalModality): + raise TypeError("produces must be a CanonicalModality object.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@runtime_checkable +class DeviceConverter(Protocol): + """Contract for turning one device's raw events into a canonical modality.""" + + @property + def schema(self) -> DeviceConverterSchema: + """Return converter metadata used for source selection.""" + ... + + def reset(self) -> None: + """Drop accumulated device state at a session or rollout boundary.""" + ... + + def convert( + self, + user_inputs: UserInputs, + window: TimeWindow, + ) -> Mapping[str, Any] | None: + """Return the modality value for ``window``, or ``None`` if inactive. + + ``user_inputs`` is already filtered to ``window``. Returning ``None`` + lets a present-but-idle device yield to a lower-priority one. + """ + ... + + +DRIVER_COMMAND = CanonicalModality( + name="driver_command", + payload_fields=frozenset({"throttle", "brake", "steer", "stop", "reverse"}), + description=( + "Normalized driving intent. throttle/brake are in [0, 1], steer is in " + "[-1, 1] with positive meaning left." + ), +) + + +class KeyboardToDriverCommand: + """Convert keyboard edges into :data:`DRIVER_COMMAND` level state. + + Mirrors the mapping the Omnidreams interactive-drive keyboard backend + already uses, so a keyboard reaches a model through the shared layer with + the same semantics it has today. + """ + + def __init__( + self, + *, + name: str = "keyboard-to-driver-command", + bindings: DriverBindings = DEFAULT_DRIVING_BINDINGS, + priority: int = 0, + ) -> None: + unknown = sorted(set(bindings) - _DRIVER_ACTIONS) + if unknown: + raise ValueError( + f"Unknown driver actions in bindings: {unknown}. " + f"Supported actions: {sorted(_DRIVER_ACTIONS)}." + ) + self._bindings = { + action: frozenset(normalize_key(key) for key in bindings.get(action, ())) + for action in _DRIVER_ACTIONS + } + # Tracked keys are derived, so they cannot drift from the bindings and + # silently make an action unreachable. + self._supported_keys = frozenset( + key for keys in self._bindings.values() for key in keys + ) + self._state = KeyboardState(supported_keys=self._supported_keys) + self._schema = DeviceConverterSchema( + name=name, + produces=DRIVER_COMMAND, + device_kind="keyboard", + priority=priority, + consumes=( + UserInputCapability( + event_type="key_down", + payload_fields=frozenset({"key"}), + ), + UserInputCapability( + event_type="key_up", + payload_fields=frozenset({"key"}), + ), + ), + ) + + @property + def schema(self) -> DeviceConverterSchema: + return self._schema + + def reset(self) -> None: + self._state = KeyboardState(supported_keys=self._supported_keys) + + def convert( + self, + user_inputs: UserInputs, + window: TimeWindow, + ) -> Mapping[str, Any] | None: + del window + for event in user_inputs.events: + if event.event_type not in {"key_down", "key_up"}: + continue + key = event.payload.get("key") + if not isinstance(key, str): + continue + self._state.apply_event( + event="keydown" if event.event_type == "key_down" else "keyup", + key=key, + ) + + pressed = {normalize_key(key) for key in self._state.snapshot()} + + def held(action: str) -> bool: + return bool(self._bindings[action] & pressed) + + steer = 0.0 + if held("steer_left"): + steer += 1.0 + if held("steer_right"): + steer -= 1.0 + return DRIVER_COMMAND.value( + { + "throttle": 1.0 if held("throttle") else 0.0, + "brake": 1.0 if held("brake") else 0.0, + "steer": steer, + "stop": held("stop"), + "reverse": held("reverse"), + } + ) + + +class ScriptedModality: + """Emit pre-authored canonical values, for benchmarks, replay, and tests. + + Mocking input should not require knowing the raw device vocabulary. This + converter consumes no raw capabilities, so it is feedable by any source + -- including an empty :class:`UserInputSchema` -- and application code is + identical between a real run and a scripted one. + + ``timeline`` is ``(start_s, value)`` pairs. Values are level-triggered and + held until the next entry begins, matching how live converters behave. An + entry applies to a window once it has begun by the window's end, and + ``None`` is returned for windows before the first entry. + """ + + def __init__( + self, + *, + modality: CanonicalModality, + timeline: Sequence[tuple[float, Mapping[str, Any]]], + name: str | None = None, + device_kind: str | None = "scripted", + priority: int = 0, + ) -> None: + entries = tuple(sorted(timeline, key=lambda entry: entry[0])) + for start_s, value in entries: + if start_s < 0: + raise ValueError("timeline start_s must be >= 0.") + modality.value(value) + self._entries = tuple( + (start_s, modality.value(value)) for start_s, value in entries + ) + self._modality = modality + self._schema = DeviceConverterSchema( + name=name or f"scripted-{modality.name}", + produces=modality, + device_kind=device_kind, + priority=priority, + ) + + @property + def schema(self) -> DeviceConverterSchema: + return self._schema + + def reset(self) -> None: + # The timeline is a pure function of the window, so replay is + # deterministic without any state to clear. + return None + + def convert( + self, + user_inputs: UserInputs, + window: TimeWindow, + ) -> Mapping[str, Any] | None: + del user_inputs + current: Mapping[str, Any] | None = None + for start_s, value in self._entries: + if start_s < window.end_s: + current = value + else: + break + return current + + +class InputCanonicalizer: + """Registry of device converters plus the raw-to-canonical rewrite. + + Registration is the whole extension point: a new device is a converter + registered against an existing modality, and a new modality is a converter + registered with a new :class:`CanonicalModality`. + """ + + def __init__(self, converters: Iterable[DeviceConverter] = ()) -> None: + self._converters: list[DeviceConverter] = [] + for converter in converters: + self.register(converter) + + def register(self, converter: DeviceConverter) -> None: + """Register one device converter.""" + if not isinstance(converter, DeviceConverter): + raise TypeError("converter must implement the DeviceConverter protocol.") + name = converter.schema.name + if any(existing.schema.name == name for existing in self._converters): + raise ValueError( + f"A device converter named {name!r} is already registered." + ) + self._converters.append(converter) + + @property + def converters(self) -> tuple[DeviceConverter, ...]: + """Return every registered converter.""" + return tuple(self._converters) + + def reset(self) -> None: + """Reset every registered converter's device state.""" + for converter in self._converters: + converter.reset() + + def converters_for( + self, + source_schema: UserInputSchema, + ) -> tuple[DeviceConverter, ...]: + """Return converters this source can feed, highest priority first.""" + feedable = [ + converter + for converter in self._converters + if all( + source_schema.supports(capability) + for capability in converter.schema.consumes + ) + ] + # Sort is stable, so equal-priority converters keep registration order. + return tuple(sorted(feedable, key=lambda each: -each.schema.priority)) + + def unavailable_converters( + self, + source_schema: UserInputSchema, + ) -> tuple[DeviceConverter, ...]: + """Return converters this source cannot feed, for diagnostics.""" + feedable = {id(converter) for converter in self.converters_for(source_schema)} + return tuple( + converter for converter in self._converters if id(converter) not in feedable + ) + + def canonical_schema( + self, + source_schema: UserInputSchema, + ) -> CanonicalInputSchema: + """Return the canonical modalities this raw source can supply. + + This is the boundary an application declares against. A mapping that + consumes ``driver_command`` then matches a keyboard source, a wheel + source, or any device registered later. + """ + modalities: list[CanonicalModality] = [] + for converter in self.converters_for(source_schema): + modality = converter.schema.produces + if modality not in modalities: + modalities.append(modality) + return CanonicalInputSchema( + modalities=tuple(modalities), + description=source_schema.description, + ) + + def canonicalize( + self, + user_inputs: UserInputs, + *, + window: TimeWindow, + source_schema: UserInputSchema, + ) -> CanonicalInputs: + """Convert one raw window into canonical inputs. + + Every feedable converter sees the window so its device state stays + current even while another device has precedence; that way unplugging + the higher-priority device does not resume from stale state. Among + converters producing the same modality, the highest-priority one that + returned a value wins. + """ + windowed = user_inputs.window(window) + values: dict[str, Any] = {} + sources: dict[str, str] = {} + for converter in self.converters_for(source_schema): + value = converter.convert(windowed, window) + modality = converter.schema.produces + if value is not None and modality.name not in values: + values[modality.name] = value + if converter.schema.device_kind is not None: + sources[modality.name] = converter.schema.device_kind + + metadata: dict[str, Any] = {} + if sources: + metadata["canonical_sources"] = freeze_mapping(sources) + return CanonicalInputs(values=values, metadata=metadata) diff --git a/flashdreams/flashdreams/runtime/config.py b/flashdreams/flashdreams/runtime/config.py new file mode 100644 index 000000000..f1c0c2a0c --- /dev/null +++ b/flashdreams/flashdreams/runtime/config.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime-facing configuration envelope.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +from flashdreams.runtime._utils import freeze_mapping + +ExecutionBackend = Literal["local", "local-distributed", "external", "hosted"] +"""Where and how inference compute is run.""" + +Precision = Literal["auto", "fp32", "fp16", "bf16"] +"""Coarse runtime precision choices.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class InferenceConfig: + """Runtime settings that affect model execution. + + Prompts, user controls, browser settings, output paths, and benchmark + directories intentionally live outside this object. The typed optimization + fields cover common cross-backend knobs; open-ended adapter-specific choices + can use :attr:`runtime_options`. + """ + + __hash__ = None + + model_id: str + """Stable identity for the model adapter or runtime integration.""" + + preset_id: str | None = None + """Optional preset identity under :attr:`model_id`.""" + + checkpoint: str | Path | None = None + """Optional checkpoint or model-asset selector understood by the adapter.""" + + backend: ExecutionBackend = "local" + """Execution placement and backend family for inference compute.""" + + device: str | None = None + """Optional device selector such as ``cuda`` or ``cuda:0``; ``None`` leaves placement to the adapter/backend.""" + + precision: Precision = "auto" + """Preferred compute precision.""" + + compile: bool | None = None + """Optional - Whether model compilation is requested or disabled. `None` means left to the adapter to decide.""" + + cuda_graph: bool | None = None + """Optional - Whether CUDA graph capture is requested or disabled. `None` means left to the adapter to decide.""" + + attention_backend: str | None = None + """Optional attention implementation selector; ``None`` leaves the choice to the adapter.""" + + cache_policy: str | None = None + """Optional cache policy selector; ``None`` leaves the choice to the adapter.""" + + seed: int | None = None + """Optional seed used when resolving deterministic demo/runtime behavior.""" + + runtime_options: Mapping[str, Any] = field(default_factory=dict) + """Adapter/backend-specific runtime options.""" + + resource_hints: Mapping[str, Any] = field(default_factory=dict) + """Resource hints for launchers, schedulers, or hosted backends.""" + + def __post_init__(self) -> None: + if not self.model_id.strip(): + raise ValueError("InferenceConfig.model_id must be non-empty.") + if self.seed is not None: + if isinstance(self.seed, bool) or not isinstance(self.seed, int): + raise TypeError("InferenceConfig.seed must be an integer.") + if self.seed < 0: + raise ValueError("InferenceConfig.seed must be >= 0.") + object.__setattr__( + self, "runtime_options", freeze_mapping(self.runtime_options) + ) + object.__setattr__(self, "resource_hints", freeze_mapping(self.resource_hints)) diff --git a/flashdreams/flashdreams/runtime/demo/__init__.py b/flashdreams/flashdreams/runtime/demo/__init__.py new file mode 100644 index 000000000..8d61e31bd --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/__init__.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Experimental shared demo API above the inference runtime API.""" + +from flashdreams.runtime.demo.drivers import ( + CLEANUP_TIMEOUT_S, + BatchSessionDriver, + DriverInvariantError, + RealtimeSessionDriver, + run_demo_session, + run_demo_session_async, + shielded_session_cleanup, + uncancel_current_task, +) +from flashdreams.runtime.demo.host import ( + ModelWarmupPlan, + RuntimeHost, + WarmupSessionInputs, +) +from flashdreams.runtime.demo.outputs import ( + Mp4OutputSink, + NullOutputSink, + OutputDecision, + OutputSink, + SessionInfo, + build_output_sink, + build_output_target, +) +from flashdreams.runtime.demo.pipeline import StepOutcome, StepPipeline +from flashdreams.runtime.demo.replay import OutputSinkFactory, run_replay_demo +from flashdreams.runtime.demo.run_modes import ( + AsyncSessionDriver, + BenchmarkErrorPolicy, + DefaultErrorPolicy, + ErrorAction, + InMemorySessionMetricsRecorder, + MetricsSnapshot, + Mp4ErrorPolicy, + NativeWindowErrorPolicy, + NoopTransportService, + NullErrorPolicy, + RunContext, + RunMode, + RunModeCapabilities, + RunModeWarmup, + RunResult, + RunSummary, + SessionDriver, + SessionEdges, + SingleSessionAdmissionPolicy, + WebRTCErrorPolicy, + build_model_warmup_plan, + warmup_run_context, +) +from flashdreams.runtime.demo.session_inputs import ( + BatchInputSource, + ControlDecision, + InputSource, + ModelInputProvider, + PreparedStep, + ProviderCapabilities, + RealtimeInputSource, + UserInputWindow, +) +from flashdreams.runtime.demo.spec import ( + DemoAdapter, + DemoSpec, + ModelWarmupAdapter, + Mp4OutputSpec, + NullOutputSpec, + OutputSpec, + PreparedScenario, + WebRTCAppResources, + WebRTCOutputSpec, +) +from flashdreams.runtime.demo.timing import ( + ActivationPolicy, + ActivationResult, + ActivationSignal, + AlwaysActiveActivationPolicy, + CatchUpDecision, + CatchUpPolicy, + DeterministicClock, + RealtimeClock, + RealtimeEventInputSource, + RealtimeEventResampler, + RealtimeWindowResult, + ResamplerRealtimeClock, + SignalActivationPolicy, + input_frame_count_from_request, +) +from flashdreams.runtime.demo.validation import ( + ResolvedRunCapabilities, + resolve_run_capabilities, + validate_resolved_run, +) + +__all__ = [ + "BatchInputSource", + "BatchSessionDriver", + "CLEANUP_TIMEOUT_S", + "ControlDecision", + "DefaultErrorPolicy", + "DemoAdapter", + "DemoSpec", + "DriverInvariantError", + "ErrorAction", + "AsyncSessionDriver", + "ActivationPolicy", + "ActivationResult", + "ActivationSignal", + "AlwaysActiveActivationPolicy", + "BenchmarkErrorPolicy", + "InMemorySessionMetricsRecorder", + "InputSource", + "CatchUpDecision", + "CatchUpPolicy", + "DeterministicClock", + "MetricsSnapshot", + "ModelWarmupAdapter", + "ModelWarmupPlan", + "ModelInputProvider", + "Mp4ErrorPolicy", + "Mp4OutputSink", + "Mp4OutputSpec", + "NativeWindowErrorPolicy", + "NoopTransportService", + "NullOutputSpec", + "NullOutputSink", + "NullErrorPolicy", + "OutputDecision", + "OutputSinkFactory", + "OutputSpec", + "OutputSink", + "PreparedScenario", + "PreparedStep", + "ProviderCapabilities", + "RealtimeInputSource", + "RealtimeClock", + "RealtimeEventInputSource", + "RealtimeEventResampler", + "RealtimeSessionDriver", + "RealtimeWindowResult", + "ResolvedRunCapabilities", + "ResamplerRealtimeClock", + "RunContext", + "RunMode", + "RunModeCapabilities", + "RunModeWarmup", + "RunResult", + "RunSummary", + "RuntimeHost", + "SessionEdges", + "SessionDriver", + "SessionInfo", + "SignalActivationPolicy", + "SingleSessionAdmissionPolicy", + "StepOutcome", + "StepPipeline", + "UserInputWindow", + "WarmupSessionInputs", + "WebRTCAppResources", + "WebRTCErrorPolicy", + "WebRTCOutputSpec", + "build_output_sink", + "build_output_target", + "build_model_warmup_plan", + "input_frame_count_from_request", + "resolve_run_capabilities", + "run_demo_session", + "run_demo_session_async", + "run_replay_demo", + "shielded_session_cleanup", + "uncancel_current_task", + "validate_resolved_run", + "warmup_run_context", +] diff --git a/flashdreams/flashdreams/runtime/demo/app.py b/flashdreams/flashdreams/runtime/demo/app.py new file mode 100644 index 000000000..b763e4eaf --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/app.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared command lifecycle for model demo applications.""" + +from __future__ import annotations + +import argparse +import sys +from abc import ABC, abstractmethod +from typing import Any + +import torch +import torch.distributed as dist + +from flashdreams.core.distributed import init as distributed_init +from flashdreams.runtime.demo.bootstrap import ( + configure_logging, + initialize_cuda_distributed, +) +from flashdreams.runtime.demo.replay import run_replay_demo +from flashdreams.runtime.demo.spec import DemoAdapter, DemoSpec + + +class DemoApplication(ABC): + """Base command application shared by model replay and WebRTC demos.""" + + def main(self, argv: list[str] | None = None) -> None: + """Parse arguments and dispatch the selected demo mode.""" + configure_logging() + args = self.parse_args(argv) + if args.command == "replay": + result = run_replay_demo( + spec=self.replay_spec(args), + adapter=self.replay_adapter(), + ) + if result.status != "completed": + reason = result.reason or ( + str(result.error) if result.error is not None else None + ) + if reason is None: + reason = f"Replay demo ended with status {result.status!r}." + print(reason, file=sys.stderr) + raise SystemExit(1) + return + if args.command == "webrtc": + context = initialize_cuda_distributed( + default_device=args.device, + distributed_init_fn=distributed_init, + configure_logging_fn=configure_logging, + torch_module=torch, + dist_module=dist, + ) + self.prepare_webrtc(args, context=context) + self.serve_webrtc(args, context=context) + return + raise AssertionError(f"Unhandled command: {args.command}") + + @abstractmethod + def parse_args(self, argv: list[str] | None = None) -> argparse.Namespace: + """Parse this model's command-line arguments.""" + + @abstractmethod + def replay_spec(self, args: argparse.Namespace) -> DemoSpec: + """Build the model-specific replay specification.""" + + @abstractmethod + def replay_adapter(self) -> DemoAdapter: + """Create the model-specific replay adapter.""" + + def prepare_webrtc(self, args: argparse.Namespace, *, context: Any) -> None: + """Perform optional model-specific setup before serving WebRTC.""" + del args, context + + @abstractmethod + def serve_webrtc(self, args: argparse.Namespace, *, context: Any) -> None: + """Build and serve the model-specific WebRTC demo.""" + + +__all__ = ["DemoApplication"] diff --git a/flashdreams/flashdreams/runtime/demo/bootstrap.py b/flashdreams/flashdreams/runtime/demo/bootstrap.py new file mode 100644 index 000000000..6b1e68f5d --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/bootstrap.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared process bootstrap for demo applications.""" + +from __future__ import annotations + +import logging +import os +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import torch +import torch.distributed as dist + + +@dataclass(frozen=True, slots=True) +class DistributedDemoContext: + """CUDA/distributed launch context for a demo process.""" + + device: torch.device + world_rank: int + world_size: int + + +def configure_logging(*, world_rank: int | None = None) -> None: + from flashdreams.core.distributed import configure_loguru_for_distributed + + configure_loguru_for_distributed(world_rank=world_rank) + for logger_name in ("aioice", "aioice.ice", "aiortc"): + logging.getLogger(logger_name).setLevel(logging.WARNING) + + +def _distributed_init() -> None: + from flashdreams.core.distributed import init as distributed_init + + distributed_init() + + +def initialize_cuda_distributed( + *, + default_device: str | torch.device = "cuda:0", + distributed_init_fn: Callable[[], object] | None = None, + configure_logging_fn: Callable[..., None] = configure_logging, + torch_module: Any = torch, + dist_module: Any = dist, +) -> DistributedDemoContext: + """Initialize CUDA and optional torch.distributed for demo serving.""" + if not torch_module.cuda.is_available(): + raise RuntimeError("CUDA is required for inference in the demo server.") + + has_rank = "RANK" in os.environ + has_world_size = "WORLD_SIZE" in os.environ + if has_rank != has_world_size: + raise RuntimeError( + "Distributed launch expects both RANK and WORLD_SIZE to be set." + ) + + distributed_launch = has_rank and has_world_size + if distributed_launch: + if distributed_init_fn is None: + distributed_init_fn = _distributed_init + distributed_init_fn() + world_rank = dist_module.get_rank() + world_size = dist_module.get_world_size() + else: + world_rank = 0 + world_size = 1 + + device_count = torch_module.cuda.device_count() + if device_count < 1: + raise RuntimeError("CUDA device count must be >= 1 for inference.") + if distributed_launch: + local_rank = world_rank % device_count + torch_device = torch_module.device(f"cuda:{local_rank}") + else: + torch_device = torch_module.device(default_device) + if torch_device.type != "cuda": + raise RuntimeError( + f"CUDA device is required for inference, got {torch_device}." + ) + if torch_device.index is None: + torch_device = torch_module.device("cuda:0") + torch_module.cuda.set_device(torch_device) + configure_logging_fn(world_rank=world_rank) + return DistributedDemoContext( + device=torch_device, + world_rank=world_rank, + world_size=world_size, + ) + + +__all__ = [ + "DistributedDemoContext", + "configure_logging", + "initialize_cuda_distributed", +] diff --git a/flashdreams/flashdreams/runtime/demo/drivers.py b/flashdreams/flashdreams/runtime/demo/drivers.py new file mode 100644 index 000000000..9bdd40db8 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/drivers.py @@ -0,0 +1,1011 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Session drivers and helpers for demo runtime vertical slices.""" + +from __future__ import annotations + +import asyncio +import inspect +from typing import Any, cast + +from flashdreams.runtime.interfaces import InferenceSession +from flashdreams.runtime.types import ( + StepRequest, + StepRequirements, + step_requirements_from_request, +) + +from .host import RuntimeHost +from .outputs import SessionInfo +from .pipeline import StepPipeline +from .run_modes import ( + DriverStatus, + RunContext, + RunMode, + RunResult, + SessionEdges, + SessionReservation, +) +from .session_inputs import BatchInputSource, ModelInputProvider +from .spec import DemoAdapter, DemoSpec, PreparedScenario +from .timing import ActivationPolicy, RealtimeClock +from .validation import resolve_run_capabilities, validate_resolved_run + +CLEANUP_TIMEOUT_S = 30.0 +_MODEL_CLEANUP_FAILED_REASON = "model-affine cleanup failed" +_MODEL_CLEANUP_TIMED_OUT_REASON = "model-affine cleanup timed out" + + +class DriverInvariantError(RuntimeError): + """A driver invariant was violated; this is a driver bug, not a run result.""" + + +class BatchSessionDriver: + """Minimal finite-session driver for Phase 2 fake-model coverage.""" + + def run_one_session( + self, + *, + host: RuntimeHost, + provider: ModelInputProvider, + session_edges: SessionEdges, + pipeline: StepPipeline, + ) -> RunResult: + session: InferenceSession | None = None + final_status: DriverStatus = "completed" + final_reason: str | None = None + final_error: Exception | None = None + invariant_closed = False + setup_ok = False + try: + try: + initial_input = host.call(provider.prepare_initial_input) + session = host.call(host.start_session, initial_input) + session_info = host.call(_session_info, session) + session_edges.output_sink.open(session_info) + setup_ok = True + except Exception as exc: + action = session_edges.error_policy.handle_setup_error(exc) + if action.drop_chunk or action.result_status == "completed": + raise DriverInvariantError( + "Setup failures must resolve to failed or skipped." + ) from exc + session_edges.metrics.record_error(exc, action) + final_status = action.result_status + final_reason = str(exc) + final_error = exc if action.result_status == "failed" else None + + input_source = cast(BatchInputSource, session_edges.input_source) + while setup_ok: + if session is None: + raise DriverInvariantError("setup_ok was set without a session.") + try: + if session_edges.input_source.is_finished(): + break + request = _next_step_requirements(host=host, session=session) + if request is None: + break + user_window = input_source.next_window(request) + outcome = host.call( + pipeline.execute_step, + request=request, + user_window=user_window, + provider=provider, + session=session, + output=session_edges.output_sink, + metrics=session_edges.metrics, + ) + if outcome.control.reset: + host.call(session.reset, outcome.control.reset_input) + if not outcome.control.provider_already_reset: + host.call(provider.reset, outcome.control.reset_input) + continue + if outcome.control.close_session: + break + if outcome.output.should_stop: + break + except DriverInvariantError: + raise + except Exception as exc: + action = session_edges.error_policy.handle(exc) + session_edges.metrics.record_error(exc, action) + if action.drop_chunk: + continue + final_status = action.result_status + final_reason = str(exc) + final_error = exc if action.result_status == "failed" else None + break + except DriverInvariantError as exc: + if session is not None: + _close_on_host_best_effort( + host=host, + close=session.close, + session_edges=session_edges, + ) + _close_on_host_best_effort( + host=host, + close=provider.close, + session_edges=session_edges, + ) + session_edges.close_result( + status="failed", + reason=str(exc), + error=exc, + ) + invariant_closed = True + raise + except Exception as exc: + final_status = "failed" + final_reason = str(exc) + final_error = exc + finally: + if not invariant_closed: + if session is not None: + _close_on_host_best_effort( + host=host, + close=session.close, + session_edges=session_edges, + ) + _close_on_host_best_effort( + host=host, + close=provider.close, + session_edges=session_edges, + ) + + return session_edges.close_result( + status=final_status, + reason=final_reason, + error=final_error, + ) + + +class RealtimeSessionDriver: + """Async realtime session driver built on shared Phase 5 primitives.""" + + cleanup_timeout_s: float + + def __init__(self, *, cleanup_timeout_s: float = CLEANUP_TIMEOUT_S) -> None: + if cleanup_timeout_s <= 0: + raise ValueError("cleanup_timeout_s must be > 0.") + self.cleanup_timeout_s = float(cleanup_timeout_s) + + async def run_one_session( + self, + *, + host: RuntimeHost, + provider: ModelInputProvider, + session_edges: SessionEdges, + pipeline: StepPipeline, + ) -> RunResult: + session: InferenceSession | None = None + final_status: DriverStatus = "completed" + final_reason: str | None = None + final_error: Exception | None = None + setup_ok = False + generation = 0 + first_step_started = False + invariant_error: DriverInvariantError | None = None + try: + activation, clock = _realtime_activation_and_clock(session_edges) + input_source = _realtime_input_source(session_edges) + activation_result = await activation.wait_until_active(clock) + if not activation_result.activated: + final_status = "not_activated" + final_reason = activation_result.reason + elif not session_edges.transport.is_active(): + final_status = "not_activated" + final_reason = "transport closed before first step" + else: + try: + initial_input = await host.call_async( + provider.prepare_initial_input + ) + session = await host.call_async(host.start_session, initial_input) + session_info = await host.call_async(_session_info, session) + session_edges.output_sink.open(session_info) + session_edges.output_sink.begin_generation(generation) + setup_ok = True + except Exception as exc: + action = session_edges.error_policy.handle_setup_error(exc) + if action.drop_chunk or action.result_status == "completed": + raise DriverInvariantError( + "Setup failures must resolve to failed or skipped." + ) from exc + session_edges.metrics.record_error(exc, action) + final_status = action.result_status + final_reason = str(exc) + final_error = exc if action.result_status == "failed" else None + + while setup_ok: + if session is None: + raise DriverInvariantError("setup_ok was set without a session.") + if not session_edges.transport.is_active(): + if not first_step_started: + final_status = "not_activated" + final_reason = "transport closed before first step" + break + try: + request = await _next_step_requirements_async( + host=host, + session=session, + ) + if request is None: + break + window_result = await input_source.next_realtime_window( + request=request, + clock=clock, + ) + session_edges.metrics.record_catch_up(window_result.catch_up) + if ( + not session_edges.transport.is_active() + and not first_step_started + ): + final_status = "not_activated" + final_reason = "transport closed before first step" + break + outcome = await host.call_async( + pipeline.execute_step, + request=request, + user_window=window_result.window, + provider=provider, + session=session, + output=session_edges.output_sink, + metrics=session_edges.metrics, + ) + first_step_started = True + if outcome.control.reset: + await host.call_async( + session.reset, + outcome.control.reset_input, + ) + if not outcome.control.provider_already_reset: + await host.call_async( + provider.reset, + outcome.control.reset_input, + ) + generation += 1 + session_edges.output_sink.begin_generation(generation) + continue + if outcome.control.close_session: + break + if outcome.output.should_stop: + break + if outcome.output.backpressure_s > 0: + await clock.apply_backpressure(outcome.output.backpressure_s) + except DriverInvariantError: + raise + except Exception as exc: + action = session_edges.error_policy.handle(exc) + session_edges.metrics.record_error(exc, action) + if action.close_session: + final_status = action.result_status + final_reason = str(exc) + final_error = exc if action.result_status == "failed" else None + break + if action.drop_chunk: + continue + final_status = "failed" + final_reason = str(exc) + final_error = exc + break + except asyncio.CancelledError: + uncancel_current_task() + final_status = "cancelled" + final_reason = ( + "cancelled before first step" if session is None else "cancelled" + ) + final_error = None + except DriverInvariantError as exc: + invariant_error = exc + final_status = "failed" + final_reason = str(exc) + final_error = exc + except Exception as exc: + final_status = "failed" + final_reason = str(exc) + final_error = exc + + result = await shielded_session_cleanup( + host=host, + session=session, + provider=provider, + session_edges=session_edges, + status=final_status, + reason=final_reason, + error=final_error, + timeout_s=self.cleanup_timeout_s, + ) + if invariant_error is not None: + raise invariant_error + return result + + +def _mark_host_cleanup_failed(host: RuntimeHost, exc: Exception | None = None) -> None: + host.mark_unhealthy(_MODEL_CLEANUP_FAILED_REASON, exc) + + +def run_demo_session( + *, + context: RunContext, + spec: DemoSpec, + scenario: PreparedScenario, + adapter: DemoAdapter, + run_mode: RunMode, + pipeline: StepPipeline, + reservation: SessionReservation | None = None, +) -> RunResult: + """Run one prepared demo session through a selected run mode.""" + if reservation is None: + reservation = context.admission.try_reserve() + if reservation is None: + result = RunResult.rejected(reason="busy") + context.run_metrics.record_session(result) + return result + + provider: Any | None = None + session_edges: SessionEdges | None = None + driver_started = False + try: + create_provider = getattr(adapter, "create_model_input_provider") + provider = context.host.call(create_provider, spec, scenario) + run_mode.validate_session( + spec=spec, + scenario=scenario, + adapter=adapter, + provider=provider, + ) + session_edges = run_mode.create_session_edges( + context=context, + spec=spec, + scenario=scenario, + provider=provider, + adapter=adapter, + ) + resolved_capabilities = resolve_run_capabilities( + spec=spec, + provider=provider, + session_edges=session_edges, + ) + validate_resolved_run( + spec=spec, + adapter=adapter, + provider=provider, + run_mode=run_mode, + session_edges=session_edges, + resolved=resolved_capabilities, + ) + if session_edges.is_closed: + raise DriverInvariantError( + "RunMode returned already closed SessionEdges; session edges " + "must not be reused." + ) + driver = run_mode.select_driver() + driver_started = True + result = _run_sync_driver( + driver=driver, + host=context.host, + provider=provider, + session_edges=session_edges, + pipeline=pipeline, + ) + context.run_metrics.record_session(result) + return result + except DriverInvariantError as exc: + _record_run_session_error(context, exc) + if provider is not None and not driver_started: + _close_partial_provider_sync( + context=context, + provider=provider, + session_edges=session_edges, + ) + if session_edges is not None and ( + driver_started or not session_edges.is_closed + ): + result = session_edges.close_result( + status="failed", + reason=str(exc), + error=exc, + ) + context.run_metrics.record_session(result) + raise + except Exception as exc: + _record_run_session_error(context, exc) + if provider is not None and not driver_started: + _close_partial_provider_sync( + context=context, + provider=provider, + session_edges=session_edges, + ) + if session_edges is not None and ( + driver_started or not session_edges.is_closed + ): + result = session_edges.close_result( + status="failed", + reason=str(exc), + error=exc, + ) + else: + result = RunResult(status="failed", reason=str(exc), error=exc) + context.run_metrics.record_session(result) + return result + finally: + reservation.release() + + +async def run_demo_session_async( + *, + context: RunContext, + spec: DemoSpec, + scenario: PreparedScenario, + adapter: DemoAdapter, + run_mode: RunMode, + pipeline: StepPipeline, + reservation: SessionReservation | None = None, +) -> RunResult: + """Run one prepared async/realtime demo session through a selected run mode.""" + if reservation is None: + reservation = context.admission.try_reserve() + if reservation is None: + result = RunResult.rejected(reason="busy") + context.run_metrics.record_session(result) + return result + + provider: Any | None = None + session_edges: SessionEdges | None = None + try: + try: + create_provider = getattr(adapter, "create_model_input_provider") + provider = await context.host.call_async(create_provider, spec, scenario) + run_mode.validate_session( + spec=spec, + scenario=scenario, + adapter=adapter, + provider=provider, + ) + session_edges = run_mode.create_session_edges( + context=context, + spec=spec, + scenario=scenario, + provider=provider, + adapter=adapter, + ) + resolved_capabilities = resolve_run_capabilities( + spec=spec, + provider=provider, + session_edges=session_edges, + ) + validate_resolved_run( + spec=spec, + adapter=adapter, + provider=provider, + run_mode=run_mode, + session_edges=session_edges, + resolved=resolved_capabilities, + ) + if session_edges.is_closed: + raise DriverInvariantError( + "RunMode returned already closed SessionEdges; session edges " + "must not be reused." + ) + driver = run_mode.select_driver() + result = await _run_async_driver( + driver=driver, + host=context.host, + provider=provider, + session_edges=session_edges, + pipeline=pipeline, + ) + context.run_metrics.record_session(result) + return result + except asyncio.CancelledError: + uncancel_current_task() + result = await _close_partial_session_async( + context=context, + provider=provider, + session_edges=session_edges, + status="cancelled", + reason="cancelled during session assembly", + error=None, + close_provider=_needs_partial_provider_cleanup(session_edges), + ) + context.run_metrics.record_session(result) + return result + except DriverInvariantError as exc: + _record_run_session_error(context, exc) + should_record_session = session_edges is not None + result = await _close_partial_session_async( + context=context, + provider=provider, + session_edges=session_edges, + status="failed", + reason=str(exc), + error=exc, + close_provider=_needs_partial_provider_cleanup(session_edges), + ) + if should_record_session: + context.run_metrics.record_session(result) + raise + except Exception as exc: + _record_run_session_error(context, exc) + result = await _close_partial_session_async( + context=context, + provider=provider, + session_edges=session_edges, + status="failed", + reason=str(exc), + error=exc, + close_provider=_needs_partial_provider_cleanup(session_edges), + ) + context.run_metrics.record_session(result) + return result + finally: + reservation.release() + + +def _session_info(session: InferenceSession) -> SessionInfo: + session_info = getattr(session, "session_info", None) + if not callable(session_info): + return SessionInfo() + value = session_info() + if not isinstance(value, SessionInfo): + raise TypeError( + "session.session_info() must return SessionInfo, " + f"got {type(value).__name__}." + ) + return value + + +def _next_step_requirements( + *, + host: RuntimeHost, + session: InferenceSession, +) -> StepRequirements | None: + next_requirements = getattr(session, "next_step_requirements", None) + if callable(next_requirements): + return _coerce_step_requirements(host.call(next_requirements)) + + next_request = getattr(session, "next_step_request", None) + if not callable(next_request): + raise TypeError( + "InferenceSession must provide next_step_requirements() or " + "legacy next_step_request()." + ) + return _coerce_step_requirements(host.call(next_request)) + + +async def _next_step_requirements_async( + *, + host: RuntimeHost, + session: InferenceSession, +) -> StepRequirements | None: + next_requirements = getattr(session, "next_step_requirements", None) + if callable(next_requirements): + return _coerce_step_requirements(await host.call_async(next_requirements)) + + next_request = getattr(session, "next_step_request", None) + if not callable(next_request): + raise TypeError( + "InferenceSession must provide next_step_requirements() or " + "legacy next_step_request()." + ) + return _coerce_step_requirements(await host.call_async(next_request)) + + +def _coerce_step_requirements(value: object) -> StepRequirements | None: + if value is None: + return None + if isinstance(value, StepRequirements): + return value + if isinstance(value, StepRequest): + return step_requirements_from_request(value) + raise TypeError( + "Session next-step method must return StepRequirements, legacy " + f"StepRequest, or None; got {type(value).__name__}." + ) + + +def _close_safely(close: Any, session_edges: SessionEdges) -> bool: + try: + close() + except Exception as exc: + session_edges.record_cleanup_error(exc) + return False + return True + + +def _close_on_host_best_effort( + *, + host: RuntimeHost, + close: Any, + session_edges: SessionEdges, +) -> bool: + try: + cleanup_succeeded = host.call(_close_safely, close, session_edges) + except Exception as exc: + # If the host/worker is already unavailable, do not fall back to calling + # model-affine cleanup directly on the caller thread. Record the loss and + # let close_result finalize output, transport, and metrics. + session_edges.record_cleanup_error(exc) + _mark_host_cleanup_failed(host, exc) + return False + if not cleanup_succeeded: + _mark_host_cleanup_failed(host) + return False + return True + + +def _close_partial_provider_sync( + *, + context: RunContext, + provider: Any, + session_edges: SessionEdges | None, +) -> None: + if session_edges is not None: + _close_on_host_best_effort( + host=context.host, + close=provider.close, + session_edges=session_edges, + ) + return + try: + cleanup_succeeded = context.host.call( + _close_run_provider_safely, + provider.close, + context, + ) + except Exception as exc: + _record_run_cleanup_error(context, exc) + _mark_host_cleanup_failed(context.host, exc) + return + if not cleanup_succeeded: + _mark_host_cleanup_failed(context.host) + + +def _close_run_provider_safely(close: Any, context: RunContext) -> bool: + try: + close() + except Exception as exc: + _record_run_cleanup_error(context, exc) + return False + return True + + +async def shielded_session_cleanup( + *, + host: RuntimeHost, + session: InferenceSession | None, + provider: ModelInputProvider, + session_edges: SessionEdges, + status: DriverStatus, + reason: str | None, + error: Exception | None, + timeout_s: float = CLEANUP_TIMEOUT_S, +) -> RunResult: + """Close realtime session resources exactly once without leaking cancellation.""" + + if timeout_s <= 0: + session_edges.record_cleanup_error(ValueError("timeout_s must be > 0.")) + return session_edges.close_result(status=status, reason=reason, error=error) + + async def cleanup() -> RunResult: + unhealthy_reason = await _close_model_resources_async( + host=host, + session=session, + provider=provider, + session_edges=session_edges, + timeout_s=timeout_s, + ) + if unhealthy_reason is not None: + host.mark_unhealthy(unhealthy_reason) + return session_edges.close_result( + status=status, + reason=reason, + error=error, + ) + + try: + cleanup_task = asyncio.create_task(cleanup()) + except RuntimeError as exc: + session_edges.record_cleanup_error(exc) + return session_edges.close_result(status=status, reason=reason, error=error) + + session_edges.cleanup_tasks.add(cleanup_task) + try: + while not cleanup_task.done(): + try: + await asyncio.shield(cleanup_task) + except asyncio.CancelledError: + uncancel_current_task() + continue + except Exception: + break + return _cleanup_result(cleanup_task, session_edges, status, reason, error) + finally: + session_edges.cleanup_tasks.discard(cleanup_task) + + +def _run_sync_driver( + *, + driver: object, + host: RuntimeHost, + provider: ModelInputProvider, + session_edges: SessionEdges, + pipeline: StepPipeline, +) -> RunResult: + run_one_session = getattr(driver, "run_one_session", None) + if not callable(run_one_session): + raise TypeError( + "RunMode.select_driver() must return an object with run_one_session(...)." + ) + result = run_one_session( + host=host, + provider=provider, + session_edges=session_edges, + pipeline=pipeline, + ) + if inspect.isawaitable(result): + raise TypeError( + "run_demo_session(...) requires a synchronous session driver; " + "use run_demo_session_async(...) for async drivers." + ) + if not isinstance(result, RunResult): + raise TypeError( + "Session driver run_one_session(...) must return RunResult, " + f"got {type(result).__name__}." + ) + return result + + +async def _run_async_driver( + *, + driver: object, + host: RuntimeHost, + provider: ModelInputProvider, + session_edges: SessionEdges, + pipeline: StepPipeline, +) -> RunResult: + run_one_session = getattr(driver, "run_one_session", None) + if not callable(run_one_session): + raise TypeError( + "RunMode.select_driver() must return an object with run_one_session(...)." + ) + result = run_one_session( + host=host, + provider=provider, + session_edges=session_edges, + pipeline=pipeline, + ) + if not inspect.isawaitable(result): + raise TypeError("run_demo_session_async(...) requires an async session driver.") + resolved = await result + if not isinstance(resolved, RunResult): + raise TypeError( + "Async session driver run_one_session(...) must return RunResult, " + f"got {type(resolved).__name__}." + ) + return resolved + + +async def _close_partial_session_async( + *, + context: RunContext, + provider: Any | None, + session_edges: SessionEdges | None, + status: DriverStatus, + reason: str | None, + error: Exception | None, + close_provider: bool, +) -> RunResult: + if provider is not None and close_provider and session_edges is not None: + return await shielded_session_cleanup( + host=context.host, + session=None, + provider=provider, + session_edges=session_edges, + status=status, + reason=reason, + error=error, + ) + if provider is not None and close_provider: + await _close_provider_async( + context=context, + provider=provider, + session_edges=session_edges, + ) + if session_edges is not None: + return session_edges.close_result(status=status, reason=reason, error=error) + return RunResult(status=status, reason=reason, error=error) + + +def _needs_partial_provider_cleanup(session_edges: SessionEdges | None) -> bool: + return session_edges is None or not session_edges.is_closed + + +async def _close_provider_async( + *, + context: RunContext, + provider: Any, + session_edges: SessionEdges | None, +) -> None: + try: + close_task = asyncio.create_task(context.host.call_async(provider.close)) + except RuntimeError as close_exc: + _record_provider_cleanup_error( + context=context, + session_edges=session_edges, + exc=close_exc, + ) + return + + while not close_task.done(): + try: + await asyncio.shield(close_task) + except asyncio.CancelledError: + uncancel_current_task() + continue + except Exception: + break + + try: + await close_task + except asyncio.CancelledError: + uncancel_current_task() + _record_provider_cleanup_error( + context=context, + session_edges=session_edges, + exc=RuntimeError("provider cleanup was cancelled"), + ) + except Exception as close_exc: + _record_provider_cleanup_error( + context=context, + session_edges=session_edges, + exc=close_exc, + ) + + +def _record_provider_cleanup_error( + *, + context: RunContext, + session_edges: SessionEdges | None, + exc: Exception, +) -> None: + if session_edges is not None: + session_edges.record_cleanup_error(exc) + else: + _record_run_cleanup_error(context, exc) + # Partial async assembly may only have a provider to close. If that + # model-affine cleanup fails, quarantine the host instead of admitting a new + # session onto a worker that may still own model resources. + _mark_host_cleanup_failed(context.host, exc) + + +def _record_run_cleanup_error(context: RunContext, exc: Exception) -> None: + try: + context.run_metrics.record_cleanup_error(exc) + except Exception: + return + + +def _record_run_session_error(context: RunContext, exc: Exception) -> None: + try: + context.run_metrics.record_session_error(exc) + except Exception: + return + + +async def _close_model_resources_async( + *, + host: RuntimeHost, + session: InferenceSession | None, + provider: ModelInputProvider, + session_edges: SessionEdges, + timeout_s: float, +) -> str | None: + try: + resources_closed = await asyncio.wait_for( + host.call_async( + _close_model_resources_safely, + session.close if session is not None else None, + provider.close, + session_edges, + ), + timeout=timeout_s, + ) + except asyncio.TimeoutError as exc: + # Keep provider cleanup ordered behind session cleanup on the model worker. + # A timed-out session close may still hold CUDA/Triton state, so running + # provider cleanup on another thread or replacing the worker is unsafe. + # The caller marks the host unhealthy so future sessions reject instead. + session_edges.record_orphaned_cleanup(exc) + return _MODEL_CLEANUP_TIMED_OUT_REASON + except Exception as exc: + session_edges.record_cleanup_error(exc) + return _MODEL_CLEANUP_FAILED_REASON + if not resources_closed: + return _MODEL_CLEANUP_FAILED_REASON + return None + + +def _close_model_resources_safely( + session_close: Any | None, + provider_close: Any, + session_edges: SessionEdges, +) -> bool: + resources_closed = True + # Session and provider close are intentionally ordered on the model worker. + # If session close hangs, timeout handling records orphaned cleanup and + # quarantines the host rather than moving provider close to another thread. + if session_close is not None: + resources_closed = _close_safely(session_close, session_edges) + return _close_safely(provider_close, session_edges) and resources_closed + + +def _cleanup_result( + cleanup_task: asyncio.Task[RunResult], + session_edges: SessionEdges, + status: DriverStatus, + reason: str | None, + error: Exception | None, +) -> RunResult: + if cleanup_task.done() and not cleanup_task.cancelled(): + exc = cleanup_task.exception() + if exc is None: + return cleanup_task.result() + if isinstance(exc, Exception): + session_edges.record_cleanup_error(exc) + else: + session_edges.record_cleanup_error( + RuntimeError(f"cleanup failed with {type(exc).__name__}") + ) + return session_edges.close_result(status=status, reason=reason, error=error) + + +def _realtime_activation_and_clock( + session_edges: SessionEdges, +) -> tuple[ActivationPolicy, RealtimeClock]: + activation = session_edges.activation + if activation is None: + raise DriverInvariantError( + "RealtimeSessionDriver requires SessionEdges.activation." + ) + clock = session_edges.clock + if not isinstance(clock, RealtimeClock): + raise DriverInvariantError("RealtimeSessionDriver requires a RealtimeClock.") + return activation, clock + + +def _realtime_input_source(session_edges: SessionEdges) -> Any: + input_source = session_edges.input_source + next_realtime_window = getattr(input_source, "next_realtime_window", None) + if not callable(next_realtime_window): + raise DriverInvariantError( + "RealtimeSessionDriver requires a RealtimeInputSource." + ) + return input_source + + +def uncancel_current_task() -> None: + task = asyncio.current_task() + if task is None: + return + uncancel = getattr(task, "uncancel", None) + if not callable(uncancel): + return + cancelling = getattr(task, "cancelling", None) + if not callable(cancelling): + return + while cancelling(): + uncancel() + + +__all__ = [ + "BatchSessionDriver", + "CLEANUP_TIMEOUT_S", + "DriverInvariantError", + "RealtimeSessionDriver", + "run_demo_session", + "run_demo_session_async", + "shielded_session_cleanup", + "uncancel_current_task", +] diff --git a/flashdreams/flashdreams/runtime/demo/host.py b/flashdreams/flashdreams/runtime/demo/host.py new file mode 100644 index 000000000..6a6e93ce5 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/host.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime host and model-execution boundary for shared demos.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import TypeVar + +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.inputs import InferenceInput +from flashdreams.runtime.interfaces import InferenceRuntime, InferenceSession +from flashdreams.runtime.worker import ModelExecutionWorker + +_T = TypeVar("_T") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WarmupSessionInputs: + """Inputs used to warm one temporary runtime session.""" + + initial_input: InferenceInput + step_inputs: Sequence[InferenceInput] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "step_inputs", tuple(self.step_inputs)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ModelWarmupPlan: + """Host-owned model warmup plan built by a demo adapter or run mode.""" + + sessions: Sequence[WarmupSessionInputs] = () + measured: bool = False + metadata: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "sessions", tuple(self.sessions)) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +class RuntimeHost: + """Own one runtime and the worker used for model-affine calls.""" + + def __init__( + self, + runtime: InferenceRuntime, + *, + worker: ModelExecutionWorker | None = None, + is_control_rank: bool = True, + worker_loop: Callable[[], None] | None = None, + ) -> None: + self._runtime = runtime + self._worker = worker or ModelExecutionWorker() + self._is_control_rank = is_control_rank + self._worker_loop = worker_loop + self._healthy = True + self._closed = False + self._unhealthy_reason: str | None = None + self._unhealthy_error: Exception | None = None + + @property + def runtime(self) -> InferenceRuntime: + """Return the hosted runtime.""" + return self._runtime + + @property + def worker(self) -> ModelExecutionWorker: + """Return the host's model-execution worker.""" + return self._worker + + @property + def is_control_rank(self) -> bool: + """Whether this process owns run modes, providers, sinks, and metrics.""" + return self._is_control_rank + + @property + def is_healthy(self) -> bool: + """Return whether admission should continue accepting sessions.""" + return self._healthy and not self._closed + + @property + def unhealthy_reason(self) -> str | None: + """Return the first latched unhealthy reason, if any.""" + return self._unhealthy_reason + + @property + def unhealthy_error(self) -> Exception | None: + """Return the first latched unhealthy error, if any.""" + return self._unhealthy_error + + def mark_unhealthy( + self, + reason: str = "marked unhealthy", + error: Exception | None = None, + ) -> None: + """Latch the host as unhealthy without overwriting the first reason.""" + if not self._healthy: + return + self._healthy = False + self._unhealthy_reason = reason + self._unhealthy_error = error + + def preload(self) -> None: + """Initialize optional distributed state and preload runtime resources.""" + self._call_optional_runtime_hook("initialize_distributed") + self._call_optional_runtime_hook("preload") + + def warmup(self, plan: ModelWarmupPlan | None = None) -> None: + """Run warmup sessions through the same worker boundary as real sessions.""" + plan = plan or ModelWarmupPlan() + for warmup_session in plan.sessions: + session = self.call(self.start_session, warmup_session.initial_input) + try: + for step_input in warmup_session.step_inputs: + self.call(session.step, step_input) + finally: + self.call(session.close) + + def call(self, func: Callable[..., _T], /, *args: object, **kwargs: object) -> _T: + """Run one model-affine callable synchronously on the worker.""" + self._require_open() + return self._worker.call_blocking(func, *args, **kwargs) + + async def call_async( + self, + func: Callable[..., _T], + /, + *args: object, + **kwargs: object, + ) -> _T: + """Run model-affine work without blocking realtime event loops.""" + self._require_open() + return await self._worker.call(func, *args, **kwargs) + + def start_session(self, inputs: InferenceInput) -> InferenceSession: + """Start one inference session through the hosted runtime.""" + self._require_open() + return self._runtime.start_session(inputs) + + def run_worker_loop(self) -> None: + """Serve control-rank work on non-control ranks until runtime shutdown.""" + worker_loop = self._worker_loop + if worker_loop is None: + worker_loop = getattr(self._runtime, "run_worker_loop", None) + if worker_loop is None: + worker_loop = getattr(self._runtime, "wait_for_termination", None) + if callable(worker_loop): + worker_loop() + + def close(self) -> None: + """Close runtime-owned state and stop the model-execution worker.""" + if self._closed: + return + try: + self._worker.call_blocking(self._runtime.close) + self._call_optional_runtime_hook("close_distributed") + finally: + self._closed = True + self._worker.close_blocking() + + def _call_optional_runtime_hook(self, name: str) -> None: + hook = getattr(self._runtime, name, None) + if callable(hook): + self.call(hook) + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("runtime host is closed") + + +__all__ = ["ModelWarmupPlan", "RuntimeHost", "WarmupSessionInputs"] diff --git a/flashdreams/flashdreams/runtime/demo/outputs.py b/flashdreams/flashdreams/runtime/demo/outputs.py new file mode 100644 index 000000000..866acbb00 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/outputs.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared demo output contracts and output construction.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Protocol, runtime_checkable + +from flashdreams.infra.postprocess import VideoTensorLayout +from flashdreams.infra.runner_io import ( + DEFAULT_RUNNER_INSTALL_HINT, + write_video_tensor, +) +from flashdreams.infra.video_output import VideoResultCollector, prepare_video_for_mp4 +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.output import NullOutputTarget, OutputArtifact, OutputTarget +from flashdreams.runtime.types import StepResult +from flashdreams.runtime.video_output import Mp4VideoOutputTarget, VideoWriter + +from .spec import Mp4OutputSpec, NullOutputSpec, OutputSpec, WebRTCOutputSpec + + +@dataclass(frozen=True, kw_only=True, slots=True) +class SessionInfo: + """Output-facing metadata known after session setup.""" + + output_layout: str | None = None + steady_output_frame_count: int | None = None + metadata: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.output_layout is not None and not self.output_layout.strip(): + raise ValueError("SessionInfo.output_layout must be non-empty when set.") + if ( + self.steady_output_frame_count is not None + and self.steady_output_frame_count < 0 + ): + raise ValueError( + "SessionInfo.steady_output_frame_count must be >= 0 when set." + ) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class OutputDecision: + """Flow-control decision returned by an output sink after one step.""" + + should_stop: bool = False + dropped: bool = False + drop_policy: Literal["none", "drop_newest", "drop_oldest"] = "none" + backpressure_s: float = 0.0 + metadata: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.drop_policy not in {"none", "drop_newest", "drop_oldest"}: + raise ValueError(f"Unsupported drop_policy={self.drop_policy!r}.") + if self.backpressure_s < 0: + raise ValueError("OutputDecision.backpressure_s must be >= 0.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@runtime_checkable +class OutputSink(Protocol): + """Consumes generated session outputs for a demo run mode.""" + + produces_artifacts: bool + + def open(self, session_info: SessionInfo) -> None: + """Prepare output resources for a session.""" + ... + + def begin_generation(self, generation: int) -> None: + """Start an output generation, discarding stale live output if needed.""" + ... + + def write(self, result: StepResult) -> OutputDecision: + """Consume one generated result and return output flow-control state.""" + ... + + def close(self) -> Sequence[OutputArtifact]: + """Finalize output resources and return produced artifacts.""" + ... + + +@dataclass(slots=True) +class NullOutputSink: + """Output sink for headless runs and fake-model vertical-slice tests.""" + + store_results: bool = False + produces_artifacts: bool = False + output_count: int = field(default=0, init=False) + results: list[Mapping[str, object]] = field(default_factory=list, init=False) + opened: bool = field(default=False, init=False) + closed: bool = field(default=False, init=False) + session_info: SessionInfo | None = field(default=None, init=False) + generation: int | None = field(default=None, init=False) + + def open(self, session_info: SessionInfo) -> None: + self.session_info = session_info + self.output_count = 0 + self.results.clear() + self.opened = True + self.closed = False + + def begin_generation(self, generation: int) -> None: + if generation < 0: + raise ValueError("generation must be >= 0.") + self.generation = generation + + def write(self, result: StepResult) -> OutputDecision: + if not self.opened or self.closed: + raise RuntimeError("Cannot write to a closed output sink.") + self.output_count += 1 + if self.store_results: + self.results.append(_result_record(result)) + return OutputDecision() + + def close(self) -> Sequence[OutputArtifact]: + self.closed = True + return () + + +@dataclass(slots=True) +class Mp4OutputSink: + """MP4 artifact sink for shared demo drivers.""" + + output_path: Path + fps: int | float + output_layout: VideoTensorLayout = "bvtchw" + writer: VideoWriter = field(default=write_video_tensor, repr=False) + install_hint: str = DEFAULT_RUNNER_INSTALL_HINT + move_to_cpu: bool = True + enabled: bool = True + produces_artifacts: bool = True + _opened: bool = field(default=False, init=False, repr=False) + _closed: bool = field(default=True, init=False, repr=False) + _collector: VideoResultCollector | None = field( + default=None, + init=False, + repr=False, + ) + _artifacts: tuple[OutputArtifact, ...] | None = field( + default=None, + init=False, + repr=False, + ) + session_info: SessionInfo | None = field(default=None, init=False) + + def __post_init__(self) -> None: + if float(self.fps) <= 0: + raise ValueError("Mp4OutputSink.fps must be > 0.") + self.output_path = Path(self.output_path) + + def open(self, session_info: SessionInfo) -> None: + self.session_info = session_info + self._collector = VideoResultCollector( + output_layout=self.output_layout, + enabled=self.enabled, + move_to_cpu=self.move_to_cpu, + ) + self._artifacts = None + self._opened = True + self._closed = False + + def begin_generation(self, generation: int) -> None: + if generation < 0: + raise ValueError("generation must be >= 0.") + + def write(self, result: StepResult) -> OutputDecision: + if not self._opened or self._closed or self._collector is None: + raise RuntimeError("Cannot write to a closed output sink.") + if result.layout is None: + raise TypeError("Mp4OutputSink requires a video StepResult with layout.") + if result.layout != self.output_layout: + raise ValueError( + "Mp4OutputSink received layout " + f"{result.layout!r}; expected {self.output_layout!r}." + ) + self._collector.add(result) + return OutputDecision() + + def close(self) -> Sequence[OutputArtifact]: + if self._artifacts is not None: + return self._artifacts + if self._collector is None: + self._opened = False + self._closed = True + self._artifacts = () + return self._artifacts + + collector = self._collector + self._collector = None + self._opened = False + self._closed = True + video = collector.finish() + if video is None: + self._artifacts = () + return self._artifacts + writable_video, writable_layout = prepare_video_for_mp4( + video, + layout=self.output_layout, + ) + path = self.writer( + writable_video, + self.output_path, + fps=self.fps, + layout=writable_layout, + install_hint=self.install_hint, + ) + self._artifacts = ( + OutputArtifact( + kind="video/mp4", + uri=str(path), + metadata={ + "fps": self.fps, + "source_layout": self.output_layout, + "shape": tuple(int(dim) for dim in video.shape), + "stats_history": tuple(collector.stats_history), + }, + ), + ) + return self._artifacts + + +def build_output_sink( + output: OutputSpec, + *, + mp4_writer: VideoWriter | None = None, +) -> OutputSink: + """Build a shared demo output sink from a demo output spec.""" + if isinstance(output, NullOutputSpec): + return NullOutputSink(store_results=output.store_results) + if isinstance(output, Mp4OutputSpec): + writer = mp4_writer or write_video_tensor + return Mp4OutputSink( + output_path=Path(output.path), + fps=output.fps, + output_layout=output.output_layout, + writer=writer, + move_to_cpu=output.move_to_cpu, + ) + if isinstance(output, WebRTCOutputSpec): + raise ValueError("WebRTC output requires a realtime transport sink.") + raise TypeError(f"Unsupported demo output spec: {type(output).__name__}.") + + +def _result_record(result: StepResult) -> Mapping[str, object]: + record: dict[str, object] = { + "step_index": result.step_index, + "frame_count": result.frame_count, + "metrics": dict(result.metrics), + "metadata": dict(result.metadata), + } + if result.layout is not None: + record["layout"] = result.layout + if result.output_window is not None: + record["output_window"] = ( + result.output_window.start_s, + result.output_window.end_s, + ) + return freeze_mapping(record) + + +def build_output_target( + output: OutputSpec, + *, + mp4_writer: VideoWriter | None = None, +) -> OutputTarget: + """Build a replay output target from a demo output spec.""" + if isinstance(output, NullOutputSpec): + return NullOutputTarget(store_results=output.store_results) + if isinstance(output, Mp4OutputSpec): + output_path = Path(output.path) + if mp4_writer is not None: + return Mp4VideoOutputTarget( + output_path=output_path, + fps=output.fps, + output_layout=output.output_layout, + writer=mp4_writer, + move_to_cpu=output.move_to_cpu, + ) + return Mp4VideoOutputTarget( + output_path=output_path, + fps=output.fps, + output_layout=output.output_layout, + move_to_cpu=output.move_to_cpu, + ) + if isinstance(output, WebRTCOutputSpec): + raise ValueError("WebRTC output does not create a replay OutputTarget.") + raise TypeError(f"Unsupported demo output spec: {type(output).__name__}.") + + +__all__ = [ + "Mp4OutputSink", + "NullOutputSink", + "OutputDecision", + "OutputSink", + "SessionInfo", + "build_output_sink", + "build_output_target", +] diff --git a/flashdreams/flashdreams/runtime/demo/pipeline.py b/flashdreams/flashdreams/runtime/demo/pipeline.py new file mode 100644 index 000000000..5a9da08b0 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/pipeline.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared per-step pipeline for demo session drivers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from flashdreams.runtime.interfaces import InferenceSession +from flashdreams.runtime.types import StepRequirements, StepResult + +from .outputs import OutputDecision, OutputSink +from .run_modes import SessionMetricsRecorder +from .session_inputs import ControlDecision, ModelInputProvider, UserInputWindow + + +@dataclass(frozen=True, kw_only=True, slots=True) +class StepOutcome: + """Combined output and control result from one shared model step.""" + + output: OutputDecision = field(default_factory=OutputDecision) + control: ControlDecision = field(default_factory=ControlDecision) + + +class StepPipeline: + """Shared invariant for provider conversion, model step, output, and metrics.""" + + def execute_step( + self, + *, + request: StepRequirements, + user_window: UserInputWindow, + provider: ModelInputProvider, + session: InferenceSession, + output: OutputSink, + metrics: SessionMetricsRecorder, + ) -> StepOutcome: + prepared = provider.prepare_step( + request=request, + user_window=user_window, + ) + if prepared.control.reset or prepared.control.close_session: + metrics.record_control( + request=request, + user_window=user_window, + control=prepared.control, + ) + return StepOutcome(control=prepared.control) + if prepared.inference_input is None: + raise RuntimeError("ModelInputProvider returned no inference input.") + + result = session.step(prepared.inference_input) + if not isinstance(result, StepResult): + raise TypeError( + "InferenceSession.step must return StepResult, " + f"got {type(result).__name__}." + ) + decision = output.write(result) + if not isinstance(decision, OutputDecision): + raise TypeError( + "OutputSink.write must return OutputDecision, " + f"got {type(decision).__name__}." + ) + metrics.record_step( + request=request, + user_window=user_window, + inference_input=prepared.inference_input, + result=result, + decision=decision, + ) + return StepOutcome(output=decision) + + +__all__ = ["StepOutcome", "StepPipeline"] diff --git a/flashdreams/flashdreams/runtime/demo/replay.py b/flashdreams/flashdreams/runtime/demo/replay.py new file mode 100644 index 000000000..fa2f507a9 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/replay.py @@ -0,0 +1,629 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared replay demo runner.""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Sequence + +from flashdreams.runtime.canonical import InputCanonicalizer +from flashdreams.runtime.config import InferenceConfig +from flashdreams.runtime.inputs import ( + CanonicalInputs, + CanonicalInputSchema, + InferenceInput, + InferenceInputSchema, + TimeWindow, + UserInputs, + UserInputSchema, +) +from flashdreams.runtime.interfaces import InferenceRuntime, InferenceSession +from flashdreams.runtime.mapping import ( + DeclaresMappingSchema, + InputMapping, + check_mapping_compatibility, +) +from flashdreams.runtime.metrics import MetricsRecorder, NullMetricsRecorder +from flashdreams.runtime.output import OutputArtifact, OutputTarget +from flashdreams.runtime.types import ( + StepRequest, + StepRequirements, + StepResult, + step_requirements_from_request, +) + +from .drivers import BatchSessionDriver, run_demo_session +from .host import ModelWarmupPlan, RuntimeHost +from .outputs import OutputDecision, OutputSink, build_output_sink, build_output_target +from .pipeline import StepPipeline +from .run_modes import ( + Mp4ErrorPolicy, + NullErrorPolicy, + RunContext, + RunModeCapabilities, + RunResult, + SessionEdges, + SingleSessionAdmissionPolicy, +) +from .session_inputs import PreparedStep, ProviderCapabilities, UserInputWindow +from .spec import ( + DemoAdapter, + DemoSpec, + OutputSpec, + PreparedScenario, + WebRTCOutputSpec, +) + +OutputTargetFactory = Callable[[OutputSpec], OutputTarget] +InferenceSessionRunner = Callable[..., Sequence[OutputArtifact]] +OutputSinkFactory = Callable[[OutputSpec], OutputSink] + + +def run_replay_demo( + *, + spec: DemoSpec, + adapter: DemoAdapter, + output_target_factory: OutputTargetFactory | None = None, + output_sink_factory: OutputSinkFactory = build_output_sink, + metrics: MetricsRecorder | None = None, + runner: InferenceSessionRunner | None = None, +) -> RunResult: + """Run one prepared replay scenario through the shared batch demo path.""" + _require_supported_mode( + mode=spec.input_mode, + supported=adapter.supported_input_modes(), + label="input_mode", + ) + if spec.input_mode != "replay": + raise ValueError( + "run_replay_demo requires input_mode='replay', " + f"got input_mode={spec.input_mode!r}." + ) + _require_supported_mode( + mode=spec.output.mode, + supported=adapter.supported_output_modes(), + label="output.mode", + ) + if isinstance(spec.output, WebRTCOutputSpec): + raise ValueError("run_replay_demo does not support WebRTC output.") + + prepared = adapter.prepare_scenario(spec) + mapping = prepared.mapping or adapter.default_input_mapping() + if mapping is None: + raise ValueError( + "Demo scenario did not provide an input mapping, and the adapter " + "has no default input mapping." + ) + if spec.config is None: + raise RuntimeError("DemoSpec.config was not initialized.") + + if runner is not None: + return _run_replay_demo_with_compat_runner( + spec=spec, + adapter=adapter, + prepared=prepared, + mapping=mapping, + output_target_factory=output_target_factory or build_output_target, + metrics=metrics, + runner=runner or _default_inference_session_runner(), + ) + + if output_target_factory is not None: + output_sink_factory = _output_target_sink_factory(output_target_factory) + + return _run_replay_demo_with_run_mode( + spec=spec, + adapter=adapter, + prepared=prepared, + mapping=mapping, + output_sink_factory=output_sink_factory, + metrics=metrics, + ) + + +def _output_target_sink_factory( + output_target_factory: OutputTargetFactory, +) -> OutputSinkFactory: + def create_output_sink(output_spec: OutputSpec) -> "_OutputTargetSink": + return _OutputTargetSink(output_target_factory(output_spec)) + + return create_output_sink + + +class _OutputTargetSink: + produces_artifacts = True + + def __init__(self, output: OutputTarget) -> None: + self._output = output + self._closed = True + self._artifacts: tuple[OutputArtifact, ...] | None = None + + def open(self, session_info: object) -> None: + del session_info + self._output.open() + self._closed = False + self._artifacts = None + + def begin_generation(self, generation: int) -> None: + del generation + + def write(self, result: StepResult) -> OutputDecision: + self._output.write(result) + return OutputDecision() + + def close(self) -> Sequence[OutputArtifact]: + if self._artifacts is not None: + return self._artifacts + if self._closed: + self._artifacts = () + return self._artifacts + self._closed = True + self._artifacts = tuple(self._output.close()) + return self._artifacts + + +def _run_replay_demo_with_compat_runner( + *, + spec: DemoSpec, + adapter: DemoAdapter, + prepared: "PreparedScenario", + mapping: InputMapping, + output_target_factory: OutputTargetFactory, + metrics: MetricsRecorder | None, + runner: InferenceSessionRunner, +) -> RunResult: + output = output_target_factory(spec.output) + metrics_recorder = metrics or NullMetricsRecorder() + artifacts = tuple( + runner( + adapter=adapter, + config=_require_config(spec), + mapping=mapping, + canonicalizer=prepared.canonicalizer, + source_schema=prepared.source_schema, + user_inputs=prepared.user_inputs, + initial_inputs=prepared.initial_inputs, + output=output, + metrics=metrics_recorder, + ) + ) + return RunResult(status="completed", artifacts=artifacts) + + +def _run_replay_demo_with_run_mode( + *, + spec: DemoSpec, + adapter: DemoAdapter, + prepared: "PreparedScenario", + mapping: InputMapping, + output_sink_factory: OutputSinkFactory, + metrics: MetricsRecorder | None, +) -> RunResult: + config = _require_config(spec) + _validate_replay_mapping( + adapter=adapter, + config=config, + mapping=mapping, + source_schema=prepared.source_schema, + canonicalizer=prepared.canonicalizer, + ) + request_state = _ReplayStepRequestState() + runtime = _ReplayRuntimeAdapter( + runtime=adapter.create_runtime(config), + request_state=request_state, + ) + host = RuntimeHost(runtime) + mode = _ReplayRunMode( + request_state=request_state, + output_sink_factory=output_sink_factory, + run_metrics=metrics or NullMetricsRecorder(), + ) + replay_adapter = _ReplayProviderAdapter( + adapter=adapter, + mapping=mapping, + request_state=request_state, + ) + context = mode.create_run_context( + spec=spec, + adapter=replay_adapter, + host=host, + model_warmup_plan=ModelWarmupPlan(), + ) + try: + return run_demo_session( + context=context, + spec=spec, + scenario=prepared, + adapter=replay_adapter, + run_mode=mode, + pipeline=StepPipeline(), + ) + finally: + context.close() + host.close() + + +class _ReplayRunMode: + name = "replay" + capabilities = RunModeCapabilities( + requires_finite_input=True, + supports_artifacts=True, + ) + + def __init__( + self, + *, + request_state: "_ReplayStepRequestState", + output_sink_factory: OutputSinkFactory, + run_metrics: MetricsRecorder, + ) -> None: + self._request_state = request_state + self._output_sink_factory = output_sink_factory + self._run_metrics = run_metrics + + def validate_run(self, *, spec: DemoSpec, adapter: DemoAdapter) -> None: + del spec, adapter + + def validate_session( + self, + *, + spec: DemoSpec, + scenario: "PreparedScenario", + adapter: DemoAdapter, + provider: object, + ) -> None: + del spec, scenario, adapter, provider + + def create_run_context( + self, + *, + spec: DemoSpec, + adapter: DemoAdapter, + host: RuntimeHost, + model_warmup_plan: ModelWarmupPlan, + ) -> RunContext: + del spec, adapter + return RunContext( + host=host, + run_metrics=self._run_metrics, + admission=SingleSessionAdmissionPolicy( + health_check=lambda: host.is_healthy + ), + model_warmup_plan=model_warmup_plan, + ) + + def create_session_edges( + self, + *, + context: RunContext, + spec: DemoSpec, + scenario: "PreparedScenario", + provider: object, + adapter: DemoAdapter, + ) -> SessionEdges: + del provider, adapter + return SessionEdges( + input_source=_ReplayBatchInputSource( + scenario=scenario, + request_state=self._request_state, + ), + output_sink=self._output_sink_factory(spec.output), + cleanup_tasks=context.cleanup_tasks, + error_policy=( + Mp4ErrorPolicy() if spec.output.mode == "mp4" else NullErrorPolicy() + ), + ) + + def select_driver(self) -> BatchSessionDriver: + return BatchSessionDriver() + + +class _ReplayProviderAdapter: + def __init__( + self, + *, + adapter: DemoAdapter, + mapping: InputMapping, + request_state: "_ReplayStepRequestState", + ) -> None: + self._adapter = adapter + self._mapping = mapping + self._request_state = request_state + + @property + def model_id(self) -> str: + return self._adapter.model_id + + @property + def inference_input_schema(self) -> InferenceInputSchema: + return self._adapter.inference_input_schema + + @property + def canonical_input_schema(self) -> CanonicalInputSchema | None: + return self._adapter.canonical_input_schema + + def default_input_mapping(self) -> InputMapping | None: + return self._adapter.default_input_mapping() + + def supported_input_modes(self) -> tuple[str, ...]: + return self._adapter.supported_input_modes() + + def supported_output_modes(self) -> tuple[str, ...]: + return self._adapter.supported_output_modes() + + def validate_config(self, config: InferenceConfig) -> None: + self._adapter.validate_config(config) + + def create_runtime(self, config: InferenceConfig) -> InferenceRuntime: + return self._adapter.create_runtime(config) + + def prepare_scenario(self, spec: DemoSpec) -> "PreparedScenario": + return self._adapter.prepare_scenario(spec) + + def create_model_input_provider( + self, + spec: DemoSpec, + scenario: "PreparedScenario", + ) -> object: + create_provider = getattr(self._adapter, "create_model_input_provider", None) + if callable(create_provider): + return create_provider(spec, scenario) + return _ReplayMappingModelInputProvider( + adapter=self._adapter, + scenario=scenario, + mapping=self._mapping, + request_state=self._request_state, + ) + + +class _ReplayRuntimeAdapter: + def __init__( + self, + *, + runtime: InferenceRuntime, + request_state: "_ReplayStepRequestState", + ) -> None: + self._runtime = runtime + self._request_state = request_state + + def start_session(self, inputs: InferenceInput) -> InferenceSession: + return _ReplaySessionAdapter( + session=self._runtime.start_session(inputs), + request_state=self._request_state, + ) + + def close(self) -> None: + self._runtime.close() + + +class _ReplaySessionAdapter: + def __init__( + self, + *, + session: InferenceSession, + request_state: "_ReplayStepRequestState", + ) -> None: + self._session = session + self._request_state = request_state + + def next_step_requirements(self) -> StepRequirements | None: + next_requirements = getattr(self._session, "next_step_requirements", None) + if callable(next_requirements): + value = next_requirements() + self._request_state.clear() + return value + + request = self._session.next_step_request() + if request is None: + self._request_state.clear() + return None + self._request_state.store(request) + return step_requirements_from_request( + request, + allow_user_input_window=True, + ) + + def next_step_request(self) -> StepRequest | None: + return self._session.next_step_request() + + def step(self, inputs: InferenceInput) -> StepResult: + return self._session.step(inputs) + + def reset(self, inputs: InferenceInput | None = None) -> None: + self._session.reset(inputs) + + def close(self) -> None: + self._session.close() + + +class _ReplayStepRequestState: + def __init__(self) -> None: + self._request: StepRequest | None = None + + def store(self, request: StepRequest) -> None: + self._request = request + + def request_for_window(self, step_index: int) -> StepRequest | None: + request = self._request + if request is None: + return None + if request.step_index != step_index: + raise RuntimeError( + "Replay input source request mismatch: " + f"expected step {request.step_index}, got {step_index}." + ) + return request + + def consume_for_step(self, request: StepRequirements) -> StepRequest: + legacy_request = self.request_for_window(request.step_index) + if legacy_request is not None: + self._request = None + return legacy_request + return StepRequest( + step_index=request.step_index, + inference_input_schema=request.inference_input_schema, + metadata=request.metadata, + ) + + def clear(self) -> None: + self._request = None + + +class _ReplayBatchInputSource: + is_finite = True + is_deterministic = True + + def __init__( + self, + *, + scenario: "PreparedScenario", + request_state: _ReplayStepRequestState, + ) -> None: + self.user_input_schema = scenario.source_schema + self._user_inputs = scenario.user_inputs + self._request_state = request_state + + def is_finished(self) -> bool: + return False + + def next_window(self, request: StepRequirements) -> UserInputWindow: + legacy_request = self._request_state.request_for_window(request.step_index) + window = ( + legacy_request.user_input_window if legacy_request is not None else None + ) + if window is None: + window = _all_user_inputs_window(self._user_inputs) + return UserInputWindow( + start_s=window.start_s, + end_s=window.end_s, + inputs=self._user_inputs, + ) + + +class _ReplayMappingModelInputProvider: + def __init__( + self, + *, + adapter: DemoAdapter, + scenario: "PreparedScenario", + mapping: InputMapping, + request_state: _ReplayStepRequestState, + ) -> None: + self.capabilities = ProviderCapabilities( + supports_recorded_input=True, + deterministic_given_inputs=True, + user_input_schema=scenario.source_schema, + inference_input_schema=adapter.inference_input_schema, + ) + self._scenario = scenario + self._mapping = mapping + self._request_state = request_state + self._step_base_inputs = InferenceInput( + step=scenario.initial_inputs.step, + metadata=scenario.initial_inputs.metadata, + ) + + def prepare_initial_input(self) -> InferenceInput: + self._scenario.canonicalizer.reset() + return self._mapping.map_global_conditioning_inputs( + canonical_inputs=CanonicalInputs(), + inference_input=self._scenario.initial_inputs, + ) + + def prepare_step( + self, + *, + request: StepRequirements, + user_window: UserInputWindow, + ) -> PreparedStep: + legacy_request = self._request_state.consume_for_step(request) + canonical_inputs = self._scenario.canonicalizer.canonicalize( + self._scenario.user_inputs, + window=TimeWindow(start_s=user_window.start_s, end_s=user_window.end_s), + source_schema=self._scenario.source_schema, + ) + return PreparedStep( + inference_input=self._mapping.map_step_inputs( + canonical_inputs=canonical_inputs, + inference_input=self._step_base_inputs, + request=legacy_request, + ) + ) + + def reset(self, inputs: InferenceInput | None = None) -> None: + del inputs + self._scenario.canonicalizer.reset() + + def close(self) -> None: + return None + + +def _validate_replay_mapping( + *, + adapter: DemoAdapter, + config: InferenceConfig, + mapping: InputMapping, + source_schema: UserInputSchema, + canonicalizer: InputCanonicalizer, +) -> None: + adapter.validate_config(config) + canonical_schema = canonicalizer.canonical_schema(source_schema) + if isinstance(mapping, DeclaresMappingSchema): + compatibility = check_mapping_compatibility( + canonical_schema=canonical_schema, + inference_input_schema=adapter.inference_input_schema, + mapping_schema=mapping.mapping_schema, + ) + compatibility.raise_if_incompatible() + mapping.validate( + canonical_schema=canonical_schema, + inference_input_schema=adapter.inference_input_schema, + ) + + +def _require_config(spec: DemoSpec) -> InferenceConfig: + if spec.config is None: + raise RuntimeError("DemoSpec.config was not initialized.") + return spec.config + + +def _all_user_inputs_window(user_inputs: UserInputs) -> TimeWindow: + if not user_inputs.events: + return TimeWindow(start_s=0.0, end_s=3600.0) + return TimeWindow( + start_s=0.0, + end_s=max( + 3600.0, + math.nextafter(user_inputs.events[-1].timestamp_s, math.inf), + ), + ) + + +def _default_inference_session_runner() -> InferenceSessionRunner: + from flashdreams.runtime.runner import run_inference_session + + return run_inference_session + + +def _require_supported_mode( + *, + mode: str, + supported: tuple[str, ...], + label: str, +) -> None: + if mode in supported: + return + supported_text = ", ".join(repr(each) for each in supported) or "" + raise ValueError( + f"Unsupported demo {label}={mode!r}; supported modes: {supported_text}." + ) + + +__all__ = [ + "InferenceSessionRunner", + "OutputSinkFactory", + "OutputTargetFactory", + "run_replay_demo", +] diff --git a/flashdreams/flashdreams/runtime/demo/run_modes.py b/flashdreams/flashdreams/runtime/demo/run_modes.py new file mode 100644 index 000000000..cb93e4e7f --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/run_modes.py @@ -0,0 +1,535 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run/session result and policy helpers for demo session drivers.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from threading import Lock +from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable + +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.metrics import ( + InMemoryMetricsRecorder, + MetricsRecorder, + MetricsSnapshot, +) +from flashdreams.runtime.output import OutputArtifact + +from .host import ModelWarmupPlan, WarmupSessionInputs +from .outputs import OutputSink + +if TYPE_CHECKING: + from .host import RuntimeHost + from .pipeline import StepPipeline + from .session_inputs import InputSource, ModelInputProvider + from .spec import DemoAdapter, DemoSpec, PreparedScenario + from .timing import ActivationPolicy, DeterministicClock, RealtimeClock + +SessionStatus = Literal[ + "completed", + "failed", + "skipped", + "cancelled", + "rejected", + "not_activated", +] + +DriverStatus = Literal[ + "completed", + "failed", + "skipped", + "cancelled", + "not_activated", +] + + +@dataclass(frozen=True, kw_only=True, slots=True) +class RunResult: + """Outcome of one demo session.""" + + __hash__ = None + + status: SessionStatus + artifacts: Sequence[OutputArtifact] = () + metrics: MetricsSnapshot | None = None + reason: str | None = None + error: Exception | None = None + + @classmethod + def rejected(cls, reason: str) -> "RunResult": + """Admission refused the session. The only no-session result helper.""" + return cls(status="rejected", reason=reason) + + def __post_init__(self) -> None: + object.__setattr__(self, "artifacts", tuple(self.artifacts)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class RunSummary: + """Summary for a run context after one or more sessions.""" + + metrics: MetricsSnapshot + sessions: Sequence[RunResult] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "sessions", tuple(self.sessions)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ErrorAction: + """Driver policy decision for an operational error.""" + + close_session: bool = True + drop_chunk: bool = False + continue_next_scenario: bool = False + result_status: Literal["completed", "failed", "skipped"] = "failed" + + +class DefaultErrorPolicy: + """Default policy: operational errors fail the current session.""" + + def handle_setup_error(self, exc: Exception) -> ErrorAction: + del exc + return ErrorAction(result_status="failed") + + def handle(self, exc: Exception) -> ErrorAction: + del exc + return ErrorAction(result_status="failed") + + +@runtime_checkable +class ErrorPolicy(Protocol): + """Maps driver-observed exceptions to session outcomes.""" + + def handle_setup_error(self, exc: Exception) -> ErrorAction: ... + + def handle(self, exc: Exception) -> ErrorAction: ... + + +class Mp4ErrorPolicy(DefaultErrorPolicy): + """Abort MP4 sessions on setup or step errors.""" + + +class NullErrorPolicy(DefaultErrorPolicy): + """Abort headless/null sessions on setup or step errors.""" + + +class NativeWindowErrorPolicy(DefaultErrorPolicy): + """Abort native-window sessions unless a future UI policy overrides it.""" + + +class BenchmarkErrorPolicy(DefaultErrorPolicy): + """Close failed scenarios while letting benchmark loops continue.""" + + def handle_setup_error(self, exc: Exception) -> ErrorAction: + del exc + return ErrorAction(result_status="failed", continue_next_scenario=True) + + def handle(self, exc: Exception) -> ErrorAction: + del exc + return ErrorAction(result_status="failed", continue_next_scenario=True) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCErrorPolicy: + """Drop configured recoverable realtime errors, otherwise close the session.""" + + recoverable_exception_types: tuple[type[Exception], ...] = () + + def handle_setup_error(self, exc: Exception) -> ErrorAction: + del exc + return ErrorAction(result_status="failed") + + def handle(self, exc: Exception) -> ErrorAction: + if self.recoverable_exception_types and isinstance( + exc, self.recoverable_exception_types + ): + return ErrorAction( + close_session=False, + drop_chunk=True, + result_status="failed", + ) + return ErrorAction(result_status="failed") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class RunModeCapabilities: + """Run-mode requirements and output/transport capabilities.""" + + realtime: bool = False + requires_finite_input: bool = False + supports_backpressure: bool = False + supports_interactive_events: bool = False + supports_artifacts: bool = False + + +SessionMetricsRecorder = MetricsRecorder +InMemorySessionMetricsRecorder = InMemoryMetricsRecorder + + +class NoopTransportService: + """Idempotent placeholder transport for batch sessions.""" + + def __init__(self) -> None: + self.closed = False + + def is_active(self) -> bool: + return not self.closed + + def close(self) -> None: + self.closed = True + + +@runtime_checkable +class TransportService(Protocol): + """Per-session transport lifecycle hook.""" + + def is_active(self) -> bool: ... + + def close(self) -> None: ... + + +@runtime_checkable +class SessionReservation(Protocol): + """Admission reservation for one session.""" + + def release(self) -> None: ... + + +class SingleSessionAdmissionPolicy: + """Atomic single-session admission policy.""" + + def __init__(self, *, health_check: Any | None = None) -> None: + self._lock = Lock() + self._reserved = False + self._health_check = health_check + + def try_reserve(self) -> SessionReservation | None: + with self._lock: + if self._reserved or not self._is_healthy(): + return None + self._reserved = True + return _SingleSessionReservation(self) + + def _release(self) -> None: + with self._lock: + self._reserved = False + + def _is_healthy(self) -> bool: + if self._health_check is None: + return True + return bool(self._health_check()) + + +class _SingleSessionReservation: + def __init__(self, policy: SingleSessionAdmissionPolicy) -> None: + self._policy = policy + self._released = False + self.release_count = 0 + + def release(self) -> None: + if self._released: + return + self._released = True + self.release_count += 1 + self._policy._release() + + +@runtime_checkable +class AdmissionPolicy(Protocol): + """Atomically reserves session capacity or rejects.""" + + def try_reserve(self) -> SessionReservation | None: ... + + +@runtime_checkable +class SessionDriver(Protocol): + """Synchronous one-session driver selected by a run mode.""" + + def run_one_session( + self, + *, + host: "RuntimeHost", + provider: "ModelInputProvider", + session_edges: "SessionEdges", + pipeline: "StepPipeline", + ) -> RunResult: ... + + +@runtime_checkable +class AsyncSessionDriver(Protocol): + """Async one-session driver selected by realtime run modes.""" + + async def run_one_session( + self, + *, + host: "RuntimeHost", + provider: "ModelInputProvider", + session_edges: "SessionEdges", + pipeline: "StepPipeline", + ) -> RunResult: ... + + +@dataclass(slots=True) +class RunContext: + """Run-scoped services shared by one or more demo sessions.""" + + host: "RuntimeHost" + run_metrics: SessionMetricsRecorder + admission: AdmissionPolicy + model_warmup_plan: ModelWarmupPlan = field(default_factory=ModelWarmupPlan) + services: Mapping[str, object] = field(default_factory=dict) + cleanup_tasks: set[asyncio.Task[RunResult]] = field(default_factory=set) + + def __post_init__(self) -> None: + self.services = freeze_mapping(self.services) + + def close(self) -> RunSummary: + if self.cleanup_tasks: + raise RuntimeError( + "Pending session cleanup tasks; async runs must await close_async()." + ) + for service in self.services.values(): + close = getattr(service, "close", None) + if callable(close): + try: + close() + except Exception as exc: + self.run_metrics.record_cleanup_error(exc) + return RunSummary( + metrics=self.run_metrics.close(), + sessions=tuple(getattr(self.run_metrics, "sessions", ())), + ) + + async def close_async(self) -> RunSummary: + while self.cleanup_tasks: + pending = tuple(self.cleanup_tasks) + await asyncio.gather(*pending, return_exceptions=True) + self.cleanup_tasks.difference_update(pending) + return self.close() + + +@dataclass(slots=True) +class SessionEdges: + """Per-session input/output/policy bundle consumed by drivers.""" + + input_source: "InputSource" + output_sink: OutputSink + cleanup_tasks: set[asyncio.Task[RunResult]] + metrics: SessionMetricsRecorder = field( + default_factory=InMemorySessionMetricsRecorder + ) + error_policy: ErrorPolicy = field(default_factory=DefaultErrorPolicy) + transport: TransportService = field(default_factory=NoopTransportService) + clock: "RealtimeClock | DeterministicClock | None" = None + activation: "ActivationPolicy | None" = None + _closed_result: RunResult | None = field(default=None, init=False, repr=False) + + @property + def is_closed(self) -> bool: + """Return whether ``close_result(...)`` has already finalized this session.""" + return self._closed_result is not None + + def record_cleanup_error(self, exc: Exception) -> None: + """Record a cleanup error without letting metrics failures block teardown.""" + try: + self.metrics.record_cleanup_error(exc) + except Exception: + return + + def record_orphaned_cleanup(self, exc: Exception) -> None: + """Record timed-out worker cleanup without blocking teardown.""" + try: + self.metrics.record_orphaned_cleanup(exc) + except Exception: + return + + def close_result( + self, + *, + status: DriverStatus = "completed", + reason: str | None = None, + error: Exception | None = None, + ) -> RunResult: + """Idempotently close output, transport, and metrics once.""" + if self._closed_result is not None: + return self._closed_result + + artifacts: Sequence[OutputArtifact] = () + try: + artifacts = tuple(self.output_sink.close()) + except Exception as exc: + self.record_cleanup_error(exc) + try: + self.transport.close() + except Exception as exc: + self.record_cleanup_error(exc) + try: + metrics = self.metrics.close() + except Exception as exc: + metrics = MetricsSnapshot(errors=(f"metrics.close failed: {exc}",)) + self._closed_result = RunResult( + status=status, + artifacts=artifacts, + metrics=metrics, + reason=reason, + error=error, + ) + return self._closed_result + + +@runtime_checkable +class RunMode(Protocol): + """Run/session construction strategy consumed by shared helpers.""" + + name: str + capabilities: RunModeCapabilities + + def validate_run( + self, + *, + spec: "DemoSpec", + adapter: "DemoAdapter", + ) -> None: ... + + def validate_session( + self, + *, + spec: "DemoSpec", + scenario: "PreparedScenario", + adapter: "DemoAdapter", + provider: "ModelInputProvider", + ) -> None: ... + + def create_run_context( + self, + *, + spec: "DemoSpec", + adapter: "DemoAdapter", + host: "RuntimeHost", + model_warmup_plan: ModelWarmupPlan, + ) -> RunContext: ... + + def create_session_edges( + self, + *, + context: RunContext, + spec: "DemoSpec", + scenario: "PreparedScenario", + provider: "ModelInputProvider", + adapter: "DemoAdapter", + ) -> SessionEdges: ... + + def select_driver(self) -> SessionDriver | AsyncSessionDriver: ... + + +@runtime_checkable +class RunModeWarmup(Protocol): + """Optional run-mode warmup for output or transport services.""" + + def warmup_context( + self, + *, + context: RunContext, + spec: "DemoSpec", + scenario: "PreparedScenario", + adapter: "DemoAdapter", + ) -> None: ... + + +def build_model_warmup_plan( + *, + host: "RuntimeHost", + adapter: "DemoAdapter", + spec: "DemoSpec", + scenario: "PreparedScenario", +) -> ModelWarmupPlan: + """Build a host-owned warmup plan through the model-affine worker.""" + + create_sessions = getattr(adapter, "create_model_warmup_sessions", None) + if create_sessions is None: + return ModelWarmupPlan() + if not callable(create_sessions): + raise TypeError( + "Demo adapter create_model_warmup_sessions attribute must be callable." + ) + sessions = host.call(create_sessions, spec, scenario) + return ModelWarmupPlan(sessions=_coerce_warmup_sessions(sessions)) + + +def warmup_run_context( + *, + context: RunContext, + spec: "DemoSpec", + scenario: "PreparedScenario", + adapter: "DemoAdapter", + run_mode: object, +) -> None: + """Run model warmup, then optional output/transport warmup for a context.""" + + context.host.warmup(context.model_warmup_plan) + warmup_context = getattr(run_mode, "warmup_context", None) + if warmup_context is None: + return + if not callable(warmup_context): + raise TypeError("RunMode.warmup_context attribute must be callable.") + warmup_context( + context=context, + spec=spec, + scenario=scenario, + adapter=adapter, + ) + + +def _coerce_warmup_sessions(value: object) -> tuple[WarmupSessionInputs, ...]: + if not isinstance(value, Sequence): + raise TypeError( + "Demo adapter create_model_warmup_sessions(...) must return a sequence " + f"of WarmupSessionInputs, got {type(value).__name__}." + ) + sessions: list[WarmupSessionInputs] = [] + for session in value: + if not isinstance(session, WarmupSessionInputs): + raise TypeError( + "Demo adapter create_model_warmup_sessions(...) must return only " + f"WarmupSessionInputs, got {type(session).__name__}." + ) + sessions.append(session) + return tuple(sessions) + + +__all__ = [ + "AdmissionPolicy", + "AsyncSessionDriver", + "BenchmarkErrorPolicy", + "DefaultErrorPolicy", + "DriverStatus", + "ErrorAction", + "ErrorPolicy", + "InMemorySessionMetricsRecorder", + "MetricsSnapshot", + "Mp4ErrorPolicy", + "NativeWindowErrorPolicy", + "NoopTransportService", + "NullErrorPolicy", + "RunContext", + "RunMode", + "RunModeCapabilities", + "RunModeWarmup", + "RunResult", + "RunSummary", + "SessionEdges", + "SessionDriver", + "SessionMetricsRecorder", + "SessionReservation", + "SessionStatus", + "SingleSessionAdmissionPolicy", + "TransportService", + "WebRTCErrorPolicy", + "build_model_warmup_plan", + "warmup_run_context", +] diff --git a/flashdreams/flashdreams/runtime/demo/session_inputs.py b/flashdreams/flashdreams/runtime/demo/session_inputs.py new file mode 100644 index 000000000..dda42995c --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/session_inputs.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Input-source and model-input-provider contracts for demo sessions.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.inputs import ( + InferenceInput, + InferenceInputSchema, + UserInputs, + UserInputSchema, +) +from flashdreams.runtime.types import StepRequirements + +if TYPE_CHECKING: + from .timing import RealtimeClock, RealtimeWindowResult + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ProviderCapabilities: + """Model-provider capabilities used to validate run-mode compatibility.""" + + supports_realtime_clock: bool = False + supports_recorded_input: bool = False + supports_reset: bool = False + deterministic_given_inputs: bool = False + user_input_schema: UserInputSchema = field(default_factory=UserInputSchema) + inference_input_schema: InferenceInputSchema = field( + default_factory=InferenceInputSchema + ) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ControlDecision: + """Provider-authored control request for the current session.""" + + reset: bool = False + close_session: bool = False + reset_input: InferenceInput | None = None + provider_already_reset: bool = False + reason: str | None = None + + def __post_init__(self) -> None: + if self.reason is not None and not self.reason.strip(): + raise ValueError("ControlDecision.reason must be non-empty when set.") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class UserInputWindow: + """User/app inputs selected by a driver for one model step.""" + + __hash__ = None + + start_s: float + end_s: float + frame_times: Sequence[float] = () + inputs: UserInputs = field(default_factory=UserInputs) + control: ControlDecision | None = None + metadata: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not math.isfinite(self.start_s) or self.start_s < 0: + raise ValueError("UserInputWindow.start_s must be finite and >= 0.") + if not math.isfinite(self.end_s) or self.end_s < self.start_s: + raise ValueError("UserInputWindow.end_s must be finite and >= start_s.") + previous = -math.inf + for frame_time in self.frame_times: + if not math.isfinite(float(frame_time)): + raise ValueError("UserInputWindow.frame_times must be finite.") + if float(frame_time) < previous: + raise ValueError( + "UserInputWindow.frame_times must be sorted in ascending order." + ) + previous = float(frame_time) + object.__setattr__( + self, "frame_times", tuple(float(t) for t in self.frame_times) + ) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class PreparedStep: + """Model-facing input plus optional provider-authored control decision.""" + + __hash__ = None + + inference_input: InferenceInput | None = None + control: ControlDecision = field(default_factory=ControlDecision) + + +@runtime_checkable +class InputSource(Protocol): + """Facts common to every demo session input source.""" + + is_finite: bool + is_deterministic: bool + user_input_schema: UserInputSchema + + def is_finished(self) -> bool: + """Return whether the driver should stop requesting windows.""" + ... + + +@runtime_checkable +class BatchInputSource(InputSource, Protocol): + """Finite input source consumed by the batch driver.""" + + def next_window(self, request: StepRequirements) -> UserInputWindow: + """Return the next batch input window for ``request``.""" + ... + + +@runtime_checkable +class RealtimeInputSource(InputSource, Protocol): + """Realtime input source consumed by a future realtime driver.""" + + async def next_realtime_window( + self, + *, + request: StepRequirements, + clock: "RealtimeClock", + ) -> "RealtimeWindowResult": + """Return the next realtime window result. + + The concrete realtime result shape lands with the realtime clock phase. + Keeping this protocol separate now prevents batch sources from stubbing + async behavior they never serve. + """ + ... + + +@runtime_checkable +class ModelInputProvider(Protocol): + """Model-owned conversion from user windows into model-facing inputs.""" + + capabilities: ProviderCapabilities + + def prepare_initial_input(self) -> InferenceInput: + """Prepare session-global model inputs.""" + ... + + def prepare_step( + self, + *, + request: StepRequirements, + user_window: UserInputWindow, + ) -> PreparedStep: + """Prepare one model step from a driver-owned user input window.""" + ... + + def reset(self, inputs: InferenceInput | None = None) -> None: + """Reset provider-owned session state. + + Implementations must be idempotent so driver cleanup and reset control + paths can safely converge after failures. + """ + ... + + def close(self) -> None: + """Release provider-owned resources. + + Implementations must be idempotent and tolerate cleanup after partial + setup or earlier reset failures. + """ + ... + + +__all__ = [ + "BatchInputSource", + "ControlDecision", + "InputSource", + "ModelInputProvider", + "PreparedStep", + "ProviderCapabilities", + "RealtimeInputSource", + "UserInputWindow", +] diff --git a/flashdreams/flashdreams/runtime/demo/spec.py b/flashdreams/flashdreams/runtime/demo/spec.py new file mode 100644 index 000000000..d7a6a309b --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/spec.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Experimental shared demo API data shapes.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any, Literal, Protocol, TypeAlias + +from flashdreams.infra.postprocess import VideoTensorLayout +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.canonical import InputCanonicalizer +from flashdreams.runtime.config import InferenceConfig +from flashdreams.runtime.inputs import InferenceInput, UserInputs, UserInputSchema +from flashdreams.runtime.interfaces import ModelAdapter +from flashdreams.runtime.mapping import InputMapping + +from .host import WarmupSessionInputs + + +@dataclass(frozen=True, kw_only=True, slots=True) +class NullOutputSpec: + """Headless/null replay output.""" + + mode: Literal["null"] = "null" + store_results: bool = False + + +@dataclass(frozen=True, kw_only=True, slots=True) +class Mp4OutputSpec: + """MP4 replay output.""" + + path: str | Path + fps: int | float + mode: Literal["mp4"] = "mp4" + output_layout: VideoTensorLayout = "bvtchw" + move_to_cpu: bool = True + + def __post_init__(self) -> None: + if float(self.fps) <= 0: + raise ValueError("Mp4OutputSpec.fps must be > 0.") + object.__setattr__(self, "path", Path(self.path)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCOutputSpec: + """Shared WebRTC serving output.""" + + mode: Literal["webrtc"] = "webrtc" + host: str = "127.0.0.1" + port: int = 8080 + fps: int = 30 + video_width: int = 1280 + video_height: int = 720 + warmup_chunks: int = 0 + warmup_timeout_s: float = 30.0 + client_liveness_timeout_s: float = 30.0 + web_dir: str | Path | None = None + request_session_path: str = "/request_session" + preload_name: str | None = None + + def __post_init__(self) -> None: + if not self.host.strip(): + raise ValueError("WebRTCOutputSpec.host must be non-empty.") + if not (0 < int(self.port) < 65536): + raise ValueError("WebRTCOutputSpec.port must be between 1 and 65535.") + if self.fps <= 0: + raise ValueError("WebRTCOutputSpec.fps must be > 0.") + if self.video_width <= 0 or self.video_height <= 0: + raise ValueError("WebRTCOutputSpec video dimensions must be > 0.") + if self.warmup_chunks < 0: + raise ValueError("WebRTCOutputSpec.warmup_chunks must be >= 0.") + if self.warmup_timeout_s <= 0: + raise ValueError("WebRTCOutputSpec.warmup_timeout_s must be > 0.") + if self.client_liveness_timeout_s <= 0: + raise ValueError("WebRTCOutputSpec.client_liveness_timeout_s must be > 0.") + if not self.request_session_path.startswith("/"): + raise ValueError( + "WebRTCOutputSpec.request_session_path must start with '/'." + ) + if self.web_dir is not None: + object.__setattr__(self, "web_dir", Path(self.web_dir)) + + +OutputSpec: TypeAlias = NullOutputSpec | Mp4OutputSpec | WebRTCOutputSpec + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCAppResources: + """Model-owned resources attached to the shared WebRTC application.""" + + model_web_resource: Any | None = None + configure_app: Callable[[Any], None] | None = None + preload_name: str | None = None + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DemoSpec: + """User-facing shared demo run description.""" + + __hash__ = None + + model_id: str + input_mode: str + output: OutputSpec + preset_id: str | None = None + scenario: Any | None = None + config: InferenceConfig | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.model_id.strip(): + raise ValueError("DemoSpec.model_id must be non-empty.") + if not self.input_mode.strip(): + raise ValueError("DemoSpec.input_mode must be non-empty.") + config = self.config + if config is None: + config = InferenceConfig( + model_id=self.model_id, + preset_id=self.preset_id, + ) + else: + if config.model_id != self.model_id: + raise ValueError( + "DemoSpec.model_id must match InferenceConfig.model_id." + ) + if self.preset_id is None: + object.__setattr__(self, "preset_id", config.preset_id) + elif config.preset_id is None: + config = replace(config, preset_id=self.preset_id) + elif config.preset_id != self.preset_id: + raise ValueError( + "DemoSpec.preset_id must match InferenceConfig.preset_id." + ) + object.__setattr__(self, "config", config) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class PreparedScenario: + """Runtime-ready scenario prepared by a model demo adapter.""" + + __hash__ = None + + initial_inputs: InferenceInput + user_inputs: UserInputs = field(default_factory=UserInputs) + source_schema: UserInputSchema = field(default_factory=UserInputSchema) + canonicalizer: InputCanonicalizer = field(default_factory=InputCanonicalizer) + mapping: InputMapping | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +class DemoAdapter(ModelAdapter, Protocol): + """Transport-neutral model adapter consumed by demo runners.""" + + def supported_input_modes(self) -> tuple[str, ...]: + """Return demo input modes this adapter can prepare.""" + ... + + def supported_output_modes(self) -> tuple[str, ...]: + """Return demo output modes this adapter can run.""" + ... + + def prepare_scenario(self, spec: DemoSpec) -> PreparedScenario: + """Validate and materialize scenario inputs before runtime creation.""" + ... + + +class ModelWarmupAdapter(Protocol): + """Optional adapter hook for model-affine runtime warmup inputs.""" + + def create_model_warmup_sessions( + self, + spec: DemoSpec, + scenario: PreparedScenario, + ) -> Sequence[WarmupSessionInputs]: + """Return temporary synthetic or loopback sessions for model warmup.""" + ... + + +__all__ = [ + "DemoAdapter", + "DemoSpec", + "ModelWarmupAdapter", + "Mp4OutputSpec", + "NullOutputSpec", + "OutputSpec", + "PreparedScenario", + "WebRTCOutputSpec", + "WebRTCAppResources", +] diff --git a/flashdreams/flashdreams/runtime/demo/timing.py b/flashdreams/flashdreams/runtime/demo/timing.py new file mode 100644 index 000000000..e6609cba9 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/timing.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Realtime activation, clock, and input-window primitives for demo run modes.""" + +from __future__ import annotations + +import asyncio +import math +import time +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, field +from typing import Literal, Protocol, runtime_checkable + +from flashdreams.runtime.inputs import UserInputs, UserInputSchema +from flashdreams.runtime.types import StepRequirements + +from .session_inputs import UserInputWindow + +CatchUpPolicy = Literal["drop", "fold", "compress"] + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CatchUpDecision: + """How a realtime clock bounded stale virtual input time.""" + + skipped_s: float = 0.0 + skipped_windows: int = 0 + input_policy: CatchUpPolicy | None = None + reason: str | None = None + + def __post_init__(self) -> None: + if not math.isfinite(self.skipped_s) or self.skipped_s < 0.0: + raise ValueError("CatchUpDecision.skipped_s must be finite and >= 0.") + if self.skipped_windows < 0: + raise ValueError("CatchUpDecision.skipped_windows must be >= 0.") + if self.input_policy not in {None, "drop", "fold", "compress"}: + raise ValueError( + f"Unsupported catch-up input_policy={self.input_policy!r}." + ) + if self.reason is not None and not self.reason.strip(): + raise ValueError("CatchUpDecision.reason must be non-empty when set.") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class RealtimeWindowResult: + """Realtime input window plus any catch-up decision that preceded it.""" + + window: UserInputWindow + catch_up: CatchUpDecision = field(default_factory=CatchUpDecision) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ActivationResult: + """Result of waiting for a realtime activation gate.""" + + activated: bool + reason: str | None = None + + def __post_init__(self) -> None: + if self.reason is not None and not self.reason.strip(): + raise ValueError("ActivationResult.reason must be non-empty when set.") + + +@runtime_checkable +class DeterministicClock(Protocol): + """Clock facts for finite deterministic run modes.""" + + is_realtime: bool + is_deterministic: bool + + +@runtime_checkable +class RealtimeClock(Protocol): + """Realtime virtual clock used by realtime drivers and input sources.""" + + is_realtime: bool + is_deterministic: bool + + def now(self) -> float: ... + + def anchor(self, wall_time_s: float) -> None: ... + + async def wait_until_window_end(self, end_s: float) -> None: ... + + async def apply_backpressure(self, requested_s: float) -> None: ... + + def catch_up( + self, + *, + request: StepRequirements, + max_lag_s: float, + policy: CatchUpPolicy, + ) -> CatchUpDecision: ... + + +@runtime_checkable +class ActivationPolicy(Protocol): + """Wait until a realtime session should start generating.""" + + timeout_s: float | None + + async def wait_until_active( + self, + clock: RealtimeClock | DeterministicClock, + ) -> ActivationResult: ... + + +@runtime_checkable +class ActivationSignal(Protocol): + """Event-like object accepted by ``SignalActivationPolicy``.""" + + def is_set(self) -> bool: ... + + async def wait(self) -> object: ... + + +@dataclass(slots=True) +class AlwaysActiveActivationPolicy: + """Activation policy for batch/null modes or already-ready realtime modes.""" + + timeout_s: float | None = None + anchor_clock: bool = False + + async def wait_until_active( + self, + clock: RealtimeClock | DeterministicClock, + ) -> ActivationResult: + _anchor_if_realtime(clock, anchor=self.anchor_clock) + return ActivationResult(activated=True) + + +@dataclass(slots=True) +class SignalActivationPolicy: + """Activate when any supplied signal fires, with optional timeout.""" + + signals: Sequence[ActivationSignal] + timeout_s: float | None = None + timeout_reason: str = "activation timed out" + anchor_clock: bool = True + + def __post_init__(self) -> None: + if not self.signals: + raise ValueError("SignalActivationPolicy.signals must be non-empty.") + self.signals = tuple(self.signals) + if self.timeout_s is not None and self.timeout_s <= 0.0: + raise ValueError("SignalActivationPolicy.timeout_s must be > 0 when set.") + if not self.timeout_reason.strip(): + raise ValueError("SignalActivationPolicy.timeout_reason must be non-empty.") + + async def wait_until_active( + self, + clock: RealtimeClock | DeterministicClock, + ) -> ActivationResult: + if any(signal.is_set() for signal in self.signals): + _anchor_if_realtime(clock, anchor=self.anchor_clock) + return ActivationResult(activated=True) + + tasks = [asyncio.create_task(signal.wait()) for signal in self.signals] + try: + done, pending = await asyncio.wait( + tasks, + timeout=self.timeout_s, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + return ActivationResult( + activated=False, + reason=self.timeout_reason, + ) + for task in done: + task.result() + _anchor_if_realtime(clock, anchor=self.anchor_clock) + return ActivationResult(activated=True) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +class _RealtimeTimeline(Protocol): + dt: float + next_chunk_start_v: float + + def reset(self, *, start_v: float) -> None: ... + + def sample_chunk( + self, + num_frames: int, + ) -> Sequence[float]: ... + + +@dataclass(slots=True) +class RealtimeEventResampler: + """Transport-neutral realtime window timeline. + + The shared driver only owns virtual time and frame sample locations. Raw + browser/native events are stored as :class:`UserInputs`; model providers + decide how to interpret them. + """ + + fps: float + start_v: float = 0.0 + next_chunk_start_v: float = field(init=False) + _dt: float = field(init=False, repr=False) + + def __post_init__(self) -> None: + if self.fps <= 0: + raise ValueError("fps must be > 0") + self._dt = 1.0 / float(self.fps) + self.next_chunk_start_v = float(self.start_v) + + @property + def dt(self) -> float: + return self._dt + + def reset(self, *, start_v: float) -> None: + self.next_chunk_start_v = float(start_v) + + def sample_chunk(self, num_frames: int) -> tuple[float, ...]: + if num_frames < 1: + raise ValueError("num_frames must be >= 1") + chunk_start_v = self.next_chunk_start_v + chunk_end_v = chunk_start_v + num_frames * self._dt + frame_times = tuple( + chunk_start_v + (index + 1) * self._dt for index in range(num_frames) + ) + self.next_chunk_start_v = chunk_end_v + return frame_times + + +@dataclass(slots=True) +class ResamplerRealtimeClock: + """Realtime clock that reuses a resampler's virtual timeline.""" + + resampler: _RealtimeTimeline + now_fn: Callable[[], float] = time.monotonic + sleep_fn: Callable[[float], Awaitable[None]] = asyncio.sleep + is_realtime: bool = True + is_deterministic: bool = False + _pending_backpressure_s: float = field(default=0.0, init=False, repr=False) + + @property + def pending_backpressure_s(self) -> float: + return self._pending_backpressure_s + + def now(self) -> float: + return float(self.now_fn()) + + def anchor(self, wall_time_s: float) -> None: + if not math.isfinite(wall_time_s): + raise ValueError("wall_time_s must be finite.") + self.resampler.next_chunk_start_v = float(wall_time_s) + self._pending_backpressure_s = 0.0 + + async def wait_until_window_end(self, end_s: float) -> None: + if not math.isfinite(end_s): + raise ValueError("end_s must be finite.") + delay_s = float(end_s) - self.now() + if delay_s > 0.0: + await self.sleep_fn(delay_s) + + async def apply_backpressure(self, requested_s: float) -> None: + if not math.isfinite(requested_s) or requested_s < 0.0: + raise ValueError("requested_s must be finite and >= 0.") + self._pending_backpressure_s += float(requested_s) + + def catch_up( + self, + *, + request: StepRequirements, + max_lag_s: float, + policy: CatchUpPolicy, + ) -> CatchUpDecision: + if policy != "fold": + raise NotImplementedError( + f"Catch-up policy {policy!r} has no existing timeline analog yet." + ) + if not math.isfinite(max_lag_s) or max_lag_s < 0.0: + raise ValueError("max_lag_s must be finite and >= 0.") + + input_frame_count = input_frame_count_from_request(request) + chunk_duration_s = input_frame_count * float(self.resampler.dt) + if chunk_duration_s <= 0.0: + raise ValueError("Realtime resampler dt must produce a positive window.") + + effective_now_s = self.now() + self._pending_backpressure_s + self._pending_backpressure_s = 0.0 + current_start_s = float(self.resampler.next_chunk_start_v) + lag_s = effective_now_s - (current_start_s + chunk_duration_s) + if lag_s <= max_lag_s: + return CatchUpDecision() + + latest_start_s = effective_now_s - chunk_duration_s + if latest_start_s <= current_start_s: + return CatchUpDecision() + + skipped_s = latest_start_s - current_start_s + skipped_windows = max(1, math.ceil(skipped_s / chunk_duration_s)) + self.resampler.next_chunk_start_v = latest_start_s + return CatchUpDecision( + skipped_s=skipped_s, + skipped_windows=skipped_windows, + input_policy=policy, + reason="lag exceeded max_lag_s", + ) + + +@dataclass(slots=True) +class RealtimeEventInputSource: + """Realtime input source backed by raw event windows.""" + + resampler: _RealtimeTimeline + max_lag_s: float | None = None + catch_up_policy: CatchUpPolicy = "fold" + is_finite: bool = False + is_deterministic: bool = False + user_input_schema: UserInputSchema = field(default_factory=UserInputSchema) + + def __post_init__(self) -> None: + if self.max_lag_s is not None and ( + not math.isfinite(self.max_lag_s) or self.max_lag_s < 0.0 + ): + raise ValueError( + "RealtimeEventInputSource.max_lag_s must be finite and >= 0." + ) + if self.catch_up_policy != "fold": + raise NotImplementedError( + f"Catch-up policy {self.catch_up_policy!r} has no existing " + "event-window analog yet." + ) + + def is_finished(self) -> bool: + return False + + def reset(self, *, start_v: float) -> None: + self.resampler.reset(start_v=start_v) + + async def next_realtime_window( + self, + *, + request: StepRequirements, + clock: RealtimeClock, + ) -> RealtimeWindowResult: + input_frame_count = input_frame_count_from_request(request) + chunk_duration_s = input_frame_count * self.resampler.dt + window_end_s = self.resampler.next_chunk_start_v + chunk_duration_s + await clock.wait_until_window_end(window_end_s) + catch_up = clock.catch_up( + request=request, + max_lag_s=self.max_lag_s + if self.max_lag_s is not None + else chunk_duration_s, + policy=self.catch_up_policy, + ) + start_s = self.resampler.next_chunk_start_v + frame_times = self.resampler.sample_chunk(input_frame_count) + end_s = self.resampler.next_chunk_start_v + window = UserInputWindow( + start_s=start_s, + end_s=end_s, + frame_times=tuple(frame_times), + inputs=UserInputs(), + ) + return RealtimeWindowResult(window=window, catch_up=catch_up) + + +def input_frame_count_from_request(request: StepRequirements) -> int: + """Return the positive input frame count declared by a step requirement.""" + + value = request.input_frame_count + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("StepRequirements.input_frame_count must be an integer.") + parsed = value + if parsed <= 0: + raise ValueError("StepRequirements.input_frame_count must be > 0.") + return parsed + + +def _anchor_if_realtime( + clock: RealtimeClock | DeterministicClock, + *, + anchor: bool, +) -> None: + if not anchor or not getattr(clock, "is_realtime", False): + return + now = getattr(clock, "now", None) + clock_anchor = getattr(clock, "anchor", None) + if callable(now) and callable(clock_anchor): + clock_anchor(float(now())) + + +__all__ = [ + "ActivationPolicy", + "ActivationResult", + "ActivationSignal", + "AlwaysActiveActivationPolicy", + "CatchUpDecision", + "CatchUpPolicy", + "DeterministicClock", + "RealtimeEventInputSource", + "RealtimeEventResampler", + "RealtimeClock", + "RealtimeWindowResult", + "ResamplerRealtimeClock", + "SignalActivationPolicy", + "input_frame_count_from_request", +] diff --git a/flashdreams/flashdreams/runtime/demo/validation.py b/flashdreams/flashdreams/runtime/demo/validation.py new file mode 100644 index 000000000..c5058b148 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/validation.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Capability resolution and validation for shared demo runs.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +from flashdreams.runtime.inputs import UserInputCapability, UserInputSchema + +from .run_modes import RunMode, RunModeCapabilities, SessionEdges +from .session_inputs import ( + BatchInputSource, + ModelInputProvider, + ProviderCapabilities, + RealtimeInputSource, +) +from .spec import DemoAdapter, DemoSpec + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ResolvedRunCapabilities: + """Capabilities of one concrete provider/run-mode/session-edges pairing.""" + + finite: bool + deterministic: bool + realtime: bool + resettable: bool + produces_artifacts: bool + + +def resolve_run_capabilities( + *, + spec: DemoSpec, + provider: ModelInputProvider, + session_edges: SessionEdges, +) -> ResolvedRunCapabilities: + """Resolve concrete run capabilities from provider, edges, and config.""" + + provider_capabilities = _provider_capabilities(provider) + clock = session_edges.clock + realtime = bool(getattr(clock, "is_realtime", False)) + deterministic_clock = ( + bool(getattr(clock, "is_deterministic", False)) if clock is not None else True + ) + config = spec.config + seeded = config is not None and config.seed is not None + return ResolvedRunCapabilities( + finite=bool(session_edges.input_source.is_finite), + deterministic=( + provider_capabilities.deterministic_given_inputs + and bool(session_edges.input_source.is_deterministic) + and deterministic_clock + and seeded + ), + realtime=realtime, + resettable=provider_capabilities.supports_reset, + produces_artifacts=bool(session_edges.output_sink.produces_artifacts), + ) + + +def validate_resolved_run( + *, + spec: DemoSpec, + adapter: DemoAdapter, + provider: ModelInputProvider, + run_mode: RunMode, + session_edges: SessionEdges, + resolved: ResolvedRunCapabilities, +) -> None: + """Reject structurally incompatible provider/input/run-mode combinations.""" + + del spec, adapter + provider_capabilities = _provider_capabilities(provider) + run_mode_capabilities = _run_mode_capabilities(run_mode) + _validate_input_source_shape( + run_mode_capabilities=run_mode_capabilities, + session_edges=session_edges, + resolved=resolved, + ) + _validate_provider_modes( + provider_capabilities=provider_capabilities, + run_mode_capabilities=run_mode_capabilities, + resolved=resolved, + ) + _validate_user_input_schema( + provider_schema=provider_capabilities.user_input_schema, + source_schema=_input_source_user_input_schema(session_edges.input_source), + ) + + +def _validate_input_source_shape( + *, + run_mode_capabilities: RunModeCapabilities, + session_edges: SessionEdges, + resolved: ResolvedRunCapabilities, +) -> None: + input_source = session_edges.input_source + if run_mode_capabilities.realtime: + if not isinstance(input_source, RealtimeInputSource): + raise ValueError("Realtime run modes require a RealtimeInputSource.") + if session_edges.clock is None or not resolved.realtime: + raise ValueError("Realtime run modes require a realtime clock.") + return + if not isinstance(input_source, BatchInputSource): + raise ValueError("Batch run modes require a BatchInputSource.") + if resolved.realtime: + raise ValueError("Batch run modes cannot use a realtime clock.") + + +def _validate_provider_modes( + *, + provider_capabilities: ProviderCapabilities, + run_mode_capabilities: RunModeCapabilities, + resolved: ResolvedRunCapabilities, +) -> None: + if run_mode_capabilities.realtime and not ( + provider_capabilities.supports_realtime_clock + ): + raise ValueError("Provider does not support realtime input.") + if run_mode_capabilities.requires_finite_input: + if not resolved.finite: + raise ValueError("Run mode requires finite input.") + if not provider_capabilities.supports_recorded_input: + raise ValueError("Provider does not support recorded input.") + if resolved.produces_artifacts and not run_mode_capabilities.supports_artifacts: + raise ValueError("Run mode does not support artifact output.") + if ( + run_mode_capabilities.supports_interactive_events + and not provider_capabilities.supports_realtime_clock + ): + raise ValueError("Interactive run mode requires realtime provider support.") + + +def _validate_user_input_schema( + *, + provider_schema: UserInputSchema, + source_schema: UserInputSchema, +) -> None: + missing = _missing_capabilities( + required=provider_schema.declared_capabilities(), + provided=source_schema, + ) + if missing: + names = ", ".join( + f"{capability.event_type}[{','.join(sorted(capability.payload_fields))}]" + for capability in missing + ) + raise ValueError( + f"Input source does not satisfy provider raw user input schema: {names}." + ) + + +def _missing_capabilities( + *, + required: Sequence[UserInputCapability], + provided: UserInputSchema, +) -> tuple[UserInputCapability, ...]: + return tuple( + capability for capability in required if not provided.supports(capability) + ) + + +def _provider_capabilities(provider: ModelInputProvider) -> ProviderCapabilities: + capabilities = getattr(provider, "capabilities", None) + if not isinstance(capabilities, ProviderCapabilities): + raise TypeError( + "ModelInputProvider.capabilities must be a ProviderCapabilities " + f"instance, got {type(capabilities).__name__}." + ) + return capabilities + + +def _run_mode_capabilities(run_mode: RunMode) -> RunModeCapabilities: + capabilities = getattr(run_mode, "capabilities", None) + if not isinstance(capabilities, RunModeCapabilities): + raise TypeError( + "RunMode.capabilities must be a RunModeCapabilities instance, " + f"got {type(capabilities).__name__}." + ) + return capabilities + + +def _input_source_user_input_schema(input_source: object) -> UserInputSchema: + schema = getattr(input_source, "user_input_schema", None) + if not isinstance(schema, UserInputSchema): + raise TypeError( + "InputSource.user_input_schema must be a UserInputSchema instance, " + f"got {type(schema).__name__}." + ) + return schema + + +__all__ = [ + "ResolvedRunCapabilities", + "resolve_run_capabilities", + "validate_resolved_run", +] diff --git a/flashdreams/flashdreams/runtime/demo/webrtc.py b/flashdreams/flashdreams/runtime/demo/webrtc.py new file mode 100644 index 000000000..2e835a6e9 --- /dev/null +++ b/flashdreams/flashdreams/runtime/demo/webrtc.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deprecated location for WebRTC demo construction helpers. + +The concrete server helper lives in the WebRTC serving package so the runtime +demo API does not depend on transport infrastructure. +""" + +from __future__ import annotations + +__all__: list[str] = [] diff --git a/flashdreams/flashdreams/runtime/inputs.py b/flashdreams/flashdreams/runtime/inputs.py new file mode 100644 index 000000000..70260de2e --- /dev/null +++ b/flashdreams/flashdreams/runtime/inputs.py @@ -0,0 +1,449 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""User- and model-input envelopes for the experimental runtime API.""" + +from __future__ import annotations + +import math +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, cast + +from flashdreams.infra.time import TimeWindow +from flashdreams.runtime._utils import freeze_mapping + +InputPhase = Literal["global_conditioning", "step"] + +INPUT_PHASES: tuple[InputPhase, ...] = ("global_conditioning", "step") + + +def validate_phase(value: str) -> InputPhase: + """Return ``value`` as a validated :data:`InputPhase`.""" + if value not in INPUT_PHASES: + raise ValueError( + f"phase must be 'global_conditioning' or 'step', got {value!r}." + ) + return cast(InputPhase, value) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class InputField: + """Lightweight schema field for user snapshots or model inputs. + + ``name`` is the model-facing input role and payload key, such as ``prompt`` + or ``negative_prompt``. ``input_modality``, ``frequency_consumed``, and + ``metadata`` are query hints only. Adapter-owned validation still decides + concrete shape, dtype, units, and tensor layout. + """ + + name: str + required: bool = True + input_modality: str | None = None + frequency_consumed: str | None = None + metadata: Mapping[str, Any] = field( + default_factory=dict, + compare=False, + hash=False, + ) + description: str = "" + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("InputField.name must be non-empty.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class UserInputCapability: + """One user event a source or mapping can provide, at payload granularity. + + ``UserInputSchema.event_types`` declares only that an event type exists. A + capability additionally pins the payload fields carried by that event, so a + mapping can state that it needs ``key_down`` events that actually carry a + ``key``. + """ + + event_type: str + input_modality: str | None = None + payload_fields: frozenset[str] = field(default_factory=frozenset) + metadata: Mapping[str, Any] = field( + default_factory=dict, + compare=False, + hash=False, + ) + description: str = "" + + def __post_init__(self) -> None: + if not self.event_type.strip(): + raise ValueError("UserInputCapability.event_type must be non-empty.") + for payload_field in self.payload_fields: + if not payload_field.strip(): + raise ValueError("payload field names must be non-empty.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + def is_satisfied_by(self, provider: "UserInputCapability") -> bool: + """Return whether ``provider`` can satisfy this consumed capability.""" + if self.event_type != provider.event_type: + return False + input_modality_ok = ( + self.input_modality is None + or provider.input_modality is None + or self.input_modality == provider.input_modality + ) + return input_modality_ok and self.payload_fields.issubset( + provider.payload_fields + ) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class UserInputSchema: + """Minimal metadata for user events a source or mapping can provide.""" + + event_types: frozenset[str] = field(default_factory=frozenset) + snapshot_fields: tuple[InputField, ...] = () + capabilities: tuple[UserInputCapability, ...] = () + description: str = "" + + def supports_event_types(self, event_types: Iterable[str]) -> bool: + """Return whether every requested event type is declared supported.""" + requested = frozenset(event_types) + if not requested: + return True + return requested.issubset(self.declared_event_types()) + + def declared_event_types(self) -> frozenset[str]: + """Return event types from ``event_types`` and from ``capabilities``.""" + return self.event_types | frozenset( + capability.event_type for capability in self.capabilities + ) + + def declared_capabilities(self) -> tuple[UserInputCapability, ...]: + """Return capabilities, widened with bare ``event_types`` entries. + + A plain ``event_types`` entry carries no payload promise, so it is + modeled as a capability with no payload fields. Coarse schemas written + before capabilities existed therefore still satisfy any consumer that + does not require specific payload fields. + """ + declared = list(self.capabilities) + covered = {capability.event_type for capability in declared} + declared.extend( + UserInputCapability(event_type=event_type) + for event_type in sorted(self.event_types - covered) + ) + return tuple(declared) + + def supports(self, capability: UserInputCapability) -> bool: + """Return whether this source can satisfy ``capability``.""" + return any( + capability.is_satisfied_by(provider) + for provider in self.declared_capabilities() + ) + + def validate_event(self, event: "UserInputEvent") -> None: + """Validate one event against the event types this source declares.""" + matching = [ + capability + for capability in self.declared_capabilities() + if capability.event_type == event.event_type + ] + if not matching: + raise ValueError( + f"User input source does not provide event type {event.event_type!r}." + ) + payload_keys = set(event.payload) + if not any( + capability.payload_fields.issubset(payload_keys) for capability in matching + ): + expected = sorted( + { + payload_field + for capability in matching + for payload_field in capability.payload_fields + } + ) + raise ValueError( + f"Event {event.event_type!r} payload is missing required " + f"fields: {expected}." + ) + + def missing_snapshot(self, inputs: "UserInputs") -> tuple[str, ...]: + """Return required snapshot fields absent from ``inputs``.""" + return _missing_required(self.snapshot_fields, inputs.snapshot) + + def require_snapshot(self, inputs: "UserInputs") -> None: + """Raise if required snapshot fields are absent.""" + missing = self.missing_snapshot(inputs) + if missing: + raise ValueError(f"Missing required user snapshot field(s): {missing}") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class InferenceInputSchema: + """Minimal metadata for global conditioning and per-step inputs.""" + + global_conditioning_fields: tuple[InputField, ...] = () + """Model inputs carried in the global conditioning slot.""" + + step_fields: tuple[InputField, ...] = () + """Model inputs required for one session step.""" + + description: str = "" + + def fields_for(self, phase: InputPhase) -> tuple[InputField, ...]: + """Return every declared field for ``phase``.""" + return ( + self.global_conditioning_fields + if validate_phase(phase) == "global_conditioning" + else self.step_fields + ) + + def required_fields( + self, + phase: InputPhase | None = None, + ) -> tuple[tuple[InputPhase, InputField], ...]: + """Return required fields as ``(phase, field)``, optionally filtered.""" + return self._select(phase, required=True) + + def optional_fields( + self, + phase: InputPhase | None = None, + ) -> tuple[tuple[InputPhase, InputField], ...]: + """Return optional fields as ``(phase, field)``, optionally filtered.""" + return self._select(phase, required=False) + + def field_for(self, *, name: str, phase: InputPhase) -> InputField | None: + """Return one declared field, if present.""" + for input_field in self.fields_for(phase): + if input_field.name == name: + return input_field + return None + + def _select( + self, + phase: InputPhase | None, + *, + required: bool, + ) -> tuple[tuple[InputPhase, InputField], ...]: + phases = INPUT_PHASES if phase is None else (validate_phase(phase),) + return tuple( + (each_phase, input_field) + for each_phase in phases + for input_field in self.fields_for(each_phase) + if input_field.required is required + ) + + def missing_global_conditioning(self, inputs: "InferenceInput") -> tuple[str, ...]: + """Return required global conditioning fields absent from ``inputs``.""" + return _missing_required( + self.global_conditioning_fields, + inputs.global_conditioning, + ) + + def missing_step(self, inputs: "InferenceInput") -> tuple[str, ...]: + """Return required per-step fields absent from ``inputs``.""" + return _missing_required(self.step_fields, inputs.step) + + def require_global_conditioning(self, inputs: "InferenceInput") -> None: + """Raise if required global conditioning fields are absent.""" + missing = self.missing_global_conditioning(inputs) + if missing: + raise ValueError( + f"Missing required global conditioning input(s): {missing}" + ) + + def require_step(self, inputs: "InferenceInput") -> None: + """Raise if required per-step fields are absent.""" + missing = self.missing_step(inputs) + if missing: + raise ValueError(f"Missing required step model input(s): {missing}") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class UserInputEvent: + """User-facing input event timestamped in seconds since session start. + + Live runtimes, transports, replay loaders, or benchmark drivers stamp events + before queuing them for input mapping. Payload schema is intentionally minimal + in T1; concrete event catalogs belong to follow-up input-mapping work. + """ + + __hash__ = None + + timestamp_s: float + event_type: str + payload: Mapping[str, Any] = field(default_factory=dict) + source: str | None = None + source_event_id: str | None = None + + def __post_init__(self) -> None: + if not math.isfinite(self.timestamp_s) or self.timestamp_s < 0: + raise ValueError("UserInputEvent.timestamp_s must be finite and >= 0.") + if not self.event_type.strip(): + raise ValueError("UserInputEvent.event_type must be non-empty.") + object.__setattr__(self, "payload", freeze_mapping(self.payload)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class UserInputs: + """Transport-neutral user input batch or window. + + Events must be in non-decreasing timestamp order. Runtimes can pass the full + input history, a drained queue batch, or a session-requested time window to an + ``InputMapping``. + """ + + __hash__ = None + + events: tuple[UserInputEvent, ...] = () + snapshot: Mapping[str, Any] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + previous_timestamp_s = -math.inf + for event in self.events: + if event.timestamp_s < previous_timestamp_s: + raise ValueError( + "UserInputs.events must be sorted by non-decreasing timestamp_s." + ) + previous_timestamp_s = event.timestamp_s + object.__setattr__(self, "snapshot", freeze_mapping(self.snapshot)) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + def window(self, time_window: TimeWindow) -> "UserInputs": + """Return inputs with events filtered to ``time_window``.""" + return UserInputs( + events=tuple( + event + for event in self.events + if time_window.contains(event.timestamp_s) + ), + snapshot=self.snapshot, + metadata=self.metadata, + ) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CanonicalModality: + """A device-independent user input an application consumes. + + This is the middle layer of ``raw input -> canonicalized input -> encoded + inference input``. Applications and benchmarks declare and consume + modalities; they never read raw device events, so adding a new device is a + converter registration rather than an application change. + + Modalities describe live user control only. Global conditioning such as a + prompt or conditioning frame is application-owned and reaches + :class:`InferenceInput` directly, without passing through this layer. + """ + + name: str + payload_fields: frozenset[str] = field(default_factory=frozenset) + metadata: Mapping[str, Any] = field( + default_factory=dict, + compare=False, + hash=False, + ) + description: str = "" + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("CanonicalModality.name must be non-empty.") + for payload_field in self.payload_fields: + if not payload_field.strip(): + raise ValueError("payload field names must be non-empty.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + def is_satisfied_by(self, provider: "CanonicalModality") -> bool: + """Return whether ``provider`` can satisfy this consumed modality.""" + return self.name == provider.name and self.payload_fields.issubset( + provider.payload_fields + ) + + def value(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + """Return ``payload`` frozen, checking it covers this modality.""" + missing = sorted(self.payload_fields - set(payload)) + if missing: + raise ValueError( + f"Canonical modality {self.name!r} requires payload fields " + f"{missing}, which the converter did not produce." + ) + return freeze_mapping(payload) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CanonicalInputSchema: + """Canonical modalities an application can be fed by a given source.""" + + modalities: tuple[CanonicalModality, ...] = () + description: str = "" + + def supports(self, modality: CanonicalModality) -> bool: + """Return whether this source can supply ``modality``.""" + return any(modality.is_satisfied_by(provided) for provided in self.modalities) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CanonicalInputs: + """Canonicalized user input for one step, keyed by modality name. + + Values are level-triggered and normally present every step: a key held down + emits no events but still means full throttle. Global conditioning does not + appear here; it is application-owned and reaches :class:`InferenceInput` + directly. + """ + + __hash__ = None + + values: Mapping[str, Any] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "values", freeze_mapping(self.values)) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class InferenceInput: + """Encoded inputs for one :class:`InferenceSession` call. + + Two conditioning slots: + + - ``global_conditioning``: values that condition the whole rollout, such as + the conditioning frame or prompt. Session start/reset establishes this + state; a step call may carry a non-empty payload to request an update when + the model supports it. + - ``step``: values needed to generate the next chunk or frame. + """ + + __hash__ = None + + global_conditioning: Mapping[str, Any] = field(default_factory=dict) + step: Mapping[str, Any] = field(default_factory=dict) + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__( + self, "global_conditioning", freeze_mapping(self.global_conditioning) + ) + object.__setattr__(self, "step", freeze_mapping(self.step)) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + def for_phase(self, phase: InputPhase) -> Mapping[str, Any]: + """Return the payload mapping for ``phase``.""" + return ( + self.global_conditioning + if validate_phase(phase) == "global_conditioning" + else self.step + ) + + +def _missing_required( + fields: tuple[InputField, ...], payload: Mapping[str, Any] +) -> tuple[str, ...]: + return tuple( + input_field.name + for input_field in fields + if input_field.required and input_field.name not in payload + ) diff --git a/flashdreams/flashdreams/runtime/interfaces.py b/flashdreams/flashdreams/runtime/interfaces.py new file mode 100644 index 000000000..5c5054dc4 --- /dev/null +++ b/flashdreams/flashdreams/runtime/interfaces.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Protocols for model adapters, reusable runtimes, and sessions.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from flashdreams.runtime.config import InferenceConfig +from flashdreams.runtime.inputs import ( + CanonicalInputSchema, + InferenceInput, + InferenceInputSchema, +) +from flashdreams.runtime.mapping import InputMapping +from flashdreams.runtime.types import StepRequest, StepResult + + +@runtime_checkable +class InferenceSession(Protocol): + """One rollout or stream with isolated model/cache state.""" + + def next_step_request(self) -> StepRequest | None: + """Return the next step's runtime request, or ``None`` when complete.""" + ... + + def step(self, inputs: InferenceInput) -> StepResult: + """Run one sequential inference step.""" + ... + + def reset(self, inputs: InferenceInput | None = None) -> None: + """Reset this session's rollout state when the backend supports it.""" + ... + + def close(self) -> None: + """Release per-session resources.""" + ... + + +@runtime_checkable +class InferenceRuntime(Protocol): + """Heavyweight reusable runtime created from :class:`InferenceConfig`.""" + + def start_session(self, inputs: InferenceInput) -> InferenceSession: + """Create an isolated session from global conditioning inputs.""" + ... + + def close(self) -> None: + """Release model/backend resources.""" + ... + + +# Do not mark ModelAdapter runtime-checkable: properties make issubclass() +# unreliable, and isinstance() would only verify attribute presence. +class ModelAdapter(Protocol): + """Model-specific boundary that declares defaults and creates runtimes. + + Adapters declare model-facing input requirements, the canonical modalities + their default mapping consumes, and an optional default mapping between the + two. Runtime, application, or benchmark code may override that mapping while + preserving the same ``CanonicalInputs`` to ``InferenceInput`` boundary. + """ + + @property + def model_id(self) -> str: + """Stable identity for the model adapter or runtime integration.""" + ... + + @property + def inference_input_schema(self) -> InferenceInputSchema: + """Model-facing global conditioning and per-step input requirements.""" + ... + + @property + def canonical_input_schema(self) -> CanonicalInputSchema | None: + """Canonical modalities the adapter's default mapping consumes.""" + ... + + def default_input_mapping(self) -> InputMapping | None: + """Return the model-provided default canonical-to-model mapping.""" + ... + + def validate_config(self, config: InferenceConfig) -> None: + """Fail early for unsupported runtime settings.""" + ... + + def create_runtime(self, config: InferenceConfig) -> InferenceRuntime: + """Initialize and return the heavyweight runtime.""" + ... diff --git a/flashdreams/flashdreams/runtime/keyboard.py b/flashdreams/flashdreams/runtime/keyboard.py new file mode 100644 index 000000000..5ef83fc82 --- /dev/null +++ b/flashdreams/flashdreams/runtime/keyboard.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keyboard state helpers shared by runtime input canonicalizers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +DEFAULT_SUPPORTED_KEYS = frozenset({"w", "a", "s", "d", "q", "e", "i", "k", "j", "l"}) +DRIVING_SUPPORTED_KEYS = frozenset( + {"w", "a", "s", "d", "up", "down", "left", "right", "space"} +) +WSAD_SUPPORTED_KEYS = frozenset({"w", "a", "s", "d"}) +KEY_ALIASES = { + "arrowup": "w", + "arrowleft": "a", + "arrowdown": "s", + "arrowright": "d", +} + + +@dataclass(frozen=True, slots=True) +class ResetRequest: + """Transport-neutral request to reset the realtime rollout.""" + + reason: str | None = None + request_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class PromptRequest: + """Transport-neutral prompt update request.""" + + prompt: str + negative_prompt: str | None = None + request_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class ImageRequest: + """Transport-neutral image update request.""" + + data: bytes + content_type: str + request_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class SparseInputSnapshot: + """Sparse input state sampled at a realtime loop boundary.""" + + timestamp_s: float + pressed_keys: frozenset[str] = field(default_factory=frozenset) + effective_keys: frozenset[str] = field(default_factory=frozenset) + reset: ResetRequest | None = None + prompt: PromptRequest | None = None + image: ImageRequest | None = None + + +def normalize_key(key: str) -> str: + normalized = key.strip().lower() + return KEY_ALIASES.get(normalized, normalized) + + +@dataclass(slots=True) +class KeyboardState: + pressed_keys: set[str] = field(default_factory=set) + supported_keys: frozenset[str] = DEFAULT_SUPPORTED_KEYS + _press_order: dict[str, int] = field(default_factory=dict) + _press_counter: int = 0 + + def apply_event(self, *, event: str, key: str) -> bool: + normalized_key = normalize_key(key) + if normalized_key not in self.supported_keys: + return False + + normalized_event = event.strip().lower() + if normalized_event == "keydown": + self.pressed_keys.add(normalized_key) + self._press_counter += 1 + self._press_order[normalized_key] = self._press_counter + return True + if normalized_event == "keyup": + self.pressed_keys.discard(normalized_key) + self._press_order.pop(normalized_key, None) + return True + return False + + def snapshot(self) -> frozenset[str]: + return frozenset(self.pressed_keys) + + def sparse_snapshot(self, *, timestamp_s: float) -> SparseInputSnapshot: + return SparseInputSnapshot( + timestamp_s=timestamp_s, + pressed_keys=self.snapshot(), + effective_keys=self.resolved_effective_keys(), + ) + + def _latest_pressed(self, keys: tuple[str, ...]) -> str | None: + latest_key: str | None = None + latest_idx = -1 + for key in keys: + if key not in self.pressed_keys: + continue + idx = self._press_order.get(key, -1) + if idx >= latest_idx: + latest_idx = idx + latest_key = key + return latest_key + + def resolved_effective_keys(self) -> frozenset[str]: + effective: set[str] = set() + for key in ( + self._latest_pressed(("w", "s")), + self._latest_pressed(("a", "d", "j", "l")), + self._latest_pressed(("q", "e")), + self._latest_pressed(("i", "k")), + ): + if key is not None: + effective.add(key) + return frozenset(key for key in effective if key in self.supported_keys) + + +__all__ = [ + "DEFAULT_SUPPORTED_KEYS", + "DRIVING_SUPPORTED_KEYS", + "ImageRequest", + "KEY_ALIASES", + "KeyboardState", + "PromptRequest", + "ResetRequest", + "SparseInputSnapshot", + "WSAD_SUPPORTED_KEYS", + "normalize_key", +] diff --git a/flashdreams/flashdreams/runtime/mapping.py b/flashdreams/flashdreams/runtime/mapping.py new file mode 100644 index 000000000..6dfb5cc45 --- /dev/null +++ b/flashdreams/flashdreams/runtime/mapping.py @@ -0,0 +1,384 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Input mapping boundary from canonical inputs to encoded inference inputs.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from typing import Any, Protocol, runtime_checkable + +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.inputs import ( + INPUT_PHASES, + CanonicalInputs, + CanonicalInputSchema, + CanonicalModality, + InferenceInput, + InferenceInputSchema, + InputField, + InputPhase, +) +from flashdreams.runtime.types import StepRequest + + +@runtime_checkable +class InputMapping(Protocol): + """Convert user-facing inputs into model-facing inputs. + + A mapping may be supplied by the model adapter as a default or by an + application/runtime override. Step mappings usually receive a timestamped + event window selected by the runner for the current model step or chunk. + """ + + def validate( + self, + *, + canonical_schema: CanonicalInputSchema | None = None, + inference_input_schema: InferenceInputSchema | None = None, + ) -> None: + """Fail early for obvious app, event-source, and model mismatches.""" + ... + + def map_global_conditioning_inputs( + self, + *, + canonical_inputs: CanonicalInputs, + inference_input: InferenceInput, + ) -> InferenceInput: + """Build global conditioning inputs for session start or reset.""" + ... + + def map_step_inputs( + self, + *, + canonical_inputs: CanonicalInputs, + inference_input: InferenceInput, + request: StepRequest, + ) -> InferenceInput: + """Build model inputs for one session step from the current input window.""" + ... + + +class IdentityInputMapping: + """No-op mapper for fixed model-input or simple generation flows.""" + + def validate( + self, + *, + canonical_schema: CanonicalInputSchema | None = None, + inference_input_schema: InferenceInputSchema | None = None, + ) -> None: + del canonical_schema, inference_input_schema + + def map_global_conditioning_inputs( + self, + *, + canonical_inputs: CanonicalInputs, + inference_input: InferenceInput, + ) -> InferenceInput: + del canonical_inputs + return inference_input + + def map_step_inputs( + self, + *, + canonical_inputs: CanonicalInputs, + inference_input: InferenceInput, + request: StepRequest, + ) -> InferenceInput: + del canonical_inputs, request + return inference_input + + +@dataclass(frozen=True, kw_only=True, slots=True) +class InputMappingSchema: + """Declarative compatibility surface for one mapping. + + ``InputMapping.validate`` fails a run late and opaquely: it raises, but it + cannot answer which optional model inputs a source would enable, or which + missing user capability is responsible for an unreachable model input. This + schema makes those questions answerable before runtime initialization. + """ + + name: str = "input-mapping" + consumes: tuple[CanonicalModality, ...] = () + produces_global_conditioning: tuple[InputField, ...] = () + produces_step: tuple[InputField, ...] = () + metadata: Mapping[str, Any] = field( + default_factory=dict, + compare=False, + hash=False, + ) + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("InputMappingSchema.name must be non-empty.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + def produces_for(self, phase: InputPhase) -> tuple[InputField, ...]: + """Return the fields this mapping produces for ``phase``.""" + return ( + self.produces_global_conditioning + if phase == "global_conditioning" + else self.produces_step + ) + + def can_produce(self, phase: InputPhase, required: InputField) -> bool: + """Return whether this mapping can produce ``required`` in ``phase``.""" + return any( + _field_matches(produced, required) for produced in self.produces_for(phase) + ) + + +def _field_matches(produced: InputField, required: InputField) -> bool: + if produced.name != required.name: + return False + input_modality_ok = ( + produced.input_modality is None + or required.input_modality is None + or produced.input_modality == required.input_modality + ) + return input_modality_ok + + +@dataclass(frozen=True, kw_only=True, slots=True) +class MappingCompatibility: + """Compatibility report for one source, model schema, and mapping set. + + Mappings whose consumed capabilities the source cannot provide are reported + in ``unavailable_mapping_schemas`` and excluded from the satisfied/available + reports, so those lists only name model inputs that can really be produced. + """ + + __hash__ = None + + canonical_schema: CanonicalInputSchema + inference_input_schema: InferenceInputSchema + mapping_schema: InputMappingSchema + missing_modalities: tuple[CanonicalModality, ...] = () + missing_required_model_fields: tuple[tuple[InputPhase, InputField], ...] = () + satisfied_required_model_fields: tuple[tuple[InputPhase, InputField], ...] = () + available_optional_model_fields: tuple[tuple[InputPhase, InputField], ...] = () + unavailable_mapping_schemas: tuple[InputMappingSchema, ...] = () + + @property + def can_drive(self) -> bool: + """Return whether this source can drive this model through the mapping. + + A mapping the source cannot feed does not block the run unless it was + the only way to produce a required model input. + """ + return not (self.missing_required_model_fields or self.missing_modalities) + + @property + def unavailable_mapping_names(self) -> tuple[str, ...]: + """Return names of mappings dropped because the source cannot feed them.""" + return tuple(schema.name for schema in self.unavailable_mapping_schemas) + + def raise_if_incompatible(self) -> None: + """Raise a compact error when this mapping cannot drive the model.""" + if self.can_drive: + return + problems: list[str] = [] + if self.missing_modalities: + missing = ", ".join(modality.name for modality in self.missing_modalities) + problems.append(f"missing canonical modalities: {missing}") + if self.missing_required_model_fields: + missing = ", ".join( + f"{phase}:{input_field.name}" + for phase, input_field in self.missing_required_model_fields + ) + problems.append(f"missing required model inputs: {missing}") + if self.unavailable_mapping_schemas: + problems.append( + "unavailable mappings: " + ", ".join(self.unavailable_mapping_names) + ) + raise ValueError( + f"Input mapping {self.mapping_schema.name!r} cannot drive this model " + f"from the selected source: " + "; ".join(problems) + ) + + +def _source_can_feed( + canonical_schema: CanonicalInputSchema, + mapping_schema: InputMappingSchema, +) -> bool: + return all( + canonical_schema.supports(modality) for modality in mapping_schema.consumes + ) + + +def combine_mapping_schemas( + mapping_schemas: Sequence[InputMappingSchema], + *, + name: str = "input-mapping-set", +) -> InputMappingSchema: + """Combine independently declared mappings into one compatibility surface. + + Duplicates are collapsed. Because ``metadata`` is excluded from equality, + the metadata of collapsed duplicates is merged rather than dropped, with the + first declaration winning on conflicting keys. + """ + consumes: list[CanonicalModality] = [] + produces: dict[InputPhase, list[InputField]] = { + "global_conditioning": [], + "step": [], + } + + def _merge(target: list[Any], value: Any) -> None: + for index, existing in enumerate(target): + if existing == value: + if value.metadata: + target[index] = replace( + existing, + metadata={**dict(value.metadata), **dict(existing.metadata)}, + ) + return + target.append(value) + + for mapping_schema in mapping_schemas: + if not isinstance(mapping_schema, InputMappingSchema): + raise TypeError("mapping_schemas must contain InputMappingSchema objects.") + for modality in mapping_schema.consumes: + _merge(consumes, modality) + for phase in INPUT_PHASES: + for input_field in mapping_schema.produces_for(phase): + _merge(produces[phase], input_field) + + return InputMappingSchema( + name=name, + consumes=tuple(consumes), + produces_global_conditioning=tuple(produces["global_conditioning"]), + produces_step=tuple(produces["step"]), + ) + + +def _build_compatibility( + *, + canonical_schema: CanonicalInputSchema, + inference_input_schema: InferenceInputSchema, + mapping_schemas: Sequence[InputMappingSchema], + reported_schema: InputMappingSchema, +) -> MappingCompatibility: + feedable: list[InputMappingSchema] = [] + unavailable: list[InputMappingSchema] = [] + for mapping_schema in mapping_schemas: + if _source_can_feed(canonical_schema, mapping_schema): + feedable.append(mapping_schema) + else: + unavailable.append(mapping_schema) + + usable = combine_mapping_schemas(feedable, name=reported_schema.name) + required = inference_input_schema.required_fields() + missing_required = tuple( + (phase, input_field) + for phase, input_field in required + if not usable.can_produce(phase, input_field) + ) + satisfied_required = tuple( + (phase, input_field) + for phase, input_field in required + if usable.can_produce(phase, input_field) + ) + available_optional = tuple( + (phase, input_field) + for phase, input_field in inference_input_schema.optional_fields() + if usable.can_produce(phase, input_field) + ) + + # Only capabilities that block a required model input make the mapping + # unusable. A dropped mapping that fed nothing but optional fields degrades + # the run instead of vetoing it. + missing_modalities: list[CanonicalModality] = [] + for mapping_schema in unavailable: + if not any( + mapping_schema.can_produce(phase, input_field) + for phase, input_field in missing_required + ): + continue + for modality in mapping_schema.consumes: + if canonical_schema.supports(modality) or modality in missing_modalities: + continue + missing_modalities.append(modality) + + return MappingCompatibility( + canonical_schema=canonical_schema, + inference_input_schema=inference_input_schema, + mapping_schema=reported_schema, + missing_modalities=tuple(missing_modalities), + missing_required_model_fields=missing_required, + satisfied_required_model_fields=satisfied_required, + available_optional_model_fields=available_optional, + unavailable_mapping_schemas=tuple(unavailable), + ) + + +def check_mapping_compatibility( + *, + canonical_schema: CanonicalInputSchema, + inference_input_schema: InferenceInputSchema, + mapping_schema: InputMappingSchema, +) -> MappingCompatibility: + """Check whether a user-input source can drive a model through a mapping.""" + if not isinstance(mapping_schema, InputMappingSchema): + raise TypeError("mapping_schema must be an InputMappingSchema object.") + return _build_compatibility( + canonical_schema=canonical_schema, + inference_input_schema=inference_input_schema, + mapping_schemas=(mapping_schema,), + reported_schema=mapping_schema, + ) + + +def check_mapping_set_compatibility( + *, + canonical_schema: CanonicalInputSchema, + inference_input_schema: InferenceInputSchema, + mapping_schemas: Sequence[InputMappingSchema], + name: str = "input-mapping-set", +) -> MappingCompatibility: + """Check compatibility for a composed set of mappings. + + Each mapping keeps its own consumes/produces link, so a mapping the source + cannot feed only costs the model inputs that mapping produced. + """ + mapping_schemas = tuple(mapping_schemas) + return _build_compatibility( + canonical_schema=canonical_schema, + inference_input_schema=inference_input_schema, + mapping_schemas=mapping_schemas, + reported_schema=combine_mapping_schemas(mapping_schemas, name=name), + ) + + +def undeclared_inference_inputs( + inputs: InferenceInput, + mapping_schema: InputMappingSchema, +) -> tuple[tuple[InputPhase, str], ...]: + """Return payload keys a mapping produced but did not declare. + + Mapping schemas are hand-written, so they drift from what + ``map_global_conditioning_inputs``/``map_step_inputs`` actually return. + Mapping tests can use this to keep the declared compatibility surface + honest. + """ + return tuple( + (phase, key) + for phase in INPUT_PHASES + for key in inputs.for_phase(phase) + if not any( + declared.name == key for declared in mapping_schema.produces_for(phase) + ) + ) + + +@runtime_checkable +class DeclaresMappingSchema(Protocol): + """Optional refinement of :class:`InputMapping` that declares its surface.""" + + @property + def mapping_schema(self) -> InputMappingSchema: + """Return the declarative compatibility surface for this mapping.""" + ... diff --git a/flashdreams/flashdreams/runtime/metrics.py b/flashdreams/flashdreams/runtime/metrics.py new file mode 100644 index 000000000..8a8fd6b1b --- /dev/null +++ b/flashdreams/flashdreams/runtime/metrics.py @@ -0,0 +1,336 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime metrics boundary for inference sessions.""" + +from __future__ import annotations + +import math +from collections import Counter, defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from flashdreams.runtime._utils import freeze_mapping + + +@dataclass(frozen=True, kw_only=True, slots=True) +class RuntimeMetricSample: + """One runtime metric sample. + + Timing samples should use seconds as their canonical unit. + """ + + __hash__ = None + + name: str + value: float | int + unit: str = "s" + step_index: int | None = None + category: str = "runtime" + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("RuntimeMetricSample.name must be non-empty.") + if isinstance(self.value, bool) or not isinstance(self.value, (int, float)): + raise TypeError("RuntimeMetricSample.value must be numeric.") + if not math.isfinite(float(self.value)): + raise ValueError("RuntimeMetricSample.value must be finite.") + if self.step_index is not None and self.step_index < 0: + raise ValueError("RuntimeMetricSample.step_index must be >= 0.") + if not self.unit.strip(): + raise ValueError("RuntimeMetricSample.unit must be non-empty.") + if self.category == "timing" and self.unit != "s": + raise ValueError("Timing metric samples must use unit='s'.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class MetricsSnapshot: + """Closed session or run metrics summary.""" + + counters: Mapping[str, int | float] = field(default_factory=dict) + timings: Mapping[str, Sequence[float]] = field(default_factory=dict) + session_statuses: Sequence[str] = () + errors: Sequence[str] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "counters", freeze_mapping(self.counters)) + object.__setattr__( + self, + "timings", + freeze_mapping( + {key: tuple(values) for key, values in self.timings.items()} + ), + ) + object.__setattr__(self, "session_statuses", tuple(self.session_statuses)) + object.__setattr__(self, "errors", tuple(self.errors)) + + +@runtime_checkable +class MetricsRecorder(Protocol): + """Collector for runtime, session, and run metrics.""" + + def record(self, sample: RuntimeMetricSample) -> None: + """Record one metric sample.""" + ... + + def record_timing( + self, + name: str, + duration_s: float, + *, + step_index: int | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> None: + """Record one timing sample in seconds.""" + ... + + def record_step( + self, + *, + request: object, + user_window: object, + inference_input: object, + result: object, + decision: object, + ) -> None: + """Record one successful model step.""" + ... + + def record_control( + self, + *, + request: object, + user_window: object, + control: object, + ) -> None: + """Record one provider-authored control decision.""" + ... + + def record_error(self, exc: Exception, action: object) -> None: + """Record a driver-observed operational error.""" + ... + + def record_catch_up(self, decision: object) -> None: + """Record a realtime catch-up decision.""" + ... + + def record_cleanup_error(self, exc: Exception) -> None: + """Record a cleanup failure without interrupting teardown.""" + ... + + def record_orphaned_cleanup(self, exc: Exception) -> None: + """Record cleanup that timed out and is still queued.""" + ... + + def record_session(self, result: object) -> None: + """Record one closed session result.""" + ... + + def record_session_error(self, exc: Exception) -> None: + """Record diagnostic session assembly failure detail.""" + ... + + def close(self) -> MetricsSnapshot: + """Finalize metric collection.""" + ... + + +@dataclass(slots=True) +class InMemoryMetricsRecorder: + """Simple metrics recorder useful for tests, smoke runs, and adapters.""" + + samples: list[RuntimeMetricSample] = field(default_factory=list) + step_count: int = 0 + control_count: int = 0 + catch_up_count: int = 0 + errors: list[str] = field(default_factory=list) + cleanup_errors: list[str] = field(default_factory=list) + orphaned_cleanup_errors: list[str] = field(default_factory=list) + session_errors: list[str] = field(default_factory=list) + sessions: list[object] = field(default_factory=list) + closed: bool = False + + def record(self, sample: RuntimeMetricSample) -> None: + if self.closed: + raise RuntimeError("Cannot record metrics after close().") + self.samples.append(sample) + + def record_timing( + self, + name: str, + duration_s: float, + *, + step_index: int | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> None: + self.record( + RuntimeMetricSample( + name=name, + value=duration_s, + unit="s", + step_index=step_index, + category="timing", + metadata={} if metadata is None else metadata, + ) + ) + + def record_step( + self, + *, + request: object, + user_window: object, + inference_input: object, + result: object, + decision: object, + ) -> None: + del request, user_window, inference_input, result, decision + if not self.closed: + self.step_count += 1 + + def record_control( + self, + *, + request: object, + user_window: object, + control: object, + ) -> None: + del request, user_window, control + if not self.closed: + self.control_count += 1 + + def record_error(self, exc: Exception, action: object) -> None: + del action + if not self.closed: + self.errors.append(str(exc)) + + def record_catch_up(self, decision: object) -> None: + del decision + if not self.closed: + self.catch_up_count += 1 + + def record_cleanup_error(self, exc: Exception) -> None: + if not self.closed: + self.cleanup_errors.append(str(exc)) + + def record_orphaned_cleanup(self, exc: Exception) -> None: + if not self.closed: + self.orphaned_cleanup_errors.append(str(exc)) + + def record_session(self, result: object) -> None: + if not self.closed: + self.sessions.append(result) + + def record_session_error(self, exc: Exception) -> None: + if not self.closed: + self.session_errors.append(str(exc)) + + def close(self) -> MetricsSnapshot: + self.closed = True + return self.snapshot() + + def snapshot(self) -> MetricsSnapshot: + timings: defaultdict[str, list[float]] = defaultdict(list) + for sample in self.samples: + if sample.category == "timing": + timings[sample.name].append(float(sample.value)) + session_statuses = tuple( + str(getattr(result, "status", "unknown")) for result in self.sessions + ) + session_status_counts = Counter(session_statuses) + return MetricsSnapshot( + counters={ + "samples": len(self.samples), + "steps": self.step_count, + "controls": self.control_count, + "catch_ups": self.catch_up_count, + "sessions": len(self.sessions), + "errors": len(self.errors), + "cleanup_errors": len(self.cleanup_errors), + "orphaned_cleanup_errors": len(self.orphaned_cleanup_errors), + "session_errors": len(self.session_errors), + **{ + f"sessions.{status}": count + for status, count in sorted(session_status_counts.items()) + }, + }, + timings=timings, + session_statuses=session_statuses, + errors=tuple( + ( + *self.errors, + *self.cleanup_errors, + *self.orphaned_cleanup_errors, + *self.session_errors, + ) + ), + ) + + +class NullMetricsRecorder: + """Metrics recorder that intentionally drops all samples.""" + + def record(self, sample: RuntimeMetricSample) -> None: + del sample + + def record_timing( + self, + name: str, + duration_s: float, + *, + step_index: int | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> None: + del name, duration_s, step_index, metadata + + def record_step( + self, + *, + request: object, + user_window: object, + inference_input: object, + result: object, + decision: object, + ) -> None: + del request, user_window, inference_input, result, decision + + def record_control( + self, + *, + request: object, + user_window: object, + control: object, + ) -> None: + del request, user_window, control + + def record_error(self, exc: Exception, action: object) -> None: + del exc, action + + def record_catch_up(self, decision: object) -> None: + del decision + + def record_cleanup_error(self, exc: Exception) -> None: + del exc + + def record_orphaned_cleanup(self, exc: Exception) -> None: + del exc + + def record_session(self, result: object) -> None: + del result + + def record_session_error(self, exc: Exception) -> None: + del exc + + def close(self) -> MetricsSnapshot: + return MetricsSnapshot() + + +__all__ = [ + "InMemoryMetricsRecorder", + "MetricsRecorder", + "MetricsSnapshot", + "NullMetricsRecorder", + "RuntimeMetricSample", +] diff --git a/flashdreams/flashdreams/runtime/output.py b/flashdreams/flashdreams/runtime/output.py new file mode 100644 index 000000000..aac341ee1 --- /dev/null +++ b/flashdreams/flashdreams/runtime/output.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Output target boundary for generated inference results.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.types import StepResult + + +@dataclass(frozen=True, kw_only=True, slots=True) +class OutputArtifact: + """Artifact produced by an output target.""" + + __hash__ = None + + kind: str + uri: str + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.kind.strip(): + raise ValueError("OutputArtifact.kind must be non-empty.") + if not self.uri.strip(): + raise ValueError("OutputArtifact.uri must be non-empty.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@runtime_checkable +class OutputTarget(Protocol): + """Consumes generated session outputs for presentation or persistence.""" + + def open(self) -> None: + """Prepare the target for a new run.""" + ... + + def write(self, result: StepResult) -> None: + """Consume one generated step result.""" + ... + + def close(self) -> Sequence[OutputArtifact]: + """Finalize and return any produced artifacts.""" + ... + + +@dataclass(slots=True) +class NullOutputTarget: + """Output target for headless runs and throughput measurements.""" + + store_results: bool = False + output_count: int = field(default=0, init=False) + results: list[StepResult] = field(default_factory=list, init=False) + _opened: bool = field(default=False, init=False, repr=False) + + @property + def closed(self) -> bool: + return not self._opened + + def open(self) -> None: + self._opened = True + self.output_count = 0 + self.results.clear() + + def write(self, result: StepResult) -> None: + if not self._opened: + raise RuntimeError("Cannot write to a closed output target.") + self.output_count += 1 + if self.store_results: + self.results.append(result) + + def close(self) -> Sequence[OutputArtifact]: + self._opened = False + return () diff --git a/flashdreams/flashdreams/runtime/runner.py b/flashdreams/flashdreams/runtime/runner.py new file mode 100644 index 000000000..f9a83411f --- /dev/null +++ b/flashdreams/flashdreams/runtime/runner.py @@ -0,0 +1,669 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal synchronous standard runner for the runtime API.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Any + +from flashdreams.runtime.canonical import InputCanonicalizer +from flashdreams.runtime.config import InferenceConfig +from flashdreams.runtime.inputs import ( + CanonicalInputs, + CanonicalInputSchema, + InferenceInput, + InferenceInputSchema, + TimeWindow, + UserInputs, + UserInputSchema, +) +from flashdreams.runtime.interfaces import ( + InferenceRuntime, + InferenceSession, + ModelAdapter, +) +from flashdreams.runtime.mapping import ( + DeclaresMappingSchema, + InputMapping, + check_mapping_compatibility, +) +from flashdreams.runtime.metrics import MetricsRecorder +from flashdreams.runtime.output import OutputArtifact, OutputTarget +from flashdreams.runtime.types import ( + StepRequest, + StepRequirements, + StepResult, + step_requirements_from_request, +) + +if TYPE_CHECKING: + from flashdreams.runtime.demo.host import RuntimeHost + from flashdreams.runtime.demo.outputs import OutputDecision, SessionInfo + from flashdreams.runtime.demo.run_modes import RunResult + from flashdreams.runtime.demo.session_inputs import PreparedStep, UserInputWindow + +_DEFAULT_SESSION_HORIZON_S = 3600.0 + + +def run_inference_session( + *, + adapter: ModelAdapter, + config: InferenceConfig, + mapping: InputMapping, + canonicalizer: InputCanonicalizer, + source_schema: UserInputSchema, + user_inputs: UserInputs, + initial_inputs: InferenceInput, + output: OutputTarget, + metrics: MetricsRecorder, +) -> tuple[OutputArtifact, ...]: + """Run one sequential inference session through the shared batch driver. + + The signature and failure semantics remain compatible with the original + replay runner while the implementation delegates the step loop to the shared + demo runtime pipeline. + """ + + return _run_inference_session_with_shared_batch( + adapter=adapter, + config=config, + mapping=mapping, + canonicalizer=canonicalizer, + source_schema=source_schema, + user_inputs=user_inputs, + initial_inputs=initial_inputs, + output=output, + metrics=metrics, + ) + + +def _run_inference_session_with_shared_batch( + *, + adapter: ModelAdapter, + config: InferenceConfig, + mapping: InputMapping, + canonicalizer: InputCanonicalizer, + source_schema: UserInputSchema, + user_inputs: UserInputs, + initial_inputs: InferenceInput, + output: OutputTarget, + metrics: MetricsRecorder, +) -> tuple[OutputArtifact, ...]: + lifecycle = _LegacyBatchLifecycle() + request_state = _LegacyStepRequestState() + runtime = _LegacyLazyRuntime( + adapter=adapter, + config=config, + lifecycle=lifecycle, + request_state=request_state, + ) + from flashdreams.runtime.demo.host import RuntimeHost + + host = RuntimeHost(runtime) + metrics_recorder = _LegacyRunnerMetricsRecorder(metrics) + output_sink = _LegacyOutputTargetSink( + output=output, + lifecycle=lifecycle, + host=host, + ) + primary_error: BaseException | None = None + + try: + _validate_legacy_runner_inputs( + adapter=adapter, + config=config, + mapping=mapping, + canonicalizer=canonicalizer, + source_schema=source_schema, + ) + provider = _LegacyMappedModelInputProvider( + mapping=mapping, + canonicalizer=canonicalizer, + source_schema=source_schema, + user_inputs=user_inputs, + initial_inputs=initial_inputs, + inference_input_schema=adapter.inference_input_schema, + request_state=request_state, + ) + input_source = _LegacyBatchInputSource( + source_schema=source_schema, + user_inputs=user_inputs, + request_state=request_state, + ) + result = _run_shared_batch_session( + host=host, + provider=provider, + input_source=input_source, + output_sink=output_sink, + metrics=metrics_recorder, + ) + _raise_legacy_runner_error( + result=result, + output_sink=output_sink, + metrics=metrics_recorder, + ) + return tuple(result.artifacts) + except BaseException as exc: + primary_error = exc + if not metrics_recorder.closed: + _close_metrics_suppressing_secondary(metrics_recorder) + raise + finally: + try: + host.close() + except BaseException: + if primary_error is None: + raise + + +def _validate_legacy_runner_inputs( + *, + adapter: ModelAdapter, + config: InferenceConfig, + mapping: InputMapping, + canonicalizer: InputCanonicalizer, + source_schema: UserInputSchema, +) -> CanonicalInputSchema: + adapter.validate_config(config) + canonical_schema = canonicalizer.canonical_schema(source_schema) + _check_declared_mapping_compatibility( + mapping=mapping, + canonical_schema=canonical_schema, + adapter=adapter, + ) + mapping.validate( + canonical_schema=canonical_schema, + inference_input_schema=adapter.inference_input_schema, + ) + return canonical_schema + + +def _run_shared_batch_session( + *, + host: RuntimeHost, + provider: "_LegacyMappedModelInputProvider", + input_source: "_LegacyBatchInputSource", + output_sink: "_LegacyOutputTargetSink", + metrics: "_LegacyRunnerMetricsRecorder", +) -> RunResult: + from flashdreams.runtime.demo.drivers import BatchSessionDriver + from flashdreams.runtime.demo.pipeline import StepPipeline + from flashdreams.runtime.demo.run_modes import SessionEdges + + return BatchSessionDriver().run_one_session( + host=host, + provider=provider, + session_edges=SessionEdges( + input_source=input_source, + output_sink=output_sink, + cleanup_tasks=set(), + metrics=metrics, + ), + pipeline=StepPipeline(), + ) + + +class _LegacyLazyRuntime: + """Create the legacy runtime only after global inputs are mapped.""" + + def __init__( + self, + *, + adapter: ModelAdapter, + config: InferenceConfig, + lifecycle: "_LegacyBatchLifecycle", + request_state: "_LegacyStepRequestState", + ) -> None: + self._adapter = adapter + self._config = config + self._lifecycle = lifecycle + self._request_state = request_state + + def start_session(self, inputs: InferenceInput) -> InferenceSession: + runtime = self._lifecycle.runtime + if runtime is None: + runtime = self._adapter.create_runtime(self._config) + self._lifecycle.set_runtime(runtime) + session = runtime.start_session(inputs) + self._lifecycle.set_session(session) + return _LegacySessionAdapter( + session=session, + request_state=self._request_state, + ) + + def close(self) -> None: + self._lifecycle.close_runtime_direct() + + +class _LegacyBatchLifecycle: + """Own legacy model resources whose close order is caller-visible.""" + + def __init__(self) -> None: + self.runtime: InferenceRuntime | None = None + self.session: InferenceSession | None = None + self._session_close_attempted = False + self._runtime_close_attempted = False + + def set_runtime(self, runtime: InferenceRuntime) -> None: + self.runtime = runtime + + def set_session(self, session: InferenceSession) -> None: + self.session = session + self._session_close_attempted = False + + def close_session_via_host(self, host: RuntimeHost) -> None: + host.call(self.close_session_direct) + + def close_runtime_via_host(self, host: RuntimeHost) -> None: + host.call(self.close_runtime_direct) + + def close_session_direct(self) -> None: + if self.session is None or self._session_close_attempted: + return + self._session_close_attempted = True + self.session.close() + + def close_runtime_direct(self) -> None: + if self.runtime is None or self._runtime_close_attempted: + return + self._runtime_close_attempted = True + self.runtime.close() + + +class _LegacySessionAdapter: + """Expose old sessions through the new StepRequirements boundary.""" + + def __init__( + self, + *, + session: InferenceSession, + request_state: "_LegacyStepRequestState", + ) -> None: + self._session = session + self._request_state = request_state + + def next_step_requirements(self) -> StepRequirements | None: + request = self._session.next_step_request() + if request is None: + self._request_state.clear() + return None + self._request_state.store(request) + return step_requirements_from_request( + request, + allow_user_input_window=True, + ) + + def next_step_request(self) -> StepRequest | None: + return self._session.next_step_request() + + def session_info(self) -> SessionInfo: + from flashdreams.runtime.demo.outputs import SessionInfo + + session_info = getattr(self._session, "session_info", None) + if not callable(session_info): + return SessionInfo() + value = session_info() + if not isinstance(value, SessionInfo): + raise TypeError( + "session.session_info() must return SessionInfo, " + f"got {type(value).__name__}." + ) + return value + + def step(self, inputs: InferenceInput) -> StepResult: + return self._session.step(inputs) + + def reset(self, inputs: InferenceInput | None = None) -> None: + self._session.reset(inputs) + + def close(self) -> None: + return None + + +class _LegacyStepRequestState: + """Share the current legacy request between the session, source, and provider.""" + + def __init__(self) -> None: + self._request: StepRequest | None = None + + def store(self, request: StepRequest) -> None: + self._request = request + + def require_for_window(self, step_index: int) -> StepRequest: + request = self._request + if request is None: + raise RuntimeError("Legacy input source has no active step request.") + if request.step_index != step_index: + raise RuntimeError( + "Legacy input source request mismatch: " + f"expected step {request.step_index}, got {step_index}." + ) + return request + + def consume_for_step(self, step_index: int) -> StepRequest: + request = self.require_for_window(step_index) + self._request = None + return request + + def clear(self) -> None: + self._request = None + + +class _LegacyBatchInputSource: + is_finite = True + is_deterministic = True + + def __init__( + self, + *, + source_schema: UserInputSchema, + user_inputs: UserInputs, + request_state: _LegacyStepRequestState, + ) -> None: + self.user_input_schema = source_schema + self._user_inputs = user_inputs + self._request_state = request_state + + def is_finished(self) -> bool: + return False + + def next_window(self, request: StepRequirements) -> UserInputWindow: + from flashdreams.runtime.demo.session_inputs import UserInputWindow + + legacy_request = self._request_state.require_for_window(request.step_index) + window = legacy_request.user_input_window or _all_user_inputs_window( + self._user_inputs + ) + return UserInputWindow( + start_s=window.start_s, + end_s=window.end_s, + inputs=self._user_inputs, + ) + + +class _LegacyMappedModelInputProvider: + def __init__( + self, + *, + mapping: InputMapping, + canonicalizer: InputCanonicalizer, + source_schema: UserInputSchema, + user_inputs: UserInputs, + initial_inputs: InferenceInput, + inference_input_schema: InferenceInputSchema, + request_state: _LegacyStepRequestState, + ) -> None: + from flashdreams.runtime.demo.session_inputs import ProviderCapabilities + + self.capabilities = ProviderCapabilities( + supports_recorded_input=True, + deterministic_given_inputs=True, + user_input_schema=source_schema, + inference_input_schema=inference_input_schema, + ) + self._mapping = mapping + self._canonicalizer = canonicalizer + self._source_schema = source_schema + self._user_inputs = user_inputs + self._initial_inputs = initial_inputs + self._request_state = request_state + self._step_base_inputs = InferenceInput( + step=initial_inputs.step, + metadata=initial_inputs.metadata, + ) + + def prepare_initial_input(self) -> InferenceInput: + self._canonicalizer.reset() + return self._mapping.map_global_conditioning_inputs( + canonical_inputs=CanonicalInputs(), + inference_input=self._initial_inputs, + ) + + def prepare_step( + self, + *, + request: StepRequirements, + user_window: UserInputWindow, + ) -> PreparedStep: + from flashdreams.runtime.demo.session_inputs import PreparedStep + + legacy_request = self._request_state.consume_for_step(request.step_index) + canonical_inputs = self._canonicalizer.canonicalize( + self._user_inputs, + window=TimeWindow(start_s=user_window.start_s, end_s=user_window.end_s), + source_schema=self._source_schema, + ) + return PreparedStep( + inference_input=self._mapping.map_step_inputs( + canonical_inputs=canonical_inputs, + inference_input=self._step_base_inputs, + request=legacy_request, + ) + ) + + def reset(self, inputs: InferenceInput | None = None) -> None: + del inputs + self._canonicalizer.reset() + + def close(self) -> None: + return None + + +class _LegacyOutputTargetSink: + produces_artifacts = True + + def __init__( + self, + *, + output: OutputTarget, + lifecycle: _LegacyBatchLifecycle, + host: RuntimeHost, + ) -> None: + self._output = output + self._lifecycle = lifecycle + self._host = host + self._opened = False + self._closed = False + self._artifacts: tuple[OutputArtifact, ...] = () + self.cleanup_error: BaseException | None = None + + def open(self, session_info: SessionInfo) -> None: + del session_info + self._output.open() + self._opened = True + self._closed = False + self._artifacts = () + self.cleanup_error = None + + def begin_generation(self, generation: int) -> None: + del generation + + def write(self, result: StepResult) -> OutputDecision: + from flashdreams.runtime.demo.outputs import OutputDecision + + self._output.write(result) + return OutputDecision() + + def close(self) -> Sequence[OutputArtifact]: + if self._closed: + if self.cleanup_error is not None: + raise self.cleanup_error + return self._artifacts + + self._closed = True + cleanup_error: BaseException | None = None + artifacts: tuple[OutputArtifact, ...] = () + + def remember_error(exc: BaseException) -> None: + nonlocal cleanup_error + if cleanup_error is None: + cleanup_error = exc + + if self._opened: + try: + artifacts = tuple(self._output.close()) + except BaseException as exc: + remember_error(exc) + try: + self._lifecycle.close_session_via_host(self._host) + except BaseException as exc: + remember_error(exc) + try: + self._lifecycle.close_runtime_via_host(self._host) + except BaseException as exc: + remember_error(exc) + + self._artifacts = artifacts + self.cleanup_error = cleanup_error + if cleanup_error is not None: + raise cleanup_error + return self._artifacts + + +class _LegacyRunnerMetricsRecorder: + def __init__(self, metrics: MetricsRecorder) -> None: + self._metrics = metrics + self.closed = False + self.close_error: BaseException | None = None + + def record(self, sample: Any) -> None: + self._metrics.record(sample) + + def record_timing( + self, + name: str, + duration_s: float, + *, + step_index: int | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> None: + self._metrics.record_timing( + name, + duration_s, + step_index=step_index, + metadata=metadata, + ) + + def record_step( + self, + *, + request: object, + user_window: object, + inference_input: object, + result: object, + decision: object, + ) -> None: + del request, user_window, inference_input, decision + if isinstance(result, StepResult): + _record_timing_metrics(self._metrics, result) + + def record_control( + self, + *, + request: object, + user_window: object, + control: object, + ) -> None: + del request, user_window, control + + def record_error(self, exc: Exception, action: object) -> None: + del exc, action + + def record_catch_up(self, decision: object) -> None: + del decision + + def record_cleanup_error(self, exc: Exception) -> None: + del exc + + def record_orphaned_cleanup(self, exc: Exception) -> None: + del exc + + def record_session(self, result: object) -> None: + del result + + def record_session_error(self, exc: Exception) -> None: + del exc + + def close(self) -> Any: + if self.closed: + if self.close_error is not None: + raise self.close_error + return None + self.closed = True + try: + return self._metrics.close() + except BaseException as exc: + self.close_error = exc + raise + + +def _raise_legacy_runner_error( + *, + result: RunResult, + output_sink: _LegacyOutputTargetSink, + metrics: _LegacyRunnerMetricsRecorder, +) -> None: + if result.error is not None: + raise result.error + if output_sink.cleanup_error is not None: + raise output_sink.cleanup_error + if metrics.close_error is not None: + raise metrics.close_error + + +def _close_metrics_suppressing_secondary( + metrics: _LegacyRunnerMetricsRecorder, +) -> None: + try: + metrics.close() + except BaseException: + return + + +def _check_declared_mapping_compatibility( + *, + mapping: InputMapping, + canonical_schema: CanonicalInputSchema, + adapter: ModelAdapter, +) -> None: + if not isinstance(mapping, DeclaresMappingSchema): + return + compatibility = check_mapping_compatibility( + canonical_schema=canonical_schema, + inference_input_schema=adapter.inference_input_schema, + mapping_schema=mapping.mapping_schema, + ) + compatibility.raise_if_incompatible() + + +def _all_user_inputs_window(user_inputs: UserInputs) -> TimeWindow: + if not user_inputs.events: + return TimeWindow(start_s=0.0, end_s=_DEFAULT_SESSION_HORIZON_S) + return TimeWindow( + start_s=0.0, + end_s=max( + _DEFAULT_SESSION_HORIZON_S, + math.nextafter(user_inputs.events[-1].timestamp_s, math.inf), + ), + ) + + +def _record_timing_metrics( + metrics: MetricsRecorder, + result: StepResult, +) -> None: + for name, value in result.metrics.items(): + if not name.endswith("_s") or isinstance(value, bool): + continue + sample_name = name[:-2] or name + metrics.record_timing( + sample_name, + float(value), + step_index=result.step_index, + ) + + +__all__ = ["run_inference_session"] diff --git a/flashdreams/flashdreams/runtime/types.py b/flashdreams/flashdreams/runtime/types.py new file mode 100644 index 000000000..49ee958a0 --- /dev/null +++ b/flashdreams/flashdreams/runtime/types.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Plain data carriers shared by runtime protocols and adapters.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +from flashdreams.infra.results import StepResult +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.inputs import InferenceInputSchema, TimeWindow + +_STEP_REQUIREMENTS_INPUT_COUNT_METADATA_KEY = "input_frame_count" +_STEP_REQUIREMENTS_STEADY_OUTPUT_COUNT_METADATA_KEY = "steady_output_frame_count" +_STEP_REQUIREMENTS_USER_INPUT_METADATA_KEYS = frozenset( + { + "input_window", + "user_input", + "user_input_window", + "user_inputs", + } +) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class StepRequirements: + """Model-authored per-step requirements consumed by shared demo drivers.""" + + __hash__ = None + + step_index: int + input_frame_count: int = 1 + steady_output_frame_count: int | None = None + inference_input_schema: InferenceInputSchema | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if isinstance(self.step_index, bool) or not isinstance(self.step_index, int): + raise TypeError("StepRequirements.step_index must be an integer.") + if self.step_index < 0: + raise ValueError("StepRequirements.step_index must be >= 0.") + if isinstance(self.input_frame_count, bool) or not isinstance( + self.input_frame_count, int + ): + raise TypeError("StepRequirements.input_frame_count must be an integer.") + if self.input_frame_count <= 0: + raise ValueError("StepRequirements.input_frame_count must be > 0.") + if self.steady_output_frame_count is not None: + if isinstance(self.steady_output_frame_count, bool) or not isinstance( + self.steady_output_frame_count, int + ): + raise TypeError( + "StepRequirements.steady_output_frame_count must be an integer." + ) + if self.steady_output_frame_count < 0: + raise ValueError( + "StepRequirements.steady_output_frame_count must be >= 0." + ) + user_input_keys = sorted( + key + for key in self.metadata + if key in _STEP_REQUIREMENTS_USER_INPUT_METADATA_KEYS + ) + if user_input_keys: + joined = ", ".join(user_input_keys) + raise ValueError( + "StepRequirements.metadata must not include driver-owned user " + f"input keys: {joined}." + ) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class StepRequest: + """Per-step runtime request emitted by an inference session. + + This is not a schema declaration. ``user_input_window`` lets a runner drain + or slice timestamped user events for the current step before invoking the + selected ``InputMapping``. + """ + + __hash__ = None + + step_index: int + inference_input_schema: InferenceInputSchema | None = None + user_input_window: TimeWindow | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.step_index < 0: + raise ValueError("StepRequest.step_index must be >= 0.") + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +def step_requirements_from_request( + request: StepRequest, + *, + allow_user_input_window: bool = False, +) -> StepRequirements: + """Adapt a legacy ``StepRequest`` that did not carry driver-owned inputs.""" + + if request.user_input_window is not None and not allow_user_input_window: + raise ValueError( + "StepRequest.user_input_window cannot be adapted to StepRequirements; " + "user input windows are driver-owned." + ) + metadata = dict(request.metadata) + input_frame_count = metadata.pop(_STEP_REQUIREMENTS_INPUT_COUNT_METADATA_KEY, 1) + steady_output_frame_count = metadata.pop( + _STEP_REQUIREMENTS_STEADY_OUTPUT_COUNT_METADATA_KEY, + None, + ) + return StepRequirements( + step_index=request.step_index, + input_frame_count=input_frame_count, + steady_output_frame_count=steady_output_frame_count, + inference_input_schema=request.inference_input_schema, + metadata=metadata, + ) + + +__all__ = [ + "StepRequest", + "StepRequirements", + "StepResult", + "step_requirements_from_request", +] diff --git a/flashdreams/flashdreams/runtime/video_output.py b/flashdreams/flashdreams/runtime/video_output.py new file mode 100644 index 000000000..c92cf2032 --- /dev/null +++ b/flashdreams/flashdreams/runtime/video_output.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Video output targets for the runtime API.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +from flashdreams.infra.postprocess import VideoTensorLayout +from flashdreams.infra.runner_io import ( + DEFAULT_RUNNER_INSTALL_HINT, + write_video_tensor, +) +from flashdreams.infra.video_output import VideoResultCollector, prepare_video_for_mp4 +from flashdreams.runtime.output import OutputArtifact +from flashdreams.runtime.types import StepResult + +VideoWriter = Callable[..., Path] + + +@dataclass(slots=True) +class Mp4VideoOutputTarget: + """Write layout-aware runtime step results to one MP4 artifact.""" + + output_path: Path + fps: int | float + output_layout: VideoTensorLayout = "bvtchw" + writer: VideoWriter = field(default=write_video_tensor, repr=False) + install_hint: str = DEFAULT_RUNNER_INSTALL_HINT + move_to_cpu: bool = True + enabled: bool = True + _opened: bool = field(default=False, init=False, repr=False) + _collector: VideoResultCollector | None = field( + default=None, + init=False, + repr=False, + ) + + @property + def closed(self) -> bool: + return not self._opened + + def open(self) -> None: + self._collector = VideoResultCollector( + output_layout=self.output_layout, + enabled=self.enabled, + move_to_cpu=self.move_to_cpu, + ) + self._opened = True + + def write(self, result: StepResult) -> None: + if not self._opened or self._collector is None: + raise RuntimeError("Cannot write to a closed output target.") + if result.layout is None: + raise TypeError( + "Mp4VideoOutputTarget requires a video StepResult with layout." + ) + if result.layout != self.output_layout: + raise ValueError( + "Mp4VideoOutputTarget received layout " + f"{result.layout!r}; expected {self.output_layout!r}." + ) + self._collector.add(result) + + def close(self) -> Sequence[OutputArtifact]: + if self._collector is None: + self._opened = False + return () + + collector = self._collector + self._collector = None + self._opened = False + video = collector.finish() + if video is None: + return () + writable_video, writable_layout = prepare_video_for_mp4( + video, layout=self.output_layout + ) + path = self.writer( + writable_video, + self.output_path, + fps=self.fps, + layout=writable_layout, + install_hint=self.install_hint, + ) + return ( + OutputArtifact( + kind="video/mp4", + uri=str(path), + metadata={ + "fps": self.fps, + "source_layout": self.output_layout, + "shape": tuple(int(dim) for dim in video.shape), + "stats_history": tuple(collector.stats_history), + }, + ), + ) + + +__all__ = ["Mp4VideoOutputTarget"] diff --git a/flashdreams/flashdreams/runtime/worker.py b/flashdreams/flashdreams/runtime/worker.py new file mode 100644 index 000000000..e523b6e6a --- /dev/null +++ b/flashdreams/flashdreams/runtime/worker.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thread-affine execution for stateful inference runtimes.""" + +from __future__ import annotations + +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Callable, TypeVar, cast + +import torch + +_T = TypeVar("_T") +_EXECUTOR_FUTURE_POLL_INTERVAL_S = 0.01 + + +class ModelExecutionWorker: + """Run ordered runtime lifecycle calls on one owned OS thread. + + CUDA graphs, Triton launchers, and some backend contexts are thread-local. + A runtime should therefore submit initialization, reset, generation, and + close operations through one worker instead of using ``asyncio.to_thread``. + + Cancelling an awaiting task does not cancel the submitted operation. The + operation remains ordered on the worker, and later calls run only after it + completes. + """ + + def __init__( + self, + *, + device: torch.device | str | None = None, + thread_name: str = "flashdreams-runtime", + ) -> None: + self._device = None if device is None else torch.device(device) + self._executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix=thread_name, + initializer=self._initialize_thread, + ) + self._state_lock = threading.Lock() + self._accepting = True + self._closed = False + self._thread_id: int | None = None + + @property + def closed(self) -> bool: + return self._closed + + @property + def worker_thread_id(self) -> int | None: + return self._thread_id + + @property + def is_worker_thread(self) -> bool: + return self._thread_id == threading.get_ident() + + async def call( + self, + func: Callable[..., _T], + /, + *args: Any, + **kwargs: Any, + ) -> _T: + """Run one callable after all previously submitted worker calls.""" + self._require_not_worker_thread() + self._require_accepting() + future = self._submit(func, args, kwargs) + try: + return await _await_executor_future(future) + except asyncio.CancelledError: + future.add_done_callback(_consume_exception) + raise + + def call_blocking( + self, + func: Callable[..., _T], + /, + *args: Any, + **kwargs: Any, + ) -> _T: + """Run one callable from synchronous code on the owned worker thread.""" + self._require_not_worker_thread() + self._require_accepting() + future = self._executor.submit(_invoke, func, args, kwargs) + return cast(_T, future.result()) + + async def close(self) -> None: + """Drain submitted work and stop accepting lifecycle calls.""" + self._require_not_worker_thread() + if not self._begin_close(): + return + try: + barrier = asyncio.wrap_future(self._executor.submit(_noop)) + await _await_executor_future(barrier) + finally: + self._finish_close() + + def close_blocking(self) -> None: + """Synchronous close for non-async setup and teardown paths.""" + self._require_not_worker_thread() + if not self._begin_close(): + return + try: + barrier = self._executor.submit(_noop) + barrier.result() + finally: + self._finish_close() + + def _submit( + self, + func: Callable[..., _T], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> asyncio.Future[_T]: + loop = asyncio.get_running_loop() + return loop.run_in_executor(self._executor, _invoke, func, args, kwargs) + + def _initialize_thread(self) -> None: + self._thread_id = threading.get_ident() + if self._device is not None and self._device.type == "cuda": + torch.cuda.set_device(self._device) + + def _require_accepting(self) -> None: + if not self._accepting: + raise RuntimeError("runtime worker is closed") + + def _require_not_worker_thread(self) -> None: + if self.is_worker_thread: + raise RuntimeError( + "Cannot dispatch to the model execution worker from its own " + "thread; call the function directly." + ) + + def _begin_close(self) -> bool: + with self._state_lock: + if self._closed: + return False + self._accepting = False + return True + + def _finish_close(self) -> None: + self._executor.shutdown(wait=True, cancel_futures=False) + with self._state_lock: + self._closed = True + + +ThreadAffineRuntimeWorker = ModelExecutionWorker + + +def _invoke( + func: Callable[..., _T], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> _T: + return func(*args, **kwargs) + + +def _noop() -> None: + return + + +async def _await_executor_future(future: asyncio.Future[_T]) -> _T: + """Await an executor future without relying on a single cross-thread wakeup.""" + while not future.done(): + await asyncio.wait( + {future}, + timeout=_EXECUTOR_FUTURE_POLL_INTERVAL_S, + ) + return future.result() + + +def _consume_exception(future: asyncio.Future[Any]) -> None: + if not future.cancelled(): + future.exception() + + +__all__ = ["ModelExecutionWorker", "ThreadAffineRuntimeWorker"] diff --git a/flashdreams/flashdreams/serving/output_targets.py b/flashdreams/flashdreams/serving/output_targets.py index b7e7be981..a5e9f961b 100644 --- a/flashdreams/flashdreams/serving/output_targets.py +++ b/flashdreams/flashdreams/serving/output_targets.py @@ -5,27 +5,19 @@ from __future__ import annotations +import importlib import runpy import shlex import sys from dataclasses import dataclass +from functools import lru_cache from pathlib import Path -from typing import Any, Literal, TypeAlias +from typing import Literal, Protocol, TypeAlias, runtime_checkable from flashdreams.infra.runner import RunnerConfig OutputMode: TypeAlias = Literal["cli", "webrtc", "local-window"] -_OMNIDREAMS_LOCAL_WINDOW_MANIFESTS = { - "omnidreams-sv-2steps-chunk2-loc6-lightvae-lighttae": ("example_world_model.yaml"), - "omnidreams-sv-2steps-chunk2-loc6-lightvae-lighttae-perf": ( - "example_world_model_perf.yaml" - ), - "omnidreams-sv-2steps-chunk2-loc6-lightvae-lighttae-native-perf": ( - "example_world_model_perf.yaml" - ), -} - class OutputTargetUnavailableError(ValueError): """Raised when a runner cannot be launched through a requested output.""" @@ -57,18 +49,39 @@ def command(self) -> str: return shlex.join(("python", "-m", self.module, *self.argv)) +@runtime_checkable +class OutputTargetAdapter(Protocol): + """Integration-owned non-CLI output capabilities for a runner config.""" + + def supported_modes( + self, + config: RunnerConfig, + options: OutputLaunchOptions, + ) -> tuple[OutputMode, ...]: ... + + def resolve( + self, + config: RunnerConfig, + *, + mode: OutputMode, + options: OutputLaunchOptions, + ) -> OutputTargetSpec | None: ... + + def available_output_modes( config: RunnerConfig, options: OutputLaunchOptions | None = None, ) -> tuple[OutputMode, ...]: """Return output modes known to support ``config``.""" options = options or OutputLaunchOptions() - modes: list[OutputMode] = ["cli"] - if _webrtc_spec(config, options) is not None: - modes.append("webrtc") - if _local_window_spec(config, options) is not None: - modes.append("local-window") - return tuple(modes) + adapter = _resolve_adapter(config) + if adapter is None: + return ("cli",) + modes = adapter.supported_modes(config, options) + invalid = [mode for mode in modes if mode == "cli"] + if invalid: + raise ValueError("Output adapters must not declare the built-in CLI mode.") + return ("cli", *dict.fromkeys(modes)) def resolve_output_target( @@ -81,10 +94,9 @@ def resolve_output_target( if mode == "cli": raise ValueError("CLI mode is run directly by the selected Runner.") options = options or OutputLaunchOptions() + adapter = _resolve_adapter(config) spec = ( - _webrtc_spec(config, options) - if mode == "webrtc" - else _local_window_spec(config, options) + None if adapter is None else adapter.resolve(config, mode=mode, options=options) ) if spec is None: supported = ", ".join(available_output_modes(config, options)) @@ -92,6 +104,10 @@ def resolve_output_target( f"Output mode {mode!r} is not available for runner " f"{config.runner_name!r}. Supported modes: {supported}." ) + if spec.mode != mode: + raise ValueError( + f"Output adapter returned mode {spec.mode!r} while resolving {mode!r}." + ) return spec @@ -105,179 +121,37 @@ def launch_output_target(spec: OutputTargetSpec) -> None: sys.argv = original_argv -def _webrtc_spec( - config: RunnerConfig, - options: OutputLaunchOptions, -) -> OutputTargetSpec | None: - name = _runner_name(config) - if _is_lingbot_runner(name): - return _lingbot_webrtc_spec(config, options) - if _is_omnidreams_runner(name) and _is_omnidreams_single_view(config): - return _omnidreams_webrtc_spec(config, options) - return None - - -def _local_window_spec( - config: RunnerConfig, - options: OutputLaunchOptions, -) -> OutputTargetSpec | None: - name = _runner_name(config) - if not _is_omnidreams_runner(name): +def _resolve_adapter(config: RunnerConfig) -> OutputTargetAdapter | None: + path = config.output_adapter + if not path: return None - manifest = options.local_window_manifest - if manifest is None: - manifest_name = _OMNIDREAMS_LOCAL_WINDOW_MANIFESTS.get(name) - if manifest_name is None: - return None - manifest_arg = manifest_name - else: - manifest_arg = str(manifest) - - argv = ["--manifest", manifest_arg] - _append_postprocess_preset(argv, config) - return OutputTargetSpec( - mode="local-window", - label="Omnidreams local interactive window", - module="omnidreams.interactive_drive", - argv=tuple(argv), - notes=( - ( - "Local-window uses the Omnidreams interactive-drive manifest for " - "scene, resolution, and runtime-specific controls." - ), - ), - ) - - -def _lingbot_webrtc_spec( - config: RunnerConfig, - options: OutputLaunchOptions, -) -> OutputTargetSpec: - argv = [ - "--config_name", - _pipeline_name(config), - "--device", - _device(config), - "--fps", - str(getattr(config, "fps", 16)), - "--video-height", - str(getattr(config, "pixel_height", 464)), - "--video-width", - str(getattr(config, "pixel_width", 832)), - ] - if _compile_network(config) is False: - argv.append("--no_compile") - example_idx = getattr(config, "example_idx", None) - if example_idx is not None: - argv.extend(("--example-idx", str(example_idx))) - _append_webrtc_bind_args(argv, options) - return OutputTargetSpec( - mode="webrtc", - label="LingBot WebRTC server", - module="lingbot.webrtc.server", - argv=tuple(argv), - ) - - -def _omnidreams_webrtc_spec( - config: RunnerConfig, - options: OutputLaunchOptions, -) -> OutputTargetSpec: - argv = [ - "--pipeline_config_name", - _pipeline_name(config), - "--device", - _device(config), - "--fps", - str(getattr(config, "output_fps", 30)), - "--video_height", - str(getattr(config, "pixel_height", 704)), - "--video_width", - str(getattr(config, "pixel_width", 1280)), - ] - seed = _diffusion_seed(config) - if seed is not None: - argv.extend(("--seed", str(seed))) - _append_postprocess_preset(argv, config) - _append_webrtc_bind_args(argv, options) - return OutputTargetSpec( - mode="webrtc", - label="Omnidreams WebRTC server", - module="omnidreams.webrtc.server", - argv=tuple(argv), - ) - + return _load_output_adapter(path) -def _append_webrtc_bind_args( - argv: list[str], - options: OutputLaunchOptions, -) -> None: - if options.host: - argv.extend(("--host", options.host)) - if options.port is not None: - argv.extend(("--port", str(options.port))) - if options.prefer_sw_encoder: - argv.append("--prefer_sw_encoder") - -def _append_postprocess_preset(argv: list[str], config: RunnerConfig) -> None: - preset = getattr(getattr(config, "postprocess", None), "preset", "") - if preset: - argv.extend(("--postprocess-preset", str(preset))) - - -def _runner_name(config: RunnerConfig) -> str: - return str(getattr(config, "runner_name", "")) - - -def _pipeline_name(config: RunnerConfig) -> str: - pipeline = getattr(config, "pipeline", None) - name = getattr(pipeline, "name", None) - return str(name or config.runner_name) - - -def _device(config: RunnerConfig) -> str: - return str(getattr(config, "device", "cuda")) - - -def _compile_network(config: RunnerConfig) -> bool | None: - transformer = _transformer_config(config) - value = getattr(transformer, "compile_network", None) - return None if value is None else bool(value) - - -def _diffusion_seed(config: RunnerConfig) -> int | None: - diffusion_model = getattr( - getattr(config, "pipeline", None), "diffusion_model", None - ) - seed = getattr(diffusion_model, "seed", None) - return None if seed is None else int(seed) - - -def _transformer_config(config: RunnerConfig) -> Any: - diffusion_model = getattr( - getattr(config, "pipeline", None), "diffusion_model", None - ) - return getattr(diffusion_model, "transformer", None) - - -def _is_lingbot_runner(name: str) -> bool: - return name.startswith("lingbot-world") - - -def _is_omnidreams_runner(name: str) -> bool: - return name.startswith("omnidreams-") - - -def _is_omnidreams_single_view(config: RunnerConfig) -> bool: - num_views = getattr(_transformer_config(config), "num_views", 1) - return int(num_views) == 1 +@lru_cache(maxsize=None) +def _load_output_adapter(path: str) -> OutputTargetAdapter: + try: + module_name, attribute = path.split(":", 1) + except ValueError as exc: + raise ValueError( + "RunnerConfig.output_adapter must use 'module:attribute' syntax; " + f"got {path!r}." + ) from exc + value = getattr(importlib.import_module(module_name), attribute) + if callable(value) and not isinstance(value, OutputTargetAdapter): + value = value() + if not isinstance(value, OutputTargetAdapter): + raise TypeError( + f"Output adapter {path!r} does not implement OutputTargetAdapter." + ) + return value __all__ = [ "OutputLaunchOptions", "OutputMode", "OutputTargetSpec", + "OutputTargetAdapter", "OutputTargetUnavailableError", "available_output_modes", "launch_output_target", diff --git a/flashdreams/flashdreams/serving/realtime/input.py b/flashdreams/flashdreams/serving/realtime/input.py index 6baf92dde..800450e6e 100644 --- a/flashdreams/flashdreams/serving/realtime/input.py +++ b/flashdreams/flashdreams/serving/realtime/input.py @@ -1,385 +1,32 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Input state and sparse-control helpers for realtime serving.""" +"""Compatibility re-exports for transport-neutral realtime input containers.""" from __future__ import annotations -from collections import deque -from dataclasses import dataclass, field -from typing import Literal - -import numpy as np - -DEFAULT_SUPPORTED_KEYS = frozenset({"w", "a", "s", "d", "q", "e", "i", "k", "j", "l"}) -DRIVING_SUPPORTED_KEYS = frozenset( - {"w", "a", "s", "d", "up", "down", "left", "right", "space"} +from flashdreams.runtime.keyboard import ( + DEFAULT_SUPPORTED_KEYS, + DRIVING_SUPPORTED_KEYS, + KEY_ALIASES, + WSAD_SUPPORTED_KEYS, + ImageRequest, + KeyboardState, + PromptRequest, + ResetRequest, + SparseInputSnapshot, + normalize_key, ) -WSAD_SUPPORTED_KEYS = frozenset({"w", "a", "s", "d"}) -KEY_ALIASES = { - "arrowup": "w", - "arrowleft": "a", - "arrowdown": "s", - "arrowright": "d", -} - - -@dataclass(frozen=True, slots=True) -class ResetRequest: - """Transport-neutral request to reset the realtime rollout.""" - - reason: str | None = None - request_id: str | None = None - - -@dataclass(frozen=True, slots=True) -class PromptRequest: - """Transport-neutral prompt update request.""" - - prompt: str - negative_prompt: str | None = None - request_id: str | None = None - - -@dataclass(frozen=True, slots=True) -class ImageRequest: - """Transport-neutral image update request.""" - - data: bytes - content_type: str - request_id: str | None = None - - -@dataclass(frozen=True, slots=True) -class SparseInputSnapshot: - """Sparse input state sampled at a realtime loop boundary.""" - - timestamp_s: float - pressed_keys: frozenset[str] = field(default_factory=frozenset) - effective_keys: frozenset[str] = field(default_factory=frozenset) - reset: ResetRequest | None = None - prompt: PromptRequest | None = None - image: ImageRequest | None = None - - -def normalize_key(key: str) -> str: - normalized = key.strip().lower() - return KEY_ALIASES.get(normalized, normalized) - - -@dataclass(slots=True) -class KeyboardState: - pressed_keys: set[str] = field(default_factory=set) - supported_keys: frozenset[str] = DEFAULT_SUPPORTED_KEYS - _press_order: dict[str, int] = field(default_factory=dict) - _press_counter: int = 0 - - def apply_event(self, *, event: str, key: str) -> bool: - normalized_key = normalize_key(key) - if normalized_key not in self.supported_keys: - return False - - normalized_event = event.strip().lower() - if normalized_event == "keydown": - self.pressed_keys.add(normalized_key) - self._press_counter += 1 - self._press_order[normalized_key] = self._press_counter - return True - if normalized_event == "keyup": - self.pressed_keys.discard(normalized_key) - self._press_order.pop(normalized_key, None) - return True - return False - - def snapshot(self) -> frozenset[str]: - return frozenset(self.pressed_keys) - - def sparse_snapshot(self, *, timestamp_s: float) -> SparseInputSnapshot: - return SparseInputSnapshot( - timestamp_s=timestamp_s, - pressed_keys=self.snapshot(), - effective_keys=self.resolved_effective_keys(), - ) - - def _latest_pressed(self, keys: tuple[str, ...]) -> str | None: - latest_key: str | None = None - latest_idx = -1 - for key in keys: - if key not in self.pressed_keys: - continue - idx = self._press_order.get(key, -1) - if idx >= latest_idx: - latest_idx = idx - latest_key = key - return latest_key - - def resolved_effective_keys(self) -> frozenset[str]: - effective: set[str] = set() - for key in ( - self._latest_pressed(("w", "s")), - self._latest_pressed(("a", "d", "j", "l")), - self._latest_pressed(("q", "e")), - self._latest_pressed(("i", "k")), - ): - if key is not None: - effective.add(key) - return frozenset(key for key in effective if key in self.supported_keys) - - -PoseSegment = tuple[float, float, frozenset[str]] - - -class KeyboardResampler: - """Resample sparse keydown/keyup edges into a chunk timeline.""" - - def __init__( - self, - *, - fps: float, - start_v: float = 0.0, - supported_keys: frozenset[str] = DEFAULT_SUPPORTED_KEYS, - ) -> None: - if fps <= 0: - raise ValueError("fps must be > 0") - self._fps = float(fps) - self._dt = 1.0 / self._fps - self._supported_keys = supported_keys - self.next_chunk_start_v = start_v - self._event_log: deque[tuple[float, dict[str, str]]] = deque() - self._carried_state = KeyboardState(supported_keys=supported_keys) - - @property - def fps(self) -> float: - return self._fps - - @property - def dt(self) -> float: - return self._dt - - def on_edge(self, *, arrival_t: float, event: str, key: str) -> None: - self._event_log.append((arrival_t, {"event": event, "key": key})) - - def sample_chunk(self, num_frames: int) -> tuple[list[PoseSegment], list[float]]: - if num_frames < 1: - raise ValueError("num_frames must be >= 1") - - chunk_start_v = self.next_chunk_start_v - chunk_end_v = chunk_start_v + num_frames * self._dt - - while self._event_log and self._event_log[0][0] < chunk_start_v: - _, payload = self._event_log.popleft() - self._carried_state.apply_event(**payload) - - segments: list[PoseSegment] = [] - prev_t = chunk_start_v - prev_state = self._carried_state.resolved_effective_keys() - while self._event_log and self._event_log[0][0] <= chunk_end_v: - event_t, payload = self._event_log.popleft() - if event_t > prev_t: - segments.append((prev_t, event_t, prev_state)) - self._carried_state.apply_event(**payload) - prev_state = self._carried_state.resolved_effective_keys() - prev_t = event_t - if prev_t < chunk_end_v: - segments.append((prev_t, chunk_end_v, prev_state)) - elif not segments: - segments.append((chunk_start_v, chunk_end_v, prev_state)) - - frame_times = [chunk_start_v + (i + 1) * self._dt for i in range(num_frames)] - self.next_chunk_start_v = chunk_end_v - return segments, frame_times - - def reset(self, *, start_v: float) -> None: - self._event_log.clear() - self._carried_state = KeyboardState(supported_keys=self._supported_keys) - self.next_chunk_start_v = start_v - - def event_log_size(self) -> int: - return len(self._event_log) - - -def _rotation_matrix(axis: str, angle_rad: float) -> np.ndarray: - cos_t = np.float32(np.cos(angle_rad)) - sin_t = np.float32(np.sin(angle_rad)) - if axis == "x": - return np.array( - [ - [1.0, 0.0, 0.0], - [0.0, cos_t, -sin_t], - [0.0, sin_t, cos_t], - ], - dtype=np.float32, - ) - if axis == "y": - return np.array( - [ - [cos_t, 0.0, sin_t], - [0.0, 1.0, 0.0], - [-sin_t, 0.0, cos_t], - ], - dtype=np.float32, - ) - if axis == "z": - return np.array( - [ - [cos_t, -sin_t, 0.0], - [sin_t, cos_t, 0.0], - [0.0, 0.0, 1.0], - ], - dtype=np.float32, - ) - return np.eye(3, dtype=np.float32) - - -@dataclass(slots=True) -class CameraPoseIntegrator: - """Integrate a piecewise-constant keyboard timeline into a camera trajectory.""" - - move_speed_per_s: float = 0.8 - rotate_speed_rad_per_s: float = float(np.deg2rad(32.0)) - pitch_limit_rad: float = float(np.deg2rad(85.0)) - coordinate_system: Literal["RDF", "FLU"] = "RDF" - _current_pose: np.ndarray = field( - default_factory=lambda: np.eye(4, dtype=np.float32), - ) - _current_pitch: float = 0.0 - - def __post_init__(self) -> None: - if self.coordinate_system not in {"RDF", "FLU"}: - raise ValueError( - "coordinate_system must be 'RDF' (right-down-forward) " - "or 'FLU' (forward-left-up)" - ) - - def reset(self, pose: np.ndarray | None = None) -> None: - if pose is None: - self._current_pose = np.eye(4, dtype=np.float32) - self._current_pitch = 0.0 - return - if pose.shape != (4, 4): - raise ValueError(f"Expected pose shape (4, 4), got {pose.shape}") - self._current_pose = pose.astype(np.float32, copy=True) - if self.coordinate_system == "FLU": - self._current_pitch = float(np.arcsin(np.clip(pose[2, 0], -1.0, 1.0))) - else: - self._current_pitch = float(np.arctan2(pose[2, 1], pose[1, 1])) - - def current_pose(self) -> np.ndarray: - return self._current_pose.copy() - - def _advance(self, *, state: frozenset[str], duration: float) -> None: - if duration <= 0: - return - - yaw_rate = 0.0 - if self.coordinate_system == "FLU": - if "a" in state or "j" in state: - yaw_rate += self.rotate_speed_rad_per_s - if "d" in state or "l" in state: - yaw_rate -= self.rotate_speed_rad_per_s - else: - if "a" in state or "j" in state: - yaw_rate -= self.rotate_speed_rad_per_s - if "d" in state or "l" in state: - yaw_rate += self.rotate_speed_rad_per_s - pitch_rate = 0.0 - if "i" in state: - pitch_rate += self.rotate_speed_rad_per_s - if "k" in state: - pitch_rate -= self.rotate_speed_rad_per_s - - yaw_delta = yaw_rate * duration - pitch_delta = pitch_rate * duration - - new_pitch = self._current_pitch + pitch_delta - if -self.pitch_limit_rad <= new_pitch <= self.pitch_limit_rad: - self._current_pitch = new_pitch - else: - pitch_delta = 0.0 - - rot = self._current_pose[:3, :3] - trans = self._current_pose[:3, 3] - if self.coordinate_system == "FLU": - rot_pitch = _rotation_matrix("y", -pitch_delta) - rot_yaw = _rotation_matrix("z", yaw_delta) - else: - rot_pitch = _rotation_matrix("x", pitch_delta) - rot_yaw = _rotation_matrix("y", yaw_delta) - rot_new = rot_yaw @ rot @ rot_pitch - - forward_rate = 0.0 - if "w" in state: - forward_rate += self.move_speed_per_s - if "s" in state: - forward_rate -= self.move_speed_per_s - right_rate = 0.0 - if "e" in state: - right_rate += self.move_speed_per_s - if "q" in state: - right_rate -= self.move_speed_per_s - - if self.coordinate_system == "FLU": - vec_forward = rot_new[:, 0] - vec_right = -rot_new[:, 1] - forward_flat = np.array( - [vec_forward[0], vec_forward[1], 0.0], dtype=np.float32 - ) - right_flat = np.array([vec_right[0], vec_right[1], 0.0], dtype=np.float32) - else: - vec_right = rot_new[:, 0] - vec_forward = rot_new[:, 2] - forward_flat = np.array( - [vec_forward[0], 0.0, vec_forward[2]], dtype=np.float32 - ) - right_flat = np.array([vec_right[0], 0.0, vec_right[2]], dtype=np.float32) - forward_norm = np.linalg.norm(forward_flat) - right_norm = np.linalg.norm(right_flat) - if forward_norm > 0: - forward_flat /= forward_norm - if right_norm > 0: - right_flat /= right_norm - - move_vec = forward_flat * (forward_rate * duration) + right_flat * ( - right_rate * duration - ) - self._current_pose = np.eye(4, dtype=np.float32) - self._current_pose[:3, :3] = rot_new - self._current_pose[:3, 3] = trans + move_vec - - def integrate_chunk( - self, - *, - segments: list[PoseSegment], - frame_times: list[float], - ) -> np.ndarray: - if not segments: - raise ValueError("segments must be non-empty") - if not frame_times: - raise ValueError("frame_times must be non-empty") - chunk_start = segments[0][0] - chunk_end = segments[-1][1] - if any( - frame_times[i] >= frame_times[i + 1] for i in range(len(frame_times) - 1) - ): - raise ValueError("frame_times must be strictly increasing") - if frame_times[0] < chunk_start - 1e-9 or frame_times[-1] > chunk_end + 1e-9: - raise ValueError( - "frame_times must lie within the chunk window " - f"[{chunk_start}, {chunk_end}]" - ) - - poses: list[np.ndarray] = [] - cur_t = chunk_start - ft_idx = 0 - for _, seg_end, seg_state in segments: - while ft_idx < len(frame_times) and frame_times[ft_idx] <= seg_end: - target_t = frame_times[ft_idx] - self._advance(state=seg_state, duration=target_t - cur_t) - cur_t = target_t - poses.append(self._current_pose.copy()) - ft_idx += 1 - if seg_end > cur_t: - self._advance(state=seg_state, duration=seg_end - cur_t) - cur_t = seg_end - return np.stack(poses, axis=0).astype(np.float32) +__all__ = [ + "DEFAULT_SUPPORTED_KEYS", + "DRIVING_SUPPORTED_KEYS", + "ImageRequest", + "KEY_ALIASES", + "KeyboardState", + "PromptRequest", + "ResetRequest", + "SparseInputSnapshot", + "WSAD_SUPPORTED_KEYS", + "normalize_key", +] diff --git a/flashdreams/flashdreams/serving/realtime/media.py b/flashdreams/flashdreams/serving/realtime/media.py index 9b93eaca1..ab63899e2 100644 --- a/flashdreams/flashdreams/serving/realtime/media.py +++ b/flashdreams/flashdreams/serving/realtime/media.py @@ -14,7 +14,15 @@ if TYPE_CHECKING: import torch -FrameLayout = Literal["hwc", "chw", "thwc", "tchw", "bvtchw"] +FrameLayout = Literal[ + "hwc", + "chw", + "thwc", + "tchw", + "btchw", + "bcthw", + "bvtchw", +] ValueRange = Literal["minus_one_one", "zero_one", "uint8"] @@ -125,6 +133,18 @@ def rgb_array_to_uint8_frames( f"[1, 1, T, 3, H, W], got {array.shape}" ) frames = np.transpose(array[0, 0], (0, 2, 3, 1)) + elif layout == "btchw": + if array.ndim != 5 or array.shape[0] != 1 or array.shape[2] != 3: + raise ValueError( + f"Expected single-batch video chunk [1, T, 3, H, W], got {array.shape}" + ) + frames = np.transpose(array[0], (0, 2, 3, 1)) + elif layout == "bcthw": + if array.ndim != 5 or array.shape[0] != 1 or array.shape[1] != 3: + raise ValueError( + f"Expected single-batch video chunk [1, 3, T, H, W], got {array.shape}" + ) + frames = np.transpose(array[0], (1, 2, 3, 0)) else: raise ValueError(f"Unsupported layout={layout!r}.") diff --git a/flashdreams/flashdreams/serving/realtime/timing.py b/flashdreams/flashdreams/serving/realtime/timing.py index 9891cfcdc..30147f97d 100644 --- a/flashdreams/flashdreams/serving/realtime/timing.py +++ b/flashdreams/flashdreams/serving/realtime/timing.py @@ -11,6 +11,8 @@ from threading import Lock from typing import Protocol +from flashdreams.runtime.metrics import MetricsRecorder + TraceComponentValue = str | int | float | bool | None @@ -402,6 +404,36 @@ def summarize_chunk_history(chunks: Iterable[ChunkTimes]) -> RecentTimingSummary ) +def record_chunk_timing_metrics( + metrics: MetricsRecorder, + chunk: ChunkTimes, +) -> None: + """Record available chunk timing durations to a session metrics recorder.""" + + _record_stage_timing_metrics( + metrics, + prefix="realtime.chunk", + durations_ms=chunk.stage_durations_ms(), + step_index=chunk.chunk_index, + ) + + +def record_video_model_timing_metrics( + metrics: MetricsRecorder, + timings: VideoModelTimings, + *, + chunk_index: int | None = None, +) -> None: + """Record backend-visible video model stage durations to session metrics.""" + + _record_stage_timing_metrics( + metrics, + prefix="realtime.model", + durations_ms=timings.stage_durations_ms(), + step_index=chunk_index, + ) + + class RollingChunkTimingSummary: def __init__(self, capacity: int) -> None: self._chunks: deque[ChunkTimes] = deque(maxlen=capacity) @@ -518,6 +550,24 @@ def _add_optional_trace_range( ) +def _record_stage_timing_metrics( + metrics: MetricsRecorder, + *, + prefix: str, + durations_ms: Mapping[str, float], + step_index: int | None, +) -> None: + for stage_name, duration_ms in durations_ms.items(): + try: + metrics.record_timing( + f"{prefix}.{stage_name}", + float(duration_ms) / 1000.0, + step_index=step_index, + ) + except Exception: + return + + def _summarize_values(values: list[float]) -> StageDurationSummary: ordered = sorted(values) count = len(ordered) diff --git a/flashdreams/flashdreams/serving/webrtc/bootstrap.py b/flashdreams/flashdreams/serving/webrtc/bootstrap.py index 9a5405a81..353e1a88c 100644 --- a/flashdreams/flashdreams/serving/webrtc/bootstrap.py +++ b/flashdreams/flashdreams/serving/webrtc/bootstrap.py @@ -6,96 +6,22 @@ from __future__ import annotations import gc -import logging -import os -from collections.abc import Callable -from dataclasses import dataclass -from typing import Any import torch import torch.distributed as dist from aiohttp import web from loguru import logger +from flashdreams.runtime.demo.bootstrap import ( + DistributedDemoContext as WebRTCDistributedContext, +) +from flashdreams.runtime.demo.bootstrap import ( + configure_logging, + initialize_cuda_distributed, +) from flashdreams.serving.webrtc.runtime import WebRTCServerLifecycle -@dataclass(frozen=True, slots=True) -class WebRTCDistributedContext: - """CUDA/distributed launch context for a WebRTC demo server.""" - - device: torch.device - world_rank: int - world_size: int - - -def configure_logging(*, world_rank: int | None = None) -> None: - from flashdreams.core.distributed import configure_loguru_for_distributed - - configure_loguru_for_distributed(world_rank=world_rank) - for logger_name in ("aioice", "aioice.ice", "aiortc"): - logging.getLogger(logger_name).setLevel(logging.WARNING) - - -def _distributed_init() -> None: - from flashdreams.core.distributed import init as distributed_init - - distributed_init() - - -def initialize_cuda_distributed( - *, - default_device: str | torch.device = "cuda:0", - distributed_init_fn: Callable[[], object] | None = None, - configure_logging_fn: Callable[..., None] = configure_logging, - torch_module: Any = torch, - dist_module: Any = dist, -) -> WebRTCDistributedContext: - """Initialize CUDA and optional torch.distributed for WebRTC serving.""" - if not torch_module.cuda.is_available(): - raise RuntimeError("CUDA is required for inference in the WebRTC server.") - - has_rank = "RANK" in os.environ - has_world_size = "WORLD_SIZE" in os.environ - if has_rank != has_world_size: - raise RuntimeError( - "Distributed launch expects both RANK and WORLD_SIZE to be set." - ) - - distributed_launch = has_rank and has_world_size - if distributed_launch: - if distributed_init_fn is None: - distributed_init_fn = _distributed_init - distributed_init_fn() - world_rank = dist_module.get_rank() - world_size = dist_module.get_world_size() - else: - world_rank = 0 - world_size = 1 - - device_count = torch_module.cuda.device_count() - if device_count < 1: - raise RuntimeError("CUDA device count must be >= 1 for inference.") - if distributed_launch: - local_rank = world_rank % device_count - torch_device = torch_module.device(f"cuda:{local_rank}") - else: - torch_device = torch_module.device(default_device) - if torch_device.type != "cuda": - raise RuntimeError( - f"CUDA device is required for inference, got {torch_device}." - ) - if torch_device.index is None: - torch_device = torch_module.device("cuda:0") - torch_module.cuda.set_device(torch_device) - configure_logging_fn(world_rank=world_rank) - return WebRTCDistributedContext( - device=torch_device, - world_rank=world_rank, - world_size=world_size, - ) - - def run_webrtc_server( *, world_rank: int, diff --git a/flashdreams/flashdreams/serving/webrtc/controls.py b/flashdreams/flashdreams/serving/webrtc/controls.py index 7f1d8461e..ecc2d3c34 100644 --- a/flashdreams/flashdreams/serving/webrtc/controls.py +++ b/flashdreams/flashdreams/serving/webrtc/controls.py @@ -5,15 +5,12 @@ from __future__ import annotations -from flashdreams.serving.realtime.input import ( +from flashdreams.runtime.keyboard import ( DEFAULT_SUPPORTED_KEYS, KEY_ALIASES, WSAD_SUPPORTED_KEYS, - CameraPoseIntegrator, ImageRequest, - KeyboardResampler, KeyboardState, - PoseSegment, PromptRequest, ResetRequest, SparseInputSnapshot, @@ -24,11 +21,8 @@ "DEFAULT_SUPPORTED_KEYS", "KEY_ALIASES", "WSAD_SUPPORTED_KEYS", - "CameraPoseIntegrator", "ImageRequest", - "KeyboardResampler", "KeyboardState", - "PoseSegment", "PromptRequest", "ResetRequest", "SparseInputSnapshot", diff --git a/flashdreams/flashdreams/serving/webrtc/demo.py b/flashdreams/flashdreams/serving/webrtc/demo.py new file mode 100644 index 000000000..0e4b281ab --- /dev/null +++ b/flashdreams/flashdreams/serving/webrtc/demo.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared WebRTC demo construction.""" + +from __future__ import annotations + +from collections.abc import Callable +from importlib.resources import files +from pathlib import Path +from typing import Any + +from aiohttp import web + +from flashdreams.runtime.demo.spec import WebRTCAppResources, WebRTCOutputSpec +from flashdreams.serving.webrtc.bootstrap import run_webrtc_server +from flashdreams.serving.webrtc.manager import BaseWebRTCSessionManager +from flashdreams.serving.webrtc.server import ( + close_package_resources, + create_packaged_webrtc_app, + create_webrtc_app, +) + +CreateWebRTCApp = Callable[..., web.Application] +RunWebRTCServer = Callable[..., None] + + +def serve_webrtc_demo( + *, + output: WebRTCOutputSpec, + model_id: str, + session_manager: BaseWebRTCSessionManager[Any, Any], + app_resources: WebRTCAppResources, + world_rank: int = 0, + create_app_fn: CreateWebRTCApp = create_webrtc_app, + server_runner: RunWebRTCServer = run_webrtc_server, +) -> web.Application | None: + """Serve a prepared model WebRTC runtime through the shared transport.""" + app = ( + _create_app( + output=output, + model_id=model_id, + app_resources=app_resources, + session_manager=session_manager, + create_app_fn=create_app_fn, + ) + if world_rank == 0 + else None + ) + server_runner( + world_rank=world_rank, + session_manager=session_manager, + app=app, + host=output.host, + port=output.port, + ) + return app + + +def _create_app( + *, + output: WebRTCOutputSpec, + model_id: str, + app_resources: WebRTCAppResources, + session_manager: BaseWebRTCSessionManager[Any, Any], + create_app_fn: CreateWebRTCApp, +) -> web.Application: + if output.web_dir is not None: + return _build_webrtc_app( + output=output, + session_manager=session_manager, + create_app_fn=create_app_fn, + preload_name=output.preload_name or app_resources.preload_name or model_id, + ) + return create_packaged_webrtc_app( + web_resource=files("flashdreams.serving.webrtc").joinpath("web"), + model_web_resource=app_resources.model_web_resource, + session_manager=session_manager, + request_session_url=_request_session_url(output), + preload_name=output.preload_name or app_resources.preload_name or model_id, + configure_app=app_resources.configure_app, + create_app_fn=create_app_fn, + cleanup_callback=close_package_resources, + ) + + +def _build_webrtc_app( + *, + output: WebRTCOutputSpec, + session_manager: BaseWebRTCSessionManager[Any, Any], + create_app_fn: CreateWebRTCApp, + preload_name: str, +) -> web.Application: + if output.web_dir is None: + raise ValueError("WebRTC app creation requires output.web_dir.") + return create_app_fn( + web_dir=Path(output.web_dir), + session_manager=session_manager, + request_session_url=_request_session_url(output), + preload_name=preload_name, + ) + + +def _request_session_url(output: WebRTCOutputSpec) -> str: + host = "127.0.0.1" if output.host in {"0.0.0.0", "::"} else output.host + return f"http://{host}:{output.port}{output.request_session_path}" + + +__all__ = [ + "CreateWebRTCApp", + "RunWebRTCServer", + "serve_webrtc_demo", +] diff --git a/flashdreams/flashdreams/serving/webrtc/encoders.py b/flashdreams/flashdreams/serving/webrtc/encoders.py index bb27c7f6c..b07d9324c 100644 --- a/flashdreams/flashdreams/serving/webrtc/encoders.py +++ b/flashdreams/flashdreams/serving/webrtc/encoders.py @@ -3,11 +3,9 @@ """Video encoder backends for the WebRTC serving path. -Integrations that opt in to hardware encoding call :func:`select_encoder` -from their own session init (omnidreams does this today via -``omnidreams.webrtc.session._initialize_video_encoder_sync``); those that -do not opt in pick up :class:`DefaultRTCEncoder` transparently through -:meth:`BaseWebRTCSessionManager._resolve_video_encoder`. +Thread-affine WebRTC runtimes call :func:`select_encoder` during shared runtime +initialization. Runtimes that do not opt in pick up :class:`DefaultRTCEncoder` +transparently through :meth:`BaseWebRTCSessionManager._resolve_video_encoder`. **This module deliberately does not import** ``PyNvVideoCodec``. The hardware encoder lives in a sibling module (:mod:`nvenc`) that @@ -22,12 +20,14 @@ import importlib.util from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Literal, Protocol, cast, runtime_checkable import torch from aiortc import MediaStreamTrack from loguru import logger +from flashdreams.runtime import StepResult + if TYPE_CHECKING: from flashdreams.serving.webrtc.media import BufferedVideoTrack, NVENCVideoTrack @@ -69,9 +69,23 @@ class VideoEncoder(Protocol): def create_track(self, *, maxsize: int) -> BufferedVideoTrack | NVENCVideoTrack: ... + def prepare_chunk_payload( + self, + result: StepResult, + track: MediaStreamTrack, + ) -> object: ... + + async def deliver_prepared_chunk( + self, + payload: object, + track: MediaStreamTrack, + *, + force_keyframe: bool = False, + ) -> ChunkDeliveryResult: ... + async def deliver_chunk( self, - chunk: torch.Tensor, + result: StepResult, track: MediaStreamTrack, *, force_keyframe: bool = False, @@ -110,9 +124,23 @@ def create_track(self, *, maxsize: int) -> BufferedVideoTrack: return BufferedVideoTrack(fps=self.fps, maxsize=maxsize) - async def deliver_chunk( + def prepare_chunk_payload( self, - chunk: torch.Tensor, + result: StepResult, + track: MediaStreamTrack, + ) -> tuple[object, ...]: + from flashdreams.serving.webrtc.media import BufferedVideoTrack + + if not isinstance(track, BufferedVideoTrack): + raise TypeError( + "DefaultRTCEncoder requires a BufferedVideoTrack; got " + f"{type(track).__name__}. Create it via encoder.create_track()." + ) + return track.prepare_result_frames(result) + + async def deliver_prepared_chunk( + self, + payload: object, track: MediaStreamTrack, *, force_keyframe: bool = False, @@ -128,7 +156,9 @@ async def deliver_chunk( "DefaultRTCEncoder requires a BufferedVideoTrack; got " f"{type(track).__name__}. Create it via encoder.create_track()." ) - enqueued = await track.enqueue_chunk(chunk) + if not isinstance(payload, tuple): + raise TypeError("DefaultRTCEncoder payload must be a tuple of RGB frames.") + enqueued = await track.enqueue_frames(cast(Any, payload)) return ChunkDeliveryResult( backend=self.backend, num_frames=enqueued, @@ -136,6 +166,19 @@ async def deliver_chunk( encode_ms=0.0, ) + async def deliver_chunk( + self, + result: StepResult, + track: MediaStreamTrack, + *, + force_keyframe: bool = False, + ) -> ChunkDeliveryResult: + return await self.deliver_prepared_chunk( + self.prepare_chunk_payload(result, track), + track, + force_keyframe=force_keyframe, + ) + def close(self) -> None: return diff --git a/flashdreams/flashdreams/serving/webrtc/manager.py b/flashdreams/flashdreams/serving/webrtc/manager.py index e752be2a4..59177742e 100644 --- a/flashdreams/flashdreams/serving/webrtc/manager.py +++ b/flashdreams/flashdreams/serving/webrtc/manager.py @@ -10,10 +10,10 @@ import inspect import json from collections import deque +from collections.abc import Callable, Mapping from collections.abc import Set as AbstractSet -from dataclasses import dataclass, field -from enum import IntEnum -from typing import Any, Generic, TypeVar +from dataclasses import dataclass, field, replace +from typing import Any, Generic, TypeVar, cast from aiortc import ( RTCConfiguration, @@ -23,10 +23,43 @@ ) from loguru import logger -from flashdreams.serving.realtime.input import KeyboardResampler +from flashdreams.runtime.demo import ( + DemoSpec, + InMemorySessionMetricsRecorder, + ModelInputProvider, + PreparedScenario, + PreparedStep, + ProviderCapabilities, + RealtimeEventResampler, + ResamplerRealtimeClock, + RunContext, + RuntimeHost, + SessionEdges, + SessionInfo, + SingleSessionAdmissionPolicy, + StepPipeline, + UserInputWindow, + WebRTCErrorPolicy, + WebRTCOutputSpec, + run_demo_session_async, +) +from flashdreams.runtime.inputs import ( + CanonicalInputSchema, + InferenceInput, + InferenceInputSchema, + TimeWindow, + UserInputEvent, + UserInputs, + UserInputSchema, +) +from flashdreams.runtime.keyboard import DEFAULT_SUPPORTED_KEYS, normalize_key +from flashdreams.runtime.mapping import InputMapping +from flashdreams.runtime.types import StepRequest, StepResult from flashdreams.serving.webrtc.encoders import ( DefaultRTCEncoder, + EncoderBackend, VideoEncoder, + select_encoder, ) from flashdreams.serving.webrtc.media import BufferedVideoTrack, NVENCVideoTrack from flashdreams.serving.webrtc.messages import ( @@ -39,12 +72,22 @@ make_event_ack_payload, ) from flashdreams.serving.webrtc.runtime import ( + WebRTCControlSignal, WebRTCRuntimeConfig, - WebRTCSessionRuntime, - WebRTCStepResult, - make_webrtc_step_result, ) from flashdreams.serving.webrtc.server import SessionBusyError +from flashdreams.serving.webrtc.services import ( + WEBRTC_SKIPPED_INPUTS_METADATA_KEY, + WEBRTC_SKIPPED_WINDOW_METADATA_KEY, + WEBRTC_USER_INPUT_SCHEMA, + ThreadSafeWebRTCOutputBridge, + WebRTCActivationPolicy, + WebRTCChunkDelivery, + WebRTCInputSource, + WebRTCOutputSink, + WebRTCRunMode, + WebRTCTransportService, +) from flashdreams.serving.webrtc.warmup import ( run_loopback_warmup_session, wait_for_ice_gathering_complete, @@ -54,8 +97,7 @@ "BaseWebRTCSessionManager", "ManagedWebRTCSession", "WebRTCControlSignal", - "WebRTCStepResult", - "make_webrtc_step_result", + "StepResult", ] # Close the active session if no client heartbeat/control message arrives @@ -65,35 +107,490 @@ # How often the liveness watchdog wakes to re-check the elapsed-since-last-message. _CLIENT_LIVENESS_CHECK_INTERVAL_S = 1.0 _DEFAULT_PERF_LOG_INTERVAL_CHUNKS = 5 - -_RuntimeT = TypeVar("_RuntimeT", bound=WebRTCSessionRuntime) +_MAX_SESSION_USER_EVENTS = 1024 +"""Maximum unconsumed raw events kept for an ``InferenceSession`` step.""" +_RELEASE_USER_EVENT_TYPES = frozenset({"key_up"}) +_KEY_USER_EVENT_TYPES = frozenset({"key_down", "key_up"}) +_SESSION_INPUT_KEY = "webrtc_session_input" +_STEP_REQUEST_KEY = "webrtc_step_request" +_SEGMENTS_KEY = "webrtc_segments" +_FRAME_TIMES_KEY = "webrtc_frame_times" +_LEGACY_SPARSE_KEY_SEGMENTS_METADATA_KEY = "sparse_key_segments" + +_RuntimeT = TypeVar("_RuntimeT") _RuntimeConfigT = TypeVar("_RuntimeConfigT", bound=WebRTCRuntimeConfig) -def _stat_float(stats: dict[str, float], name: str, default: float = 0.0) -> float: +class _InferenceSessionExhausted(RuntimeError): + """Raised when an ``InferenceSession`` reports normal completion.""" + + +def _summarize_sdp_candidates(sdp: str) -> str: + candidates = [ + line.removeprefix("a=candidate:") + for line in sdp.splitlines() + if line.startswith("a=candidate:") + ] + if not candidates: + return "0 candidates" + + protocols: dict[str, int] = {} + addresses: set[str] = set() + endpoints: list[str] = [] + for candidate in candidates: + parts = candidate.split() + if len(parts) >= 5: + protocols[parts[2].lower()] = protocols.get(parts[2].lower(), 0) + 1 + addresses.add(parts[4]) + if len(parts) >= 6: + endpoints.append(f"{parts[2].lower()}://{parts[4]}:{parts[5]}") + protocol_summary = ",".join( + f"{key}={value}" for key, value in sorted(protocols.items()) + ) + address_summary = ",".join(sorted(addresses)[:8]) + if len(addresses) > 8: + address_summary += f",+{len(addresses) - 8} more" + endpoint_summary = ",".join(endpoints[:12]) + if len(endpoints) > 12: + endpoint_summary += f",+{len(endpoints) - 12} more" + return ( + f"{len(candidates)} candidates protocols=[{protocol_summary}] " + f"addresses=[{address_summary}] endpoints=[{endpoint_summary}]" + ) + + +def _stat_float( + stats: Mapping[str, float | int], name: str, default: float = 0.0 +) -> float: value = stats.get(name) if value is None: return default return float(value) -def _stat_ms(stats: dict[str, float], name: str, default_ms: float = 0.0) -> float: +def _stat_ms( + stats: Mapping[str, float | int], name: str, default_ms: float = 0.0 +) -> float: return _stat_float(stats, name, default_ms / 1e3) * 1e3 -def _stat_int(stats: dict[str, float], name: str) -> int: +def _stat_int(stats: Mapping[str, float | int], name: str) -> int: return int(round(_stat_float(stats, name))) -class WebRTCControlSignal(IntEnum): - """Rank-orchestration signals shared by the single-session runtimes.""" +def _runtime_drives_inference_session(runtime: Any) -> bool: + return callable(getattr(runtime, "start_inference_session", None)) + + +def _run_on_event_loop(loop: asyncio.AbstractEventLoop, awaitable: Any) -> Any: + """Run one legacy async WebRTC runtime call from a RuntimeHost worker.""" + return asyncio.run_coroutine_threadsafe(awaitable, loop).result() + + +def _step_request_from_requirements( + request: Any, + *, + window: TimeWindow, +) -> StepRequest: + metadata = dict(getattr(request, "metadata", {})) + metadata["input_frame_count"] = request.input_frame_count + steady_output_frame_count = getattr(request, "steady_output_frame_count", None) + if steady_output_frame_count is not None: + metadata["steady_output_frame_count"] = steady_output_frame_count + return StepRequest( + step_index=request.step_index, + inference_input_schema=getattr(request, "inference_input_schema", None), + user_input_window=window, + metadata=metadata, + ) + + +def _encoder_backend_from_config(value: object) -> EncoderBackend: + backend = str(value) + if backend not in {"auto", "default", "nvenc"}: + raise ValueError( + f"encoder_backend must be 'auto', 'default', or 'nvenc', got {backend!r}." + ) + return cast(EncoderBackend, backend) + + +def _gpu_id_from_device_spec(device_spec: str) -> int: + if not device_spec.startswith("cuda"): + return 0 + _prefix, separator, index = device_spec.partition(":") + if not separator or not index: + return 0 + try: + return int(index) + except ValueError: + return 0 + + +class _LegacyWebRTCRuntimeAdapter: + """Shared compatibility adapter from old async WebRTC runtimes to RuntimeHost.""" + + def __init__(self, *, runtime: Any, loop: asyncio.AbstractEventLoop) -> None: + self._runtime = runtime + self._loop = loop + + def reset_for_new_session(self, session_input: Any = None) -> None: + _run_on_event_loop( + self._loop, + self._runtime.reset_for_new_session(session_input=session_input), + ) + + def start_session(self, inputs: InferenceInput) -> "_LegacyWebRTCSessionAdapter": + del inputs + inference_session = None + if _runtime_drives_inference_session(self._runtime): + inference_session = _run_on_event_loop( + self._loop, + self._runtime.start_inference_session(), + ) + return _LegacyWebRTCSessionAdapter( + runtime=self._runtime, + inference_session=inference_session, + loop=self._loop, + ) + + def close(self) -> None: + # The underlying async runtime is owned by BaseWebRTCSessionManager and + # closed from shutdown(); RuntimeHost only owns this adapter's worker. + return + - INITIALIZE = 0 - RESET_SESSION = 1 - ACTION_STEP = 2 - CLOSE = 3 - EVENT = 4 - EXIT = 99 +class _LegacyWebRTCSessionAdapter: + """RuntimeHost-facing session view over a legacy WebRTC runtime/session.""" + + def __init__( + self, + *, + runtime: Any, + inference_session: Any | None, + loop: asyncio.AbstractEventLoop, + ) -> None: + self._runtime = runtime + self._inference_session = inference_session + self._loop = loop + + def session_info(self) -> SessionInfo: + steady_frames: int | None = None + try: + steady_frames = int(self._runtime.peek_steady_output_num_frames()) + except Exception: + steady_frames = None + return SessionInfo(steady_output_frame_count=steady_frames) + + def next_step_request(self) -> StepRequest | None: + if self._inference_session is not None: + return self._inference_session.next_step_request() + return self._runtime.next_step_request() + + def step(self, inputs: InferenceInput) -> StepResult: + if self._inference_session is not None: + result = self._inference_session.step(inputs) + else: + result = _run_on_event_loop( + self._loop, + self._runtime.step( + request=inputs.step[_STEP_REQUEST_KEY], + segments=list(inputs.step[_SEGMENTS_KEY]), + frame_times=list(inputs.step[_FRAME_TIMES_KEY]), + ), + ) + request = inputs.step[_STEP_REQUEST_KEY] + if result.step_index != request.step_index: + raise RuntimeError( + "Runtime result step does not match its request: " + f"requested {request.step_index}, got {result.step_index}." + ) + if not isinstance(result, StepResult): + raise TypeError( + "WebRTC session steps must produce StepResult, got " + f"{type(result).__name__}." + ) + return result + + def reset(self, inputs: InferenceInput | None = None) -> None: + session_input = None + if inputs is not None: + session_input = inputs.global_conditioning.get(_SESSION_INPUT_KEY) + _run_on_event_loop( + self._loop, + self._runtime.reset_for_new_session(session_input=session_input), + ) + if self._inference_session is not None: + self._inference_session = _run_on_event_loop( + self._loop, + self._runtime.start_inference_session(), + ) + + def close(self) -> None: + close = getattr(self._inference_session, "close", None) + if callable(close): + close() + + +class _LegacyWebRTCModelInputProvider: + """Shared provider used until model-specific WebRTC providers land.""" + + def __init__(self, *, runtime: Any, session_input: Any = None) -> None: + self._runtime = runtime + self._session_input = session_input + self._uses_inference_session = _runtime_drives_inference_session(runtime) + self._session_input_state_advanced = False + self.capabilities = ProviderCapabilities( + supports_realtime_clock=True, + supports_reset=True, + deterministic_given_inputs=False, + user_input_schema=self._user_input_schema(), + ) + + def prepare_initial_input(self) -> InferenceInput: + if self._session_input is None: + return InferenceInput() + return InferenceInput( + global_conditioning={_SESSION_INPUT_KEY: self._session_input} + ) + + def prepare_step( + self, + *, + request: Any, + user_window: UserInputWindow, + ) -> PreparedStep: + if self._uses_inference_session: + return PreparedStep( + inference_input=self._prepare_inference_session_step( + request=request, + user_window=user_window, + ) + ) + return PreparedStep( + inference_input=self._prepare_segment_step( + request=request, + user_window=user_window, + ) + ) + + def reset(self, inputs: InferenceInput | None = None) -> None: + del inputs + self._session_input_state_advanced = False + + def close(self) -> None: + return + + def _user_input_schema(self) -> UserInputSchema: + schema = getattr(self._runtime, "input_source_schema", None) + if isinstance(schema, UserInputSchema): + return schema + return WEBRTC_USER_INPUT_SCHEMA + + def _prepare_inference_session_step( + self, + *, + request: Any, + user_window: UserInputWindow, + ) -> InferenceInput: + self._advance_skipped_input_state(user_window) + window_start = user_window.start_s + if request.step_index == 0 and not self._session_input_state_advanced: + window_start = 0.0 + window = TimeWindow(start_s=window_start, end_s=user_window.end_s) + canonical_inputs = self._runtime.input_canonicalizer.canonicalize( + user_window.inputs, + window=window, + source_schema=self._runtime.input_source_schema, + ) + mapping = self._runtime.input_mapping + inference_input = InferenceInput( + metadata={ + **dict(user_window.metadata), + "frame_times": tuple(user_window.frame_times), + "window_start_s": window.start_s, + "window_end_s": window.end_s, + } + ) + return mapping.map_step_inputs( + canonical_inputs=canonical_inputs, + inference_input=inference_input, + request=_step_request_from_requirements(request, window=window), + ) + + def _advance_skipped_input_state(self, user_window: UserInputWindow) -> None: + skipped_inputs = user_window.metadata.get(WEBRTC_SKIPPED_INPUTS_METADATA_KEY) + skipped_window = user_window.metadata.get(WEBRTC_SKIPPED_WINDOW_METADATA_KEY) + if not isinstance(skipped_inputs, UserInputs): + return + if not isinstance(skipped_window, tuple) or len(skipped_window) != 2: + return + start_value, end_value = skipped_window + if not isinstance(start_value, int | float) or not isinstance( + end_value, + int | float, + ): + return + start_s = float(start_value) + end_s = float(end_value) + if end_s <= start_s: + return + self._runtime.input_canonicalizer.canonicalize( + skipped_inputs, + window=TimeWindow(start_s=start_s, end_s=end_s), + source_schema=self._runtime.input_source_schema, + ) + self._session_input_state_advanced = True + + @staticmethod + def _prepare_segment_step( + *, + request: Any, + user_window: UserInputWindow, + ) -> InferenceInput: + segments = user_window.metadata.get(_LEGACY_SPARSE_KEY_SEGMENTS_METADATA_KEY) + if not isinstance(segments, tuple): + raise RuntimeError("WebRTC user window is missing resampled key segments.") + window = TimeWindow(start_s=user_window.start_s, end_s=user_window.end_s) + return InferenceInput( + step={ + _STEP_REQUEST_KEY: _step_request_from_requirements( + request, + window=window, + ), + _SEGMENTS_KEY: tuple(segments), + _FRAME_TIMES_KEY: tuple(user_window.frame_times), + } + ) + + +class _LegacyWebRTCDemoAdapter: + """Minimal adapter for the shared helper while WebRTC providers migrate.""" + + model_id: str + inference_input_schema = InferenceInputSchema() + canonical_input_schema = CanonicalInputSchema() + + def __init__( + self, + *, + runtime: Any, + identity: str, + session_input: Any = None, + ) -> None: + self._runtime = runtime + self.model_id = identity + self._session_input = session_input + + def supported_input_modes(self) -> tuple[str, ...]: + return ("webrtc",) + + def supported_output_modes(self) -> tuple[str, ...]: + return ("webrtc",) + + def default_input_mapping(self) -> InputMapping | None: + return None + + def validate_config(self, config: Any) -> None: + if config.model_id != self.model_id: + raise ValueError( + f"Expected WebRTC model_id={self.model_id!r}, got {config.model_id!r}." + ) + + def create_runtime(self, config: Any) -> Any: + self.validate_config(config) + return self._runtime + + def prepare_scenario(self, spec: Any) -> PreparedScenario: + del spec + return PreparedScenario(initial_inputs=self._initial_inputs()) + + def create_model_input_provider( + self, + spec: Any, + scenario: PreparedScenario, + ) -> _LegacyWebRTCModelInputProvider: + del spec, scenario + return _LegacyWebRTCModelInputProvider( + runtime=self._runtime, + session_input=self._session_input, + ) + + def _initial_inputs(self) -> InferenceInput: + if self._session_input is None: + return InferenceInput() + return InferenceInput( + global_conditioning={_SESSION_INPUT_KEY: self._session_input} + ) + + +class _ManagedWebRTCSessionEdgeFactory: + """Build shared realtime edges for one negotiated peer connection.""" + + def __init__( + self, + *, + manager: "BaseWebRTCSessionManager[Any, Any]", + managed_session: "ManagedWebRTCSession", + loop: asyncio.AbstractEventLoop, + ) -> None: + self._manager = manager + self._managed_session = managed_session + self._loop = loop + + def create_session_edges( + self, + *, + context: RunContext, + spec: Any, + scenario: PreparedScenario, + provider: ModelInputProvider, + adapter: Any, + ) -> SessionEdges: + del spec, scenario, provider, adapter + input_source = self._managed_session.input_source + transport = self._managed_session.transport + if input_source is None or transport is None: + raise RuntimeError("Managed WebRTC session is missing shared edges.") + bridge = ThreadSafeWebRTCOutputBridge( + loop=self._loop, + video_encoder=self._managed_session.video_encoder, + video_track=self._managed_session.video_track, + on_chunk_delivery=self._on_chunk_delivery, + on_error=self._on_delivery_error, + ) + return SessionEdges( + input_source=input_source, + output_sink=WebRTCOutputSink(bridge=bridge), + cleanup_tasks=context.cleanup_tasks, + metrics=InMemorySessionMetricsRecorder(), + error_policy=WebRTCErrorPolicy(), + transport=transport, + clock=ResamplerRealtimeClock( + resampler=self._managed_session.resampler, + now_fn=self._loop.time, + sleep_fn=asyncio.sleep, + ), + activation=WebRTCActivationPolicy( + input_source=input_source, + transport=transport, + ), + ) + + def _on_chunk_delivery(self, chunk: WebRTCChunkDelivery) -> None: + self._manager._handle_shared_chunk_delivery( + managed_session=self._managed_session, + chunk=chunk, + ) + + def _on_delivery_error(self, exc: BaseException) -> None: + self._manager._handle_shared_delivery_error( + managed_session=self._managed_session, + exc=exc, + ) + if self._manager.fatal_generation_errors: + self._loop.call_soon_threadsafe( + lambda: asyncio.create_task(self._manager.close_active_session()) + ) @dataclass(slots=True) @@ -104,11 +601,23 @@ class ManagedWebRTCSession: video_track: BufferedVideoTrack | NVENCVideoTrack video_encoder: VideoEncoder peer_connection: Any - resampler: KeyboardResampler + resampler: RealtimeEventResampler + legacy_segment_resampler: Any | None = None control_channel: Any | None = None generation_task: asyncio.Task[Any] | None = None first_action_received: asyncio.Event = field(default_factory=asyncio.Event) + input_source: WebRTCInputSource | None = None + transport: WebRTCTransportService | None = None + reservation: Any | None = None pending_action_arrivals: deque[float] = field(default_factory=deque) + inference_session: Any | None = None + """Active ``InferenceSession``; ``None`` means call ``runtime.generate_chunk``.""" + session_steps_completed: int = 0 + session_input_state_advanced: bool = False + user_events: deque[UserInputEvent] = field(default_factory=deque) + """Raw user events awaiting canonicalization, oldest first.""" + coalesced_release_events: dict[str, UserInputEvent] = field(default_factory=dict) + """Overflow key releases, coalesced by normalized key.""" last_client_message_at: float = 0.0 liveness_task: asyncio.Task[Any] | None = None closed: bool = False @@ -137,6 +646,11 @@ async def close(self) -> None: self.generation_task.cancel() with contextlib.suppress(asyncio.CancelledError): await self.generation_task + if self.generation_task is None or self.generation_task.done(): + reservation = self.reservation + self.reservation = None + if reservation is not None: + reservation.release() self.generation_task = None await self.video_track.close() @@ -146,11 +660,6 @@ async def close(self) -> None: class BaseWebRTCSessionManager(Generic[_RuntimeT, _RuntimeConfigT]): """Owns one active WebRTC session and forwards actions into a model runtime.""" - _busy_message: str = "A WebRTC session is already active." - _warmup_label: str = "WebRTC" - _runtime_error_types: tuple[type[Exception], ...] = (RuntimeError,) - _close_session_on_generation_error: bool = False - _resampler_supported_keys: AbstractSet[str] | None = None _perf_log_interval_chunks: int = _DEFAULT_PERF_LOG_INTERVAL_CHUNKS def __init__( @@ -159,12 +668,33 @@ def __init__( runtime: _RuntimeT, runtime_config: _RuntimeConfigT, fps: int, + identity: str, + busy_message: str = "A WebRTC session is already active.", + warmup_label: str = "WebRTC", + supported_control_keys: AbstractSet[str] | None = None, + fatal_generation_errors: bool = False, client_liveness_timeout_s: float = DEFAULT_CLIENT_LIVENESS_TIMEOUT_S, + shared_host: RuntimeHost | None = None, + shared_adapter: Any | None = None, + shared_spec: DemoSpec | None = None, + shared_spec_factory: Callable[[Any], DemoSpec] | None = None, + shared_scenario: PreparedScenario | None = None, + shared_pipeline_factory: Callable[[], StepPipeline] | None = None, + legacy_segment_resampler_factory: Callable[..., Any] | None = None, ) -> None: if client_liveness_timeout_s <= 0: raise ValueError("client_liveness_timeout_s must be > 0") self.runtime_config = runtime_config self.fps = fps + self.identity = identity + self.busy_message = busy_message + self.warmup_label = warmup_label + self.supported_control_keys = ( + None + if supported_control_keys is None + else frozenset(supported_control_keys) + ) + self.fatal_generation_errors = fatal_generation_errors self.client_liveness_timeout_s = client_liveness_timeout_s self._runtime = runtime self._runtime_ready = False @@ -172,36 +702,75 @@ def __init__( self._active_session: ManagedWebRTCSession | None = None self._preload_lock = asyncio.Lock() self._session_lock = asyncio.Lock() - - def _model_name(self) -> str: - """Human-readable model identifier reported in ``chunk_done``.""" - raise NotImplementedError - - def _peek_pending_session_input(self) -> Any: - """Session input applied to the next ``create_answer`` (or ``None``).""" - return None - - def _clear_pending_session_input(self) -> None: - """Clear the pending session input after a successful answer.""" - - async def _reset_runtime_for_session(self, session_input: Any) -> None: - """Reset the runtime for a new rollout, honoring ``session_input``.""" - await self._runtime.reset_for_new_session() - - def _make_resampler(self, *, start_v: float) -> KeyboardResampler: + self._pending_session_input: Any = None + self._shared_runtime_adapter: _LegacyWebRTCRuntimeAdapter | None = None + self._shared_host: RuntimeHost | None = shared_host + self._owns_shared_host = shared_host is not None + self._shared_context: RunContext | None = None + self._shared_adapter = shared_adapter + self._shared_spec = shared_spec + self._shared_spec_factory = shared_spec_factory + self._shared_scenario = shared_scenario + self._shared_pipeline_factory = shared_pipeline_factory + self._shared_video_encoder: VideoEncoder | None = None + self._legacy_segment_resampler_factory = legacy_segment_resampler_factory + + @property + def pending_session_input(self) -> Any: + """Input that will be applied to the next successfully negotiated session.""" + return self._pending_session_input + + @property + def runtime(self) -> _RuntimeT: + """Model runtime driven by this transport manager.""" + return self._runtime + + def set_pending_session_input(self, session_input: Any) -> None: + """Store validated model input for the next session.""" + if self.has_active_session(): + raise SessionBusyError(self.busy_message) + self._pending_session_input = session_input + + def _make_resampler(self, *, start_v: float) -> RealtimeEventResampler: return self._make_resampler_at_fps(start_v=start_v, fps=self.fps) def _make_resampler_at_fps( self, *, start_v: float, fps: float - ) -> KeyboardResampler: - if self._resampler_supported_keys is None: - return KeyboardResampler(fps=fps, start_v=start_v) - return KeyboardResampler( - fps=fps, - start_v=start_v, - supported_keys=frozenset(self._resampler_supported_keys), + ) -> RealtimeEventResampler: + return RealtimeEventResampler(fps=fps, start_v=start_v) + + def _make_legacy_segment_resampler_at_fps( + self, *, start_v: float, fps: float + ) -> Any: + factory = self._legacy_segment_resampler_factory + if factory is None: + raise RuntimeError( + "Legacy WebRTC segment runtimes require " + "legacy_segment_resampler_factory." + ) + supported_control_keys = self._effective_supported_control_keys() + kwargs: dict[str, object] = { + "fps": fps, + "start_v": start_v, + } + if supported_control_keys is not None: + kwargs["supported_keys"] = supported_control_keys + return factory(**kwargs) + + def _needs_legacy_segment_metadata(self) -> bool: + return self._shared_adapter is None and not _runtime_drives_inference_session( + self._runtime ) + def _effective_supported_control_keys(self) -> frozenset[str] | None: + supported_control_keys = self.supported_control_keys + if supported_control_keys is not None: + return frozenset(supported_control_keys) + legacy_supported_keys = getattr(self, "_resampler_supported_keys", None) + if legacy_supported_keys is None: + return None + return frozenset(legacy_supported_keys) + @staticmethod def _positive_int_runtime_value(value: Any, *, label: str) -> int: try: @@ -223,55 +792,110 @@ def _positive_float_runtime_value(value: Any, *, label: str) -> float: return parsed def _runtime_input_fps(self, runtime: Any) -> float: - method = getattr(runtime, "peek_input_fps", None) - if callable(method): - return self._positive_float_runtime_value( - method(), - label="peek_input_fps", - ) - return float(self.fps) + peek_input_fps = getattr(runtime, "peek_input_fps", None) + if not callable(peek_input_fps): + return float(self.fps) + return self._positive_float_runtime_value( + peek_input_fps(), + label="peek_input_fps", + ) - def _runtime_next_input_num_frames(self, runtime: Any) -> int: - method = getattr(runtime, "peek_next_input_num_frames", None) - if callable(method): - return self._positive_int_runtime_value( - method(), - label="peek_next_input_num_frames", + def _runtime_next_step_request(self, runtime: Any) -> tuple[StepRequest, int]: + request = runtime.next_step_request() + if not isinstance(request, StepRequest): + raise TypeError( + "next_step_request must return StepRequest, " + f"got {type(request).__name__}." ) - return self._positive_int_runtime_value( - runtime.peek_next_chunk_num_frames(), - label="peek_next_chunk_num_frames", + input_num_frames = self._positive_int_runtime_value( + request.metadata.get("input_frame_count"), + label="StepRequest.metadata['input_frame_count']", ) + return request, input_num_frames def _runtime_steady_output_num_frames(self, runtime: Any) -> int: - method = getattr(runtime, "peek_steady_output_num_frames", None) - if callable(method): + peek_output_frames = getattr(runtime, "peek_steady_output_num_frames", None) + if callable(peek_output_frames): return self._positive_int_runtime_value( - method(), + peek_output_frames(), label="peek_steady_output_num_frames", ) + pipeline = getattr(runtime, "pipeline", None) + get_num_frames = getattr(pipeline, "get_num_frames", None) + if callable(get_num_frames): + return self._positive_int_runtime_value( + get_num_frames(1), + label="pipeline.get_num_frames(1)", + ) return self._positive_int_runtime_value( - runtime.peek_steady_chunk_num_frames(), - label="peek_steady_chunk_num_frames", + 1, + label="fallback steady output frame count", ) - def _register_extra_peer_handlers(self, peer_connection: Any) -> None: - """Register optional extra peer-connection event handlers.""" - def _resolve_video_encoder(self) -> VideoEncoder: """Return the encoder to use for the next session. Default: read ``runtime.video_encoder`` if the runtime provides - one (omnidreams does, via ``_initialize_video_encoder_sync``); + one through the shared thread-affine runtime; otherwise construct a session-scope :class:`DefaultRTCEncoder`. Runtimes that do not participate in encoder selection transparently get the software path without having to opt in. """ encoder = getattr(self._runtime, "video_encoder", None) + if encoder is None: + encoder = self._shared_video_encoder if encoder is None: encoder = DefaultRTCEncoder(fps=self.fps) return encoder + def _shared_run_context(self, loop: asyncio.AbstractEventLoop) -> RunContext: + if self._shared_context is not None: + return self._shared_context + host = self._shared_host + if host is None: + runtime_adapter = _LegacyWebRTCRuntimeAdapter( + runtime=self._runtime, + loop=loop, + ) + host = RuntimeHost(runtime_adapter) + self._shared_runtime_adapter = runtime_adapter + self._shared_host = host + self._shared_context = RunContext( + host=host, + run_metrics=InMemorySessionMetricsRecorder(), + admission=SingleSessionAdmissionPolicy( + health_check=lambda: host.is_healthy + ), + ) + return self._shared_context + + def _shared_demo_spec(self) -> DemoSpec: + return DemoSpec( + model_id=self.identity, + input_mode="webrtc", + output=WebRTCOutputSpec( + fps=self.fps, + video_width=self.runtime_config.video_width, + video_height=self.runtime_config.video_height, + warmup_chunks=self.runtime_config.warmup_chunks, + warmup_timeout_s=self.runtime_config.warmup_timeout_s, + client_liveness_timeout_s=self.client_liveness_timeout_s, + ), + ) + + async def _reset_runtime_for_session( + self, + *, + context: RunContext, + session_input: Any, + ) -> None: + reset = getattr(context.host.runtime, "reset_for_new_session", None) + if not callable(reset): + if self._shared_adapter is not None: + return + raise RuntimeError("WebRTC runtime adapter cannot reset sessions.") + await context.host.call_async(reset, session_input) + def _prefer_h264_video_codec(self, *, transceiver: Any) -> None: """Constrain the transceiver's codec preferences to H.264 variants. @@ -329,7 +953,7 @@ async def _enforce_h264_or_fallback( # is drained. Otherwise ``ManagedWebRTCSession.close()`` would # only ever see the fallback track and never clean this one up. # The hardware encoder itself is owned by the runtime (created - # once in ``_initialize_video_encoder_sync`` and reused across + # once during runtime initialization and reused across # sessions), so it is intentionally NOT closed here — subsequent # sessions read the same object via ``runtime.video_encoder`` # and expect it live. Runtime shutdown releases it. @@ -341,16 +965,6 @@ async def _enforce_h264_or_fallback( managed_session.video_encoder = fallback_encoder managed_session.video_track = fallback_track - def _on_offer_received(self, offer_sdp: str) -> None: - """Hook invoked with the remote offer SDP before negotiation.""" - - def _on_answer_created(self, answer_sdp: str) -> None: - """Hook invoked with the local answer SDP after negotiation.""" - - def _chunk_done_extra(self) -> dict[str, Any]: - """Extra fields merged into every ``chunk_done`` payload.""" - return {} - async def _handle_event_message( self, *, @@ -375,6 +989,43 @@ async def _handle_event_message( ) return False + if managed_session.inference_session is not None: + # On the session branch a text event is just another user event: + # the mapping turns it into a session-global conditioning update + # applied by the next step, so there is no separate runtime call. + clears = state in clear_states + try: + event_payload = self._validate_user_event_payload( + managed_session=managed_session, + event_type="text_event", + payload={ + "event_id": None if clears else event_id, + "state": state, + }, + ) + self._record_user_event( + managed_session=managed_session, + timestamp_s=asyncio.get_running_loop().time(), + event_type="text_event", + payload=event_payload, + ) + except Exception as exc: + if channel is not None: + self._send_json(channel, make_error_payload(str(exc))) + return False + if channel is not None: + active_event_id = event_payload.get("event_id") + ack_event_id = None if active_event_id is None else str(active_event_id) + self._send_json( + channel, + make_event_ack_payload( + event_id=ack_event_id, + state=str(event_payload.get("state", state)), + result={"active_event_id": ack_event_id}, + ), + ) + return True + trigger_event = getattr(managed_session.runtime, "trigger_event", None) if not callable(trigger_event): if channel is not None: @@ -404,6 +1055,273 @@ async def _handle_event_message( ) return True + @staticmethod + def _drives_inference_session(runtime: Any) -> bool: + """Return whether ``runtime`` should be driven through ``InferenceSession``.""" + return _runtime_drives_inference_session(runtime) + + def _record_user_event( + self, + *, + managed_session: ManagedWebRTCSession, + timestamp_s: float, + event_type: str, + payload: dict[str, Any], + ) -> None: + """Buffer one raw user event for the session branch. + + Timestamps use the same monotonic clock as the realtime resampler so + chunk ``TimeWindow`` filtering and raw data-channel events agree. + """ + if event_type in _KEY_USER_EVENT_TYPES and not self._supports_key_payload( + payload + ): + return + if len(managed_session.user_events) >= _MAX_SESSION_USER_EVENTS: + if event_type in _RELEASE_USER_EVENT_TYPES: + made_room = self._make_room_for_release_event( + managed_session=managed_session, + event_type=event_type, + payload=payload, + ) + if not made_room: + self._record_coalesced_release_event( + managed_session=managed_session, + timestamp_s=timestamp_s, + event_type=event_type, + payload=payload, + ) + return + else: + raise RuntimeError( + "Too many queued WebRTC user events; wait for inference to catch up." + ) + managed_session.user_events.append( + UserInputEvent( + timestamp_s=timestamp_s, + event_type=event_type, + payload=payload, + source="webrtc", + ) + ) + + def _make_room_for_release_event( + self, + *, + managed_session: ManagedWebRTCSession, + event_type: str, + payload: dict[str, Any], + ) -> bool: + events = managed_session.user_events + if not events: + return False + if event_type == "key_up": + released_key = payload.get("key") + normalized_released_key = ( + normalize_key(released_key) if isinstance(released_key, str) else None + ) + if normalized_released_key is not None: + for index, queued_event in enumerate(events): + queued_key = queued_event.payload.get("key") + if ( + queued_event.event_type == "key_down" + and isinstance(queued_key, str) + and normalize_key(queued_key) == normalized_released_key + ): + del events[index] + return True + for index, queued_event in enumerate(events): + queued_key = queued_event.payload.get("key") + if ( + queued_event.event_type == "key_up" + and isinstance(queued_key, str) + and normalize_key(queued_key) == normalized_released_key + ): + del events[index] + return True + return False + + def _record_coalesced_release_event( + self, + *, + managed_session: ManagedWebRTCSession, + timestamp_s: float, + event_type: str, + payload: dict[str, Any], + ) -> None: + if event_type != "key_up": + return + key = payload.get("key") + if not isinstance(key, str): + return + managed_session.coalesced_release_events[normalize_key(key)] = UserInputEvent( + timestamp_s=timestamp_s, + event_type=event_type, + payload=payload, + source="webrtc", + ) + + def _supported_key_names(self) -> frozenset[str]: + supported_keys = self._effective_supported_control_keys() + if supported_keys is None: + supported_keys = DEFAULT_SUPPORTED_KEYS + return frozenset(normalize_key(key) for key in supported_keys) + + def _supports_key_payload(self, payload: dict[str, Any]) -> bool: + key = payload.get("key") + return ( + isinstance(key, str) and normalize_key(key) in self._supported_key_names() + ) + + @staticmethod + def _pending_user_events( + managed_session: ManagedWebRTCSession, + ) -> tuple[UserInputEvent, ...]: + return tuple( + sorted( + ( + *managed_session.user_events, + *managed_session.coalesced_release_events.values(), + ), + key=lambda event: event.timestamp_s, + ) + ) + + def _catch_up_input_clock( + self, + *, + managed_session: ManagedWebRTCSession, + now: float, + chunk_duration: float, + ) -> None: + """Skip stale input windows without skipping session input state.""" + resampler = managed_session.resampler + lag = now - (resampler.next_chunk_start_v + chunk_duration) + if lag <= chunk_duration: + return + latest_chunk_start = now - chunk_duration + if managed_session.inference_session is not None: + catch_up_start = ( + 0.0 + if managed_session.session_steps_completed == 0 + else resampler.next_chunk_start_v + ) + if latest_chunk_start > catch_up_start: + self._advance_inference_input_state( + managed_session=managed_session, + window=TimeWindow( + start_s=catch_up_start, + end_s=latest_chunk_start, + ), + ) + resampler.next_chunk_start_v = latest_chunk_start + + def _advance_inference_input_state( + self, + *, + managed_session: ManagedWebRTCSession, + window: TimeWindow, + ) -> None: + """Advance session input converters over a skipped raw-event window.""" + if managed_session.inference_session is None or window.end_s <= window.start_s: + return + runtime = managed_session.runtime + runtime.input_canonicalizer.canonicalize( + UserInputs(events=self._pending_user_events(managed_session)), + window=window, + source_schema=runtime.input_source_schema, + ) + managed_session.session_input_state_advanced = True + self._prune_consumed_user_events( + managed_session, + before_s=window.end_s, + ) + + def _validate_user_event_payload( + self, + *, + managed_session: ManagedWebRTCSession, + event_type: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + """Return a runtime-validated user-event payload.""" + validate = getattr(managed_session.runtime, "validate_user_event", None) + if not callable(validate): + return payload + result = validate(event_type=event_type, payload=dict(payload)) + if result is None: + return payload + if not isinstance(result, dict): + raise TypeError( + "validate_user_event must return a payload dict or None, got " + f"{type(result).__name__}." + ) + return result + + @staticmethod + def _prune_consumed_user_events( + managed_session: ManagedWebRTCSession, *, before_s: float + ) -> None: + """Drop events already folded into converter state.""" + events = managed_session.user_events + while events and events[0].timestamp_s < before_s: + events.popleft() + for key, event in tuple(managed_session.coalesced_release_events.items()): + if event.timestamp_s < before_s: + del managed_session.coalesced_release_events[key] + + async def _step_inference_session( + self, + *, + managed_session: ManagedWebRTCSession, + window: TimeWindow, + ) -> StepResult: + """Map this chunk's events into model inputs and run one session step.""" + session: Any = managed_session.inference_session + if session is None: + raise RuntimeError("Session branch invoked without an inference session.") + request = session.next_step_request() + if request is None: + raise _InferenceSessionExhausted() + if request.step_index == 0 and not managed_session.session_input_state_advanced: + window = TimeWindow(start_s=0.0, end_s=window.end_s) + request = replace(request, user_input_window=window) + step_inputs = self._build_step_inputs( + managed_session=managed_session, + request=request, + window=window, + ) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(None, session.step, step_inputs) + if not isinstance(result, StepResult): + raise TypeError( + "Inference session steps must produce StepResult, got " + f"{type(result).__name__}." + ) + self._prune_consumed_user_events(managed_session, before_s=window.start_s) + managed_session.session_steps_completed += 1 + return result + + def _build_step_inputs( + self, + *, + managed_session: ManagedWebRTCSession, + request: Any, + window: TimeWindow, + ) -> InferenceInput: + """Canonicalize this chunk's events and map them into model inputs.""" + runtime = managed_session.runtime + canonical_inputs = runtime.input_canonicalizer.canonicalize( + UserInputs(events=self._pending_user_events(managed_session)), + window=window, + source_schema=runtime.input_source_schema, + ) + return runtime.input_mapping.map_step_inputs( + canonical_inputs=canonical_inputs, + inference_input=InferenceInput(), + request=request, + ) + def has_active_session(self) -> bool: return self._active_session is not None and not self._active_session.closed @@ -413,29 +1331,61 @@ def is_runtime_ready(self) -> bool: async def preload_runtime(self) -> None: async with self._preload_lock: if not self._runtime_ready: - await self._runtime.initialize() + initialize = getattr(self._runtime, "initialize", None) + if callable(initialize): + result = initialize() + if inspect.isawaitable(result): + await result + elif self._shared_host is not None: + await asyncio.to_thread(self._shared_host.preload) self._runtime_ready = True + self._initialize_shared_video_encoder() if not self._warmup_complete: await self._run_loopback_warmup_session( num_chunks=self.runtime_config.warmup_chunks ) self._warmup_complete = True + def _initialize_shared_video_encoder(self) -> None: + if self._shared_video_encoder is not None: + return + if getattr(self._runtime, "video_encoder", None) is not None: + return + encoder_backend = getattr(self.runtime_config, "encoder_backend", None) + if encoder_backend is None: + return + backend = _encoder_backend_from_config(encoder_backend) + device_spec = str(getattr(self.runtime_config, "device", "")) + device_type = device_spec.split(":", maxsplit=1)[0] + if device_type != "cuda" and backend == "auto": + backend = "default" + if device_type != "cuda" and backend == "nvenc": + raise RuntimeError("encoder_backend='nvenc' requires a CUDA device.") + self._shared_video_encoder = select_encoder( + backend=backend, + width=self.runtime_config.video_width, + height=self.runtime_config.video_height, + fps=self.fps, + bitrate=int(getattr(self.runtime_config, "encoder_bitrate_bps", 6_000_000)), + gpu_id=_gpu_id_from_device_spec(device_spec), + gop=int(getattr(self.runtime_config, "encoder_gop", self.fps)), + ) + async def create_answer(self, *, offer_sdp: str, offer_type: str) -> dict[str, str]: if not self._runtime_ready or not self._warmup_complete: await self.preload_runtime() async with self._session_lock: if self._active_session is not None and not self._active_session.closed: - raise SessionBusyError(self._busy_message) + raise SessionBusyError(self.busy_message) - session_input = self._peek_pending_session_input() + session_input = self._pending_session_input answer = await self._create_answer_with_runtime_ready_locked( offer_sdp=offer_sdp, offer_type=offer_type, session_input=session_input, ) - self._clear_pending_session_input() + self._pending_session_input = None return answer async def _create_answer_with_runtime_ready_locked( @@ -448,45 +1398,80 @@ async def _create_answer_with_runtime_ready_locked( enable_liveness_watchdog: bool = True, ) -> dict[str, str]: if self._active_session is not None and not self._active_session.closed: - raise SessionBusyError(self._busy_message) + raise SessionBusyError(self.busy_message) if not self._runtime_ready: - raise self._runtime_error_types[0]("Runtime is not initialized.") - - await self._reset_runtime_for_session(session_input) - - peer_connection = RTCPeerConnection(rtc_configuration) - # Bounded queue sized to one *steady-state* chunk so the producer - # is throttled to the consumer's drain rate. AR step 0 emits fewer - # frames than steady state; sizing to it would force a per-chunk - # stall, so we size to the steady-state count. - num_frames = self._runtime_steady_output_num_frames(self._runtime) - video_encoder = self._resolve_video_encoder() - video_track = video_encoder.create_track(maxsize=num_frames) - # Use ``addTransceiver`` (not ``addTrack``) so we can constrain the - # SDP m-line's codec list via ``setCodecPreferences`` when the - # encoder emits pre-encoded H.264 packets. - video_transceiver = peer_connection.addTransceiver( - video_track, - direction="sendonly", - ) - if video_encoder.prefers_codec == "h264": - self._prefer_h264_video_codec(transceiver=video_transceiver) - # Start the resampler's virtual clock at 0; the real anchor is set - # in the ``on_datachannel`` handler so chunk 0's window starts when - # input can actually arrive. - resampler = self._make_resampler_at_fps( - start_v=0.0, - fps=self._runtime_input_fps(self._runtime), - ) + raise RuntimeError("Runtime is not initialized.") + loop = asyncio.get_running_loop() - managed_session = ManagedWebRTCSession( - runtime=self._runtime, - video_track=video_track, - video_encoder=video_encoder, - peer_connection=peer_connection, - resampler=resampler, - last_client_message_at=loop.time(), - ) + context = self._shared_run_context(loop) + reservation = context.admission.try_reserve() + if reservation is None: + raise SessionBusyError(self.busy_message) + try: + await self._reset_runtime_for_session( + context=context, + session_input=session_input, + ) + except Exception: + reservation.release() + raise + + try: + peer_connection = RTCPeerConnection(rtc_configuration) + # Bounded queue sized to one *steady-state* chunk so the producer + # is throttled to the consumer's drain rate. AR step 0 emits fewer + # frames than steady state; sizing to it would force a per-chunk + # stall, so we size to the steady-state count. + num_frames = self._runtime_steady_output_num_frames(self._runtime) + video_encoder = self._resolve_video_encoder() + video_track = video_encoder.create_track(maxsize=num_frames) + # Use ``addTransceiver`` (not ``addTrack``) so we can constrain the + # SDP m-line's codec list via ``setCodecPreferences`` when the + # encoder emits pre-encoded H.264 packets. + video_transceiver = peer_connection.addTransceiver( + video_track, + direction="sendonly", + ) + if video_encoder.prefers_codec == "h264": + self._prefer_h264_video_codec(transceiver=video_transceiver) + # Start the resampler's virtual clock at 0; the real anchor is set + # in the ``on_datachannel`` handler so chunk 0's window starts when + # input can actually arrive. + resampler = self._make_resampler_at_fps( + start_v=0.0, + fps=self._runtime_input_fps(self._runtime), + ) + legacy_segment_resampler = None + if self._needs_legacy_segment_metadata(): + legacy_segment_resampler = self._make_legacy_segment_resampler_at_fps( + start_v=0.0, + fps=self._runtime_input_fps(self._runtime), + ) + input_source = WebRTCInputSource( + resampler=resampler, + legacy_segment_resampler=legacy_segment_resampler, + legacy_segments_metadata_key=( + _LEGACY_SPARSE_KEY_SEGMENTS_METADATA_KEY + if legacy_segment_resampler is not None + else None + ), + ) + transport = WebRTCTransportService(loop=loop) + managed_session = ManagedWebRTCSession( + runtime=self._runtime, + video_track=video_track, + video_encoder=video_encoder, + peer_connection=peer_connection, + resampler=resampler, + legacy_segment_resampler=legacy_segment_resampler, + input_source=input_source, + transport=transport, + reservation=reservation, + last_client_message_at=loop.time(), + ) + except Exception: + reservation.release() + raise self._active_session = managed_session if enable_liveness_watchdog: managed_session.liveness_task = asyncio.create_task( @@ -497,10 +1482,13 @@ async def _create_answer_with_runtime_ready_locked( def on_datachannel(channel: Any) -> None: managed_session.control_channel = channel # Re-anchor the resampler at channel open. The real - # virtual-clock anchor happens in ``_generation_worker`` once - # the first keyboard event arrives. + # virtual-clock anchor happens in ``WebRTCActivationPolicy`` once + # the first browser event activates the shared realtime driver. channel_open_v = asyncio.get_running_loop().time() - managed_session.resampler.reset(start_v=channel_open_v) + if managed_session.input_source is not None: + managed_session.input_source.reset(start_v=channel_open_v) + else: + managed_session.resampler.reset(start_v=channel_open_v) @channel.on("message") def on_message(message: Any) -> None: @@ -511,15 +1499,21 @@ def on_message(message: Any) -> None: ) ) - # Spawn the generation worker once the channel is wired up so + # Spawn the shared realtime session once the channel is wired up so # ``chunk_done`` notifications have a channel to land on. managed_session.generation_task = asyncio.create_task( - self._generation_worker(managed_session=managed_session) + self._run_realtime_driver_session( + managed_session=managed_session, + context=context, + session_input=session_input, + ) ) @channel.on("close") def on_close() -> None: logger.info("Control data channel closed; closing active session.") + if managed_session.transport is not None: + managed_session.transport.disconnect("data channel closed") asyncio.create_task(self.close_active_session()) @peer_connection.on("connectionstatechange") @@ -531,11 +1525,26 @@ async def on_connectionstatechange() -> None: }: await self.close_active_session() - self._register_extra_peer_handlers(peer_connection) + @peer_connection.on("iceconnectionstatechange") + def on_iceconnectionstatechange() -> None: + logger.info( + "Peer ICE connection state changed: {}", + peer_connection.iceConnectionState, + ) + + @peer_connection.on("icegatheringstatechange") + def on_icegatheringstatechange() -> None: + logger.debug( + "Peer ICE gathering state changed: {}", + peer_connection.iceGatheringState, + ) try: offer = RTCSessionDescription(sdp=offer_sdp, type=offer_type) - self._on_offer_received(offer_sdp) + logger.info( + "Received WebRTC offer with {}.", + _summarize_sdp_candidates(offer_sdp), + ) await peer_connection.setRemoteDescription(offer) answer = await peer_connection.createAnswer() await peer_connection.setLocalDescription(answer) @@ -549,7 +1558,10 @@ async def on_connectionstatechange() -> None: local_description = peer_connection.localDescription if local_description is None: raise RuntimeError("Peer connection did not produce local description.") - self._on_answer_created(local_description.sdp) + logger.info( + "Created WebRTC answer with {}.", + _summarize_sdp_candidates(local_description.sdp), + ) return {"sdp": local_description.sdp, "type": local_description.type} except Exception: logger.exception("WebRTC negotiation failed while creating an answer.") @@ -559,13 +1571,13 @@ async def on_connectionstatechange() -> None: async def _run_loopback_warmup_session(self, *, num_chunks: int) -> None: if not self._runtime_ready: - raise self._runtime_error_types[0]("Runtime is not initialized.") + raise RuntimeError("Runtime is not initialized.") await run_loopback_warmup_session( num_chunks=num_chunks, warmup_timeout_s=self.runtime_config.warmup_timeout_s, create_answer=self._create_loopback_warmup_answer, close_active_session=self.close_active_session, - label=self._warmup_label, + label=self.warmup_label, logger=logger, ) @@ -614,15 +1626,37 @@ async def _client_liveness_watchdog( async def shutdown(self) -> None: await self.close_active_session() - await self._runtime.close() + if self._shared_context is not None: + await self._shared_context.close_async() + if self._shared_host is not None: + await asyncio.to_thread(self._shared_host.close) + self._shared_context = None + self._shared_host = None + self._shared_runtime_adapter = None + if self._shared_video_encoder is not None: + self._shared_video_encoder.close() + self._shared_video_encoder = None + if not self._owns_shared_host: + close = getattr(self._runtime, "close", None) + if callable(close): + result = close() + if inspect.isawaitable(result): + await result self._runtime_ready = False self._warmup_complete = False def wait_for_termination(self) -> None: - self._runtime.wait_for_termination() + wait = getattr(self._runtime, "wait_for_termination", None) + if callable(wait): + wait() + return + if self._shared_host is not None: + self._shared_host.run_worker_loop() def send_exit_signal(self) -> None: - self._runtime.send_exit_signal() + send = getattr(self._runtime, "send_exit_signal", None) + if callable(send): + send() async def _handle_datachannel_message( self, @@ -634,6 +1668,10 @@ async def _handle_datachannel_message( if channel is None or managed_session.closed: return managed_session.last_client_message_at = asyncio.get_running_loop().time() + if managed_session.transport is not None: + managed_session.transport.mark_client_message( + managed_session.last_client_message_at + ) if not isinstance(raw_message, str): self._send_json(channel, make_error_payload("Expected text payload.")) @@ -650,6 +1688,12 @@ async def _handle_datachannel_message( channel, make_error_payload("Payload must be a JSON object.") ) return + if managed_session.input_source is not None: + await self._handle_shared_datachannel_payload( + managed_session=managed_session, + payload=payload, + ) + return message_type = str(payload.get("type", "")).strip().lower() if message_type == MESSAGE_TYPE_HEARTBEAT: return @@ -705,28 +1749,240 @@ async def _handle_datachannel_message( ) return - # Stamp arrival on the same monotonic clock that seeds the - # resampler's ``next_chunk_start_v`` so virtual-time comparisons in - # ``KeyboardResampler.sample_chunk`` are well-defined. + # Stamp arrival on the same monotonic clock that seeds the realtime + # window clock so user-input windows can be compared directly. arrival_t = asyncio.get_running_loop().time() - managed_session.resampler.on_edge(arrival_t=arrival_t, event=event, key=key) + if managed_session.inference_session is not None: + try: + self._record_user_event( + managed_session=managed_session, + timestamp_s=arrival_t, + event_type="key_down" if event == "keydown" else "key_up", + payload={"key": key}, + ) + except Exception as exc: + self._send_json(channel, make_error_payload(str(exc))) + if event != "keyup": + return + legacy_resampler = managed_session.legacy_segment_resampler + if legacy_resampler is not None: + legacy_resampler.on_edge(arrival_t=arrival_t, event=event, key=key) managed_session.pending_action_arrivals.append(arrival_t) # Releases the generation worker, which blocks on this until the # user actually interacts. Idempotent once already set. managed_session.first_action_received.set() + async def _handle_shared_datachannel_payload( + self, + *, + managed_session: ManagedWebRTCSession, + payload: dict[str, Any], + ) -> None: + channel = managed_session.control_channel + input_source = managed_session.input_source + if channel is None or input_source is None: + return + message_type = str(payload.get("type", "")).strip().lower() + if message_type == MESSAGE_TYPE_HEARTBEAT: + return + if message_type == MESSAGE_TYPE_DISCONNECT: + logger.info("Client requested disconnect; closing active session.") + if managed_session.transport is not None: + managed_session.transport.disconnect("client disconnected") + await self.close_active_session() + return + if message_type == MESSAGE_TYPE_EVENT: + handled = self._record_shared_event_payload( + managed_session=managed_session, + payload=payload, + ) + if handled: + managed_session.first_action_received.set() + return + result = input_source.handle_browser_payload( + payload, + timestamp_s=asyncio.get_running_loop().time(), + ) + if result.kind == "error": + self._send_json(channel, make_error_payload(result.error or "Bad input.")) + return + if result.activated: + managed_session.first_action_received.set() + + def _record_shared_event_payload( + self, + *, + managed_session: ManagedWebRTCSession, + payload: dict[str, Any], + ) -> bool: + channel = managed_session.control_channel + input_source = managed_session.input_source + if channel is None or input_source is None: + return False + event_id = str(payload.get("event_id", payload.get("id", ""))).strip() + state = str(payload.get("state", "trigger")).strip().lower() or "trigger" + clear_states = {"clear", "release", "off", "none"} + if not event_id and state not in clear_states: + self._send_json( + channel, + make_error_payload( + ( + "Event payload must include non-empty 'event_id' " + "unless state clears the active event." + ), + ), + ) + return False + clears = state in clear_states + try: + event_payload = self._validate_user_event_payload( + managed_session=managed_session, + event_type="text_event", + payload={ + "event_id": None if clears else event_id, + "state": state, + }, + ) + active_event_id = event_payload.get("event_id") + source_event_id = None if active_event_id is None else str(active_event_id) + input_source.record_user_event( + timestamp_s=asyncio.get_running_loop().time(), + event_type="text_event", + payload=event_payload, + source_event_id=source_event_id, + ) + except Exception as exc: + self._send_json(channel, make_error_payload(str(exc))) + return False + active_event_id = event_payload.get("event_id") + ack_event_id = None if active_event_id is None else str(active_event_id) + self._send_json( + channel, + make_event_ack_payload( + event_id=ack_event_id, + state=str(event_payload.get("state", state)), + result={"active_event_id": ack_event_id}, + ), + ) + return True + + async def _run_realtime_driver_session( + self, + *, + managed_session: ManagedWebRTCSession, + context: RunContext, + session_input: Any, + ) -> None: + adapter = self._shared_adapter + spec = self._shared_spec + scenario = self._shared_scenario + spec_factory = self._shared_spec_factory + if adapter is None or spec is None: + adapter = _LegacyWebRTCDemoAdapter( + runtime=self._runtime, + identity=self.identity, + session_input=session_input, + ) + spec = self._shared_demo_spec() + elif spec_factory is not None and session_input is not None: + spec = spec_factory(session_input) + scenario = None + if scenario is None: + scenario = adapter.prepare_scenario(spec) + run_mode = WebRTCRunMode( + edge_factory=_ManagedWebRTCSessionEdgeFactory( + manager=self, + managed_session=managed_session, + loop=asyncio.get_running_loop(), + ) + ) + try: + result = await run_demo_session_async( + context=context, + spec=spec, + scenario=scenario, + adapter=adapter, + run_mode=run_mode, + pipeline=( + self._shared_pipeline_factory() + if self._shared_pipeline_factory is not None + else StepPipeline() + ), + reservation=managed_session.reservation, + ) + if result.status == "completed": + logger.info("Shared WebRTC session completed.") + else: + logger.warning( + "Shared WebRTC session ended with status={} reason={}", + result.status, + result.reason, + ) + if result.status != "completed" and result.reason: + channel = managed_session.control_channel + if channel is not None: + self._send_json(channel, make_error_payload(result.reason)) + finally: + managed_session.reservation = None + if self._active_session is managed_session: + await self.close_active_session() + + def _handle_shared_chunk_delivery( + self, + *, + managed_session: ManagedWebRTCSession, + chunk: WebRTCChunkDelivery, + ) -> None: + channel = managed_session.control_channel + if channel is None or managed_session.closed: + return + delivery = chunk.delivery + enqueued_frames = int(getattr(delivery, "num_frames", chunk.frame_count)) + encode_ms = float(getattr(delivery, "encode_ms", 0.0)) + play_ms = chunk.frame_count * 1000.0 / managed_session.video_track.fps + queue_depth = managed_session.video_track.qsize() + self._send_json( + channel, + make_chunk_done_payload( + chunk_index=chunk.step_index, + num_frames=chunk.frame_count, + enqueued_frames=enqueued_frames, + fps=managed_session.video_track.fps, + width=self.runtime_config.video_width, + height=self.runtime_config.video_height, + model=self.identity, + gen_ms=_stat_ms(chunk.metrics, "model_step_s"), + enqueue_ms=encode_ms, + play_ms=play_ms, + queue_depth=queue_depth, + lag_ms=0.0, + control_latency_ms=None, + consumed_actions=0, + extra=chunk.metadata, + ), + ) + + def _handle_shared_delivery_error( + self, + *, + managed_session: ManagedWebRTCSession, + exc: BaseException, + ) -> None: + channel = managed_session.control_channel + if channel is not None: + self._send_json(channel, make_error_payload(str(exc))) + async def _generation_worker( self, *, managed_session: ManagedWebRTCSession ) -> None: - """Drive back-to-back chunk generation aligned to the resampler clock. + """Drive back-to-back chunk generation aligned to the realtime clock. Sits idle until the first keyboard event arrives, then drives the chunk loop. Each iteration waits for wallclock to catch up to the - *end* of the next chunk's virtual window, samples the chunk's - piecewise-constant timeline, hands segments and frame times to the - runtime, and pushes the generated frames into the video track. The - track's bounded queue then paces the loop to playback via - backpressure on ``BufferedVideoTrack.enqueue_chunk``. + *end* of the next chunk's virtual window, hands legacy segment data and + frame times to the runtime, and pushes generated frames into the video + track. The track's bounded queue then paces the loop to playback via + backpressure on ``BufferedVideoTrack.enqueue_result``. """ loop = asyncio.get_running_loop() runtime = managed_session.runtime @@ -758,8 +2014,8 @@ async def _generation_worker( try: while not managed_session.closed: try: - input_num_frames = self._runtime_next_input_num_frames(runtime) - except self._runtime_error_types: + request, input_num_frames = self._runtime_next_step_request(runtime) + except RuntimeError: logger.exception("Runtime not ready; stopping generation worker.") return # Trigger when wallclock reaches the chunk's window end. @@ -773,17 +2029,34 @@ async def _generation_worker( # Catch the virtual clock up to wall if it has fallen more # than one chunk behind so end-to-end latency stays bounded. - # Held-key continuity is preserved because ``sample_chunk`` - # folds every event below the new window start into the - # carried state. + # The segment branch folds skipped edges through the resampler; + # the session branch first advances its input canonicalizer + # across the skipped raw-event window. now = loop.time() - lag = now - (resampler.next_chunk_start_v + chunk_duration) - if lag > chunk_duration: - resampler.next_chunk_start_v = now - chunk_duration + self._catch_up_input_clock( + managed_session=managed_session, + now=now, + chunk_duration=chunk_duration, + ) t_before_gen = loop.time() - segments, frame_times = resampler.sample_chunk(input_num_frames) + chunk_start_v = resampler.next_chunk_start_v + frame_times = list(resampler.sample_chunk(input_num_frames)) chunk_end_v = resampler.next_chunk_start_v + segments: list[Any] = [] + legacy_resampler = managed_session.legacy_segment_resampler + if legacy_resampler is not None: + legacy_resampler.next_chunk_start_v = chunk_start_v + segments, frame_times = legacy_resampler.sample_chunk( + input_num_frames + ) + segment_request = replace( + request, + user_input_window=TimeWindow( + start_s=chunk_start_v, + end_s=chunk_end_v, + ), + ) consumed_action_arrivals: list[float] = [] while ( managed_session.pending_action_arrivals @@ -793,21 +2066,44 @@ async def _generation_worker( managed_session.pending_action_arrivals.popleft() ) try: - result = await runtime.generate_chunk( - segments=segments, frame_times=frame_times + if managed_session.inference_session is not None: + result = await self._step_inference_session( + managed_session=managed_session, + window=TimeWindow( + start_s=chunk_start_v, + end_s=chunk_end_v, + ), + ) + else: + result = await runtime.step( + request=segment_request, + segments=segments, + frame_times=frame_times, + ) + if result.step_index != segment_request.step_index: + raise RuntimeError( + "Runtime result step does not match its request: " + f"requested {segment_request.step_index}, " + f"got {result.step_index}." + ) + except _InferenceSessionExhausted: + logger.info( + "Inference session reported completion; closing WebRTC session." ) + await self.close_active_session() + return except Exception as exc: logger.exception("Chunk generation failed.") channel = managed_session.control_channel if channel is not None: self._send_json(channel, make_error_payload(str(exc))) - if self._close_session_on_generation_error: + if self.fatal_generation_errors: await self.close_active_session() return continue t_after_gen = loop.time() delivery = await video_encoder.deliver_chunk( - result.video_chunk, + result, video_track, force_keyframe=False, ) @@ -816,7 +2112,7 @@ async def _generation_worker( gen_ms = (t_after_gen - t_before_gen) * 1e3 enqueue_ms = (t_after_enqueue - t_after_gen) * 1e3 - play_ms = result.num_frames * 1000.0 / video_track.fps + play_ms = result.frame_count * 1000.0 / video_track.fps lag_ms = (t_after_enqueue - resampler.next_chunk_start_v) * 1e3 control_latency_ms = ( (t_after_enqueue - consumed_action_arrivals[0]) * 1e3 @@ -824,17 +2120,16 @@ async def _generation_worker( else None ) perf_window_chunks += 1 - perf_window_frames += result.num_frames - if result.chunk_index == 0 or ( - perf_log_interval > 0 - and result.chunk_index % perf_log_interval == 0 + perf_window_frames += result.frame_count + if result.step_index == 0 or ( + perf_log_interval > 0 and result.step_index % perf_log_interval == 0 ): interval_s = max(t_after_enqueue - perf_window_start, 1.0e-6) interval_fps = perf_window_frames / interval_s - gen_fps = result.num_frames / max( + gen_fps = result.frame_count / max( t_after_gen - t_before_gen, 1.0e-6 ) - stats = result.stats or {} + stats = result.metrics logger.info( "WebRTC perf chunk={} interval_chunks={} frames={} " "gen_fps={:.1f} interval_fps={:.1f} playback_fps={} " @@ -845,7 +2140,7 @@ async def _generation_worker( "queue_depth={} lag_ms={:.0f} control_latency_ms={} " "compile_active={} compile_start_step={} cuda_graph={} " "cache_frames={} cache_tokens={}", - result.chunk_index, + result.step_index, perf_window_chunks, perf_window_frames, gen_fps, @@ -880,9 +2175,9 @@ async def _generation_worker( "segments={} enqueued={} " "gen_ms={:.1f} enqueue_ms={:.1f} play_ms={:.1f} queue_depth={} " "lag_ms={:.1f}", - result.chunk_index, + result.step_index, input_num_frames, - result.num_frames, + result.frame_count, len(segments), enqueued, gen_ms, @@ -897,13 +2192,13 @@ async def _generation_worker( self._send_json( channel, make_chunk_done_payload( - chunk_index=result.chunk_index, - num_frames=result.num_frames, + chunk_index=result.step_index, + num_frames=result.frame_count, enqueued_frames=enqueued, fps=video_track.fps, width=self.runtime_config.video_width, height=self.runtime_config.video_height, - model=self._model_name(), + model=self.identity, gen_ms=gen_ms, enqueue_ms=enqueue_ms, play_ms=play_ms, @@ -911,7 +2206,7 @@ async def _generation_worker( lag_ms=lag_ms, control_latency_ms=control_latency_ms, consumed_actions=len(consumed_action_arrivals), - extra=self._chunk_done_extra(), + extra=result.metadata, ), ) except asyncio.CancelledError: diff --git a/flashdreams/flashdreams/serving/webrtc/media.py b/flashdreams/flashdreams/serving/webrtc/media.py index 2912a042f..25fa09437 100644 --- a/flashdreams/flashdreams/serving/webrtc/media.py +++ b/flashdreams/flashdreams/serving/webrtc/media.py @@ -7,7 +7,7 @@ import contextlib from collections.abc import Callable, Sequence from fractions import Fraction -from typing import TYPE_CHECKING +from typing import cast import numpy as np from aiortc import MediaStreamTrack @@ -16,17 +16,31 @@ from av.packet import Packet from loguru import logger -from flashdreams.serving.realtime.media import tensor_chunk_to_rgb_frames - -if TYPE_CHECKING: - import torch +from flashdreams.runtime import StepResult +from flashdreams.serving.realtime.media import ( + FrameLayout, + ValueRange, + rgb_array_to_uint8_frames, +) +from flashdreams.serving.realtime.media import ( + tensor_chunk_to_rgb_frames as tensor_chunk_to_rgb_frames, +) _STALL_THRESHOLD_MS = 1.0 _PACING_LAG_LOG_MS = 5.0 -def _default_frame_converter(video_chunk: torch.Tensor) -> list[np.ndarray]: - return tensor_chunk_to_rgb_frames(video_chunk, sync_device=True) +def _default_frame_converter(result: StepResult) -> list[np.ndarray]: + video_chunk = result.video_chunk + value_range: ValueRange = ( + "minus_one_one" if video_chunk.is_floating_point() else "uint8" + ) + return rgb_array_to_uint8_frames( + video_chunk, + layout=cast(FrameLayout, result.layout), + value_range=value_range, + sync_device=True, + ) class BufferedVideoTrack(MediaStreamTrack): @@ -39,7 +53,7 @@ def __init__( *, fps: int, maxsize: int, - frame_converter: Callable[[torch.Tensor], list[np.ndarray]] | None = None, + frame_converter: Callable[[StepResult], list[np.ndarray]] | None = None, ) -> None: super().__init__() if fps <= 0: @@ -67,16 +81,37 @@ def maxsize(self) -> int: def qsize(self) -> int: return self._frames.qsize() - async def enqueue_chunk(self, video_chunk: torch.Tensor) -> int: + def prepare_result_frames(self, result: StepResult) -> tuple[np.ndarray, ...]: + if self._closed: + return () + return tuple(self._frame_converter(result)) + + async def enqueue_frames(self, frames: Sequence[np.ndarray]) -> int: if self._closed: return 0 - frames = await asyncio.to_thread(self._frame_converter, video_chunk) for i, frame in enumerate(frames): if self._closed: return i await self._frames.put(frame) return len(frames) + async def enqueue_result(self, result: StepResult) -> int: + if self._closed: + return 0 + frames = await asyncio.to_thread(self.prepare_result_frames, result) + return await self.enqueue_frames(frames) + + async def flush(self) -> None: + """Drop queued frames while keeping the RTP timestamp sequence alive.""" + if self._closed: + return + while True: + try: + self._frames.get_nowait() + except asyncio.QueueEmpty: + break + self._next_deadline_s = None + async def recv(self) -> VideoFrame: if self._closed: raise MediaStreamError @@ -220,6 +255,17 @@ def enqueue_encoded_packet_nowait(self, packet: Packet) -> bool: self._packets.put_nowait(packet) return True + async def flush(self) -> None: + """Drop queued encoded packets while preserving the open media track.""" + if self._closed: + return + while True: + try: + self._packets.get_nowait() + except asyncio.QueueEmpty: + break + self._next_deadline_s = None + async def recv(self) -> Packet: if self._closed: raise MediaStreamError diff --git a/flashdreams/flashdreams/serving/webrtc/nvenc.py b/flashdreams/flashdreams/serving/webrtc/nvenc.py index ade33cb49..94a52200f 100644 --- a/flashdreams/flashdreams/serving/webrtc/nvenc.py +++ b/flashdreams/flashdreams/serving/webrtc/nvenc.py @@ -23,6 +23,7 @@ import contextlib import time from collections.abc import Callable +from dataclasses import dataclass from fractions import Fraction from typing import TYPE_CHECKING, Any @@ -30,7 +31,9 @@ from aiortc import MediaStreamTrack from av.packet import Packet from loguru import logger +from torch import Tensor +from flashdreams.runtime import StepResult from flashdreams.serving.webrtc.encoders import ChunkDeliveryResult # Runtime imports ``PyNvVideoCodec`` unconditionally (the isolation @@ -56,6 +59,13 @@ _RTP_VIDEO_CLOCK = 90_000 +@dataclass(frozen=True, slots=True) +class NVENCChunkPayload: + """Encoder-owned CUDA frames prepared before async delivery is scheduled.""" + + frames: Tensor + + def _payload_contains_nal_type(payload: bytes, nal_type: int) -> bool: """Scan an Annex-B H.264 payload for the presence of a specific NAL type.""" i = 0 @@ -71,13 +81,12 @@ def _payload_contains_nal_type(payload: bytes, nal_type: int) -> bool: i = nal_start + 1 -def _chunk_to_abgr_cuda_frames(chunk: torch.Tensor) -> torch.Tensor: - """Convert a model-output chunk to NVENC-``ABGR``-formatted CUDA frames. +def _result_to_abgr_frames(result: StepResult) -> torch.Tensor: + """Convert a declared video result to NVENC-``ABGR``-formatted frames. - Accepts ``[T, 3, H, W]`` or ``[1, 1, T, 3, H, W]`` (the shape produced - by the omnidreams runtime) in either ``uint8`` or float dtype - (float assumed to be in ``[-1, 1]``). Returns a contiguous - ``[T, H, W, 4]`` ``uint8`` CUDA tensor with alpha=255. + The result layout selects the time, channel, batch, and view axes; tensor + rank is never used to guess the model's output contract. The returned + contiguous ``[T, H, W, 4]`` uint8 tensor stays on the source device. **NVENC ``NV_ENC_BUFFER_FORMAT_ABGR`` is a word-ordered token, not memory-ordered.** From ``nvEncodeAPI.h``: "a pixel is represented by @@ -93,25 +102,7 @@ def _chunk_to_abgr_cuda_frames(chunk: torch.Tensor) -> torch.Tensor: conversion handles the colour transform, sparing us a bespoke NV12 kernel. """ - if not chunk.is_cuda: - raise ValueError("expected CUDA tensor for hardware encode path") - if chunk.ndim == 6: - if chunk.shape[0] != 1 or chunk.shape[1] != 1: - raise ValueError( - "expected single-batch, single-view chunk [1, 1, T, 3, H, W]; " - f"got {tuple(chunk.shape)}" - ) - chunk = chunk[0, 0] - if chunk.ndim != 4 or chunk.shape[1] != 3: - raise ValueError( - "expected chunk shape [T, 3, H, W] or [1, 1, T, 3, H, W]; " - f"got {tuple(chunk.shape)}" - ) - if chunk.dtype == torch.uint8: - rgb = chunk.permute(0, 2, 3, 1) - else: - rgb = ((chunk.float() + 1.0) / 2.0 * 255.0).clamp(0, 255).byte() - rgb = rgb.permute(0, 2, 3, 1) + rgb = result.video_hwc_uint8() t, h, w, _ = rgb.shape a = torch.full((t, h, w, 1), 255, dtype=torch.uint8, device=rgb.device) # Channel-last [R, G, B, A] → little-endian bytes [R, G, B, A] → @@ -258,9 +249,38 @@ def create_track(self, *, maxsize: int) -> NVENCVideoTrack: return NVENCVideoTrack(fps=self.fps, maxsize=maxsize) + def prepare_chunk_payload( + self, + result: StepResult, + track: MediaStreamTrack, + ) -> NVENCChunkPayload: + from flashdreams.serving.webrtc.media import NVENCVideoTrack + + if not isinstance(track, NVENCVideoTrack): + raise TypeError( + "PyNvHardwareEncoder requires an NVENCVideoTrack; got " + f"{type(track).__name__}. Create it via encoder.create_track()." + ) + return NVENCChunkPayload(frames=_result_to_abgr_frames(result)) + + async def deliver_prepared_chunk( + self, + payload: object, + track: MediaStreamTrack, + *, + force_keyframe: bool = False, + ) -> ChunkDeliveryResult: + if not isinstance(payload, NVENCChunkPayload): + raise TypeError("PyNvHardwareEncoder payload must be an NVENCChunkPayload.") + return await self._deliver_prepared_frames( + payload.frames, + track, + force_keyframe=force_keyframe, + ) + async def deliver_chunk( self, - chunk: torch.Tensor, + result: StepResult, track: MediaStreamTrack, *, force_keyframe: bool = False, @@ -297,7 +317,64 @@ def _stream(packet: Packet) -> None: _num_frames, num_keyframes, encode_ms = await asyncio.to_thread( self.encode_chunk_sync, - chunk, + result, + force_keyframe=force_keyframe, + on_packet=_stream, + ) + if enqueued < emitted: + logger.debug( + "NVENC track closed while enqueueing encoded chunk; " + "enqueued {} of {} packet(s).", + enqueued, + emitted, + ) + return ChunkDeliveryResult( + backend=self.backend, + num_frames=enqueued, + num_keyframes=num_keyframes, + encode_ms=encode_ms, + ) + + async def _deliver_prepared_frames( + self, + frames: Tensor, + track: MediaStreamTrack, + *, + force_keyframe: bool = False, + ) -> ChunkDeliveryResult: + from flashdreams.serving.webrtc.media import NVENCVideoTrack + + if not isinstance(track, NVENCVideoTrack): + raise TypeError( + "PyNvHardwareEncoder requires an NVENCVideoTrack; got " + f"{type(track).__name__}. Create it via encoder.create_track()." + ) + loop = asyncio.get_running_loop() + emitted = 0 + enqueued = 0 + + def _stream(packet: Packet) -> None: + nonlocal emitted, enqueued + emitted += 1 + enqueue = track.enqueue_encoded_packet(packet) + try: + future = asyncio.run_coroutine_threadsafe( + enqueue, + loop, + ) + except RuntimeError: + enqueue.close() + return + try: + accepted = future.result() + except Exception: + return + if accepted: + enqueued += 1 + + _num_frames, num_keyframes, encode_ms = await asyncio.to_thread( + self.encode_frames_sync, + frames, force_keyframe=force_keyframe, on_packet=_stream, ) @@ -317,18 +394,34 @@ def _stream(packet: Packet) -> None: def encode_chunk_sync( self, - chunk: torch.Tensor, + result: StepResult, *, force_keyframe: bool = False, on_packet: Callable[[Packet], None] | None = None, ) -> tuple[int, int, float]: - """Encode a chunk synchronously; returns ``(num_frames, num_keyframes, encode_ms)``. + """Encode a result and return frame, keyframe, and timing counts. Kept public because callers (e.g. tests) that already run on a worker thread should not have to route through :meth:`deliver_chunk` just to get access to the emitted packets. """ - frames = _chunk_to_abgr_cuda_frames(chunk) + frames = _result_to_abgr_frames(result) + return self.encode_frames_sync( + frames, + force_keyframe=force_keyframe, + on_packet=on_packet, + ) + + def encode_frames_sync( + self, + frames: Tensor, + *, + force_keyframe: bool = False, + on_packet: Callable[[Packet], None] | None = None, + ) -> tuple[int, int, float]: + """Encode preconverted ``ABGR`` frames for prepared async delivery.""" + if not frames.is_cuda: + raise ValueError("expected CUDA tensor for hardware encode path") num_frames = frames.shape[0] num_keyframes = 0 start_s = time.perf_counter() diff --git a/flashdreams/flashdreams/serving/webrtc/runtime.py b/flashdreams/flashdreams/serving/webrtc/runtime.py index 79fa9e0e2..1472807ca 100644 --- a/flashdreams/flashdreams/serving/webrtc/runtime.py +++ b/flashdreams/flashdreams/serving/webrtc/runtime.py @@ -1,47 +1,43 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Runtime contracts for shared WebRTC demo serving.""" +"""Runtime contracts and thread-affine execution for shared WebRTC serving.""" from __future__ import annotations -from collections.abc import Awaitable, Mapping -from typing import Any, Protocol +import asyncio +from abc import ABC, abstractmethod +from collections.abc import Awaitable +from enum import IntEnum +from typing import Any, Generic, Protocol, TypeVar import torch - -from flashdreams.infra.postprocess import VideoTensorLayout -from flashdreams.infra.video_output import VideoStepResult, infer_video_num_frames -from flashdreams.serving.realtime.input import PoseSegment - - -class WebRTCStepResult(VideoStepResult): - """One generated chunk handed back by a WebRTC model runtime.""" - - -def make_webrtc_step_result( - *, - chunk_index: int, - video_chunk: torch.Tensor, - layout: VideoTensorLayout, - stats: dict[str, float] | None = None, - sync_device: torch.device | str | None = None, - metadata: Mapping[str, Any] | None = None, -) -> WebRTCStepResult: - """Package a generated chunk for WebRTC without forcing a host copy.""" - if sync_device is not None: - device = torch.device(sync_device) - if device.type == "cuda": - torch.cuda.current_stream(device).synchronize() - - return WebRTCStepResult( - chunk_index=chunk_index, - num_frames=infer_video_num_frames(video_chunk, layout=layout), - video_chunk=video_chunk.detach(), - stats=stats, - layout=layout, - metadata=dict(metadata or {}), - ) +import torch.distributed as dist + +from flashdreams.core.distributed.rank_orchestration import ( + RankCoordinator, + distributed_op, +) +from flashdreams.runtime.types import StepRequest, StepResult +from flashdreams.runtime.worker import ThreadAffineRuntimeWorker +from flashdreams.serving.webrtc.encoders import ( + EncoderBackend, + VideoEncoder, + select_encoder, +) + + +class WebRTCControlSignal(IntEnum): + """Rank-orchestration signals shared by WebRTC runtimes.""" + + INITIALIZE = 0 + RESET_SESSION = 1 + ACTION_STEP = 2 + CLOSE = 3 + EVENT = 4 + SESSION_STEP = 5 + SESSION_CLOSE = 6 + EXIT = 99 class WebRTCRuntimeConfig(Protocol): @@ -53,37 +49,49 @@ class WebRTCRuntimeConfig(Protocol): warmup_timeout_s: float -class WebRTCGenerationRuntime(Protocol): - """Generation lifecycle for one shared WebRTC session. +class ThreadAffineWebRTCRuntimeConfig(WebRTCRuntimeConfig, Protocol): + """Configuration consumed by the shared runtime execution layer.""" + + device: str + fps: int + encoder_backend: EncoderBackend + encoder_bitrate_bps: int + encoder_gop: int + + +class WebRTCServerLifecycle(Protocol): + """Distributed worker lifecycle used by the shared WebRTC serve loop.""" + + def send_exit_signal(self) -> None: ... + + def wait_for_termination(self) -> None: ... + + +class WebRTCSessionRuntime(WebRTCServerLifecycle, Protocol): + """Complete runtime contract consumed by the shared session manager. Integrations keep their model-specific state, checkpoints, conditioning, and cache logic inside their concrete runtime. The shared manager only needs this lifecycle and chunk-generation surface. - - By default, ``peek_next_chunk_num_frames`` and - ``peek_steady_chunk_num_frames`` are used for both input sampling and - output queue sizing. Runtimes whose model input clock differs from their - output video clock may also implement these optional methods: - - - ``peek_input_fps() -> float`` for the control/input sampling clock. - - ``peek_next_input_num_frames() -> int`` for the length of ``frame_times``. - - ``peek_steady_output_num_frames() -> int`` for video queue sizing. """ async def initialize(self) -> None: ... - async def reset_for_new_session(self) -> None: ... + async def reset_for_new_session(self, *, session_input: Any = None) -> None: ... + + def peek_input_fps(self) -> float: ... - def peek_steady_chunk_num_frames(self) -> int: ... + def next_step_request(self) -> StepRequest: ... - def peek_next_chunk_num_frames(self) -> int: ... + def peek_steady_output_num_frames(self) -> int: ... - async def generate_chunk( + async def step( self, *, - segments: list[PoseSegment], + request: StepRequest, + segments: list[Any], frame_times: list[float], - ) -> WebRTCStepResult: ... + ) -> StepResult: ... async def close(self) -> None: ... @@ -96,13 +104,229 @@ def trigger_event( ) -> dict[str, Any] | Awaitable[dict[str, Any]]: ... -class WebRTCServerLifecycle(Protocol): - """Distributed worker lifecycle used by the shared WebRTC serve loop.""" +_ConfigT = TypeVar("_ConfigT", bound=ThreadAffineWebRTCRuntimeConfig) +_SessionInputT = TypeVar("_SessionInputT") - def send_exit_signal(self) -> None: ... - def wait_for_termination(self) -> None: ... +class ThreadAffineDistributedWebRTCRuntime( + ABC, + Generic[_ConfigT, _SessionInputT], +): + """Coordinate one thread-affine, distributed WebRTC model runtime. + + Subclasses own model construction, rollout state, conditioning, and chunk + generation. This base owns the identical async-to-thread dispatch, rank + signaling, step ordering, and video-encoder lifecycle used by integrations. + """ + + MASTER_RANK = 0 + + def __init__( + self, + *, + config: _ConfigT, + runtime_error_type: type[RuntimeError], + thread_name: str, + ) -> None: + self.config = config + self.rank = 0 if not dist.is_initialized() else dist.get_rank() + self._runtime_error_type = runtime_error_type + self._device = self._resolve_device(config.device) + self._closed = False + self._video_encoder: VideoEncoder | None = None + self._worker = ThreadAffineRuntimeWorker( + device=self._device, + thread_name=thread_name, + ) + self._step_lock = asyncio.Lock() + self.rank_coordinator = RankCoordinator( + device=self._device, + signal_type=WebRTCControlSignal, + is_master=self.is_master, + master_rank=self.MASTER_RANK, + ) + self.rank_coordinator.register_distributed_ops(self) + + @staticmethod + def _resolve_device(device_spec: str | torch.device) -> torch.device: + device = torch.device(device_spec) + if device.type == "cuda" and device.index is None: + device = torch.device( + f"cuda:{torch.cuda.current_device()}" + if torch.cuda.is_available() + else "cuda:0" + ) + return device + + @property + def is_master(self) -> bool: + return self.rank == self.MASTER_RANK + + @property + def video_encoder(self) -> VideoEncoder: + """Return the encoder selected during runtime initialization.""" + if self._video_encoder is None: + raise self._runtime_error( + "Video encoder is not initialized; call runtime.initialize() first." + ) + return self._video_encoder + + def wait_for_termination(self) -> None: + self.rank_coordinator.worker_loop(exit_signal=WebRTCControlSignal.EXIT) + + def send_exit_signal(self) -> None: + if self.is_master: + self.rank_coordinator.send_exit(exit_signal=WebRTCControlSignal.EXIT) + + async def initialize(self) -> None: + if self._is_runtime_initialized(): + return + await self._worker.call(self._initialize_sync_all_ranks) + + async def reset_for_new_session( + self, session_input: _SessionInputT | None = None + ) -> None: + self._require_open_and_initialized() + await self._worker.call(self._reset_rollout_sync_all_ranks, session_input) + + async def close(self) -> None: + if self._closed: + return + self._closed = True + try: + await self._worker.call(self._close_sync_all_ranks) + finally: + await self._worker.close() + + async def step( + self, + *, + request: StepRequest, + segments: list[Any], + frame_times: list[float], + ) -> StepResult: + self._require_open_and_initialized(session=True) + expected_step = self._runtime_step_index() + if request.step_index != expected_step: + raise self._runtime_error( + f"Expected request step {expected_step}, got {request.step_index}." + ) + + async with self._step_lock: + self._require_open_and_initialized(session=True) + return await self._worker.call( + self._generate_chunk_sync_all_ranks, + segments, + frame_times, + ) + + def peek_input_fps(self) -> float: + return float(self.config.fps) + + def next_step_request(self) -> StepRequest: + self._require_open_and_initialized() + return StepRequest( + step_index=self._runtime_step_index(), + metadata={"input_frame_count": self._next_input_frame_count()}, + ) + + def peek_steady_output_num_frames(self) -> int: + self._require_open_and_initialized() + return self._steady_output_frame_count() + + def _runtime_error(self, message: str) -> RuntimeError: + return self._runtime_error_type(message) + + def _require_open_and_initialized(self, *, session: bool = False) -> None: + if self._closed: + noun = "Session" if session else "Runtime" + raise self._runtime_error(f"{noun} is closed.") + if not self._is_runtime_initialized(): + raise self._runtime_error("Runtime is not initialized.") + + def _initialize_video_encoder_sync(self) -> None: + """Select the master rank's encoder on the model runtime thread.""" + if not self.is_master: + return + if self._video_encoder is not None: + self._video_encoder.close() + self._video_encoder = None + + backend = self.config.encoder_backend + if self._device.type != "cuda" and backend == "auto": + backend = "default" + if self._device.type != "cuda" and backend == "nvenc": + raise self._runtime_error( + "encoder_backend='nvenc' requires a CUDA runtime device." + ) + gpu_id = self._device.index if self._device.index is not None else 0 + self._video_encoder = select_encoder( + backend=backend, + width=self.config.video_width, + height=self.config.video_height, + fps=self.config.fps, + bitrate=self.config.encoder_bitrate_bps, + gpu_id=gpu_id, + gop=self.config.encoder_gop, + ) + + def _close_video_encoder_sync(self) -> None: + if self._video_encoder is not None: + self._video_encoder.close() + self._video_encoder = None + + @distributed_op(WebRTCControlSignal.INITIALIZE) + def _initialize_sync_all_ranks(self) -> None: + self._initialize_sync() + + @distributed_op(WebRTCControlSignal.RESET_SESSION) + def _reset_rollout_sync_all_ranks( + self, session_input: _SessionInputT | None = None + ) -> None: + self._reset_rollout_sync(session_input=session_input) + + @distributed_op(WebRTCControlSignal.ACTION_STEP) + def _generate_chunk_sync_all_ranks( + self, + segments: list[Any], + frame_times: list[float], + ) -> StepResult: + return self._generate_one_chunk_sync(segments=segments, frame_times=frame_times) + + @distributed_op(WebRTCControlSignal.CLOSE) + def _close_sync_all_ranks(self) -> None: + try: + self._close_sync() + finally: + self._close_video_encoder_sync() + + @abstractmethod + def _is_runtime_initialized(self) -> bool: ... + + @abstractmethod + def _runtime_step_index(self) -> int: ... + + @abstractmethod + def _next_input_frame_count(self) -> int: ... + + @abstractmethod + def _steady_output_frame_count(self) -> int: ... + @abstractmethod + def _initialize_sync(self) -> None: ... + + @abstractmethod + def _reset_rollout_sync( + self, session_input: _SessionInputT | None = None + ) -> None: ... + + @abstractmethod + def _generate_one_chunk_sync( + self, + *, + segments: list[Any], + frame_times: list[float], + ) -> StepResult: ... -class WebRTCSessionRuntime(WebRTCGenerationRuntime, WebRTCServerLifecycle, Protocol): - """Complete runtime contract consumed by the shared session manager.""" + @abstractmethod + def _close_sync(self) -> None: ... diff --git a/flashdreams/flashdreams/serving/webrtc/server.py b/flashdreams/flashdreams/serving/webrtc/server.py index a9e386533..860640a6f 100644 --- a/flashdreams/flashdreams/serving/webrtc/server.py +++ b/flashdreams/flashdreams/serving/webrtc/server.py @@ -34,6 +34,7 @@ async def shutdown(self) -> None: ... def create_webrtc_app( *, web_dir: Path, + model_web_dir: Path | None = None, session_manager: WebRTCSessionManager, request_session_url: str, index_filename: str = "request_session.html", @@ -89,6 +90,14 @@ async def healthz(request: web.Request) -> web.StreamResponse: } ) + async def ui_config(_: web.Request) -> web.StreamResponse: + payload: dict[str, str | None] = {"adapter_module": None} + if model_web_dir is not None and (model_web_dir / "adapter.js").is_file(): + payload["adapter_module"] = "/model-static/adapter.js?v=model-ui-v4" + if model_web_dir is not None and (model_web_dir / "adapter.css").is_file(): + payload["model_stylesheet"] = "/model-static/adapter.css?v=model-ui-v4" + return web.json_response(payload) + async def on_startup(app: web.Application) -> None: manager = app[SESSION_MANAGER_KEY] logger.info("Preloading {} runtime on startup.", preload_name) @@ -104,7 +113,10 @@ async def on_shutdown(app: web.Application) -> None: app.router.add_get("/request_session", request_session_page) app.router.add_post("/api/webrtc/offer", offer) app.router.add_get("/healthz", healthz) + app.router.add_get("/api/ui/config", ui_config) app.router.add_static("/static/", web_dir, show_index=False) + if model_web_dir is not None: + app.router.add_static("/model-static/", model_web_dir, show_index=False) app.on_startup.append(on_startup) app.on_shutdown.append(on_shutdown) return app @@ -117,6 +129,7 @@ async def close_package_resources(app: web.Application) -> None: def create_packaged_webrtc_app( *, web_resource: Any, + model_web_resource: Any | None = None, session_manager: WebRTCSessionManager, request_session_url: str, preload_name: str, @@ -136,13 +149,18 @@ def create_packaged_webrtc_app( resource_stack = ExitStack() try: web_dir = resource_stack.enter_context(as_file_fn(web_resource)) - app = create_app_fn( - web_dir=web_dir, - session_manager=session_manager, - preload_name=preload_name, - request_session_url=request_session_url, - index_filename=index_filename, - ) + create_kwargs: dict[str, Any] = { + "web_dir": web_dir, + "session_manager": session_manager, + "preload_name": preload_name, + "request_session_url": request_session_url, + "index_filename": index_filename, + } + if model_web_resource is not None: + create_kwargs["model_web_dir"] = resource_stack.enter_context( + as_file_fn(model_web_resource) + ) + app = create_app_fn(**create_kwargs) if configure_app is not None: configure_app(app) app[PACKAGE_RESOURCE_STACK_KEY] = resource_stack diff --git a/flashdreams/flashdreams/serving/webrtc/services.py b/flashdreams/flashdreams/serving/webrtc/services.py new file mode 100644 index 000000000..cfcb11029 --- /dev/null +++ b/flashdreams/flashdreams/serving/webrtc/services.py @@ -0,0 +1,1232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Session-edge services for the shared WebRTC demo run mode. + +These classes are the Phase 12 decomposition layer: they translate WebRTC +transport facts into the shared demo runtime contracts without owning model +execution. The production manager still uses its legacy execution hook until +the realtime-driver adoption phase. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import json +import math +import threading +from collections import deque +from collections.abc import Callable, Coroutine, Mapping, MutableSet, Sequence +from concurrent.futures import Future +from dataclasses import dataclass, field +from typing import Any, Literal, Protocol, runtime_checkable + +from flashdreams.runtime import ( + StepRequirements, + StepResult, + UserInputCapability, + UserInputEvent, + UserInputs, + UserInputSchema, +) +from flashdreams.runtime._utils import freeze_mapping +from flashdreams.runtime.demo import ( + AsyncSessionDriver, + DemoAdapter, + DemoSpec, + InMemorySessionMetricsRecorder, + ModelInputProvider, + ModelWarmupPlan, + OutputDecision, + PreparedScenario, + RealtimeSessionDriver, + RealtimeWindowResult, + RunContext, + RunModeCapabilities, + RunResult, + RuntimeHost, + SessionEdges, + SessionInfo, + SingleSessionAdmissionPolicy, + StepPipeline, + UserInputWindow, + WebRTCErrorPolicy, + input_frame_count_from_request, + run_demo_session_async, +) +from flashdreams.runtime.demo.timing import ( + ActivationResult, + CatchUpPolicy, + DeterministicClock, + RealtimeClock, +) + +from .messages import ( + MESSAGE_TYPE_ACTION, + MESSAGE_TYPE_DISCONNECT, + MESSAGE_TYPE_EVENT, + MESSAGE_TYPE_HEARTBEAT, +) +from .server import SessionBusyError + +WebRTCMessageKind = Literal[ + "action", + "disconnect", + "event", + "heartbeat", + "error", +] + +WebRTCDropPolicy = Literal["none", "drop_newest", "drop_oldest"] + +_CLEAR_EVENT_STATES = frozenset({"clear", "release", "off", "none"}) +WEBRTC_SKIPPED_INPUTS_METADATA_KEY = "webrtc_skipped_inputs" +WEBRTC_SKIPPED_WINDOW_METADATA_KEY = "webrtc_skipped_window" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCMessageResult: + """Result of translating one browser data-channel message.""" + + kind: WebRTCMessageKind + activated: bool = False + error: str | None = None + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCOfferRequest: + """Browser SDP offer passed to the shared offer/session handler.""" + + sdp: str + type: str + + def __post_init__(self) -> None: + if not self.sdp.strip(): + raise ValueError("WebRTCOfferRequest.sdp must be non-empty.") + if not self.type.strip(): + raise ValueError("WebRTCOfferRequest.type must be non-empty.") + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCOutputBridgeDecision: + """Immediate delivery decision from a nonblocking WebRTC output bridge.""" + + accepted: bool = True + should_stop: bool = False + dropped: bool = False + drop_policy: WebRTCDropPolicy = "none" + backpressure_s: float = 0.0 + metadata: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.drop_policy not in {"none", "drop_newest", "drop_oldest"}: + raise ValueError(f"Unsupported drop_policy={self.drop_policy!r}.") + if not math.isfinite(self.backpressure_s) or self.backpressure_s < 0.0: + raise ValueError( + "WebRTCOutputBridgeDecision.backpressure_s must be finite and >= 0." + ) + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class WebRTCChunkDelivery: + """Completed WebRTC chunk delivery plus the model chunk summary.""" + + delivery: object + step_index: int + frame_count: int + generation: int + force_keyframe: bool + metadata: Mapping[str, object] = field(default_factory=dict) + metrics: Mapping[str, float | int] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) + object.__setattr__(self, "metrics", freeze_mapping(self.metrics)) + + +@runtime_checkable +class WebRTCOfferAnswerer(Protocol): + """Creates an SDP answer after the shared session task has been scheduled.""" + + async def create_answer( + self, + *, + offer: WebRTCOfferRequest, + session_task: asyncio.Task[RunResult], + ) -> Mapping[str, str]: ... + + +@runtime_checkable +class BlockingPreparationService(Protocol): + """Runs blocking scenario preparation outside the aiohttp event loop.""" + + async def run( + self, + func: Callable[..., object], + *args: object, + **kwargs: object, + ) -> object: ... + + +@runtime_checkable +class WebRTCOutputBridge(Protocol): + """Thread-safe bridge from model-worker output writes to WebRTC delivery.""" + + def begin_generation(self, generation: int) -> None: ... + + def submit_chunk( + self, + result: StepResult, + *, + generation: int, + force_keyframe: bool = False, + ) -> WebRTCOutputBridgeDecision: ... + + def close(self) -> None: ... + + +@runtime_checkable +class WebRTCSessionEdgeFactory(Protocol): + """Builds per-peer shared session edges on the WebRTC control rank.""" + + def create_session_edges( + self, + *, + context: RunContext, + spec: DemoSpec, + scenario: PreparedScenario, + provider: ModelInputProvider, + adapter: DemoAdapter, + ) -> SessionEdges: ... + + +class AsyncioBlockingPreparationService: + """Default blocking-prep service backed by ``asyncio.to_thread``.""" + + async def run( + self, + func: Callable[..., object], + *args: object, + **kwargs: object, + ) -> object: + return await asyncio.to_thread(func, *args, **kwargs) + + +class WebRTCTransportService: + """Idempotent per-peer transport lifecycle for realtime session edges.""" + + def __init__( + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + on_close: Callable[[str | None], None] | None = None, + ) -> None: + self._loop = loop + self._on_close = on_close + self._closed_signal = _ThreadSafeActivationSignal(loop=loop) + self._lock = threading.Lock() + self._closed = False + self._close_reason: str | None = None + self._close_count = 0 + self.last_client_message_at: float | None = None + + @property + def close_count(self) -> int: + """Number of effective transport closes, after idempotency.""" + + return self._close_count + + @property + def close_reason(self) -> str | None: + return self._close_reason + + @property + def closed_signal(self) -> "_ThreadSafeActivationSignal": + return self._closed_signal + + def mark_client_message(self, timestamp_s: float) -> None: + if not math.isfinite(timestamp_s) or timestamp_s < 0.0: + raise ValueError("timestamp_s must be finite and >= 0.") + self.last_client_message_at = float(timestamp_s) + + def is_active(self) -> bool: + return not self._closed + + def disconnect(self, reason: str = "client disconnected") -> None: + self.close(reason=reason) + + def close(self, reason: str | None = None) -> None: + callback: Callable[[str | None], None] | None = None + with self._lock: + if self._closed: + return + self._closed = True + self._close_reason = reason + self._close_count += 1 + callback = self._on_close + self._closed_signal.set() + if callback is not None: + callback(reason) + + +@dataclass(slots=True) +class WebRTCActivationPolicy: + """Activate on the first browser action/event or stop on disconnect.""" + + input_source: "WebRTCInputSource" + transport: WebRTCTransportService + timeout_s: float | None = None + timeout_reason: str = "activation timed out" + anchor_clock: bool = True + + def __post_init__(self) -> None: + if self.timeout_s is not None and self.timeout_s <= 0.0: + raise ValueError("timeout_s must be > 0 when set.") + if not self.timeout_reason.strip(): + raise ValueError("timeout_reason must be non-empty.") + + async def wait_until_active( + self, + clock: RealtimeClock | DeterministicClock, + ) -> ActivationResult: + if self.input_source.activation_signal.is_set(): + self._anchor(clock) + return ActivationResult(activated=True) + if not self.transport.is_active(): + return ActivationResult( + activated=False, + reason=self.transport.close_reason or "transport closed", + ) + + activation_task = asyncio.create_task( + self.input_source.activation_signal.wait() + ) + closed_task = asyncio.create_task(self.transport.closed_signal.wait()) + try: + done, pending = await asyncio.wait( + {activation_task, closed_task}, + timeout=self.timeout_s, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + return ActivationResult( + activated=False, + reason=self.timeout_reason, + ) + for task in done: + task.result() + if not self.transport.is_active(): + return ActivationResult( + activated=False, + reason=self.transport.close_reason or "transport closed", + ) + self._anchor(clock) + return ActivationResult(activated=True) + finally: + for task in (activation_task, closed_task): + if not task.done(): + task.cancel() + await asyncio.gather( + activation_task, + closed_task, + return_exceptions=True, + ) + + def _anchor(self, clock: RealtimeClock | DeterministicClock) -> None: + if not self.anchor_clock or not clock.is_realtime: + return + now = getattr(clock, "now", None) + anchor = getattr(clock, "anchor", None) + if callable(now) and callable(anchor): + anchor(now()) + + +@dataclass(slots=True) +class WebRTCInputSource: + """Realtime source fed by browser data-channel events.""" + + resampler: Any + legacy_segment_resampler: Any | None = None + legacy_segments_metadata_key: str | None = None + max_lag_s: float | None = None + catch_up_policy: CatchUpPolicy = "fold" + user_input_schema: UserInputSchema = field( + default_factory=lambda: WEBRTC_USER_INPUT_SCHEMA + ) + is_finite: bool = False + is_deterministic: bool = False + _activation_signal: "_ThreadSafeActivationSignal" = field( + default_factory=lambda: _ThreadSafeActivationSignal(), + init=False, + repr=False, + ) + _events: deque[UserInputEvent] = field( + default_factory=deque, + init=False, + repr=False, + ) + + def __post_init__(self) -> None: + if self.max_lag_s is not None and ( + not math.isfinite(self.max_lag_s) or self.max_lag_s < 0.0 + ): + raise ValueError("max_lag_s must be finite and >= 0.") + if self.catch_up_policy != "fold": + raise NotImplementedError( + f"Catch-up policy {self.catch_up_policy!r} has no WebRTC analog yet." + ) + + @property + def activation_signal(self) -> "_ThreadSafeActivationSignal": + return self._activation_signal + + def is_finished(self) -> bool: + return False + + def reset(self, *, start_v: float) -> None: + self.resampler.reset(start_v=start_v) + if self.legacy_segment_resampler is not None: + self.legacy_segment_resampler.reset(start_v=start_v) + self._events.clear() + self._activation_signal.clear() + + def handle_browser_message( + self, + raw_message: object, + *, + timestamp_s: float, + ) -> WebRTCMessageResult: + """Translate one browser data-channel message into typed user inputs.""" + + if not isinstance(raw_message, str): + return WebRTCMessageResult(kind="error", error="Expected text payload.") + try: + payload = json.loads(raw_message) + except json.JSONDecodeError: + return WebRTCMessageResult(kind="error", error="Invalid JSON payload.") + if not isinstance(payload, dict): + return WebRTCMessageResult( + kind="error", + error="Payload must be a JSON object.", + ) + return self.handle_browser_payload(payload, timestamp_s=timestamp_s) + + def handle_browser_payload( + self, + payload: Mapping[str, object], + *, + timestamp_s: float, + ) -> WebRTCMessageResult: + message_type = str(payload.get("type", "")).strip().lower() + if message_type == MESSAGE_TYPE_HEARTBEAT: + return WebRTCMessageResult(kind="heartbeat") + if message_type == MESSAGE_TYPE_DISCONNECT: + return WebRTCMessageResult(kind="disconnect") + if message_type == MESSAGE_TYPE_EVENT: + return self._record_text_event(payload, timestamp_s=timestamp_s) + if message_type == MESSAGE_TYPE_ACTION: + action_payload = payload.get("action", payload) + if not isinstance(action_payload, Mapping): + return WebRTCMessageResult( + kind="error", + error="'action' must be an object.", + ) + return self._record_action( + {str(key): value for key, value in action_payload.items()}, + timestamp_s=timestamp_s, + ) + return WebRTCMessageResult( + kind="error", + error=( + "Unsupported message type, expected " + "'action', 'event', 'heartbeat', or 'disconnect'." + ), + ) + + def record_user_event( + self, + *, + timestamp_s: float, + event_type: str, + payload: Mapping[str, object], + source_event_id: str | None = None, + activate: bool = True, + ) -> None: + event = UserInputEvent( + timestamp_s=timestamp_s, + event_type=event_type, + payload=dict(payload), + source="webrtc", + source_event_id=source_event_id, + ) + self.user_input_schema.validate_event(event) + self._events.append(event) + if activate: + self._activation_signal.set() + + async def next_realtime_window( + self, + *, + request: StepRequirements, + clock: RealtimeClock, + ) -> RealtimeWindowResult: + input_frame_count = input_frame_count_from_request(request) + chunk_duration_s = input_frame_count * float(self.resampler.dt) + if chunk_duration_s <= 0.0: + raise ValueError("Realtime resampler dt must produce a positive window.") + + window_end_s = float(self.resampler.next_chunk_start_v) + chunk_duration_s + await clock.wait_until_window_end(window_end_s) + pre_catch_up_start_s = float(self.resampler.next_chunk_start_v) + catch_up = clock.catch_up( + request=request, + max_lag_s=self.max_lag_s + if self.max_lag_s is not None + else chunk_duration_s, + policy=self.catch_up_policy, + ) + start_s = float(self.resampler.next_chunk_start_v) + frame_times = tuple(self.resampler.sample_chunk(input_frame_count)) + end_s = float(self.resampler.next_chunk_start_v) + metadata: dict[str, object] = {} + legacy_resampler = self.legacy_segment_resampler + legacy_metadata_key = self.legacy_segments_metadata_key + if legacy_resampler is not None and legacy_metadata_key is not None: + legacy_resampler.next_chunk_start_v = start_s + segments, legacy_frame_times = legacy_resampler.sample_chunk( + input_frame_count + ) + metadata[legacy_metadata_key] = tuple(segments) + frame_times = tuple(legacy_frame_times) + if start_s > pre_catch_up_start_s: + metadata[WEBRTC_SKIPPED_INPUTS_METADATA_KEY] = UserInputs( + events=self._events_for_window(pre_catch_up_start_s, start_s) + ) + metadata[WEBRTC_SKIPPED_WINDOW_METADATA_KEY] = ( + pre_catch_up_start_s, + start_s, + ) + window = RealtimeWindowResult( + window=_user_input_window( + start_s=start_s, + end_s=end_s, + frame_times=tuple(frame_times), + inputs=UserInputs(events=self._events_for_window(start_s, end_s)), + metadata=metadata, + ), + catch_up=catch_up, + ) + self._prune_events(before_s=start_s) + return window + + def _record_action( + self, + payload: Mapping[str, object], + *, + timestamp_s: float, + ) -> WebRTCMessageResult: + event = str(payload.get("event", "")).strip().lower() + if event == "step": + self._activation_signal.set() + return WebRTCMessageResult(kind="action", activated=True) + if event not in {"keydown", "keyup"}: + return WebRTCMessageResult( + kind="error", + error=f"Unsupported event={event!r}; expected 'keydown' or 'keyup'.", + ) + key = str(payload.get("key", "")).strip() + if not key: + return WebRTCMessageResult( + kind="error", + error="Action payload must include non-empty 'key'.", + ) + if self.legacy_segment_resampler is not None: + self.legacy_segment_resampler.on_edge( + arrival_t=timestamp_s, + event=event, + key=key, + ) + self.record_user_event( + timestamp_s=timestamp_s, + event_type="key_down" if event == "keydown" else "key_up", + payload={"key": key}, + ) + return WebRTCMessageResult(kind="action", activated=True) + + def _record_text_event( + self, + payload: Mapping[str, object], + *, + timestamp_s: float, + ) -> WebRTCMessageResult: + state = str(payload.get("state", "trigger")).strip().lower() or "trigger" + event_id = str(payload.get("event_id", payload.get("id", ""))).strip() + clears = state in _CLEAR_EVENT_STATES + if not event_id and not clears: + return WebRTCMessageResult( + kind="error", + error=( + "Event payload must include non-empty 'event_id' unless state " + "clears the active event." + ), + ) + active_event_id = None if clears else event_id + self.record_user_event( + timestamp_s=timestamp_s, + event_type="text_event", + payload={"event_id": active_event_id, "state": state}, + source_event_id=active_event_id, + ) + return WebRTCMessageResult(kind="event", activated=True) + + def _events_for_window( + self, + start_s: float, + end_s: float, + ) -> tuple[UserInputEvent, ...]: + return tuple( + sorted( + ( + event + for event in self._events + if start_s <= event.timestamp_s < end_s + ), + key=lambda event: event.timestamp_s, + ) + ) + + def _prune_events(self, *, before_s: float) -> None: + self._events = deque( + event for event in self._events if event.timestamp_s >= before_s + ) + + +class WebRTCOutputSink: + """Output sink that schedules WebRTC media delivery without blocking.""" + + produces_artifacts = False + + def __init__(self, *, bridge: WebRTCOutputBridge) -> None: + self._bridge = bridge + self._opened = False + self._closed = True + self._bridge_closed = False + self._generation = 0 + self._force_keyframe = False + self.session_info: SessionInfo | None = None + + def open(self, session_info: SessionInfo) -> None: + self.session_info = session_info + self._opened = True + self._closed = False + self._generation = 0 + self._force_keyframe = True + self._bridge.begin_generation(0) + + def begin_generation(self, generation: int) -> None: + if generation < 0: + raise ValueError("generation must be >= 0.") + self._generation = generation + self._force_keyframe = True + self._bridge.begin_generation(generation) + + def write(self, result: StepResult) -> OutputDecision: + if not self._opened or self._closed: + raise RuntimeError("Cannot write to a closed output sink.") + decision = self._bridge.submit_chunk( + result, + generation=self._generation, + force_keyframe=self._force_keyframe, + ) + self._force_keyframe = False + return OutputDecision( + should_stop=decision.should_stop, + dropped=decision.dropped, + drop_policy=decision.drop_policy, + backpressure_s=decision.backpressure_s, + metadata=decision.metadata, + ) + + def close(self) -> Sequence[Any]: + if self._bridge_closed: + return () + self._closed = True + self._opened = False + self._bridge.close() + self._bridge_closed = True + return () + + +class ThreadSafeWebRTCOutputBridge: + """Schedule async encoder delivery from any thread without blocking writes.""" + + def __init__( + self, + *, + loop: asyncio.AbstractEventLoop, + video_encoder: Any, + video_track: Any, + max_pending_chunks: int = 2, + close_track: bool = True, + on_delivery: Callable[[object], None] | None = None, + on_chunk_delivery: Callable[[WebRTCChunkDelivery], None] | None = None, + on_error: Callable[[BaseException], None] | None = None, + ) -> None: + if max_pending_chunks <= 0: + raise ValueError("max_pending_chunks must be > 0.") + self._loop = loop + self._video_encoder = video_encoder + self._video_track = video_track + self._max_pending_chunks = max_pending_chunks + self._close_track = close_track + self._on_delivery = on_delivery + self._on_chunk_delivery = on_chunk_delivery + self._on_error = on_error + self._pending: dict[Future[WebRTCChunkDelivery], int] = {} + self._delivery_lock = asyncio.Lock() + self._lock = threading.Lock() + self._closed = False + self._generation = 0 + + @property + def pending_count(self) -> int: + with self._lock: + return len(self._pending) + + def begin_generation(self, generation: int) -> None: + if generation < 0: + raise ValueError("generation must be >= 0.") + with self._lock: + if self._closed or generation <= self._generation: + return + self._generation = generation + stale = tuple( + future + for future, future_generation in self._pending.items() + if future_generation < generation + ) + for future in stale: + future.cancel() + self._schedule_track_flush() + + def submit_chunk( + self, + result: StepResult, + *, + generation: int, + force_keyframe: bool = False, + ) -> WebRTCOutputBridgeDecision: + prepare = getattr(self._video_encoder, "prepare_chunk_payload", None) + deliver = getattr(self._video_encoder, "deliver_prepared_chunk", None) + if not callable(prepare) or not callable(deliver): + raise TypeError( + "ThreadSafeWebRTCOutputBridge requires a video encoder with " + "prepare_chunk_payload(...) and deliver_prepared_chunk(...)." + ) + with self._lock: + if self._closed: + return WebRTCOutputBridgeDecision( + accepted=False, + should_stop=True, + dropped=True, + drop_policy="drop_newest", + metadata={"reason": "closed"}, + ) + if generation < self._generation: + return WebRTCOutputBridgeDecision( + accepted=False, + dropped=True, + drop_policy="drop_newest", + metadata={"reason": "stale generation"}, + ) + if len(self._pending) >= self._max_pending_chunks: + stale = self._pop_pending_locked() + else: + stale = () + if stale: + self._cancel_stale_deliveries(stale) + self._schedule_track_flush() + payload = prepare(result, self._video_track) + chunk = WebRTCChunkDelivery( + delivery=None, + step_index=result.step_index, + frame_count=result.frame_count, + generation=generation, + force_keyframe=force_keyframe, + metadata=result.metadata, + metrics=result.metrics, + ) + with self._lock: + if self._closed: + return WebRTCOutputBridgeDecision( + accepted=False, + should_stop=True, + dropped=True, + drop_policy="drop_newest", + metadata={"reason": "closed"}, + ) + if generation < self._generation: + return WebRTCOutputBridgeDecision( + accepted=False, + dropped=True, + drop_policy="drop_newest", + metadata={"reason": "stale generation"}, + ) + if len(self._pending) >= self._max_pending_chunks: + stale = self._pop_pending_locked() + else: + stale = () + if stale: + self._cancel_stale_deliveries(stale) + self._schedule_track_flush() + with self._lock: + if self._closed: + return WebRTCOutputBridgeDecision( + accepted=False, + should_stop=True, + dropped=True, + drop_policy="drop_newest", + metadata={"reason": "closed"}, + ) + if generation < self._generation: + return WebRTCOutputBridgeDecision( + accepted=False, + dropped=True, + drop_policy="drop_newest", + metadata={"reason": "stale generation"}, + ) + future = asyncio.run_coroutine_threadsafe( + self._deliver( + payload, + chunk=chunk, + generation=generation, + force_keyframe=force_keyframe, + ), + self._loop, + ) + self._pending[future] = generation + future.add_done_callback(self._on_done) + + return WebRTCOutputBridgeDecision( + accepted=True, + backpressure_s=self._track_backpressure_s(), + ) + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + pending = tuple(self._pending) + for future in pending: + future.cancel() + if self._close_track: + self._schedule_track_close() + + async def _deliver( + self, + payload: object, + *, + chunk: WebRTCChunkDelivery, + generation: int, + force_keyframe: bool, + ) -> WebRTCChunkDelivery: + with self._lock: + if self._closed or generation < self._generation: + raise asyncio.CancelledError + async with self._delivery_lock: + with self._lock: + if self._closed or generation < self._generation: + raise asyncio.CancelledError + await self._flush_full_track_queue(frame_count=chunk.frame_count) + delivery = await self._video_encoder.deliver_prepared_chunk( + payload, + self._video_track, + force_keyframe=force_keyframe, + ) + with self._lock: + if self._closed or generation < self._generation: + stale_after_delivery = True + else: + stale_after_delivery = False + if stale_after_delivery: + self._schedule_track_flush() + raise asyncio.CancelledError + return WebRTCChunkDelivery( + delivery=delivery, + step_index=chunk.step_index, + frame_count=chunk.frame_count, + generation=chunk.generation, + force_keyframe=chunk.force_keyframe, + metadata=chunk.metadata, + metrics=chunk.metrics, + ) + + def _on_done(self, future: Future[WebRTCChunkDelivery]) -> None: + with self._lock: + self._pending.pop(future, None) + if future.cancelled(): + return + try: + result = future.result() + except BaseException as exc: + if self._on_error is not None: + self._on_error(exc) + return + if self._on_delivery is not None: + self._on_delivery(result.delivery) + if self._on_chunk_delivery is not None: + self._on_chunk_delivery(result) + + def _pop_pending_locked(self) -> tuple[Future[WebRTCChunkDelivery], ...]: + pending = tuple(self._pending) + self._pending.clear() + return pending + + @staticmethod + def _cancel_stale_deliveries( + futures: Sequence[Future[WebRTCChunkDelivery]], + ) -> None: + for future in futures: + future.cancel() + + def _track_backpressure_s(self) -> float: + qsize = getattr(self._video_track, "qsize", None) + fps = getattr(self._video_track, "fps", None) or getattr( + self._video_encoder, + "fps", + None, + ) + if not callable(qsize) or fps is None: + return 0.0 + try: + queue_depth = int(qsize()) + frames_per_second = float(fps) + except (TypeError, ValueError): + return 0.0 + if frames_per_second <= 0.0: + return 0.0 + return max(0.0, queue_depth / frames_per_second) + + async def _flush_full_track_queue(self, *, frame_count: int) -> None: + if frame_count <= 0: + return + qsize = getattr(self._video_track, "qsize", None) + flush = getattr(self._video_track, "flush", None) + if not callable(qsize) or not callable(flush): + return + try: + queue_depth = int(qsize()) + except (TypeError, ValueError): + return + if queue_depth < frame_count: + return + + # WebRTC is interactive: a full track queue means a whole generated + # chunk is stale relative to the latest input window. Drop that queued + # media before enqueueing the current chunk so visual latency stays + # bounded instead of preserving every frame. + result = flush() + if inspect.isawaitable(result): + await result + + def _schedule_track_close(self) -> None: + close = getattr(self._video_track, "close", None) + if not callable(close): + return + try: + result = close() + if inspect.isawaitable(result): + asyncio.run_coroutine_threadsafe(result, self._loop) + except BaseException as exc: + if self._on_error is not None: + self._on_error(exc) + + def _schedule_track_flush(self) -> None: + flush = getattr(self._video_track, "flush", None) + if not callable(flush): + return + try: + result = flush() + if inspect.isawaitable(result): + asyncio.run_coroutine_threadsafe(result, self._loop) + except BaseException as exc: + if self._on_error is not None: + self._on_error(exc) + + +class WebRTCRunMode: + """Shared realtime run mode that delegates peer-specific edges to WebRTC.""" + + name = "webrtc" + capabilities = RunModeCapabilities( + realtime=True, + supports_backpressure=True, + supports_interactive_events=True, + ) + + def __init__( + self, + *, + edge_factory: WebRTCSessionEdgeFactory, + blocking_preparation: BlockingPreparationService | None = None, + driver: AsyncSessionDriver | None = None, + error_policy: WebRTCErrorPolicy | None = None, + ) -> None: + self._edge_factory = edge_factory + self._blocking_preparation = ( + blocking_preparation or AsyncioBlockingPreparationService() + ) + self._driver = driver or RealtimeSessionDriver() + self._error_policy = error_policy or WebRTCErrorPolicy() + + @property + def blocking_preparation(self) -> BlockingPreparationService: + return self._blocking_preparation + + @property + def error_policy(self) -> WebRTCErrorPolicy: + return self._error_policy + + def validate_run(self, *, spec: DemoSpec, adapter: DemoAdapter) -> None: + del adapter + if spec.output.mode != "webrtc": + raise ValueError("WebRTCRunMode requires WebRTC output.") + + def validate_session( + self, + *, + spec: DemoSpec, + scenario: PreparedScenario, + adapter: DemoAdapter, + provider: ModelInputProvider, + ) -> None: + del spec, scenario, adapter + if not provider.capabilities.supports_realtime_clock: + raise ValueError("WebRTC providers must support realtime clocks.") + + def create_run_context( + self, + *, + spec: DemoSpec, + adapter: DemoAdapter, + host: RuntimeHost, + model_warmup_plan: ModelWarmupPlan, + ) -> RunContext: + del spec, adapter + services: dict[str, object] = {} + if host.is_control_rank: + services["blocking_preparation"] = self._blocking_preparation + return RunContext( + host=host, + run_metrics=InMemorySessionMetricsRecorder(), + admission=SingleSessionAdmissionPolicy( + health_check=lambda: host.is_control_rank and host.is_healthy + ), + model_warmup_plan=model_warmup_plan, + services=services, + ) + + def create_session_edges( + self, + *, + context: RunContext, + spec: DemoSpec, + scenario: PreparedScenario, + provider: ModelInputProvider, + adapter: DemoAdapter, + ) -> SessionEdges: + if not context.host.is_control_rank: + raise RuntimeError("WebRTC session edges are control-rank only.") + edges = self._edge_factory.create_session_edges( + context=context, + spec=spec, + scenario=scenario, + provider=provider, + adapter=adapter, + ) + if not isinstance(edges, SessionEdges): + raise TypeError( + "WebRTC edge factory must return SessionEdges, " + f"got {type(edges).__name__}." + ) + return edges + + def select_driver(self) -> AsyncSessionDriver: + return self._driver + + +class WebRTCSessionOfferHandler: + """Reserve, prepare, and launch one WebRTC session before SDP negotiation.""" + + def __init__( + self, + *, + context: RunContext, + spec: DemoSpec, + adapter: DemoAdapter, + run_mode: WebRTCRunMode, + answerer: WebRTCOfferAnswerer, + pipeline: StepPipeline | None = None, + session_helper: Callable[..., Coroutine[Any, Any, RunResult]] | None = None, + busy_message: str = "Another WebRTC session is already active.", + session_tasks: MutableSet[asyncio.Task[RunResult]] | None = None, + ) -> None: + self._context = context + self._spec = spec + self._adapter = adapter + self._run_mode = run_mode + self._answerer = answerer + self._pipeline = pipeline or StepPipeline() + self._session_helper = session_helper or run_demo_session_async + self._busy_message = busy_message + self._session_tasks = session_tasks if session_tasks is not None else set() + + async def handle_offer( + self, + *, + offer_sdp: str, + offer_type: str, + ) -> Mapping[str, str]: + if not self._context.host.is_control_rank: + raise RuntimeError("WebRTC offers are handled only on the control rank.") + reservation = self._context.admission.try_reserve() + if reservation is None: + raise SessionBusyError(self._busy_message) + + task: asyncio.Task[RunResult] | None = None + try: + scenario = await self._run_blocking_prepare(self._spec) + task = asyncio.create_task( + self._session_helper( + context=self._context, + spec=self._spec, + scenario=scenario, + adapter=self._adapter, + run_mode=self._run_mode, + pipeline=self._pipeline, + reservation=reservation, + ) + ) + self._track_task(task) + answer = await self._answerer.create_answer( + offer=WebRTCOfferRequest(sdp=offer_sdp, type=offer_type), + session_task=task, + ) + return dict(answer) + except Exception: + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + reservation.release() + raise + + async def _run_blocking_prepare(self, spec: DemoSpec) -> PreparedScenario: + service = self._run_mode.blocking_preparation + result = await service.run(self._adapter.prepare_scenario, spec) + if not isinstance(result, PreparedScenario): + raise TypeError( + "DemoAdapter.prepare_scenario must return PreparedScenario, " + f"got {type(result).__name__}." + ) + return result + + def _track_task(self, task: asyncio.Task[RunResult]) -> None: + self._session_tasks.add(task) + task.add_done_callback(self._discard_task) + + def _discard_task(self, task: asyncio.Task[RunResult]) -> None: + self._session_tasks.discard(task) + if task.cancelled(): + return + with contextlib.suppress(Exception): + task.exception() + + +WEBRTC_USER_INPUT_SCHEMA = UserInputSchema( + capabilities=( + UserInputCapability( + event_type="key_down", + input_modality="keyboard", + payload_fields=frozenset({"key"}), + ), + UserInputCapability( + event_type="key_up", + input_modality="keyboard", + payload_fields=frozenset({"key"}), + ), + UserInputCapability( + event_type="text_event", + input_modality="text", + payload_fields=frozenset({"event_id", "state"}), + ), + ), + description="browser WebRTC data-channel events", +) + + +def _user_input_window( + *, + start_s: float, + end_s: float, + frame_times: Sequence[float], + inputs: UserInputs, + metadata: Mapping[str, object], +) -> UserInputWindow: + return UserInputWindow( + start_s=start_s, + end_s=end_s, + frame_times=frame_times, + inputs=inputs, + metadata=metadata, + ) + + +class _ThreadSafeActivationSignal: + def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._loop = loop + self._event = asyncio.Event() + + def is_set(self) -> bool: + return self._event.is_set() + + async def wait(self) -> object: + return await self._event.wait() + + def set(self) -> None: + if self._event.is_set(): + return + if self._loop is not None and self._loop.is_running(): + self._loop.call_soon_threadsafe(self._event.set) + return + self._event.set() + + def clear(self) -> None: + self._event.clear() + + +__all__ = [ + "AsyncioBlockingPreparationService", + "BlockingPreparationService", + "ThreadSafeWebRTCOutputBridge", + "WEBRTC_USER_INPUT_SCHEMA", + "WEBRTC_SKIPPED_INPUTS_METADATA_KEY", + "WEBRTC_SKIPPED_WINDOW_METADATA_KEY", + "WebRTCActivationPolicy", + "WebRTCInputSource", + "WebRTCMessageResult", + "WebRTCOfferAnswerer", + "WebRTCOfferRequest", + "WebRTCOutputBridge", + "WebRTCOutputBridgeDecision", + "WebRTCChunkDelivery", + "WebRTCOutputSink", + "WebRTCRunMode", + "WebRTCSessionEdgeFactory", + "WebRTCSessionOfferHandler", + "WebRTCTransportService", +] diff --git a/flashdreams/flashdreams/serving/webrtc/warmup.py b/flashdreams/flashdreams/serving/webrtc/warmup.py index 05f3d1c10..5f59c4a86 100644 --- a/flashdreams/flashdreams/serving/webrtc/warmup.py +++ b/flashdreams/flashdreams/serving/webrtc/warmup.py @@ -13,6 +13,8 @@ from aiortc.mediastreams import MediaStreamError from loguru import logger as loguru_logger +from .messages import MESSAGE_TYPE_CHUNK_DONE, MESSAGE_TYPE_ERROR + class CreateAnswerCallback(Protocol): async def __call__(self, *, offer_sdp: str, offer_type: str) -> dict[str, str]: ... @@ -47,14 +49,30 @@ async def run_loopback_warmup_session( client_peer.addTransceiver("video", direction="recvonly") channel_open = asyncio.Event() warmup_done = asyncio.Event() + warmup_failure: str | None = None received_chunks = 0 drain_tasks: set[asyncio.Task[Any]] = set() heartbeat_task: asyncio.Task[Any] | None = None + def fail_warmup(reason: str) -> None: + nonlocal warmup_failure + if warmup_done.is_set(): + return + warmup_failure = reason + warmup_done.set() + @control_channel.on("open") def on_open() -> None: channel_open.set() + @control_channel.on("close") + def on_close() -> None: + if received_chunks < num_chunks: + fail_warmup( + f"{label} loopback warmup data channel closed before warmup " + f"completed ({received_chunks}/{num_chunks} chunk(s))." + ) + @control_channel.on("message") def on_message(message: Any) -> None: nonlocal received_chunks @@ -64,7 +82,14 @@ def on_message(message: Any) -> None: payload = json.loads(message) except json.JSONDecodeError: return - if not isinstance(payload, dict) or payload.get("type") != "chunk_done": + if not isinstance(payload, dict): + return + message_type = payload.get("type") + if message_type == MESSAGE_TYPE_ERROR: + message_text = str(payload.get("message", "unknown error")) + fail_warmup(f"{label} loopback warmup failed: {message_text}") + return + if message_type != MESSAGE_TYPE_CHUNK_DONE: return received_chunks += 1 logger.info( @@ -109,6 +134,8 @@ def on_track(track: Any) -> None: for action_payload in action_payloads: control_channel.send(json.dumps(action_payload)) await asyncio.wait_for(warmup_done.wait(), timeout=warmup_timeout_s) + if warmup_failure is not None: + raise RuntimeError(warmup_failure) finally: if heartbeat_task is not None: heartbeat_task.cancel() diff --git a/integrations/omnidreams/omnidreams/webrtc/web/__init__.py b/flashdreams/flashdreams/serving/webrtc/web/__init__.py similarity index 100% rename from integrations/omnidreams/omnidreams/webrtc/web/__init__.py rename to flashdreams/flashdreams/serving/webrtc/web/__init__.py diff --git a/integrations/lingbot/lingbot/webrtc/web/assets/horizontal-dark.svg b/flashdreams/flashdreams/serving/webrtc/web/assets/horizontal-dark.svg similarity index 100% rename from integrations/lingbot/lingbot/webrtc/web/assets/horizontal-dark.svg rename to flashdreams/flashdreams/serving/webrtc/web/assets/horizontal-dark.svg diff --git a/integrations/lingbot/lingbot/webrtc/web/assets/horizontal-light.svg b/flashdreams/flashdreams/serving/webrtc/web/assets/horizontal-light.svg similarity index 100% rename from integrations/lingbot/lingbot/webrtc/web/assets/horizontal-light.svg rename to flashdreams/flashdreams/serving/webrtc/web/assets/horizontal-light.svg diff --git a/flashdreams/flashdreams/serving/webrtc/web/mock_ui_server.py b/flashdreams/flashdreams/serving/webrtc/web/mock_ui_server.py new file mode 100644 index 000000000..867dd3e02 --- /dev/null +++ b/flashdreams/flashdreams/serving/webrtc/web/mock_ui_server.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import argparse +import json +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from importlib.resources import as_file, files +from os import PathLike +from pathlib import Path +from socket import socket +from socketserver import BaseServer +from urllib.parse import urlsplit + +WEB_DIR_RESOURCE = files("flashdreams.serving.webrtc").joinpath("web") + + +class MockUIRequestHandler(SimpleHTTPRequestHandler): + """Serve the static viewer without preloading a model runtime.""" + + def __init__( + self, + request: socket | tuple[bytes, socket], + client_address: tuple[str, int], + server: BaseServer, + *, + directory: str | PathLike[str] | None = None, + model_web_dir: Path | None = None, + ) -> None: + self.model_web_dir = model_web_dir + super().__init__( + request, + client_address, + server, + directory=directory, + ) + + def _rewrite_path(self) -> bool: + path = urlsplit(self.path).path + if path == "/": + self.send_response(302) + self.send_header("Location", "/request_session?mock=1") + self.end_headers() + return True + if path == "/request_session": + self.path = "/request_session.html" + elif path.startswith("/static/"): + self.path = "/" + path.removeprefix("/static/") + return False + + def do_GET(self) -> None: + if self._serve_ui_config(): + return + if self._serve_model_asset(head_only=False): + return + if self._rewrite_path(): + return + super().do_GET() + + def do_HEAD(self) -> None: + if self._serve_ui_config(): + return + if self._serve_model_asset(head_only=True): + return + if self._rewrite_path(): + return + super().do_HEAD() + + def _serve_ui_config(self) -> bool: + if urlsplit(self.path).path != "/api/ui/config": + return False + ui_config: dict[str, str | None] = {"adapter_module": None} + if self.model_web_dir is not None: + if (self.model_web_dir / "adapter.js").is_file(): + ui_config["adapter_module"] = "/model-static/adapter.js?v=model-ui-v4" + if (self.model_web_dir / "adapter.css").is_file(): + ui_config["model_stylesheet"] = ( + "/model-static/adapter.css?v=model-ui-v4" + ) + payload = json.dumps(ui_config).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(payload) + return True + + def _serve_model_asset(self, *, head_only: bool) -> bool: + path = urlsplit(self.path).path + if not path.startswith("/model-static/") or self.model_web_dir is None: + return False + relative = Path(path.removeprefix("/model-static/")) + if relative.is_absolute() or ".." in relative.parts: + self.send_error(404) + return True + original_directory = self.directory + original_path = self.path + try: + self.directory = str(self.model_web_dir) + self.path = "/" + relative.as_posix() + if head_only: + super().do_HEAD() + else: + super().do_GET() + finally: + self.directory = original_directory + self.path = original_path + return True + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Serve the shared WebRTC mock UI.") + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("--port", type=int, default=8090) + parser.add_argument( + "--model-web-dir", + type=Path, + default=None, + help="Optional integration web directory containing adapter.js.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + with as_file(WEB_DIR_RESOURCE) as web_dir: + handler = partial( + MockUIRequestHandler, + directory=str(web_dir), + model_web_dir=args.model_web_dir, + ) + server = ThreadingHTTPServer((args.host, args.port), handler) + print( + f"Serving shared mock UI at http://{args.host}:{args.port}/request_session?mock=1" + ) + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopping mock UI server.") + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/integrations/omnidreams/omnidreams/webrtc/web/request_session.css b/flashdreams/flashdreams/serving/webrtc/web/request_session.css similarity index 97% rename from integrations/omnidreams/omnidreams/webrtc/web/request_session.css rename to flashdreams/flashdreams/serving/webrtc/web/request_session.css index 890c36147..53325e01b 100644 --- a/integrations/omnidreams/omnidreams/webrtc/web/request_session.css +++ b/flashdreams/flashdreams/serving/webrtc/web/request_session.css @@ -88,6 +88,22 @@ select { transition: opacity 220ms ease; } +.modelStageSlot { + position: absolute; + z-index: 2; + inset: 0; + pointer-events: none; +} + +.modelPanelSlot { + display: contents; +} + +.modelStatusSlot:empty, +.modelControlSlot:empty { + display: none; +} + body.has-video .stageVideo { opacity: 1; } @@ -282,6 +298,10 @@ body[data-status="generating"] .connectButton { padding: 18px 20px 20px; } +.controlCard[hidden] { + display: none; +} + .controlCard h2, .logCard h2 { display: flex; diff --git a/integrations/omnidreams/omnidreams/webrtc/web/request_session.html b/flashdreams/flashdreams/serving/webrtc/web/request_session.html similarity index 70% rename from integrations/omnidreams/omnidreams/webrtc/web/request_session.html rename to flashdreams/flashdreams/serving/webrtc/web/request_session.html index 263a1b299..bd2bdcd86 100644 --- a/integrations/omnidreams/omnidreams/webrtc/web/request_session.html +++ b/flashdreams/flashdreams/serving/webrtc/web/request_session.html @@ -8,16 +8,17 @@ - Omnidreams WebRTC Drive - + FlashDreams WebRTC Drive + - - Omnidreams WebRTC Drive + + FlashDreams WebRTC Drive + @@ -30,32 +31,26 @@ Omnidreams WebRTC Drive Idle - Connect Session + Connect Session Post-process Off + Flow waiting - + + + Controls - - - - W - A - S - D - - Drive / Turn - - + + @@ -86,12 +81,12 @@ Client Logs World Model - Omnidreams + World Model - +