[crazy-robotaxi] Live-edit abilities: skin swap, coins, spawned obstacles, weather (stacked on #463)

apps/crazy_robotaxi/README.md

@@ -32,3 +32,59 @@ crazy-robotaxi --help
 
 Use `--stream-mjpeg HOST:PORT` with either entry point to run the browser HUD
 instead of opening a local Vulkan window.
+
+## Live-edit abilities
+
+Flag-gated, off by default (`--live-edit-*`; see `crazy-robotaxi --help`,
+group "live edit"):
+
+- **Coins** (`--live-edit-coins`, toggle with `C`): collectible coins
+  composited along the route. Pixel-only — no model hooks; when the model
+  frame is CUDA-resident (native window fast path, MJPEG pre-download) the
+  sprites/HUD chips are blended on the GPU with pre-uploaded textures
+  (< 1 ms per frame), so coins are perf-neutral and work under
+  `native_dit_acceleration`.
+- **Skins** (`--live-edit-style` + `--live-edit-style-lora`, cycle with
+  `K`): prompt swaps realized by a pre-merged text-edit LoRA. The window
+  runs single-branch (no per-step extra forward). All skin/weather prompts
+  are pre-encoded once at session start, so a swap injects cached
+  embeddings — the swap-boundary chunk no longer pays the 0.5-1 s text
+  re-encode. `--live-edit-skin-duration-chunks N` turns skins into timed
+  power-ups: an activation auto-reverts to the base world after N chunks
+  (11 ≈ 3 s), the HUD chip counts down the remaining time, and `K` keeps
+  its cycle semantics during an active power-up — next skin, fresh timer
+  (the default 0 holds a skin until cycled).
+- **Weather** (`--live-edit-weather`, cycle with `V`): guided prompt swaps
+  with no LoRA, deployed land-then-release. **Transient cost: the landing
+  window costs ~2x per chunk** (a second forward per denoise step) for
+  `--live-edit-weather-guidance-chunks` chunks (default 6); the weather
+  then holds unguided at ~1x — it persists through the KV history and the
+  swapped text (A/B'd: a 27-chunk unguided hold matches the always-guided
+  policy). `--live-edit-weather-maintain-interval N` optionally re-opens a
+  short rebased window (`--live-edit-weather-maintain-chunks`, default 2)
+  every N chunks; the default 0 never does (measured unnecessary).
+- **Obstacle / traffic** (`--live-edit-obstacle`, spawn with `O`): cloned
+  scene vehicles. `--live-edit-obstacle-count N` turns one key press into a
+  traffic burst of N distinct crossing/oncoming clones staggered ahead of
+  the ego (`--live-edit-obstacle-stagger-chunks` apart in time; each
+  despawns after its own pass). Conditioning-only unless
+  `--live-edit-obstacle-guide-scale > 0`, which adds a second forward per
+  step while an event is active. The guidance is CUDA-graph safe (guided
+  steps replay the captured graph twice with box/no-box conditioning
+  staged in), so the accelerated pipeline stays on; event chunks cost ~2x
+  model time, non-event chunks are unchanged.
+- **Drift correctors** (`--live-edit-style-corrector`,
+  `--live-edit-base-corrector`, `--live-edit-weather-corrector`): optional
+  per-state weight-merged correctors. `--live-edit-corrector-mode off`
+  disables all of them (no transformer weights are snapshotted or touched)
+  even when checkpoints are configured; `fused` (default) keeps CUDA graphs
+  and `compile_network` on; `unfused` is the eager fallback (slow,
+  graph-free pipeline).
+
+Native-DIT interaction: the prompt-swap abilities (skins, weather) need the
+Python transformer forward — `replace_text_embeddings` is not wired for the
+native optimized-DiT executor, and the LoRA/corrector weight toggles never
+reach the native fp8 weight snapshot — so running them requires
+`native_dit_acceleration: disabled` in the world-model manifest. Coins (and
+obstacle without guidance) do not touch the model and run at full speed
+under native DIT.

apps/crazy_robotaxi/crazy_robotaxi/app.py

@@ -55,6 +55,9 @@
 from crazy_robotaxi.input import (
     CrazyRobotaxiKeyboardState,
 )
+from crazy_robotaxi.live_edit.coin_ability import CoinAbility
+from crazy_robotaxi.live_edit.config import LiveEditConfig
+from crazy_robotaxi.navigation import NavigationLane
 from crazy_robotaxi.passengers import (
     build_pickup_passenger_trajectories,
 )
@@ -75,17 +78,25 @@ def __init__(
         self,
         controller: TaxiGameController,
         keyboard: CrazyRobotaxiKeyboardState,
+        *,
+        style_ability: Any | None = None,
+        coin_ability: CoinAbility | None = None,
+        obstacle_ability: Any | None = None,
     ) -> None:
         self._controller = controller
         self._keyboard = keyboard
+        self._style_ability = style_ability
+        self._coin_ability = coin_ability
+        self._obstacle_ability = obstacle_ability
 
     @property
     def is_running(self) -> bool:
         """Return whether the game is still accepting simulation chunks."""
         return self._controller.is_playing
 
     def process_events(self, state: VehicleState) -> None:
-        """Consume a pending high-score name submission."""
+        """Drain live-edit key requests and a pending high-score name."""
+        self._process_live_edit_requests()
         submitted_name = self._keyboard.consume_taxi_name_submission()
         if submitted_name is None:
             return
@@ -95,18 +106,43 @@ def process_events(self, state: VehicleState) -> None:
             logger.warning(f"[crazy-robotaxi] ignored high-score submission: {exc}")
         self.publish_boundary(state)
 
+    def _process_live_edit_requests(self) -> None:
+        """Consume rising-edge skin-cycle / coins-toggle key requests."""
+        requests = getattr(self._keyboard, "live_edit", None)
+        if requests is None:
+            return
+        if requests.consume_skin_cycle() and self._style_ability is not None:
+            self._style_ability.request_cycle()
+        if requests.consume_weather_cycle() and self._style_ability is not None:
+            self._style_ability.request_weather_cycle()
+        if requests.consume_obstacle_spawn() and self._obstacle_ability is not None:
+            self._obstacle_ability.request_spawn()
+        if requests.consume_coins_toggle() and self._coin_ability is not None:
+            enabled = self._coin_ability.toggle()
+            logger.info(f"[live-edit] coins {'on' if enabled else 'off'}")
+
     def advance_frames(
         self, trajectory: TrajectoryChunk, frame_interval_s: float
     ) -> ApplicationChunkUpdate:
         """Advance the game and add passengers synchronized to pickup state."""
         snapshots = tuple(self._controller.advance_frames(trajectory, frame_interval_s))
+        if self._coin_ability is not None:
+            picked = self._coin_ability.advance_frames(trajectory.vehicle_states)
+            if picked:
+                logger.info(
+                    f"[live-edit] coin pickup +{picked} "
+                    f"total={self._coin_ability.collected_count}"
+                )
         passengers = build_pickup_passenger_trajectories(
             snapshots, trajectory.timestamps_us
         )
+        obstacles: tuple[Any, ...] = ()
+        if self._obstacle_ability is not None:
+            obstacles = self._obstacle_ability.advance_frames(trajectory)
         return ApplicationChunkUpdate(
             trajectory=replace(
                 trajectory,
-                dynamic_actors=(*trajectory.dynamic_actors, *passengers),
+                dynamic_actors=(*trajectory.dynamic_actors, *passengers, *obstacles),
             ),
             frame_application_states=snapshots,
         )
@@ -124,6 +160,9 @@ def __init__(
         config: TaxiGameConfig,
         keyboard: CrazyRobotaxiKeyboardState,
         presenter_config: Any,
+        *,
+        live_edit: LiveEditConfig | None = None,
+        style_ability: Any | None = None,
     ) -> None:
         self._config = config
         self._keyboard = keyboard
@@ -133,6 +172,14 @@ def __init__(
         self._ground_snapper: GroundSnapper | None = None
         self._map_bounds: MapBounds | None = None
         self._enclosure_segments_world = np.empty((0, 2, 3), dtype=np.float32)
+        self._live_edit = live_edit or LiveEditConfig()
+        self._style_ability = style_ability
+        self._live_edit_presenter: Any | None = None
+        self._coin_lanes: tuple[NavigationLane, ...] = ()
+
+    def attach_live_edit_presenter(self, presenter: Any) -> None:
+        """Bind the live-edit presenter so coin abilities reach the pixels."""
+        self._live_edit_presenter = presenter
 
     def configure_presenter(self, presenter: Any) -> None:
         """Configure application presentation before scene loading."""
@@ -148,6 +195,16 @@ def load_scene(self, scene: SceneBundle, map_bounds: MapBounds | None) -> None:
         self._enclosure_segments_world = scene_data.enclosure_segments_world
         self._ground_snapper = _build_taxi_ground_snapper(scene)
         self._map_bounds = map_bounds
+        if self._live_edit.coins.enabled:
+            # Lay coins along the driving-lane graph; legacy scenes without
+            # mapped lanes fall back to the recorded ego route.
+            self._coin_lanes = scene_data.navigation_lanes or (
+                NavigationLane(
+                    centerline_world=np.asarray(
+                        scene_data.reference_route_world, dtype=np.float32
+                    )
+                ),
+            )
         logger.info(
             "[crazy-robotaxi] play-area enclosure: perimeter_segments={}",
             len(scene_data.perimeter_segments_world),
@@ -203,7 +260,33 @@ def create_runtime(
             initial_camera=scene.selected_camera,
             map_bounds=self._map_bounds,
         )
-        return CrazyRobotaxiRuntime(controller, self._keyboard)
+        coin_ability: CoinAbility | None = None
+        if self._live_edit.coins.enabled and self._coin_lanes:
+            # Rebuilt per rollout so a reset restores the full course.
+            coin_ability = CoinAbility.from_lanes(
+                self._coin_lanes, self._live_edit.coins
+            )
+            logger.info(
+                f"[live-edit] coin course laid out: {coin_ability.remaining_count} coins"
+            )
+        obstacle_ability: Any | None = None
+        if self._live_edit.obstacle.enabled:
+            from crazy_robotaxi.live_edit.obstacle_ability import ObstacleAbility
+
+            # Rebuilt per rollout so events/hits reset with the game.
+            obstacle_ability = ObstacleAbility.from_scene(
+                scene, self._live_edit.obstacle
+            )
+        if self._live_edit_presenter is not None:
+            self._live_edit_presenter.set_coin_ability(coin_ability)
+            self._live_edit_presenter.set_obstacle_ability(obstacle_ability)
+        return CrazyRobotaxiRuntime(
+            controller,
+            self._keyboard,
+            style_ability=self._style_ability,
+            coin_ability=coin_ability,
+            obstacle_ability=obstacle_ability,
+        )
 
 
 class CrazyRobotaxiApp(InteractiveDriveApp):
@@ -219,8 +302,30 @@ def __init__(
         alignment_diagnostics_root: Path | None = None,
         trace_sink: TraceSink | None = None,
         close_presenter_on_exit: bool = True,
+        live_edit_config: LiveEditConfig | None = None,
     ) -> None:
         keyboard = CrazyRobotaxiKeyboardState()
+        live_edit = live_edit_config or LiveEditConfig()
+        style_ability = None
+        if live_edit.obstacle.enabled and live_edit.obstacle.guide_scale > 0.0:
+            from crazy_robotaxi.live_edit.obstacle_ability import (
+                install_obstacle_guidance_on_backend,
+            )
+
+            # Before the style install so both share one CUDA-graph-free
+            # session swap (marker-coordinated) and one warmup wrap each.
+            install_obstacle_guidance_on_backend(backend, live_edit.obstacle)
+        if live_edit.style.enabled or live_edit.weather.enabled:
+            from crazy_robotaxi.live_edit.style_ability import (
+                StyleAbility,
+                install_style_ability_on_backend,
+            )
+
+            # Must run before super().__init__ starts model warmup on the
+            # pipeline worker: the corrector needs a CUDA-graph-free session
+            # and the LoRA attach is deferred to the end of warmup_model.
+            style_ability = StyleAbility(live_edit.style, live_edit.weather)
+            install_style_ability_on_backend(backend, style_ability)
         if alignment_diagnostics_root is not None:
             from crazy_robotaxi.alignment_diagnostics import (
                 AlignmentDiagnosticPresenter,
@@ -237,16 +342,33 @@ def __init__(
             logger.info(
                 f"[crazy-robotaxi] alignment diagnostics -> {presenter.output_dir}"
             )
+        application = CrazyRobotaxiApplication(
+            taxi_config,
+            keyboard,
+            config.bev,
+            live_edit=live_edit,
+            style_ability=style_ability,
+        )
         super().__init__(
             config=config,
             backend=backend,
             presenter=presenter,
             trace_sink=trace_sink,
             close_presenter_on_exit=close_presenter_on_exit,
             keyboard=keyboard,
-            application=CrazyRobotaxiApplication(taxi_config, keyboard, config.bev),
+            application=application,
         )
-        self._presenter = CausalFrameAlignmentPresenter(self._presenter)
+        inner_presenter = self._presenter
+        if live_edit.any_enabled:
+            from crazy_robotaxi.live_edit.presenter import LiveEditPresenter
+
+            # Inside the alignment wrapper so composites see the
+            # frame-synchronized rig_to_world matching the generated RGB.
+            inner_presenter = LiveEditPresenter(
+                inner_presenter, live_edit, style_ability=style_ability
+            )
+            application.attach_live_edit_presenter(inner_presenter)
+        self._presenter = CausalFrameAlignmentPresenter(inner_presenter)
 
 
 def taxi_config_from_args(args: argparse.Namespace) -> TaxiGameConfig:

apps/crazy_robotaxi/crazy_robotaxi/cli.py

@@ -925,6 +925,23 @@ def _run_streaming(args: argparse.Namespace) -> None:
             parse_bind,
         )
 
+    stream_token = (getattr(args, "stream_token", None) or "").strip() or None
+    if stream_token is not None and not getattr(args, "taxi_game", False):
+        # The upstream engine presenter has no token gate; failing loudly
+        # beats silently serving an open stream the user believes is gated.
+        raise SystemExit(
+            "--stream-token requires the taxi-game streaming presenter; "
+            "the upstream engine presenter would serve an ungated stream."
+        )
+    stream_jpeg_quality = int(getattr(args, "stream_jpeg_quality", 85))
+    stream_scale = float(getattr(args, "stream_scale", 1.0))
+    if stream_scale != 1.0 and not getattr(args, "taxi_game", False):
+        # The upstream engine presenter has no scale knob; failing loudly
+        # beats silently streaming full-size frames the user asked to shrink.
+        raise SystemExit(
+            "--stream-scale requires the taxi-game streaming presenter."
+        )
+
     _apply_cuda_visible_devices_inplace(args.cuda_visible_devices)
     _resolve_demo_paths(args)
     _materialize_synthetic_scene_for_picker(args)
@@ -982,8 +999,11 @@ def _run_streaming(args: argparse.Namespace) -> None:
         keyboard=placeholder_keyboard,
         bind_host=bind_host,
         bind_port=bind_port,
+        jpeg_quality=stream_jpeg_quality,
         scenes=scenes_payload,
         thumbnails=thumbnails,
+        **({"stream_scale": stream_scale} if stream_scale != 1.0 else {}),
+        **({"stream_token": stream_token} if stream_token is not None else {}),
     )
 
     # Build the backend + engine once so the model warms up (on the
@@ -1087,6 +1107,7 @@ def _build_application(
             CrazyRobotaxiApp,
             taxi_config_from_args,
         )
+        from crazy_robotaxi.live_edit.config import live_edit_config_from_args
 
         return CrazyRobotaxiApp(
             config=config,
@@ -1097,6 +1118,7 @@ def _build_application(
                 args, "taxi_alignment_diagnostics", None
             ),
             close_presenter_on_exit=close_presenter_on_exit,
+            live_edit_config=live_edit_config_from_args(args),
         )
     return InteractiveDriveApp(
         config=config,

apps/crazy_robotaxi/crazy_robotaxi/hud_presenter.py

@@ -2842,6 +2842,10 @@ def _build_key_codes(self) -> dict[str, Any]:
             "d": _lookup_key(spy.KeyCode, "d"),
             "r": _lookup_key(spy.KeyCode, "r"),
             "x": _lookup_key(spy.KeyCode, "x"),
+            "k": _lookup_key(spy.KeyCode, "k"),
+            "c": _lookup_key(spy.KeyCode, "c"),
+            "v": _lookup_key(spy.KeyCode, "v"),
+            "o": _lookup_key(spy.KeyCode, "o"),
             "space": _lookup_key(spy.KeyCode, "space"),
             "up": _lookup_key(spy.KeyCode, "up", "arrow_up"),
             "down": _lookup_key(spy.KeyCode, "down", "arrow_down"),
@@ -2964,6 +2968,28 @@ def _on_keyboard_event(self, event: Any) -> None:
             self._keyboard.request_reset()
         elif self._key_matches(key, "x"):
             self.exit_scene()
+        elif self._key_matches(key, "k"):
+            self._request_live_edit("skin")
+        elif self._key_matches(key, "c"):
+            self._request_live_edit("coins")
+        elif self._key_matches(key, "v"):
+            self._request_live_edit("weather")
+        elif self._key_matches(key, "o"):
+            self._request_live_edit("obstacle")
+
+    def _request_live_edit(self, ability: str) -> None:
+        """Raise a live-edit key request; no-op when the flags are off."""
+        requests = getattr(self._keyboard, "live_edit", None)
+        if requests is None:
+            return
+        if ability == "skin":
+            requests.request_skin_cycle()
+        elif ability == "weather":
+            requests.request_weather_cycle()
+        elif ability == "obstacle":
+            requests.request_obstacle_spawn()
+        else:
+            requests.request_coins_toggle()
 
     def _expire_pending_drive_releases(self) -> None:
         """Commit any debounced release whose grace window has passed.

apps/crazy_robotaxi/crazy_robotaxi/input.py

@@ -26,6 +26,7 @@
 from crazy_robotaxi.high_scores import (
     validate_player_name,
 )
+from crazy_robotaxi.live_edit.input_hooks import LiveEditRequests
 from flashdreams.serving.realtime.input import normalize_key
 
 
@@ -44,6 +45,9 @@ def __init__(self) -> None:
         self._name_submission: str | None = None
         self._keyboard_steer = 0.0
         self._last_command_s = time.monotonic()
+        # One-shot live-edit key requests (skin cycle / coins toggle) raised
+        # by the presenters and drained by ``CrazyRobotaxiRuntime`` per tick.
+        self.live_edit = LiveEditRequests()
 
     def submit_taxi_name(self, name: str) -> bool:
         """Validate and queue one high-score name submission."""