From c5e08c7bcf8e739c7e8c04b01cf2c435e8e8d5b2 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Nov 2025 12:22:27 -0600 Subject: [PATCH 1/8] fix wording in reference config --- docs/docs/configuration/reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 907bda21e9..10a4803933 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -700,9 +700,9 @@ genai: # Optional: Configuration for audio transcription # NOTE: only the enabled option can be overridden at the camera level audio_transcription: - # Optional: Enable license plate recognition (default: shown below) + # Optional: Enable audio transcription (default: shown below) enabled: False - # Optional: The device to run the models on (default: shown below) + # Optional: The device to run the models on. (default: shown below) device: CPU # Optional: Set the model size used for transcription. (default: shown below) model_size: small From 9242997079ff9c4fd8e2482b0877312d5a7846f3 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Nov 2025 12:25:00 -0600 Subject: [PATCH 2/8] spacing tweaks --- web/src/views/settings/AuthenticationView.tsx | 2 +- web/src/views/settings/CameraManagementView.tsx | 2 +- web/src/views/settings/CameraReviewSettingsView.tsx | 2 +- web/src/views/settings/EnrichmentsSettingsView.tsx | 2 +- web/src/views/settings/FrigatePlusSettingsView.tsx | 2 +- web/src/views/settings/MasksAndZonesView.tsx | 2 +- web/src/views/settings/MotionTunerView.tsx | 2 +- web/src/views/settings/NotificationsSettingsView.tsx | 4 ++-- web/src/views/settings/ObjectSettingsView.tsx | 6 +++--- web/src/views/settings/TriggerView.tsx | 2 +- web/src/views/settings/UiSettingsView.tsx | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/web/src/views/settings/AuthenticationView.tsx b/web/src/views/settings/AuthenticationView.tsx index 5c11d89143..19f157b467 100644 --- a/web/src/views/settings/AuthenticationView.tsx +++ b/web/src/views/settings/AuthenticationView.tsx @@ -784,7 +784,7 @@ export default function AuthenticationView({ return (
-
+
{section === "users" && UsersSection} {section === "roles" && RolesSection} {!section && ( diff --git a/web/src/views/settings/CameraManagementView.tsx b/web/src/views/settings/CameraManagementView.tsx index 1a626fa022..8f1b5eae58 100644 --- a/web/src/views/settings/CameraManagementView.tsx +++ b/web/src/views/settings/CameraManagementView.tsx @@ -65,7 +65,7 @@ export default function CameraManagementView({ closeButton />
-
+
{viewMode === "settings" ? ( <> diff --git a/web/src/views/settings/CameraReviewSettingsView.tsx b/web/src/views/settings/CameraReviewSettingsView.tsx index 47ea5c22aa..7a7b92e4e3 100644 --- a/web/src/views/settings/CameraReviewSettingsView.tsx +++ b/web/src/views/settings/CameraReviewSettingsView.tsx @@ -298,7 +298,7 @@ export default function CameraReviewSettingsView({ <>
-
+
{t("cameraReview.title")} diff --git a/web/src/views/settings/EnrichmentsSettingsView.tsx b/web/src/views/settings/EnrichmentsSettingsView.tsx index e3b0626b96..6aba50dd30 100644 --- a/web/src/views/settings/EnrichmentsSettingsView.tsx +++ b/web/src/views/settings/EnrichmentsSettingsView.tsx @@ -244,7 +244,7 @@ export default function EnrichmentsSettingsView({ return (
-
+
{t("enrichments.title")} diff --git a/web/src/views/settings/FrigatePlusSettingsView.tsx b/web/src/views/settings/FrigatePlusSettingsView.tsx index 52af94354d..80d98b1975 100644 --- a/web/src/views/settings/FrigatePlusSettingsView.tsx +++ b/web/src/views/settings/FrigatePlusSettingsView.tsx @@ -211,7 +211,7 @@ export default function FrigatePlusSettingsView({ <>
-
+
{t("frigatePlus.title")} diff --git a/web/src/views/settings/MasksAndZonesView.tsx b/web/src/views/settings/MasksAndZonesView.tsx index 27c542e875..efeaa9be08 100644 --- a/web/src/views/settings/MasksAndZonesView.tsx +++ b/web/src/views/settings/MasksAndZonesView.tsx @@ -434,7 +434,7 @@ export default function MasksAndZonesView({ {cameraConfig && editingPolygons && (
-
+
{editPane == "zone" && ( -
+
{t("motionDetectionTuner.title")} diff --git a/web/src/views/settings/NotificationsSettingsView.tsx b/web/src/views/settings/NotificationsSettingsView.tsx index 6280ca6a88..77da163861 100644 --- a/web/src/views/settings/NotificationsSettingsView.tsx +++ b/web/src/views/settings/NotificationsSettingsView.tsx @@ -331,7 +331,7 @@ export default function NotificationView({ if (!("Notification" in window) || !window.isSecureContext) { return ( -
+
@@ -385,7 +385,7 @@ export default function NotificationView({ <>
-
+
-
+
{t("debug.title")} @@ -434,7 +434,7 @@ function ObjectList({ cameraConfig, objects }: ObjectListProps) { {t("debug.objectShapeFilterDrawing.area")}

{obj.area ? ( - <> +
px: {obj.area.toString()}
@@ -448,7 +448,7 @@ function ObjectList({ cameraConfig, objects }: ObjectListProps) { .toFixed(4) .toString()}
- +
) : ( "-" )} diff --git a/web/src/views/settings/TriggerView.tsx b/web/src/views/settings/TriggerView.tsx index 0b004fd823..a0e19f5b2a 100644 --- a/web/src/views/settings/TriggerView.tsx +++ b/web/src/views/settings/TriggerView.tsx @@ -440,7 +440,7 @@ export default function TriggerView({ return (
-
+
{!isSemanticSearchEnabled ? (
diff --git a/web/src/views/settings/UiSettingsView.tsx b/web/src/views/settings/UiSettingsView.tsx index 8ec484aa3d..34df0ddc82 100644 --- a/web/src/views/settings/UiSettingsView.tsx +++ b/web/src/views/settings/UiSettingsView.tsx @@ -108,7 +108,7 @@ export default function UiSettingsView() { <>
-
+
{t("general.title")} From bf4f63e50e71c0f7403e01f12ddbda140c02a0d4 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Nov 2025 12:25:23 -0600 Subject: [PATCH 3/8] make live view settings drawer scrollable --- web/src/views/live/LiveCameraView.tsx | 598 +++++++++++++------------- 1 file changed, 306 insertions(+), 292 deletions(-) diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 65257326fe..ada72bee37 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -1376,329 +1376,343 @@ function FrigateCameraFeatures({ title={t("cameraSettings.title", { camera })} /> - -
- {isAdmin && ( - <> - - sendEnabled(enabledState == "ON" ? "OFF" : "ON") - } - /> - - sendDetect(detectState == "ON" ? "OFF" : "ON") - } - /> - {recordingEnabled && ( - - sendRecord(recordState == "ON" ? "OFF" : "ON") - } - /> - )} - - sendSnapshot(snapshotState == "ON" ? "OFF" : "ON") - } - /> - {audioDetectEnabled && ( + +
+ <> + {isAdmin && ( + <> - sendAudio(audioState == "ON" ? "OFF" : "ON") + sendEnabled(enabledState == "ON" ? "OFF" : "ON") } /> - )} - {audioDetectEnabled && transcriptionEnabled && ( - sendTranscription(transcriptionState == "ON" ? "OFF" : "ON") + sendDetect(detectState == "ON" ? "OFF" : "ON") } /> - )} - {autotrackingEnabled && ( + {recordingEnabled && ( + + sendRecord(recordState == "ON" ? "OFF" : "ON") + } + /> + )} - sendAutotracking(autotrackingState == "ON" ? "OFF" : "ON") + sendSnapshot(snapshotState == "ON" ? "OFF" : "ON") } /> - )} - - )} -
+ {audioDetectEnabled && ( + + sendAudio(audioState == "ON" ? "OFF" : "ON") + } + /> + )} + {audioDetectEnabled && transcriptionEnabled && ( + + sendTranscription( + transcriptionState == "ON" ? "OFF" : "ON", + ) + } + /> + )} + {autotrackingEnabled && ( + + sendAutotracking(autotrackingState == "ON" ? "OFF" : "ON") + } + /> + )} + + )} -
- {!isRestreamed && ( -
- -
- -
- {t("streaming.restreaming.disabled", { - ns: "components/dialog", - })} -
- - -
- - - {t("button.info", { ns: "common" })} - -
- - - {t("streaming.restreaming.desc.title", { - ns: "components/dialog", - })} -
- - {t("readTheDocumentation", { ns: "common" })} - - +
+ {!isRestreamed && ( +
+ +
+ +
+ {t("streaming.restreaming.disabled", { + ns: "components/dialog", + })}
- - -
-
- )} - {isRestreamed && Object.values(camera.live.streams).length > 0 && ( -
-
{t("stream.title")}
- - - {debug && ( -
- <> - -
{t("stream.debug.picker")}
- + + +
+ + + {t("button.info", { ns: "common" })} + +
+ + + {t("streaming.restreaming.desc.title", { + ns: "components/dialog", + })} +
+ + {t("readTheDocumentation", { ns: "common" })} + + +
+ + +
)} + {isRestreamed && + Object.values(camera.live.streams).length > 0 && ( +
+
{t("stream.title")}
+ + + {debug && ( +
+ <> + +
{t("stream.debug.picker")}
+ +
+ )} - {preferredLiveMode != "jsmpeg" && !debug && isRestreamed && ( -
- {supportsAudioOutput ? ( - <> - -
{t("stream.audio.available")}
- - ) : ( - <> - -
{t("stream.audio.unavailable")}
- - -
- - - {t("button.info", { ns: "common" })} - -
- - - {t("stream.audio.tips.title")} -
- - {t("readTheDocumentation", { ns: "common" })} - - + {preferredLiveMode != "jsmpeg" && + !debug && + isRestreamed && ( +
+ {supportsAudioOutput ? ( + <> + +
{t("stream.audio.available")}
+ + ) : ( + <> + +
{t("stream.audio.unavailable")}
+ + +
+ + + {t("button.info", { ns: "common" })} + +
+ + + {t("stream.audio.tips.title")} +
+ + {t("readTheDocumentation", { + ns: "common", + })} + + +
+ + + + )} +
+ )} + {preferredLiveMode != "jsmpeg" && + !debug && + isRestreamed && + supportsAudioOutput && ( +
+ {supports2WayTalk ? ( + <> + +
{t("stream.twoWayTalk.available")}
+ + ) : ( + <> + +
{t("stream.twoWayTalk.unavailable")}
+ + +
+ + + {t("button.info", { ns: "common" })} + +
+ + + {t("stream.twoWayTalk.tips")} +
+ + {t("readTheDocumentation", { + ns: "common", + })} + + +
+ + + + )} +
+ )} + {preferredLiveMode == "jsmpeg" && isRestreamed && ( +
+
+ +

+ {t("stream.lowBandwidth.tips")} +

+
+
- )} - {preferredLiveMode != "jsmpeg" && - !debug && - isRestreamed && - supportsAudioOutput && ( -
- {supports2WayTalk ? ( - <> - -
{t("stream.twoWayTalk.available")}
- - ) : ( - <> - -
{t("stream.twoWayTalk.unavailable")}
- - -
- - - {t("button.info", { ns: "common" })} - -
- - - {t("stream.twoWayTalk.tips")} -
- - {t("readTheDocumentation", { ns: "common" })} - - -
- - - + +
)}
)} - {preferredLiveMode == "jsmpeg" && isRestreamed && ( -
-
- -

{t("stream.lowBandwidth.tips")}

-
+
+
+ {t("manualRecording.title")} +
+
+
- )} -
- )} -
-
- {t("manualRecording.title")} -
-
- - -
-

- {t("manualRecording.tips")} -

-
- {isRestreamed && ( - <> -
- { - setPlayInBackground(checked); - }} - disabled={debug} - /> -

- {t("manualRecording.playInBackground.desc")} +

+ {t("manualRecording.tips")}

-
+ {isRestreamed && ( + <> +
+ { + setPlayInBackground(checked); + }} + disabled={debug} + /> +

+ {t("manualRecording.playInBackground.desc")} +

+
+
+ { + setShowStats(checked); + }} + disabled={debug} + /> +

+ {t("manualRecording.showStats.desc")} +

+
+ + )} +
{ - setShowStats(checked); - }} - disabled={debug} + label={t("streaming.debugView", { ns: "components/dialog" })} + isChecked={debug} + onCheckedChange={(checked) => setDebug(checked)} /> -

- {t("manualRecording.showStats.desc")} -

- - )} -
- setDebug(checked)} - /> -
+
+
From 42fdadecd943228c2186614f2852eae999822f46 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Nov 2025 12:35:35 -0600 Subject: [PATCH 4/8] clarify audio transcription docs --- docs/docs/configuration/audio_detectors.md | 8 +++++++- docs/docs/configuration/reference.md | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/docs/configuration/audio_detectors.md b/docs/docs/configuration/audio_detectors.md index bf71f8d819..3bf57b1a78 100644 --- a/docs/docs/configuration/audio_detectors.md +++ b/docs/docs/configuration/audio_detectors.md @@ -144,4 +144,10 @@ In order to use transcription and translation for past events, you must enable a The transcribed/translated speech will appear in the description box in the Tracked Object Details pane. If Semantic Search is enabled, embeddings are generated for the transcription text and are fully searchable using the description search type. -Recorded `speech` events will always use a `whisper` model, regardless of the `model_size` config setting. Without a GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient. +:::note + +Only one `speech` event may be transcribed at a time. Frigate does not automatically transcribe `speech` events or implement a queue for long-running transcription model inference. + +::: + +Recorded `speech` events will always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient. diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 10a4803933..f8b49303f0 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -700,11 +700,11 @@ genai: # Optional: Configuration for audio transcription # NOTE: only the enabled option can be overridden at the camera level audio_transcription: - # Optional: Enable audio transcription (default: shown below) + # Optional: Enable live and speech event audio transcription (default: shown below) enabled: False - # Optional: The device to run the models on. (default: shown below) + # Optional: The device to run the models on for live transcription. (default: shown below) device: CPU - # Optional: Set the model size used for transcription. (default: shown below) + # Optional: Set the model size used for live transcription. (default: shown below) model_size: small # Optional: Set the language used for transcription translation. (default: shown below) # List of language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10 From 2c893aa1256660e2fe7c44fd0ba921aa7a6bcc08 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 24 Nov 2025 06:45:09 -0600 Subject: [PATCH 5/8] change audio transcription icon to activity indicator when transcription is in progress the backend doesn't implement any kind of queueing for speech event transcription --- frigate/api/classification.py | 1 + frigate/comms/dispatcher.py | 17 ++++++++++ frigate/const.py | 1 + .../post/audio_transcription.py | 5 +++ web/src/api/ws.tsx | 34 +++++++++++++++++++ .../overlay/detail/SearchDetailDialog.tsx | 20 +++++++++-- 6 files changed, 76 insertions(+), 2 deletions(-) diff --git a/frigate/api/classification.py b/frigate/api/classification.py index a2aec68982..9b116be107 100644 --- a/frigate/api/classification.py +++ b/frigate/api/classification.py @@ -542,6 +542,7 @@ def transcribe_audio(request: Request, body: AudioTranscriptionBody): status_code=409, # 409 Conflict ) else: + logger.debug(f"Failed to transcribe audio, response: {response}") return JSONResponse( content={ "success": False, diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 235693c8c5..0c2ba5a898 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -23,6 +23,7 @@ NOTIFICATION_TEST, REQUEST_REGION_GRID, UPDATE_AUDIO_ACTIVITY, + UPDATE_AUDIO_TRANSCRIPTION_STATE, UPDATE_BIRDSEYE_LAYOUT, UPDATE_CAMERA_ACTIVITY, UPDATE_EMBEDDINGS_REINDEX_PROGRESS, @@ -61,6 +62,7 @@ def __init__( self.model_state: dict[str, ModelStatusTypesEnum] = {} self.embeddings_reindex: dict[str, Any] = {} self.birdseye_layout: dict[str, Any] = {} + self.audio_transcription_state: str = "idle" self._camera_settings_handlers: dict[str, Callable] = { "audio": self._on_audio_command, "audio_transcription": self._on_audio_transcription_command, @@ -178,6 +180,19 @@ def handle_update_model_state() -> None: def handle_model_state() -> None: self.publish("model_state", json.dumps(self.model_state.copy())) + def handle_update_audio_transcription_state() -> None: + if payload: + self.audio_transcription_state = payload + self.publish( + "audio_transcription_state", + json.dumps(self.audio_transcription_state), + ) + + def handle_audio_transcription_state() -> None: + self.publish( + "audio_transcription_state", json.dumps(self.audio_transcription_state) + ) + def handle_update_embeddings_reindex_progress() -> None: self.embeddings_reindex = payload self.publish( @@ -264,10 +279,12 @@ def handle_notification_test() -> None: UPDATE_MODEL_STATE: handle_update_model_state, UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress, UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout, + UPDATE_AUDIO_TRANSCRIPTION_STATE: handle_update_audio_transcription_state, NOTIFICATION_TEST: handle_notification_test, "restart": handle_restart, "embeddingsReindexProgress": handle_embeddings_reindex_progress, "modelState": handle_model_state, + "audioTranscriptionState": handle_audio_transcription_state, "birdseyeLayout": handle_birdseye_layout, "onConnect": handle_on_connect, } diff --git a/frigate/const.py b/frigate/const.py index 5710966bf9..11e89886fb 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -113,6 +113,7 @@ UPDATE_CAMERA_ACTIVITY = "update_camera_activity" UPDATE_AUDIO_ACTIVITY = "update_audio_activity" EXPIRE_AUDIO_ACTIVITY = "expire_audio_activity" +UPDATE_AUDIO_TRANSCRIPTION_STATE = "update_audio_transcription_state" UPDATE_EVENT_DESCRIPTION = "update_event_description" UPDATE_REVIEW_DESCRIPTION = "update_review_description" UPDATE_MODEL_STATE = "update_model_state" diff --git a/frigate/data_processing/post/audio_transcription.py b/frigate/data_processing/post/audio_transcription.py index 870c340686..b7b6cb021b 100644 --- a/frigate/data_processing/post/audio_transcription.py +++ b/frigate/data_processing/post/audio_transcription.py @@ -13,6 +13,7 @@ from frigate.const import ( CACHE_DIR, MODEL_CACHE_DIR, + UPDATE_AUDIO_TRANSCRIPTION_STATE, UPDATE_EVENT_DESCRIPTION, ) from frigate.data_processing.types import PostProcessDataEnum @@ -190,6 +191,8 @@ def _transcription_wrapper(self, event: dict[str, any]) -> None: self.transcription_running = False self.transcription_thread = None + self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "idle") + def handle_request(self, topic: str, request_data: dict[str, any]) -> str | None: if topic == "transcribe_audio": event = request_data["event"] @@ -203,6 +206,8 @@ def handle_request(self, topic: str, request_data: dict[str, any]) -> str | None # Mark as running and start the thread self.transcription_running = True + self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "processing") + self.transcription_thread = threading.Thread( target=self._transcription_wrapper, args=(event,), daemon=True ) diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index 302f3f2632..44d45ea2f0 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -461,6 +461,40 @@ export function useEmbeddingsReindexProgress( return { payload: data }; } +export function useAudioTranscriptionProcessState( + revalidateOnFocus: boolean = true, +): { payload: string } { + const { + value: { payload }, + send: sendCommand, + } = useWs("audio_transcription_state", "audioTranscriptionState"); + + const data = useDeepMemo( + payload ? (JSON.parse(payload as string) as string) : "idle", + ); + + useEffect(() => { + let listener = undefined; + if (revalidateOnFocus) { + sendCommand("audioTranscriptionState"); + listener = () => { + if (document.visibilityState == "visible") { + sendCommand("audioTranscriptionState"); + } + }; + addEventListener("visibilitychange", listener); + } + return () => { + if (listener) { + removeEventListener("visibilitychange", listener); + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [revalidateOnFocus]); + + return { payload: data || "idle" }; +} + export function useBirdseyeLayout(revalidateOnFocus: boolean = true): { payload: string; } { diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 6b716a5637..467008e929 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -92,6 +92,7 @@ import { DialogPortal } from "@radix-ui/react-dialog"; import { useDetailStream } from "@/context/detail-stream-context"; import { PiSlidersHorizontalBold } from "react-icons/pi"; import { HiSparkles } from "react-icons/hi"; +import { useAudioTranscriptionProcessState } from "@/api/ws"; const SEARCH_TABS = ["snapshot", "tracking_details"] as const; export type SearchTab = (typeof SEARCH_TABS)[number]; @@ -1076,6 +1077,11 @@ function ObjectDetailsTab({ }); }, [search, t]); + // audio transcription processing state + + const { payload: audioTranscriptionProcessState } = + useAudioTranscriptionProcessState(); + // frigate+ submission type SubmissionState = "reviewing" | "uploading" | "submitted"; @@ -1431,10 +1437,20 @@ function ObjectDetailsTab({ From 086330a542842206fcd703cfd41fcbf2bb70ecf2 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 24 Nov 2025 06:54:22 -0600 Subject: [PATCH 6/8] tracking details tweaks - Add attribute box overlay and area - Add score - Throttle swr revalidation during video component rerendering --- frigate/timeline.py | 1 + .../components/overlay/ObjectTrackOverlay.tsx | 30 ++- .../overlay/detail/TrackingDetails.tsx | 214 ++++++++++++------ web/src/types/timeline.ts | 1 + 4 files changed, 173 insertions(+), 73 deletions(-) diff --git a/frigate/timeline.py b/frigate/timeline.py index 8e6aedc679..a2d59b88ee 100644 --- a/frigate/timeline.py +++ b/frigate/timeline.py @@ -109,6 +109,7 @@ def handle_object_detection( event_data["region"], ), "attribute": "", + "score": event_data["score"], }, } diff --git a/web/src/components/overlay/ObjectTrackOverlay.tsx b/web/src/components/overlay/ObjectTrackOverlay.tsx index 7e548af2e8..8f78adcd74 100644 --- a/web/src/components/overlay/ObjectTrackOverlay.tsx +++ b/web/src/components/overlay/ObjectTrackOverlay.tsx @@ -42,6 +42,7 @@ type ObjectData = { pathPoints: PathPoint[]; currentZones: string[]; currentBox?: number[]; + currentAttributeBox?: number[]; }; export default function ObjectTrackOverlay({ @@ -105,6 +106,12 @@ export default function ObjectTrackOverlay({ selectedObjectIds.length > 0 ? ["event_ids", { ids: selectedObjectIds.join(",") }] : null, + null, + { + revalidateOnFocus: false, + revalidateOnReconnect: false, + dedupingInterval: 30000, + }, ); // Fetch timeline data for each object ID using fixed number of hooks @@ -112,7 +119,12 @@ export default function ObjectTrackOverlay({ selectedObjectIds.length > 0 ? `timeline?source_id=${selectedObjectIds.join(",")}&limit=1000` : null, - { revalidateOnFocus: false }, + null, + { + revalidateOnFocus: false, + revalidateOnReconnect: false, + dedupingInterval: 30000, + }, ); const getZonesFriendlyNames = (zones: string[], config: FrigateConfig) => { @@ -270,6 +282,7 @@ export default function ObjectTrackOverlay({ ); const currentBox = nearbyTimelineEvent?.data?.box; + const currentAttributeBox = nearbyTimelineEvent?.data?.attribute_box; return { objectId, @@ -278,6 +291,7 @@ export default function ObjectTrackOverlay({ pathPoints: combinedPoints, currentZones, currentBox, + currentAttributeBox, }; }) .filter((obj: ObjectData) => obj.pathPoints.length > 0); // Only include objects with path data @@ -482,6 +496,20 @@ export default function ObjectTrackOverlay({ /> )} + {objData.currentAttributeBox && showBoundingBoxes && ( + + + + )} ); })} diff --git a/web/src/components/overlay/detail/TrackingDetails.tsx b/web/src/components/overlay/detail/TrackingDetails.tsx index 727dd45527..c6e10f8c2b 100644 --- a/web/src/components/overlay/detail/TrackingDetails.tsx +++ b/web/src/components/overlay/detail/TrackingDetails.tsx @@ -75,12 +75,15 @@ export function TrackingDetails({ setIsVideoLoading(true); }, [event.id]); - const { data: eventSequence } = useSWR([ - "timeline", + const { data: eventSequence } = useSWR( + ["timeline", { source_id: event.id }], + null, { - source_id: event.id, + revalidateOnFocus: false, + revalidateOnReconnect: false, + dedupingInterval: 30000, }, - ]); + ); const { data: config } = useSWR("config"); @@ -104,6 +107,12 @@ export function TrackingDetails({ }, ] : null, + null, + { + revalidateOnFocus: false, + revalidateOnReconnect: false, + dedupingInterval: 30000, + }, ); // Convert a timeline timestamp to actual video player time, accounting for @@ -714,53 +723,6 @@ export function TrackingDetails({ )}
{eventSequence.map((item, idx) => { - const isActive = - Math.abs( - (effectiveTime ?? 0) - (item.timestamp ?? 0), - ) <= 0.5; - const formattedEventTimestamp = config - ? formatUnixTimestampToDateTime(item.timestamp ?? 0, { - timezone: config.ui.timezone, - date_format: - config.ui.time_format == "24hour" - ? t( - "time.formattedTimestampHourMinuteSecond.24hour", - { ns: "common" }, - ) - : t( - "time.formattedTimestampHourMinuteSecond.12hour", - { ns: "common" }, - ), - time_style: "medium", - date_style: "medium", - }) - : ""; - - const ratio = - Array.isArray(item.data.box) && - item.data.box.length >= 4 - ? ( - aspectRatio * - (item.data.box[2] / item.data.box[3]) - ).toFixed(2) - : "N/A"; - const areaPx = - Array.isArray(item.data.box) && - item.data.box.length >= 4 - ? Math.round( - (config.cameras[event.camera]?.detect?.width ?? - 0) * - (config.cameras[event.camera]?.detect - ?.height ?? 0) * - (item.data.box[2] * item.data.box[3]), - ) - : undefined; - const areaPct = - Array.isArray(item.data.box) && - item.data.box.length >= 4 - ? (item.data.box[2] * item.data.box[3]).toFixed(4) - : undefined; - return (
handleLifecycleClick(item)} setSelectedZone={setSelectedZone} getZoneColor={getZoneColor} @@ -798,11 +756,7 @@ export function TrackingDetails({ type LifecycleIconRowProps = { item: TrackingDetailsSequence; - isActive?: boolean; - formattedEventTimestamp: string; - ratio: string; - areaPx?: number; - areaPct?: string; + event: Event; onClick: () => void; setSelectedZone: (z: string) => void; getZoneColor: (zoneName: string) => number[] | undefined; @@ -812,11 +766,7 @@ type LifecycleIconRowProps = { function LifecycleIconRow({ item, - isActive, - formattedEventTimestamp, - ratio, - areaPx, - areaPct, + event, onClick, setSelectedZone, getZoneColor, @@ -826,9 +776,101 @@ function LifecycleIconRow({ const { t } = useTranslation(["views/explore", "components/player"]); const { data: config } = useSWR("config"); const [isOpen, setIsOpen] = useState(false); - const navigate = useNavigate(); + const aspectRatio = useMemo(() => { + if (!config) { + return 16 / 9; + } + + return ( + config.cameras[event.camera].detect.width / + config.cameras[event.camera].detect.height + ); + }, [config, event]); + + const isActive = useMemo( + () => Math.abs((effectiveTime ?? 0) - (item.timestamp ?? 0)) <= 0.5, + [effectiveTime, item.timestamp], + ); + + const formattedEventTimestamp = useMemo( + () => + config + ? formatUnixTimestampToDateTime(item.timestamp ?? 0, { + timezone: config.ui.timezone, + date_format: + config.ui.time_format == "24hour" + ? t("time.formattedTimestampHourMinuteSecond.24hour", { + ns: "common", + }) + : t("time.formattedTimestampHourMinuteSecond.12hour", { + ns: "common", + }), + time_style: "medium", + date_style: "medium", + }) + : "", + [config, item.timestamp, t], + ); + + const ratio = useMemo( + () => + Array.isArray(item.data.box) && item.data.box.length >= 4 + ? (aspectRatio * (item.data.box[2] / item.data.box[3])).toFixed(2) + : "N/A", + [aspectRatio, item.data.box], + ); + + const areaPx = useMemo( + () => + Array.isArray(item.data.box) && item.data.box.length >= 4 + ? Math.round( + (config?.cameras[event.camera]?.detect?.width ?? 0) * + (config?.cameras[event.camera]?.detect?.height ?? 0) * + (item.data.box[2] * item.data.box[3]), + ) + : undefined, + [config, event.camera, item.data.box], + ); + + const attributeAreaPx = useMemo( + () => + Array.isArray(item.data.attribute_box) && + item.data.attribute_box.length >= 4 + ? Math.round( + (config?.cameras[event.camera]?.detect?.width ?? 0) * + (config?.cameras[event.camera]?.detect?.height ?? 0) * + (item.data.attribute_box[2] * item.data.attribute_box[3]), + ) + : undefined, + [config, event.camera, item.data.attribute_box], + ); + + const attributeAreaPct = useMemo( + () => + Array.isArray(item.data.attribute_box) && + item.data.attribute_box.length >= 4 + ? (item.data.attribute_box[2] * item.data.attribute_box[3]).toFixed(4) + : undefined, + [item.data.attribute_box], + ); + + const areaPct = useMemo( + () => + Array.isArray(item.data.box) && item.data.box.length >= 4 + ? (item.data.box[2] * item.data.box[3]).toFixed(4) + : undefined, + [item.data.box], + ); + + const score = useMemo(() => { + if (item.data.score !== undefined) { + return (item.data.score * 100).toFixed(0) + "%"; + } + return "N/A"; + }, [item.data.score]); + return (
{getLifecycleItemDescription(item)}
-
-
+
+
+ + {t("trackingDetails.lifecycleItemDesc.header.score")} + + {score} +
+
{t("trackingDetails.lifecycleItemDesc.header.ratio")} {ratio}
-
+
- {t("trackingDetails.lifecycleItemDesc.header.area")} + {t("trackingDetails.lifecycleItemDesc.header.area")}{" "} + {attributeAreaPx !== undefined && + attributeAreaPct !== undefined && ( + + ({getTranslatedLabel(item.data.label)}) + + )} {areaPx !== undefined && areaPct !== undefined ? ( @@ -876,9 +930,25 @@ function LifecycleIconRow({ N/A )}
+ {attributeAreaPx !== undefined && + attributeAreaPct !== undefined && ( +
+ + {t("trackingDetails.lifecycleItemDesc.header.area")} ( + {getTranslatedLabel(item.data.attribute)}) + + + {t("information.pixels", { + ns: "common", + area: attributeAreaPx, + })}{" "} + ยท {attributeAreaPct}% + +
+ )} {item.data?.zones && item.data.zones.length > 0 && ( -
+
{item.data.zones.map((zone, zidx) => { const color = getZoneColor(zone)?.join(",") ?? "0,0,0"; return ( diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts index c8e5f7543a..0de0674062 100644 --- a/web/src/types/timeline.ts +++ b/web/src/types/timeline.ts @@ -16,6 +16,7 @@ export type TrackingDetailsSequence = { data: { camera: string; label: string; + score: number; sub_label: string; box?: [number, number, number, number]; region: [number, number, number, number]; From a76c4c180e80f753e2f5c9ccd1a4a95a874b4354 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 24 Nov 2025 07:11:43 -0600 Subject: [PATCH 7/8] add mse codecs to console debug on errors --- web/src/components/player/MsePlayer.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx index 7b78b53bc4..8e4c1b601a 100644 --- a/web/src/components/player/MsePlayer.tsx +++ b/web/src/components/player/MsePlayer.tsx @@ -82,6 +82,7 @@ function MSEPlayer({ [key: string]: (msg: { value: string; type: string }) => void; }>({}); const msRef = useRef(null); + const mseCodecRef = useRef(null); const wsURL = useMemo(() => { return `${baseUrl.replace(/^http/, "ws")}live/mse/api/ws?src=${camera}`; @@ -93,6 +94,10 @@ function MSEPlayer({ console.error( `${camera} - MSE error '${error}': ${description} See the documentation: https://docs.frigate.video/configuration/live/#live-player-error-messages`, ); + if (mseCodecRef.current) { + // eslint-disable-next-line no-console + console.error(`MSE codec in use: ${mseCodecRef.current}`); + } onError?.(error); }, [camera, onError], @@ -299,6 +304,9 @@ function MSEPlayer({ onmessageRef.current["mse"] = (msg) => { if (msg.type !== "mse") return; + // Store the codec value for error logging + mseCodecRef.current = msg.value; + let sb: SourceBuffer | undefined; try { sb = msRef.current?.addSourceBuffer(msg.value); From 1b5a37ed6deb29d03c2c5bdf3ae341b293b7e3ac Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 24 Nov 2025 07:15:14 -0600 Subject: [PATCH 8/8] add camera name --- web/src/components/player/MsePlayer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx index 8e4c1b601a..576fc93d61 100644 --- a/web/src/components/player/MsePlayer.tsx +++ b/web/src/components/player/MsePlayer.tsx @@ -96,7 +96,7 @@ function MSEPlayer({ ); if (mseCodecRef.current) { // eslint-disable-next-line no-console - console.error(`MSE codec in use: ${mseCodecRef.current}`); + console.error(`${camera} - MSE codec in use: ${mseCodecRef.current}`); } onError?.(error); },