Add support for raw RTP over UDP video

README.md

@@ -104,6 +104,7 @@ Below is a table summarizing the current status, but in general, you can expect
 |-------------|---------------|----------------|
 | **Joystick Support** | Only when tab and window are in focus  | ✅ Window can be unfocused and in the background |
 | **Video** | Needs to be downloaded and merged into a working video using the Desktop app | ✅ Final MP4 file saved directly to your folders |
+| **Direct Video Sources** | ❌ Not available (browsers can't open RTSP or raw UDP video) | ✅ Add RTSP cameras, and UDP H.264/H.265 streams as sent by ArduPilot/PX4 companion computers |
 | **Snapshots** | Needs to be downloaded | ✅ Saved directly to your folders |
 | **Vehicle Discovery** | ❌ Not available | ✅ Auto-scan for vehicles in the network|
 | **External Serial GNSS** | ❌ Not available (browsers can't access serial devices outside a secure context) | ✅ Read one or more USB/serial NMEA GNSS receivers into the data-lake |

src/components/VideoPlayerStatsForNerds.vue

@@ -42,9 +42,10 @@ const props = defineProps({
   },
 })
 
-const isRtspStream = (): boolean => {
+// Streams the go2rtc sidecar ingests report their stats through it, rather than through WebRTC's own stats.
+const isGo2rtcStream = (): boolean => {
   if (!props.streamName) return false
-  return videoStore.getStreamProtocol(props.streamName) === 'rtsp'
+  return videoStore.getStreamProtocol(props.streamName) !== 'webrtc'
 }
 
 const canvasRef = ref(null)
@@ -123,8 +124,8 @@ function draw(): void {
   ctx.font = '10px Arial'
 
   // Draw stats and plots for both types of streams.
-  // The stats and plots are different from one to another since RTSP and WebRTC streams provide different types of information.
-  if (isRtspStream()) {
+  // The stats and plots are different from one to another since go2rtc-backed and WebRTC streams provide different types of information.
+  if (isGo2rtcStream()) {
     drawPlot(rtspBitrateData.value, 'rgb(255, 165, 0)', maxRtspBitrate)
     drawPlot(rtspPacketRateData.value, 'rgb(100, 200, 255)', maxRtspPacketRate)
     drawPlot(rtspStallData.value, 'rgb(255, 0, 0)', 1)
@@ -211,7 +212,7 @@ watch(videoStore.activeStreams, (streams): void => {
 })
 
 const fetchRtspInfo = async (): Promise<void> => {
-  if (!isRtspStream() || !window.electronAPI) return
+  if (!isGo2rtcStream() || !window.electronAPI) return
   try {
     const allInfo = await window.electronAPI.go2rtcGetStreamsInfo()
     const info = allInfo[props.streamName]
@@ -240,7 +241,7 @@ onMounted(() => {
   intervalId = setInterval(update, props.updateInterval)
   draw()
 
-  if (isRtspStream()) {
+  if (isGo2rtcStream()) {
     fetchRtspInfo()
     rtspInfoInterval = setInterval(fetchRtspInfo, 100)
   }

src/electron/preload.ts

@@ -1,5 +1,6 @@
 import { contextBridge, ipcRenderer } from 'electron'
 
+import type { RtpSourceConfig } from '@/libs/rtp-source'
 import type { ElectronSDLJoystickControllerStateEventData } from '@/types/joystick'
 import type { FileDialogOptions, FileStats } from '@/types/storage'
 
@@ -63,6 +64,8 @@ contextBridge.exposeInMainWorld('electronAPI', {
   },
   finalizeVideoRecording: (processId: string) => ipcRenderer.invoke('finalize-video-recording', processId),
   go2rtcAddStream: (name: string, rtspUrl: string) => ipcRenderer.invoke('go2rtc-add-stream', name, rtspUrl),
+  go2rtcAddRtpStream: (name: string, config: RtpSourceConfig) =>
+    ipcRenderer.invoke('go2rtc-add-rtp-stream', name, config),
   go2rtcRemoveStream: (name: string) => ipcRenderer.invoke('go2rtc-remove-stream', name),
   go2rtcGetStreamsInfo: () => ipcRenderer.invoke('go2rtc-get-streams-info'),
   go2rtcGetPort: () => ipcRenderer.invoke('go2rtc-get-port'),

src/electron/services/go2rtc.ts

@@ -5,10 +5,17 @@ import { promises as fs } from 'fs'
 import http from 'http'
 import { createServer } from 'net'
 import { tmpdir } from 'os'
-import { join } from 'path'
+import { delimiter, dirname, join } from 'path'
 
 import type { Go2RTCStreamInfo } from '@/types/video'
 
+import {
+  type RtpSourceConfig,
+  buildRtpGo2rtcSource,
+  rtpInputTemplate,
+  rtpInputTemplateName,
+} from '../../libs/rtp-source'
+import { getFFmpegPath } from './ffmpeg-path'
 import { getGo2RTCPath } from './go2rtc-path'
 
 let go2rtcProcess: ChildProcess | null = null
@@ -90,17 +97,45 @@ const startGo2RTC = async (): Promise<number> => {
     return go2rtcPort
   }
 
-  const port = await getFreeTcpPort()
+  // Allocated together so every socket is bound before any is released, which keeps the ports distinct.
+  const [port, rtspPort, webrtcPort] = await Promise.all([getFreeTcpPort(), getFreeTcpPort(), getFreeTcpPort()])
   const configDir = join(tmpdir(), 'cockpit-go2rtc')
   await fs.mkdir(configDir, { recursive: true })
 
   const configPath = join(configDir, 'go2rtc.yaml')
-  const config = `api:\n  listen: "127.0.0.1:${port}"\n  origin: "*"\n`
+  const config = [
+    'api:',
+    `  listen: "127.0.0.1:${port}"`,
+    '  origin: "*"',
+    // Left on its defaults go2rtc serves RTSP on :8554 and WebRTC on :8555 across every interface,
+    // colliding with MediaMTX or with a second go2rtc.
+    'rtsp:',
+    `  listen: "127.0.0.1:${rtspPort}"`,
+    // ponytail: the WebRTC port carries UDP too, but is picked by probing TCP, so a port free on TCP and
+    // taken on UDP would break ICE. Upgrade path is probing both, once anything reports it in the wild.
+    'webrtc:',
+    `  listen: ":${webrtcPort}"`,
+    // ffmpeg-backed sources publish into the RTSP server above.
+    'ffmpeg:',
+    `  ${rtpInputTemplateName}: '${rtpInputTemplate}'`,
+    '',
+  ].join('\n')
   await fs.writeFile(configPath, config, 'utf-8')
 
+  let ffmpegDir: string | undefined
+  try {
+    ffmpegDir = dirname(getFFmpegPath())
+  } catch (error) {
+    console.warn('[go2rtc] Bundled FFmpeg not found, UDP video streams will not work:', error)
+  }
+
   const binaryPath = getGo2RTCPath()
   const proc = spawn(binaryPath, ['-config', configPath], {
     stdio: ['ignore', 'pipe', 'pipe'],
+    // go2rtc resolves `ffmpeg` through PATH. Naming the bundled binary in its `ffmpeg.bin` setting instead
+    // would break on any install path containing a space, since go2rtc splits the command it builds on
+    // whitespace while probing the binary without that splitting.
+    env: ffmpegDir ? { ...process.env, PATH: `${ffmpegDir}${delimiter}${process.env.PATH ?? ''}` } : process.env,
   })
 
   proc.stdout?.on('data', (data) => {
@@ -174,6 +209,16 @@ const addStream = async (name: string, rtspUrl: string): Promise<void> => {
   console.log(`[go2rtc] Added stream '${name}' -> ${rtspUrl}`)
 }
 
+/**
+ * Register a raw RTP over UDP stream with go2rtc, ingested through the bundled FFmpeg
+ * @param {string} name - Unique stream name
+ * @param {RtpSourceConfig} config - Address and port to listen on, and the codec to expect there
+ * @returns {Promise<void>}
+ */
+const addRtpStream = async (name: string, config: RtpSourceConfig): Promise<void> => {
+  await addStream(name, buildRtpGo2rtcSource(config))
+}
+
 /**
  * Remove an RTSP stream from go2rtc
  * @param {string} name - Stream name to remove
@@ -418,6 +463,15 @@ export const setupGo2RTCService = (): void => {
     }
   })
 
+  ipcMain.handle('go2rtc-add-rtp-stream', async (_, name: string, config: RtpSourceConfig) => {
+    try {
+      await addRtpStream(name, config)
+    } catch (error) {
+      console.error('[go2rtc] Error adding RTP stream:', error)
+      throw error
+    }
+  })
+
   ipcMain.handle('go2rtc-remove-stream', async (_, name: string) => {
     try {
       await removeStream(name)

src/libs/cosmos.ts

@@ -519,6 +519,12 @@ declare global {
        * @param rtspUrl - Full RTSP URL including optional credentials
        */
       go2rtcAddStream: (name: string, rtspUrl: string) => Promise<void>
+      /**
+       * Register a raw RTP over UDP stream with the go2rtc sidecar for WebRTC consumption
+       * @param name - Unique stream name used for WebRTC signaling
+       * @param config - Address and port to listen on, and the codec to expect there
+       */
+      go2rtcAddRtpStream: (name: string, config: import('@/libs/rtp-source').RtpSourceConfig) => Promise<void>
       /**
        * Remove an RTSP stream from the go2rtc sidecar
        * @param name - The stream name to remove

src/libs/rtp-source.ts

@@ -0,0 +1,125 @@
+/**
+ * Raw RTP over UDP video sources, as emitted by ArduPilot/PX4 companion computers.
+ *
+ * RTP carries no description of what it transports, so a receiver has to be told the codec and clock rate
+ * out of band. We synthesize a minimal SDP for that, the same information QGroundControl hardcodes into its
+ * GStreamer caps, and hand it to ffmpeg through the go2rtc sidecar.
+ */
+
+import { isValidIpv4Address } from './utils'
+
+export const rtpCodecs = ['h264', 'h265'] as const
+
+export type RtpCodec = (typeof rtpCodecs)[number]
+
+export type RtpSourceConfig = {
+  /**
+   * Local IPv4 address to listen on, or 0.0.0.0 for every network interface
+   */
+  host: string
+  /**
+   * Local UDP port to listen on
+   */
+  port: number
+  /**
+   * Codec the sender packetizes into RTP
+   */
+  codec: RtpCodec
+}
+
+// First dynamic RTP payload type, and the default of both ffmpeg's RTP muxer and GStreamer's rtph264pay,
+// so it is what companion computers overwhelmingly emit.
+const rtpPayloadType = 96
+
+/**
+ * Name of the ffmpeg input template that go2rtc.yaml must declare for RTP sources to resolve.
+ *
+ * go2rtc has no native RTP source, and it refuses sources created through its API that either use the
+ * `exec:` scheme or contain whitespace. So the ffmpeg arguments cannot travel in the source string and
+ * live in a named template in the config file instead, leaving the per-stream source to just reference it.
+ */
+export const rtpInputTemplateName = 'cockpit-rtp'
+
+/** ffmpeg input arguments for the template named {@link rtpInputTemplateName}. */
+export const rtpInputTemplate = '-fflags nobuffer -flags low_delay -protocol_whitelist data,udp,rtp -f sdp -i {input}'
+
+/**
+ * Check whether a config describes a stream Cockpit can actually listen for.
+ * @param {RtpSourceConfig} config The config to check
+ * @returns {string | undefined} A user-facing description of the problem, or undefined when the config is valid
+ */
+export const rtpConfigError = (config: RtpSourceConfig): string | undefined => {
+  // The SDP describes the listen address as `IN IP4`, so a hostname or IPv6 literal would not be honest there.
+  if (!isValidIpv4Address(config.host)) {
+    return 'Address must be an IPv4 address, such as 0.0.0.0 to listen on every network interface.'
+  }
+  // Ports under 1024 need administrator privileges to bind, which would fail with no video and no clear reason.
+  if (!Number.isInteger(config.port) || config.port < 1024 || config.port > 65535) {
+    return 'Port must be a whole number between 1024 and 65535.'
+  }
+  if (!rtpCodecs.includes(config.codec)) {
+    return `Codec must be one of: ${rtpCodecs.join(', ')}.`
+  }
+  return undefined
+}
+
+/**
+ * Build the canonical URI that identifies an RTP stream throughout Cockpit.
+ * @param {RtpSourceConfig} config The stream to identify
+ * @returns {string} A `rtp://host:port?codec=` URI
+ */
+export const buildRtpUrl = (config: RtpSourceConfig): string =>
+  `rtp://${config.host}:${config.port}?codec=${config.codec}`
+
+/**
+ * Recover the config behind a URI built by {@link buildRtpUrl}.
+ * @param {string} url The URI to parse
+ * @returns {RtpSourceConfig | undefined} The config, or undefined if the URI is not a valid RTP source URI
+ */
+export const parseRtpUrl = (url: string): RtpSourceConfig | undefined => {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    return undefined
+  }
+  if (parsed.protocol !== 'rtp:') return undefined
+
+  const config = {
+    host: parsed.hostname,
+    port: Number(parsed.port),
+    codec: parsed.searchParams.get('codec') as RtpCodec,
+  }
+  return rtpConfigError(config) === undefined ? config : undefined
+}
+
+/**
+ * Build the SDP that tells ffmpeg where to listen and what it will find there.
+ * @param {RtpSourceConfig} config The stream to describe
+ * @returns {string} An SDP session description
+ */
+export const buildRtpSdp = (config: RtpSourceConfig): string =>
+  [
+    'v=0',
+    `o=- 0 0 IN IP4 ${config.host}`,
+    's=Cockpit RTP Stream',
+    `c=IN IP4 ${config.host}`,
+    't=0 0',
+    `m=video ${config.port} RTP/AVP ${rtpPayloadType}`,
+    `a=rtpmap:${rtpPayloadType} ${config.codec.toUpperCase()}/90000`,
+    '',
+  ].join('\r\n')
+
+/**
+ * Build the go2rtc source string that makes the sidecar ingest an RTP stream through ffmpeg.
+ *
+ * The SDP travels inline as a `data:` URI rather than as a temp file, because go2rtc rejects sources
+ * containing whitespace and so could never reference a path with a space in it.
+ *
+ * Both codecs are passed through untouched. H.265 reaches the renderer only because `src/electron/main.ts`
+ * turns on Chromium's HEVC decoding and WebRTC HEVC switches, so removing those breaks H.265 here too.
+ * @param {RtpSourceConfig} config The stream to ingest
+ * @returns {string} A go2rtc `ffmpeg:` source string
+ */
+export const buildRtpGo2rtcSource = (config: RtpSourceConfig): string =>
+  `ffmpeg:data:application/sdp;base64,${btoa(buildRtpSdp(config))}#input=${rtpInputTemplateName}#video=copy`

src/libs/video-stream-protocol.ts

@@ -0,0 +1,16 @@
+import type { VideoStreamProtocol } from '@/types/video'
+
+import { parseRtpUrl } from './rtp-source'
+
+/**
+ * Infer which protocol an external stream id refers to, from the id alone.
+ *
+ * Needed wherever no correspondency entry exists to carry the protocol, such as the ignored-streams list.
+ * @param {string} externalId - The external stream identifier
+ * @returns {VideoStreamProtocol} The protocol the id describes
+ */
+export const protocolFromExternalId = (externalId: string): VideoStreamProtocol => {
+  if (externalId.startsWith('rtsp://') || externalId.startsWith('rtsps://')) return 'rtsp'
+  if (parseRtpUrl(externalId) !== undefined) return 'rtp'
+  return 'webrtc'
+}