From 18c9ea9a10dcdc4858960edffabb56d146bed083 Mon Sep 17 00:00:00 2001 From: Shaohong Date: Fri, 6 Mar 2026 14:34:49 -0800 Subject: [PATCH 1/4] added heuristics to improve asr for demo event --- src/inputs/plugins/riva_asr.py | 65 ++++++++++++++++++++++++++++- src/inputs/plugins/riva_asr_rtsp.py | 20 ++++++++- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/inputs/plugins/riva_asr.py b/src/inputs/plugins/riva_asr.py index 6b3d31b7a6..5e677bdae3 100644 --- a/src/inputs/plugins/riva_asr.py +++ b/src/inputs/plugins/riva_asr.py @@ -1,6 +1,7 @@ import asyncio import json import logging +import re import time from typing import Dict, List, Optional from uuid import uuid4 @@ -56,6 +57,55 @@ class RivaASRSensorConfig(SensorConfig): ) +# Keywords that suggest speech is directed at the robot (greeter context) +# All lowercase — matching is case-insensitive +# Avoid generic words (is, are, do, will, etc.) that appear in any conversation +_DIRECTED_KEYWORDS = { + # Addressing the robot directly + "you", "your", "yours", "yourself", + "bits", "robot", "dog", "puppy", "buddy", + "openmind", "om1", + # Greetings and social + "hello", "hi", "hey", "howdy", "greetings", + "goodbye", "bye", + "thanks", "thank", "please", + # Questions — only question words, not auxiliaries + "what", "how", "why", "who", "where", "when", "which", + # Requests directed at the robot + "tell", "show", "explain", "describe", "help", + # Conference / demo context + "gtc", "nvidia", "unitree", "conference", "demo", "booth", "exhibit", + # Product questions + "name", "company", "product", "price", "cost", "buy", "available", + "software", "ai", "autonomous", "platform", +} + + +# Common ASR misrecognitions → correct text (case-insensitive) +_ASR_CORRECTIONS = [ + (re.compile(r"\b(?:om one|ol one|on one|om 1|ol 1|oh and one|o one|oh one)\b", re.IGNORECASE), "OM1"), + (re.compile(r"\b(?:open mind|pokemon)\b", re.IGNORECASE), "OpenMind"), + (re.compile(r"\bunit tree\b", re.IGNORECASE), "Unitree"), +] + + +def _normalize_asr_text(text: str) -> str: + """Fix common ASR misrecognitions.""" + for pattern, replacement in _ASR_CORRECTIONS: + text = pattern.sub(replacement, text) + return text + + +def _seems_directed_at_robot(text: str) -> bool: + """Check if the transcript seems directed at the robot rather than overheard chatter.""" + words = set(text.lower().split()) + if words & _DIRECTED_KEYWORDS: + return True + if text.rstrip().endswith("?"): + return True + return False + + class RivaASRInput(FuserInput[RivaASRSensorConfig, Optional[str]]): """ Automatic Speech Recognition (ASR) input handler. @@ -85,6 +135,10 @@ def __init__(self, config: RivaASRSensorConfig): # Message buffer for incoming ASR messages self.message_buffer: asyncio.Queue[str] = asyncio.Queue() + # Cooldown after a message is accepted: ignore ASR during robot response + self._cooldown_until: float = 0.0 + self._cooldown_seconds: float = 2.0 + # Initialize ASR provider api_key = self.config.api_key rate = self.config.rate @@ -140,9 +194,16 @@ def _handle_asr_message(self, raw_message: str): try: json_message: Dict = json.loads(raw_message) if "asr_reply" in json_message: - asr_reply = json_message["asr_reply"] - if len(asr_reply.split()) > 1: + asr_reply = _normalize_asr_text(json_message["asr_reply"]) + if len(asr_reply.split()) > 2: + if time.time() < self._cooldown_until: + logging.info("ASR suppressed during cooldown: %s", asr_reply) + return + if not _seems_directed_at_robot(asr_reply): + logging.info("ASR filtered as overheard chatter: %s", asr_reply) + return self.message_buffer.put_nowait(asr_reply) + self._cooldown_until = time.time() + self._cooldown_seconds logging.info("Detected ASR message: %s", asr_reply) except json.JSONDecodeError: pass diff --git a/src/inputs/plugins/riva_asr_rtsp.py b/src/inputs/plugins/riva_asr_rtsp.py index 87c5656e09..04c726badc 100644 --- a/src/inputs/plugins/riva_asr_rtsp.py +++ b/src/inputs/plugins/riva_asr_rtsp.py @@ -1,6 +1,7 @@ import asyncio import json import logging +import re import time from typing import Dict, List, Optional from uuid import uuid4 @@ -49,6 +50,10 @@ class RivaASRRTSPSensorConfig(SensorConfig): ) +# Import shared ASR filters from riva_asr +from inputs.plugins.riva_asr import _normalize_asr_text, _seems_directed_at_robot + + class RivaASRRTSPInput(FuserInput[RivaASRRTSPSensorConfig, Optional[str]]): """ Automatic Speech Recognition (ASR) input handler. @@ -78,6 +83,10 @@ def __init__(self, config: RivaASRRTSPSensorConfig): # Message buffer for incoming ASR messages self.message_buffer: asyncio.Queue[str] = asyncio.Queue() + # Cooldown after a message is accepted: ignore ASR during robot response + self._cooldown_until: float = 0.0 + self._cooldown_seconds: float = 2.0 + # Initialize ASR provider api_key = self.config.api_key rtsp_url = self.config.rtsp_url @@ -128,9 +137,16 @@ def _handle_asr_message(self, raw_message: str): try: json_message: Dict = json.loads(raw_message) if "asr_reply" in json_message: - asr_reply = json_message["asr_reply"] - if len(asr_reply.split()) > 1: + asr_reply = _normalize_asr_text(json_message["asr_reply"]) + if len(asr_reply.split()) > 2: + if time.time() < self._cooldown_until: + logging.info("ASR suppressed during cooldown: %s", asr_reply) + return + if not _seems_directed_at_robot(asr_reply): + logging.info("ASR filtered as overheard chatter: %s", asr_reply) + return self.message_buffer.put_nowait(asr_reply) + self._cooldown_until = time.time() + self._cooldown_seconds logging.info("Detected ASR message: %s", asr_reply) except json.JSONDecodeError: pass From 7a2c2b1a89dd883fd81822bf58e6ef179c1b3eec Mon Sep 17 00:00:00 2001 From: Shaohong Date: Fri, 6 Mar 2026 14:46:57 -0800 Subject: [PATCH 2/4] remove asr cooldown --- src/inputs/plugins/riva_asr.py | 8 -------- src/inputs/plugins/riva_asr_rtsp.py | 8 -------- 2 files changed, 16 deletions(-) diff --git a/src/inputs/plugins/riva_asr.py b/src/inputs/plugins/riva_asr.py index 5e677bdae3..497e766a1e 100644 --- a/src/inputs/plugins/riva_asr.py +++ b/src/inputs/plugins/riva_asr.py @@ -135,10 +135,6 @@ def __init__(self, config: RivaASRSensorConfig): # Message buffer for incoming ASR messages self.message_buffer: asyncio.Queue[str] = asyncio.Queue() - # Cooldown after a message is accepted: ignore ASR during robot response - self._cooldown_until: float = 0.0 - self._cooldown_seconds: float = 2.0 - # Initialize ASR provider api_key = self.config.api_key rate = self.config.rate @@ -196,14 +192,10 @@ def _handle_asr_message(self, raw_message: str): if "asr_reply" in json_message: asr_reply = _normalize_asr_text(json_message["asr_reply"]) if len(asr_reply.split()) > 2: - if time.time() < self._cooldown_until: - logging.info("ASR suppressed during cooldown: %s", asr_reply) - return if not _seems_directed_at_robot(asr_reply): logging.info("ASR filtered as overheard chatter: %s", asr_reply) return self.message_buffer.put_nowait(asr_reply) - self._cooldown_until = time.time() + self._cooldown_seconds logging.info("Detected ASR message: %s", asr_reply) except json.JSONDecodeError: pass diff --git a/src/inputs/plugins/riva_asr_rtsp.py b/src/inputs/plugins/riva_asr_rtsp.py index 04c726badc..07f8f8fa8e 100644 --- a/src/inputs/plugins/riva_asr_rtsp.py +++ b/src/inputs/plugins/riva_asr_rtsp.py @@ -83,10 +83,6 @@ def __init__(self, config: RivaASRRTSPSensorConfig): # Message buffer for incoming ASR messages self.message_buffer: asyncio.Queue[str] = asyncio.Queue() - # Cooldown after a message is accepted: ignore ASR during robot response - self._cooldown_until: float = 0.0 - self._cooldown_seconds: float = 2.0 - # Initialize ASR provider api_key = self.config.api_key rtsp_url = self.config.rtsp_url @@ -139,14 +135,10 @@ def _handle_asr_message(self, raw_message: str): if "asr_reply" in json_message: asr_reply = _normalize_asr_text(json_message["asr_reply"]) if len(asr_reply.split()) > 2: - if time.time() < self._cooldown_until: - logging.info("ASR suppressed during cooldown: %s", asr_reply) - return if not _seems_directed_at_robot(asr_reply): logging.info("ASR filtered as overheard chatter: %s", asr_reply) return self.message_buffer.put_nowait(asr_reply) - self._cooldown_until = time.time() + self._cooldown_seconds logging.info("Detected ASR message: %s", asr_reply) except json.JSONDecodeError: pass From 64c361620a0d64bcd84f0f5ed254d9dd86076d53 Mon Sep 17 00:00:00 2001 From: Shaohong Date: Fri, 6 Mar 2026 15:08:23 -0800 Subject: [PATCH 3/4] fix formats --- src/inputs/plugins/riva_asr.py | 70 ++++++++++++++++++++++++----- src/inputs/plugins/riva_asr_rtsp.py | 6 +-- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/inputs/plugins/riva_asr.py b/src/inputs/plugins/riva_asr.py index 497e766a1e..1169ad3b3e 100644 --- a/src/inputs/plugins/riva_asr.py +++ b/src/inputs/plugins/riva_asr.py @@ -62,28 +62,74 @@ class RivaASRSensorConfig(SensorConfig): # Avoid generic words (is, are, do, will, etc.) that appear in any conversation _DIRECTED_KEYWORDS = { # Addressing the robot directly - "you", "your", "yours", "yourself", - "bits", "robot", "dog", "puppy", "buddy", - "openmind", "om1", + "you", + "your", + "yours", + "yourself", + "bits", + "robot", + "dog", + "puppy", + "buddy", + "openmind", + "om1", # Greetings and social - "hello", "hi", "hey", "howdy", "greetings", - "goodbye", "bye", - "thanks", "thank", "please", + "hello", + "hi", + "hey", + "howdy", + "greetings", + "goodbye", + "bye", + "thanks", + "thank", + "please", # Questions — only question words, not auxiliaries - "what", "how", "why", "who", "where", "when", "which", + "what", + "how", + "why", + "who", + "where", + "when", + "which", # Requests directed at the robot - "tell", "show", "explain", "describe", "help", + "tell", + "show", + "explain", + "describe", + "help", # Conference / demo context - "gtc", "nvidia", "unitree", "conference", "demo", "booth", "exhibit", + "gtc", + "nvidia", + "unitree", + "conference", + "demo", + "booth", + "exhibit", # Product questions - "name", "company", "product", "price", "cost", "buy", "available", - "software", "ai", "autonomous", "platform", + "name", + "company", + "product", + "price", + "cost", + "buy", + "available", + "software", + "ai", + "autonomous", + "platform", } # Common ASR misrecognitions → correct text (case-insensitive) _ASR_CORRECTIONS = [ - (re.compile(r"\b(?:om one|ol one|on one|om 1|ol 1|oh and one|o one|oh one)\b", re.IGNORECASE), "OM1"), + ( + re.compile( + r"\b(?:om one|ol one|on one|om 1|ol 1|oh and one|o one|oh one)\b", + re.IGNORECASE, + ), + "OM1", + ), (re.compile(r"\b(?:open mind|pokemon)\b", re.IGNORECASE), "OpenMind"), (re.compile(r"\bunit tree\b", re.IGNORECASE), "Unitree"), ] diff --git a/src/inputs/plugins/riva_asr_rtsp.py b/src/inputs/plugins/riva_asr_rtsp.py index 07f8f8fa8e..9f5751d14f 100644 --- a/src/inputs/plugins/riva_asr_rtsp.py +++ b/src/inputs/plugins/riva_asr_rtsp.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import re import time from typing import Dict, List, Optional from uuid import uuid4 @@ -10,6 +9,7 @@ from inputs.base import Message, SensorConfig from inputs.base.loop import FuserInput +from inputs.plugins.riva_asr import _normalize_asr_text, _seems_directed_at_robot from providers.asr_rtsp_provider import ASRRTSPProvider from providers.io_provider import IOProvider from providers.sleep_ticker_provider import SleepTickerProvider @@ -50,10 +50,6 @@ class RivaASRRTSPSensorConfig(SensorConfig): ) -# Import shared ASR filters from riva_asr -from inputs.plugins.riva_asr import _normalize_asr_text, _seems_directed_at_robot - - class RivaASRRTSPInput(FuserInput[RivaASRRTSPSensorConfig, Optional[str]]): """ Automatic Speech Recognition (ASR) input handler. From 6a462b0918a306c682d6fdad722c0470489f8313 Mon Sep 17 00:00:00 2001 From: Shaohong Zhong Date: Mon, 9 Mar 2026 10:34:07 -0700 Subject: [PATCH 4/4] fix tests for asr scripts --- tests/inputs/plugins/test_riva_asr.py | 100 ++++++++++++++++++++- tests/inputs/plugins/test_riva_asr_rtsp.py | 81 +++++++++++++++++ 2 files changed, 177 insertions(+), 4 deletions(-) diff --git a/tests/inputs/plugins/test_riva_asr.py b/tests/inputs/plugins/test_riva_asr.py index 6a635543e8..9d4388e5ae 100644 --- a/tests/inputs/plugins/test_riva_asr.py +++ b/tests/inputs/plugins/test_riva_asr.py @@ -3,7 +3,102 @@ import pytest from inputs.base import Message -from inputs.plugins.riva_asr import RivaASRInput, RivaASRSensorConfig +from inputs.plugins.riva_asr import ( + RivaASRInput, + RivaASRSensorConfig, + _normalize_asr_text, + _seems_directed_at_robot, +) + + +class TestNormalizeAsrText: + """Tests for _normalize_asr_text.""" + + def test_corrects_om_one_variants(self): + assert "OM1" in _normalize_asr_text("tell me about om one") + assert "OM1" in _normalize_asr_text("what is ol one") + assert "OM1" in _normalize_asr_text("I like om 1") + + def test_corrects_open_mind(self): + assert "OpenMind" in _normalize_asr_text("this is open mind") + + def test_corrects_unit_tree(self): + assert "Unitree" in _normalize_asr_text("the unit tree robot") + + def test_no_correction_needed(self): + assert _normalize_asr_text("hello world") == "hello world" + + +class TestSeemsDirectedAtRobot: + """Tests for _seems_directed_at_robot.""" + + def test_directed_keyword_match(self): + assert _seems_directed_at_robot("hello how are you") is True + + def test_question_mark_detected(self): + assert _seems_directed_at_robot("something random stuff?") is True + + def test_overheard_chatter_filtered(self): + assert _seems_directed_at_robot("yeah totally agree man") is False + + def test_case_insensitive(self): + assert _seems_directed_at_robot("HELLO there friend") is True + + +def test_handle_asr_message_filters_overheard_chatter(): + """Test that _handle_asr_message filters messages not directed at robot.""" + with ( + patch("inputs.plugins.riva_asr.IOProvider"), + patch("inputs.plugins.riva_asr.ASRProvider") as mock_asr, + patch("inputs.plugins.riva_asr.SleepTickerProvider"), + ): + mock_asr_instance = MagicMock() + mock_asr.return_value = mock_asr_instance + + config = RivaASRSensorConfig() + sensor = RivaASRInput(config=config) + + # Message with >2 words but not directed at robot + raw_message = '{"asr_reply": "yeah totally agree man"}' + sensor._handle_asr_message(raw_message) + assert sensor.message_buffer.qsize() == 0 + + +def test_handle_asr_message_accepts_directed_speech(): + """Test that _handle_asr_message accepts messages directed at robot.""" + with ( + patch("inputs.plugins.riva_asr.IOProvider"), + patch("inputs.plugins.riva_asr.ASRProvider") as mock_asr, + patch("inputs.plugins.riva_asr.SleepTickerProvider"), + ): + mock_asr_instance = MagicMock() + mock_asr.return_value = mock_asr_instance + + config = RivaASRSensorConfig() + sensor = RivaASRInput(config=config) + + raw_message = '{"asr_reply": "hello how are you"}' + sensor._handle_asr_message(raw_message) + assert sensor.message_buffer.qsize() == 1 + + +def test_handle_asr_message_normalizes_text(): + """Test that _handle_asr_message applies ASR text normalization.""" + with ( + patch("inputs.plugins.riva_asr.IOProvider"), + patch("inputs.plugins.riva_asr.ASRProvider") as mock_asr, + patch("inputs.plugins.riva_asr.SleepTickerProvider"), + ): + mock_asr_instance = MagicMock() + mock_asr.return_value = mock_asr_instance + + config = RivaASRSensorConfig() + sensor = RivaASRInput(config=config) + + raw_message = '{"asr_reply": "tell me about om one please"}' + sensor._handle_asr_message(raw_message) + assert sensor.message_buffer.qsize() == 1 + assert "OM1" in sensor.message_buffer.get_nowait() def test_initialization(): @@ -13,7 +108,6 @@ def test_initialization(): patch("inputs.plugins.riva_asr.ASRProvider") as mock_asr, patch("inputs.plugins.riva_asr.SleepTickerProvider"), ): - mock_asr_instance = MagicMock() mock_asr.return_value = mock_asr_instance @@ -32,7 +126,6 @@ async def test_poll(): patch("inputs.plugins.riva_asr.ASRProvider"), patch("inputs.plugins.riva_asr.SleepTickerProvider"), ): - config = RivaASRSensorConfig() sensor = RivaASRInput(config=config) @@ -48,7 +141,6 @@ def test_formatted_latest_buffer(): patch("inputs.plugins.riva_asr.ASRProvider"), patch("inputs.plugins.riva_asr.SleepTickerProvider"), ): - config = RivaASRSensorConfig() sensor = RivaASRInput(config=config) diff --git a/tests/inputs/plugins/test_riva_asr_rtsp.py b/tests/inputs/plugins/test_riva_asr_rtsp.py index 9cfe83102c..27fc79eced 100644 --- a/tests/inputs/plugins/test_riva_asr_rtsp.py +++ b/tests/inputs/plugins/test_riva_asr_rtsp.py @@ -319,6 +319,87 @@ def test_handle_asr_message_ignores_json_without_asr_reply( assert final_size == initial_size +def test_handle_asr_message_filters_overheard_chatter( + mock_io_provider, + mock_asr_provider, + mock_sleep_ticker_provider, + mock_teleops_conversation_provider, + mock_zenoh, +): + """Test that _handle_asr_message filters messages not directed at robot.""" + _, mock_asr_instance = mock_asr_provider + _, mock_sleep_ticker_instance = mock_sleep_ticker_provider + _, mock_teleops_conv_instance = mock_teleops_conversation_provider + + config = RivaASRRTSPSensorConfig() + with ( + patch("inputs.plugins.riva_asr_rtsp.IOProvider", return_value=mock_io_provider), + patch( + "inputs.plugins.riva_asr_rtsp.ASRRTSPProvider", + return_value=mock_asr_instance, + ), + patch( + "inputs.plugins.riva_asr_rtsp.SleepTickerProvider", + return_value=mock_sleep_ticker_instance, + ), + patch( + "inputs.plugins.riva_asr_rtsp.TeleopsConversationProvider", + return_value=mock_teleops_conv_instance, + ), + patch( + "inputs.plugins.riva_asr_rtsp.open_zenoh_session", + mock_zenoh["open_session"], + ), + ): + instance = RivaASRRTSPInput(config=config) + + # Message with >2 words but not directed at robot should be filtered + raw_message = '{"asr_reply": "yeah totally agree man"}' + initial_size = instance.message_buffer.qsize() + instance._handle_asr_message(raw_message) + assert instance.message_buffer.qsize() == initial_size + + +def test_handle_asr_message_normalizes_text( + mock_io_provider, + mock_asr_provider, + mock_sleep_ticker_provider, + mock_teleops_conversation_provider, + mock_zenoh, +): + """Test that _handle_asr_message applies ASR text normalization.""" + _, mock_asr_instance = mock_asr_provider + _, mock_sleep_ticker_instance = mock_sleep_ticker_provider + _, mock_teleops_conv_instance = mock_teleops_conversation_provider + + config = RivaASRRTSPSensorConfig() + with ( + patch("inputs.plugins.riva_asr_rtsp.IOProvider", return_value=mock_io_provider), + patch( + "inputs.plugins.riva_asr_rtsp.ASRRTSPProvider", + return_value=mock_asr_instance, + ), + patch( + "inputs.plugins.riva_asr_rtsp.SleepTickerProvider", + return_value=mock_sleep_ticker_instance, + ), + patch( + "inputs.plugins.riva_asr_rtsp.TeleopsConversationProvider", + return_value=mock_teleops_conv_instance, + ), + patch( + "inputs.plugins.riva_asr_rtsp.open_zenoh_session", + mock_zenoh["open_session"], + ), + ): + instance = RivaASRRTSPInput(config=config) + + raw_message = '{"asr_reply": "tell me about om one please"}' + instance._handle_asr_message(raw_message) + assert instance.message_buffer.qsize() == 1 + assert "OM1" in instance.message_buffer.get_nowait() + + def test_handle_asr_message_ignores_json_with_asr_reply_shorter_than_two_words( mock_io_provider, mock_asr_provider,