diff --git a/CHANGELOG.md b/CHANGELOG.md index 6828198a8..e40fabb70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Breaking Changes +### `sarvam` plugin: drop `sarvam-30b` from supported LLM models (#636) + +`sarvam.LLM` no longer accepts `sarvam-30b`. Use `sarvam-m` or `sarvam-105b` instead. + ### `deepgram` plugin: TTS defaults to Flux (`/v2/speak`) `deepgram.TTS` now streams Flux TTS on `wss://api.deepgram.com/v2/speak` and defaults to `flux-haley-en`. Aura model strings (`aura-*`) are rejected with `ValueError`. Call sites that passed an Aura voice must switch to a Flux model (`flux-{voice}-en`). See the [Flux voice catalog](https://developers.deepgram.com/docs/flux-tts/voices). diff --git a/plugins/elevenlabs/tests/test_elevenlabs_stt.py b/plugins/elevenlabs/tests/test_elevenlabs_stt.py index db8f6f34e..0cc553b13 100644 --- a/plugins/elevenlabs/tests/test_elevenlabs_stt.py +++ b/plugins/elevenlabs/tests/test_elevenlabs_stt.py @@ -11,6 +11,74 @@ load_dotenv() +class TestElevenLabsSTTCallbacks: + """Unit coverage for translating provider callbacks into turn events.""" + + @pytest.fixture + def participant(self) -> Participant: + return Participant({}, user_id="test-user", id="test-user") + + @pytest.fixture + def stt(self, participant: Participant) -> elevenlabs.STT: + stt = elevenlabs.STT(api_key="test-key") + stt._current_participant = participant + return stt + + def test_duplicate_partials_emit_one_turn_started( + self, stt: elevenlabs.STT + ) -> None: + stt._on_partial_transcript({"text": "Hello"}) + stt._on_partial_transcript({"text": "Hello world"}) + + turn_events = [ + item for item in stt.output.peek() if isinstance(item, TurnStarted) + ] + assert len(turn_events) == 1 + + def test_multiple_committed_utterances_emit_balanced_ordered_turns( + self, stt: elevenlabs.STT + ) -> None: + stt._on_partial_transcript({"text": "First"}) + stt._on_committed_transcript({"text": "First utterance"}) + stt._on_partial_transcript({"text": "Second"}) + stt._on_committed_transcript({"text": "Second utterance"}) + + turn_events = [ + item + for item in stt.output.peek() + if isinstance(item, (TurnStarted, TurnEnded)) + ] + assert len(turn_events) == 4 + assert all( + isinstance(turn_events[index], TurnStarted) + and isinstance(turn_events[index + 1], TurnEnded) + for index in range(0, len(turn_events), 2) + ) + + def test_empty_commit_ends_active_turn(self, stt: elevenlabs.STT) -> None: + stt._on_partial_transcript({"text": "Speech"}) + stt._on_committed_transcript({"text": ""}) + + turn_events = [ + item + for item in stt.output.peek() + if isinstance(item, (TurnStarted, TurnEnded)) + ] + assert len(turn_events) == 2 + assert isinstance(turn_events[0], TurnStarted) + assert isinstance(turn_events[1], TurnEnded) + + def test_keepalive_empty_commit_does_not_emit_turn_events( + self, stt: elevenlabs.STT + ) -> None: + stt._on_committed_transcript({"text": ""}) + stt._on_committed_transcript({"text": " "}) + + assert not any( + isinstance(item, (TurnStarted, TurnEnded)) for item in stt.output.peek() + ) + + class TestElevenLabsSTT: """Integration tests for ElevenLabs Scribe v2 STT""" @@ -87,14 +155,24 @@ async def test_turn_detection_enabled(self, stt): @pytest.mark.integration async def test_turn_events_emitted(self, stt, mia_audio_16khz, participant): - """One TurnStarted and exactly one TurnEnded per utterance.""" + """Every provider-detected utterance has ordered start and end events.""" await stt.process_audio(mia_audio_16khz, participant=participant) items = await stt.output.collect(timeout=10.0) - turn_started = [i for i in items if isinstance(i, TurnStarted)] - turn_ended = [i for i in items if isinstance(i, TurnEnded)] - assert len(turn_started) == 1 - assert len(turn_ended) == 1 + turn_events = [ + item for item in items if isinstance(item, (TurnStarted, TurnEnded)) + ] + turn_started = [item for item in turn_events if isinstance(item, TurnStarted)] + turn_ended = [item for item in turn_events if isinstance(item, TurnEnded)] + + assert turn_started + assert len(turn_started) == len(turn_ended) + assert all( + isinstance(turn_events[index], TurnStarted) + and isinstance(turn_events[index + 1], TurnEnded) + for index in range(0, len(turn_events), 2) + ) + assert all(event.participant == participant for event in turn_events) @pytest.mark.integration async def test_multiple_audio_segments( diff --git a/plugins/huggingface/tests/test_transformers_vlm.py b/plugins/huggingface/tests/test_transformers_vlm.py index 5126a7686..80ed087b6 100644 --- a/plugins/huggingface/tests/test_transformers_vlm.py +++ b/plugins/huggingface/tests/test_transformers_vlm.py @@ -1,7 +1,10 @@ """Tests for TransformersVLM - local vision-language model inference.""" +import asyncio import fractions import os +import threading +import time from unittest.mock import MagicMock import numpy as np @@ -11,7 +14,6 @@ from conftest import skip_blockbuster, skip_if_huggingface_model_unavailable from vision_agents.testing import collect_simple_response from vision_agents.core.agents.conversation import InMemoryConversation -from vision_agents.core.llm.llm import LLMResponseFinal from vision_agents.plugins.huggingface.transformers_vlm import ( TransformersVLM, VLMResources, @@ -122,6 +124,40 @@ async def test_generation_error(self, vlm, conversation): assert final.text == "" assert deltas == [] + async def test_interrupt_stops_in_flight_generation(self, vlm): + generation_started = threading.Event() + generated_token_counts: list[int] = [] + + def cancellable_generate(*args, **kwargs): + input_ids = kwargs["input_ids"] + output = input_ids.clone() + stopping_criteria = kwargs["stopping_criteria"] + generation_started.set() + + for _ in range(500): + if stopping_criteria(output, None): + break + output = torch.cat( + (output, torch.ones((1, 1), dtype=output.dtype)), dim=1 + ) + time.sleep(0.001) + + generated_token_counts.append(output.shape[-1] - input_ids.shape[-1]) + return output + + vlm._resources.model.generate.side_effect = cancellable_generate + response_task = asyncio.create_task( + collect_simple_response(vlm.simple_response(text="describe")) + ) + + started = await asyncio.to_thread(generation_started.wait, 1) + assert started + await vlm.interrupt() + + _, final = await asyncio.wait_for(response_task, timeout=1) + assert final.text == "A cat on a couch" + assert generated_token_counts[0] < 20 + async def test_processor_fallback(self, vlm): """When apply_chat_template fails, falls back to direct processor call.""" processor = vlm._resources.processor @@ -297,22 +333,44 @@ async def test_interrupt_stops_generation(self): vlm._frame_buffer.append(_random_video_frame()) - deltas = [] - final = None - async for item in vlm.simple_response( - text="Describe in extreme detail every single object you can see" - ): - if isinstance(item, LLMResponseFinal): - final = item - else: - deltas.append(item) - if len(deltas) == 1: - await vlm.interrupt() - - assert final is not None - # Without interrupt the response would run for hundreds of tokens. - # With interrupt fired after the first delta, the rest of the run - # produces only a handful more tokens before generation exits. - assert len(deltas) < 20 + generation_started = threading.Event() + generated_token_counts: list[int] = [] + original_generate = resources.model.generate + + def signaling_generate(*args, **kwargs): + generation_started.set() + output = original_generate(*args, **kwargs) + generated_token_counts.append( + output.shape[-1] - kwargs["input_ids"].shape[-1] + ) + return output + + setattr(resources.model, "generate", signaling_generate) + + response_task = asyncio.create_task( + collect_simple_response( + vlm.simple_response( + text="Describe in extreme detail every single object you can see" + ) + ) + ) - vlm.unload() + try: + started = await asyncio.to_thread(generation_started.wait, 10) + assert started, "model.generate did not start within 10 seconds" + + await vlm.interrupt() + _, final = await asyncio.wait_for(response_task, timeout=10) + + assert final is not None + assert generated_token_counts + assert generated_token_counts[0] < 20 + finally: + if not response_task.done(): + await vlm.interrupt() + try: + await asyncio.wait_for(response_task, timeout=10) + except asyncio.TimeoutError: + response_task.cancel() + await asyncio.gather(response_task, return_exceptions=True) + vlm.unload() diff --git a/plugins/sarvam/README.md b/plugins/sarvam/README.md index b072a7f90..a0973e126 100644 --- a/plugins/sarvam/README.md +++ b/plugins/sarvam/README.md @@ -9,8 +9,8 @@ AI models built for Indian languages. Activity Detection for turn events. - **TTS**: WebSocket streaming text-to-speech (Bulbul) with configurable speaker, pace, and language. -- **LLM**: OpenAI-compatible chat completions (Sarvam-30B / Sarvam-105B / - Sarvam-M) via the existing `ChatCompletionsLLM` from the OpenAI plugin. +- **LLM**: OpenAI-compatible chat completions (Sarvam-105B / Sarvam-M) via + the existing `ChatCompletionsLLM` from the OpenAI plugin. ## Installation @@ -28,7 +28,7 @@ agent = Agent( edge=getstream.Edge(), agent_user=User(name="Sarvam AI"), instructions="Reply in Hindi or English, whichever the user speaks", - llm=sarvam.LLM(model="sarvam-30b"), + llm=sarvam.LLM(model="sarvam-105b"), stt=sarvam.STT(language="hi-IN"), tts=sarvam.TTS(speaker="shubh"), turn_detection=smart_turn.TurnDetection(), diff --git a/plugins/sarvam/tests/test_sarvam_llm.py b/plugins/sarvam/tests/test_sarvam_llm.py index 2e07cc8cb..a085e905b 100644 --- a/plugins/sarvam/tests/test_sarvam_llm.py +++ b/plugins/sarvam/tests/test_sarvam_llm.py @@ -69,8 +69,8 @@ async def test_default_model(self): assert llm.model == "sarvam-m" async def test_custom_model(self): - llm = LLM(api_key="sk_test", model="sarvam-30b") - assert llm.model == "sarvam-30b" + llm = LLM(api_key="sk_test", model="sarvam-105b") + assert llm.model == "sarvam-105b" async def test_base_url_points_to_sarvam(self): llm = LLM(api_key="sk_test") @@ -89,7 +89,7 @@ class TestSarvamLLMIntegration: @pytest.fixture async def llm(self): - llm = LLM(model="sarvam-30b") + llm = LLM(model="sarvam-105b") llm.set_conversation(InMemoryConversation("be friendly", [])) return llm diff --git a/plugins/sarvam/vision_agents/plugins/sarvam/llm.py b/plugins/sarvam/vision_agents/plugins/sarvam/llm.py index 92a097ade..c255dccc6 100644 --- a/plugins/sarvam/vision_agents/plugins/sarvam/llm.py +++ b/plugins/sarvam/vision_agents/plugins/sarvam/llm.py @@ -26,7 +26,7 @@ SARVAM_BASE_URL = "https://api.sarvam.ai/v1" DEFAULT_MODEL = "sarvam-m" -SUPPORTED_MODELS = {"sarvam-m", "sarvam-30b", "sarvam-105b"} +SUPPORTED_MODELS = {"sarvam-m", "sarvam-105b"} _THINK_RE = re.compile(r".*?", re.DOTALL) @@ -97,7 +97,7 @@ class SarvamLLM(ChatCompletionsLLM): Examples: from vision_agents.plugins import sarvam - llm = sarvam.LLM(model="sarvam-30b") + llm = sarvam.LLM(model="sarvam-105b") """ provider_name = "sarvam" @@ -113,7 +113,7 @@ def __init__( Args: model: The Sarvam model id. Defaults to ``sarvam-m``. Supported: - ``sarvam-m``, ``sarvam-30b``, ``sarvam-105b``. + ``sarvam-m``, ``sarvam-105b``. api_key: Sarvam API key. Defaults to ``SARVAM_API_KEY`` env var. base_url: API base URL. Defaults to ``https://api.sarvam.ai/v1``. client: Optional pre-configured ``AsyncOpenAI`` client. Takes