From 8f5ad57422d1fcb96dac09354288b177dc43de74 Mon Sep 17 00:00:00 2001 From: Liben Wei Date: Thu, 17 Oct 2024 17:07:41 -0700 Subject: [PATCH 1/2] one work one not --- .../CallAutomation_LiveTranscription.csproj | 2 +- .../CallAutomationLiveTranscription/Helper.cs | 85 ++++++++++++++----- .../Program.cs | 66 +++++++++++--- .../appsettings.json | 12 +-- 4 files changed, 124 insertions(+), 41 deletions(-) diff --git a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/CallAutomation_LiveTranscription.csproj b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/CallAutomation_LiveTranscription.csproj index 849daf11..7014e29a 100644 --- a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/CallAutomation_LiveTranscription.csproj +++ b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/CallAutomation_LiveTranscription.csproj @@ -8,7 +8,7 @@ - + diff --git a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs index 0945ea44..da10e2c6 100644 --- a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs +++ b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs @@ -1,6 +1,7 @@ using Azure.Communication.CallAutomation; using System.Net.WebSockets; using System.Text; +using System.Text.Json; namespace CallAutomation_LiveTranscription { @@ -14,11 +15,11 @@ public static async Task ProcessRequest(WebSocket webSocket) { try { - while (webSocket.State == WebSocketState.Open || webSocket.State == WebSocketState.CloseSent) - { + //while (webSocket.State == WebSocketState.Open || webSocket.State == WebSocketState.CloseSent) + //{ var buffer = new byte[1024 * 4]; - var cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(60)).Token; - WebSocketReceiveResult receiveResult = await webSocket.ReceiveAsync(new ArraySegment(buffer), cancellationToken); + //var cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(60)).Token; + WebSocketReceiveResult receiveResult = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); while (!receiveResult.CloseStatus.HasValue) { @@ -27,31 +28,22 @@ public static async Task ProcessRequest(WebSocket webSocket) var response = StreamingDataParser.Parse(msg); if (response != null) - { - if (response is TranscriptionMetadata transcriptionMetadata) - { - Console.WriteLine("***************************************************************************************"); - Console.WriteLine("TRANSCRIPTION SUBSCRIPTION ID-->" + transcriptionMetadata.TranscriptionSubscriptionId); - Console.WriteLine("LOCALE-->" + transcriptionMetadata.Locale); - Console.WriteLine("CALL CONNECTION ID--?" + transcriptionMetadata.CallConnectionId); - Console.WriteLine("CORRELATION ID-->" + transcriptionMetadata.CorrelationId); - Console.WriteLine("***************************************************************************************"); - } + { if (response is TranscriptionData transcriptionData) { Console.WriteLine("***************************************************************************************"); Console.WriteLine("TEXT-->" + transcriptionData.Text); Console.WriteLine("FORMAT-->" + transcriptionData.Format); - Console.WriteLine("OFFSET-->" + transcriptionData.Offset.Ticks); - Console.WriteLine("DURATION-->" + transcriptionData.Duration.Ticks); Console.WriteLine("PARTICIPANT-->" + transcriptionData.Participant.RawId); Console.WriteLine("CONFIDENCE-->" + transcriptionData.Confidence); - Console.WriteLine("RESULT STATUS-->" + transcriptionData.ResultState); + //Console.WriteLine("OFFSET-->" + transcriptionData.Offset.Ticks); + //Console.WriteLine("DURATION-->" + transcriptionData.Duration.Ticks); + //Console.WriteLine("RESULT STATUS-->" + transcriptionData.ResultState); foreach (var word in transcriptionData.Words) { Console.WriteLine("WORDS TEXT-->" + word.Text); - Console.WriteLine("WORDS OFFSET-->" + word.Offset.Ticks); - Console.WriteLine("WORDS DURATION-->" + word.Duration.Ticks); + //Console.WriteLine("WORDS OFFSET-->" + word.Offset.Ticks); + //Console.WriteLine("WORDS DURATION-->" + word.Duration.Ticks); } Console.WriteLine("***************************************************************************************"); } @@ -66,7 +58,7 @@ await webSocket.SendAsync( } await webSocket.CloseAsync(receiveResult.CloseStatus.Value, receiveResult.CloseStatusDescription, CancellationToken.None); - } + //} } catch (Exception ex) { @@ -76,5 +68,58 @@ await webSocket.SendAsync( { } } + + public static async Task Echo(WebSocket webSocket) + { + var buffer = new byte[1024 * 4]; + var receiveResult = await webSocket.ReceiveAsync( new ArraySegment(buffer), CancellationToken.None); + + while (!receiveResult.CloseStatus.HasValue) + { + + string msg = Encoding.UTF8.GetString(buffer, 0, receiveResult.Count); + Console.WriteLine($"Received: {msg}"); + + var response = StreamingDataParser.Parse(msg); + + if (response != null) + { + if (response is TranscriptionData transcriptionData) + { + Console.WriteLine("***************************************************************************************"); + Console.WriteLine("TEXT-->" + transcriptionData.Text); + Console.WriteLine("FORMAT-->" + transcriptionData.Format); + Console.WriteLine("PARTICIPANT-->" + transcriptionData.Participant.RawId); + Console.WriteLine("CONFIDENCE-->" + transcriptionData.Confidence); + //Console.WriteLine("OFFSET-->" + transcriptionData.Offset.Ticks); + //Console.WriteLine("DURATION-->" + transcriptionData.Duration.Ticks); + //Console.WriteLine("RESULT STATUS-->" + transcriptionData.ResultState); + foreach (var word in transcriptionData.Words) + { + Console.WriteLine("WORDS TEXT-->" + word.Text); + //Console.WriteLine("WORDS OFFSET-->" + word.Offset.Ticks); + //Console.WriteLine("WORDS DURATION-->" + word.Duration.Ticks); + } + Console.WriteLine("***************************************************************************************"); + } + + } + + await webSocket.SendAsync( + new ArraySegment(buffer, 0, receiveResult.Count), + receiveResult.MessageType, + receiveResult.EndOfMessage, + CancellationToken.None); + + receiveResult = await webSocket.ReceiveAsync( + new ArraySegment(buffer), CancellationToken.None); + } + + await webSocket.CloseAsync( + receiveResult.CloseStatus.Value, + receiveResult.CloseStatusDescription, + CancellationToken.None); + } + } } \ No newline at end of file diff --git a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Program.cs b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Program.cs index 212a6c9d..cde80971 100644 --- a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Program.cs +++ b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Program.cs @@ -87,8 +87,10 @@ logger.LogInformation($"Incoming call - correlationId: {incomingCallEventData.CorrelationId}, " + $"Callback url: {callbackUri}, transport Url: {transportUrl}"); - TranscriptionOptions transcriptionOptions = new TranscriptionOptions(new Uri(transportUrl), - locale, false, TranscriptionTransport.Websocket); + TranscriptionOptions transcriptionOptions = new TranscriptionOptions(new Uri(transportUrl),TranscriptionTransport.Websocket,locale,false); + + MediaStreamingOptions mediaStreamingOptions = new MediaStreamingOptions(new Uri(transportUrl), MediaStreamingTransport.Websocket, MediaStreamingContent.Audio, MediaStreamingAudioChannel.Mixed); + var options = new AnswerCallOptions(incomingCallEventData.IncomingCallContext, callbackUri) { @@ -96,8 +98,16 @@ TranscriptionOptions = transcriptionOptions }; + + var options1 = new AnswerCallOptions(incomingCallEventData.IncomingCallContext, callbackUri) + { + CallIntelligenceOptions = new CallIntelligenceOptions() { CognitiveServicesEndpoint = new Uri(cognitiveServicesEndpoint) }, + MediaStreamingOptions = mediaStreamingOptions + }; + AnswerCallResult answerCallResult = await client.AnswerCallAsync(options); var callConnectionMedia = answerCallResult.CallConnection.GetCallMedia(); + /* Use EventProcessor to process CallConnected event */ var answer_result = await answerCallResult.WaitForEventProcessorAsync(); @@ -112,11 +122,8 @@ recordingId = recordingResult.Value.RecordingId; logger.LogInformation($"Recording started. RecordingId: {recordingId}"); - /* Start the Transcription */ - await InitiateTranscription(callConnectionMedia); - logger.LogInformation("Transcription initiated."); - await PauseOrStopTranscriptionAndRecording(callConnectionMedia, logger, false, recordingId); + // await PauseOrStopTranscriptionAndRecording(callConnectionMedia, logger, false, recordingId); /* Play hello prompt to user */ await HandleDtmfRecognizeAsync(callConnectionMedia, callerId, helpIVRPrompt, "hellocontext"); @@ -128,16 +135,24 @@ if (playCompletedEvent.OperationContext == addAgentContext) { + // Add Agent - var callInvite = new CallInvite(new PhoneNumberIdentifier(agentPhoneNumber), - new PhoneNumberIdentifier(acsPhoneNumber)); + var callInvite = new CallInvite(new MicrosoftTeamsUserIdentifier("d4397adf-fa9d-4797-8f68-1e5d43ab58b8")) { }; + + var addParticipantResult = await answerCallResult.CallConnection.AddParticipantAsync(callInvite); + logger.LogInformation($"Adding agent to the call: {addParticipantResult.Value?.InvitationId}"); + + + //// Add Agent + //var callInvite = new CallInvite(new PhoneNumberIdentifier(agentPhoneNumber), + // new PhoneNumberIdentifier(acsPhoneNumber)); - var addParticipantOptions = new AddParticipantOptions(callInvite) - { - OperationContext = addAgentContext - }; + //var addParticipantOptions = new AddParticipantOptions(callInvite) + //{ + // OperationContext = addAgentContext + //}; - var addParticipantResult = await answerCallResult.CallConnection.AddParticipantAsync(addParticipantOptions); + //var addParticipantResult = await answerCallResult.CallConnection.AddParticipantAsync(addParticipantOptions); logger.LogInformation($"Adding agent to the call: {addParticipantResult.Value?.InvitationId}"); } else if (playCompletedEvent.OperationContext == goodbyeContext || playCompletedEvent.OperationContext == addParticipantFailureContext) @@ -160,7 +175,7 @@ Match match = regex.Match(tones); if (match.Success) { - await ResumeTranscriptionAndRecording(callConnectionMedia, logger, recordingId); + // await ResumeTranscriptionAndRecording(callConnectionMedia, logger, recordingId); await HandlePlayAsync(callConnectionMedia, addAgentPrompt, addAgentContext); } else @@ -173,6 +188,11 @@ answerCallResult.CallConnection.CallConnectionId, async (addParticipantSucceededEvent) => { logger.LogInformation($"Received call event: {addParticipantSucceededEvent.GetType()}, context: {addParticipantSucceededEvent.OperationContext}"); + + /* Start the Transcription */ + await InitiateTranscription(callConnectionMedia); + logger.LogInformation("Transcription initiated."); + }); client.GetEventProcessor().AttachOngoingEventProcessor( answerCallResult.CallConnection.CallConnectionId, async (participantsUpdatedEvent) => @@ -240,6 +260,16 @@ logger.LogInformation($"Received transcription event: {TranscriptionFailed.GetType()}, CorrelationId: {TranscriptionFailed.CorrelationId}, " + $"SubCode: {TranscriptionFailed?.ResultInformation?.SubCode}, Message: {TranscriptionFailed?.ResultInformation?.Message}"); }); + + client.GetEventProcessor().AttachOngoingEventProcessor( + answerCallResult.CallConnection.CallConnectionId, async (CallDisconnected) => + { + logger.LogInformation($"Received transcription event: {CallDisconnected.GetType()}, CorrelationId: {CallDisconnected.CorrelationId}, " + + $"SubCode: {CallDisconnected?.ResultInformation?.SubCode}, Message: {CallDisconnected?.ResultInformation?.Message}"); + await client.GetCallRecording().StopAsync(recordingId); + logger.LogInformation("Transcription stopped."); + + }); } } return Results.Ok(); @@ -288,6 +318,13 @@ return Results.Ok(); }); + +app.MapGet("/stopRecording", (ILogger logger) => +{ + client.GetCallRecording().Stop(recordingId); + return Results.Ok(); +}); + if (app.Environment.IsDevelopment()) { app.UseSwagger(); @@ -379,6 +416,7 @@ async Task InitiateTranscription(CallMedia callConnectionMedia) { using var webSocket = await context.WebSockets.AcceptWebSocketAsync(); await Helper.ProcessRequest(webSocket); + //await Helper.Echo(webSocket); } else { diff --git a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/appsettings.json b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/appsettings.json index d897420b..29411441 100644 --- a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/appsettings.json +++ b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/appsettings.json @@ -6,11 +6,11 @@ } }, "AllowedHosts": "*", - "CallbackUriHost": "MY_NGROK_TUNNEL_URI", - "CognitiveServiceEndpoint": "MY_COGNITIVE_SERVICES_ENDPOINT", - "AcsConnectionString": "MY_ACS_CONNECTION_STRING", - "AcsPhoneNumber": "ACS_PHONE_NUMBER", - "TransportUrl": "MY_TRANSPORT_URL", - "Locale": "TRANSCRIPTION_LOCALE", + "CallbackUriHost": "https://b2be-2001-4898-a000-2a-c923-b49e-36cb-1302.ngrok-free.app", + "CognitiveServiceEndpoint": "https://acsairesource.cognitiveservices.azure.com/", + "AcsConnectionString": "endpoint=https://acscallingrtc.canada.communication.azure.com/;accesskey=4/NPNeEVGe4WBukcuwn1xTWfJxAeyZFqT/BbN+Zskstj698c8yXSNity9n79nXpRezoM+C11baBRxLEf/rOiCQ==", + "AcsPhoneNumber": "+14253862821", + "TransportUrl": "wss://6c1f-2001-4898-a000-2a-c923-b49e-36cb-1302.ngrok-free.app/ws", + "Locale": "en-US", "AgentPhoneNumber": "AGENT_PHONE_NUMBER" } From 36b7d7aa123ea5aa83600f89565022647b1d2b31 Mon Sep 17 00:00:00 2001 From: Liben Wei Date: Fri, 18 Oct 2024 09:13:47 -0700 Subject: [PATCH 2/2] Update CallAutomationLiveTranscription Helper class websocket to continue receive data from while loop, so that transcript data will show out from console --- .../CallAutomationLiveTranscription/Helper.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs index 0945ea44..fe48cc52 100644 --- a/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs +++ b/CallAutomation_CallLiveTranscription/CallAutomationLiveTranscription/Helper.cs @@ -17,8 +17,7 @@ public static async Task ProcessRequest(WebSocket webSocket) while (webSocket.State == WebSocketState.Open || webSocket.State == WebSocketState.CloseSent) { var buffer = new byte[1024 * 4]; - var cancellationToken = new CancellationTokenSource(TimeSpan.FromSeconds(60)).Token; - WebSocketReceiveResult receiveResult = await webSocket.ReceiveAsync(new ArraySegment(buffer), cancellationToken); + WebSocketReceiveResult receiveResult = await webSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); while (!receiveResult.CloseStatus.HasValue) { @@ -63,6 +62,9 @@ await webSocket.SendAsync( receiveResult.MessageType, receiveResult.EndOfMessage, CancellationToken.None); + + receiveResult = await webSocket.ReceiveAsync( + new ArraySegment(buffer), CancellationToken.None); } await webSocket.CloseAsync(receiveResult.CloseStatus.Value, receiveResult.CloseStatusDescription, CancellationToken.None);