[Modify] some fixes & optimize memory & optimize online (#26)

1. fix some errors in cyberdog_audio
2. optimize memory using in cyberdog_audio while recording and playing
3. update audio online work mode

Signed-off-by: northdk <[email protected]>

Author: northdk

CHANGELOG.md

@@ -4,14 +4,16 @@
 
 Xiaomi CyberDog's first release.
 
-## 1.0.1 (2021-09-16)
+### 1.0.0.1 (2021-09-16)
 
 Move the repositories to GitHub and replace the athena keyword with cyberdog in batches.
 
-## 1.0.2 (2021-09-27)
+### 1.0.0.2 (2021-09-27)
 
 Add grpc_vendor, lcm_vendor, mpg123_vendor, sdl2main_vendor, sdl2mixer_vendor and opencv_vendor to constrain the building environment.
 
-## 1.0.3 (2021-10-14)
+### 1.0.0.3 (2021-10-14)
 
-Remove opencv_vendor, combine sdl2main_vendor & sdl2mixer_vendor into sdl2_vendor. Remove vision pkgs.
+- Remove opencv_vendor, combine sdl2main_vendor & sdl2mixer_vendor into sdl2_vendor. Remove vision pkgs.
+- Add behaviortreecppv3_vendor
+- Optimize audio pkgs

cyberdog_interaction/cyberdog_audio/audio_assistant/include/audio_assistant/combiner.hpp

@@ -17,10 +17,13 @@
 
 #include <stdlib.h>
 #include <cstdlib>
+#include <memory>
 
 #include "sys/stat.h"
 #include "sys/types.h"
 
+#include "audio_base/audio_player.hpp"
+#include "audio_base/audio_queue.hpp"
 
 #ifdef __cplusplus
 extern "C"
@@ -30,6 +33,9 @@ extern "C"
 #ifdef __cplusplus
 }
 #endif  // __cplusplus
+
+#define DELAY_TIMES 10
+
 extern bool player_end;
 
 void recorder_work_loop(void);

cyberdog_interaction/cyberdog_audio/audio_assistant/src/ai_keyword.cpp

@@ -19,6 +19,7 @@
 #include "rclcpp/rclcpp.hpp"
 #include "xiaoai_sdk/vpm/vpm_sdk.h"
 #include "audio_base/debug/ai_debugger.hpp"
+#include "audio_base/audio_queue.hpp"
 #include "audio_assistant/mp3decoder.hpp"
 
 #define VPM_CONFIG_FILE_PATH "/opt/ros2/cyberdog/ai_conf/xaudio_engine.conf"
@@ -36,10 +37,9 @@ typedef struct
 {
   unsigned char * start;
   unsigned int length;
-  std::mutex mutex;
 } vpm_input_buf_t;
 
-extern vpm_input_buf_t vpm_input_buffer;
+extern athena_audio::MsgQueue<vpm_input_buf_t> vpm_msg_queue;
 extern bool vpm_buf_ready;
 extern void ai_nativeasr_data_handler(asr_msg_type_t msg, unsigned char * buf, int buf_size);
 void ai_vpm_work_loop();
@@ -154,9 +154,9 @@ static void vpm_wakeup_data_handle(
     return;
   }
 
+  // close interrupt while audio playing
   // std::shared_ptr<mp3decoder> mp3decoder_ptr = mp3decoder::GetInstance();
-  // if ( mp3decoder_ptr->tts_player_is_playing() == true )
-  // {
+  // if ( mp3decoder_ptr->tts_player_is_playing() == true ) {
   // aivs_set_interrupt();
   // tts_stream_flush();
   // mp3decoder_ptr->tts_player_stop_now();
@@ -341,6 +341,7 @@ static int setParas(int type, int value)
 
 void ai_vpm_work_loop()
 {
+  fprintf(stdout, "vc: ai vp work loop on call\n");
   for (;; ) {
     if (getAiWakeupRunStatus() == false) {
       printf("vc:vpm: getAiWakeupRunStatus shitch off\n");
@@ -352,12 +353,12 @@ void ai_vpm_work_loop()
     if (vpm_buf_ready == false) {
       continue;
     }
-    // will check here for open/close input to wakeup sdk
-    vpm_input_buffer.mutex.lock();
-    input.raw = reinterpret_cast<void *>(vpm_input_buffer.start);
-    input.size = vpm_input_buffer.length;
+
+    vpm_input_buf_t vpm_data;
+    vpm_msg_queue.DeQueue(vpm_data);
+    input.raw = reinterpret_cast<void *>(vpm_data.start);
+    input.size = vpm_data.length;
     vpm_process(&input);
-    vpm_input_buffer.mutex.unlock();
   }
 
   /*release vpm*/

cyberdog_interaction/cyberdog_audio/audio_assistant/src/ai_online/AivsMain.cpp

@@ -50,6 +50,7 @@
 #include "xiaoai_sdk/aivs/ClientLoggerHooker.h"
 #include "xiaoai_sdk/audio_config/config.h"
 #include "audio_assistant/audio_assitant.hpp"
+#include "audio_base/audio_queue.hpp"
 #include "audio_assistant/mp3decoder.hpp"
 
 #include "xiaoai_sdk/aivs/RobotController.h"
@@ -96,6 +97,13 @@ typedef struct tts_info
   tts_type_t type;
   unsigned char * address;
   unsigned int length;
+  void free_mem()
+  {
+    if (address != nullptr) {
+      fprintf(stdout, "vc: free ttsinfo memory\n");
+      free(address);
+    }
+  }
 } tts_info_t;
 
 typedef struct tts_queue
@@ -105,6 +113,8 @@ typedef struct tts_queue
 } tts_queue_t;
 static tts_queue_t mMsgQueueTts;
 
+static athena_audio::MsgQueue<tts_info_t> TTSMsgQueue;
+
 bool getAivsTtsRunStatus(void)
 {
   return mAivsTtsRunStatus;
@@ -139,12 +149,7 @@ int tts_player_decode_paly(void);
 
 int tts_stream_flush(void)
 {
-  // will check here
-  std::cout << "vc: befor clear, queue.size " << mMsgQueueTts.queue.size() << std::endl;
-  mMsgQueueTts.mutex.lock();    // queue lock
-  mMsgQueueTts.queue.clear();
-  mMsgQueueTts.mutex.unlock();    // queue unlock
-  std::cout << "vc: befor clear, queue.size " << mMsgQueueTts.queue.size() << std::endl;
+  TTSMsgQueue.Clear();
   return 0;
 }
 
@@ -159,47 +164,38 @@ int aivs_set_interrupt(void)
   return 0;
 }
 
-int ai_push_msg_playback(tts_info_t ttsInfo)
+int ai_push_msg_playback(const tts_info_t & ttsInfo)
 {
-  std::cout << "vc: enter ai_push_msg_playback()" << std::endl;
-  tts_info_t ttsInfoPlayback = {TTS_TYPE_INVALID, NULL, 0};
-  tts_info_t ttsInfoPlayStart = {TTS_TYPE_INVALID, NULL, 0};
-  tts_info_t ttsInfoPlayStop = {TTS_TYPE_INVALID, NULL, 0};
+  std::cout << "vc: enter ai_push_msg_playback(), type: " << ttsInfo.type << std::endl;
+  tts_info_t ttsInfoPlay = {ttsInfo.type, NULL, ttsInfo.length};
 
   switch (ttsInfo.type) {
     case TTS_TYPE_STREAM:
       std::cout << "vc: TTS_TYPE_STREAM Case !!" << std::endl;
 
       if (ttsInfo.address == NULL || ttsInfo.length == 0) {
-        std::cout << "vc: ai_push_msg_playback invalid paramter !!!" << std::endl;
+        std::cout << "vc: ai_push_msg_playback invalid paramter, stream addr is null !!!" <<
+          std::endl;
         return -1;
       } else {
-        mMsgQueueTts.mutex.lock();    // queue lock
-        ttsInfoPlayback.type = ttsInfo.type;
-        ttsInfoPlayback.length = ttsInfo.length;
-        ttsInfoPlayback.address = (unsigned char *)malloc(ttsInfo.length);
-        if (ttsInfoPlayback.address != NULL) {
-          memcpy(ttsInfoPlayback.address, ttsInfo.address, ttsInfoPlayback.length);
-          mMsgQueueTts.queue.push_front(ttsInfoPlayback);
+        ttsInfoPlay.address = (unsigned char *)malloc(ttsInfo.length);
+        if (ttsInfoPlay.address != NULL) {
+          memcpy(ttsInfoPlay.address, ttsInfo.address, ttsInfoPlay.length);
+          TTSMsgQueue.EnQueue(ttsInfoPlay);
+        } else {
+          std::cout << "vc: ai_push_msg_playback failed, cannot allocate memory!" << std::endl;
         }
-        mMsgQueueTts.mutex.unlock();    // queue unlock
       }
       break;
 
     case TTS_TYPE_SYNC_START:
       std::cout << "vc: TTS_TYPE_SYNC_START Case !!" << std::endl;
-      mMsgQueueTts.mutex.lock();  // queue lock
-      ttsInfoPlayStart.type = ttsInfo.type;
-      mMsgQueueTts.queue.push_front(ttsInfoPlayStart);
-      mMsgQueueTts.mutex.unlock();  // queue unlock
+      TTSMsgQueue.EnQueue(ttsInfoPlay);
       break;
 
     case TTS_TYPE_SYNC_STOP:
       std::cout << "vc: TTS_TYPE_SYNC_STOP Case !!" << std::endl;
-      mMsgQueueTts.mutex.lock();  // queue lock
-      ttsInfoPlayStop.type = ttsInfo.type;
-      mMsgQueueTts.queue.push_front(ttsInfoPlayStop);
-      mMsgQueueTts.mutex.unlock();  // queue unlock
+      TTSMsgQueue.EnQueue(ttsInfoPlay);
       break;
 
     default:
@@ -222,51 +218,46 @@ int aivsTtsHandler(void)
       continue;
     }
 
-    mMsgQueueTts.mutex.lock();    // queue lock
-    if (!mMsgQueueTts.queue.empty()) {
-      ttsInfoPlayback = mMsgQueueTts.queue.back();
-      mMsgQueueTts.queue.pop_back();
-      mMsgQueueTts.mutex.unlock();      // queue unlock
-      switch (ttsInfoPlayback.type) {
-        case TTS_TYPE_STREAM:
-          if (ttsInfoPlayback.address == NULL || ttsInfoPlayback.length == 0) {
-            usleep(1000 * 100);
-            continue;
-          } else {
-            std::cout << "vc: Process TTS_TYPE_STREAM " << std::endl;
-            cyberdog_audio::mp3decoder::GetInstance()->tts_player_accumulate(
-              ttsInfoPlayback.address,
-              ttsInfoPlayback.length);
-            free(ttsInfoPlayback.address);
-          }
-          break;
+    if (!TTSMsgQueue.DeQueue(ttsInfoPlayback)) {
+      std::cout << "vc: aivsTtsHandler get msg with empty, skip once!" << std::endl;
+      continue;
+    }
 
-        case TTS_TYPE_SYNC_START:
-          std::cout << "vc: Process TTS_TYPE_SYNC_START " << std::endl;
-          cyberdog_audio::mp3decoder::GetInstance()->tts_player_init();
-          ai_push_msg(101);      /*led control for start-dialog status*/
-          break;
+    switch (ttsInfoPlayback.type) {
+      case TTS_TYPE_STREAM:
+        if (ttsInfoPlayback.address == NULL || ttsInfoPlayback.length == 0) {
+          usleep(1000 * 100);
+          continue;
+        } else {
+          std::cout << "vc: Process TTS_TYPE_STREAM " << std::endl;
+          cyberdog_audio::mp3decoder::GetInstance()->tts_player_accumulate(
+            ttsInfoPlayback.address,
+            ttsInfoPlayback.length);
+          free(ttsInfoPlayback.address);
+        }
+        break;
 
-        case TTS_TYPE_SYNC_STOP:
-          std::cout << "vc: Process TTS_TYPE_SYNC_STOP " << std::endl;
+      case TTS_TYPE_SYNC_START:
+        std::cout << "vc: Process TTS_TYPE_SYNC_START " << std::endl;
+        cyberdog_audio::mp3decoder::GetInstance()->tts_player_init();
+        ai_push_msg(101);      /*led control for start-dialog status*/
+        break;
 
-          if (cyberdog_audio::mp3decoder::GetInstance() == nullptr) {
-            fprintf(stderr, "Process TTS_TYPE_SYNC_STOP Error with invalid pointer!\n");
-            break;
-          }
+      case TTS_TYPE_SYNC_STOP:
+        std::cout << "vc: Process TTS_TYPE_SYNC_STOP " << std::endl;
 
-          cyberdog_audio::mp3decoder::GetInstance()->tts_player_decode_paly();
-          // ai_push_msg(104);/*led control for end -dialog status*/
+        if (cyberdog_audio::mp3decoder::GetInstance() == nullptr) {
+          fprintf(stderr, "Process TTS_TYPE_SYNC_STOP Error with invalid pointer!\n");
           break;
+        }
 
-        default:
-          std::cout << "vc: Process default case !!" << std::endl;
-          break;
-      }
-    } else {
-      mMsgQueueTts.mutex.unlock();      // queue unlock
-      usleep(1000 * 100);
-      continue;
+        cyberdog_audio::mp3decoder::GetInstance()->tts_player_decode_paly();
+        // ai_push_msg(104);/*led control for end -dialog status*/
+        break;
+
+      default:
+        std::cout << "vc: Process default case !!" << std::endl;
+        break;
     }
   }
 }
@@ -778,6 +769,8 @@ void initAivsDeviceOAuth()
   clientInfo->setDeviceId(DEVICE_OAUTH_DEVICE_ID);
 
   auto config = getAudioConfig();
+  config->putBoolean(aivs::AivsConfig::Auth::REQ_TOKEN_HYBRID, true);
+  config->putBoolean(aivs::AivsConfig::Auth::REQ_TOKEN_BY_SDK, true);
   gEngine = aivs::Engine::create(config, clientInfo, aivs::Engine::ENGINE_AUTH_DEVICE_OAUTH);
 }
 

cyberdog_interaction/cyberdog_audio/audio_assistant/src/ai_online/StorageCapabilityImpl.cpp

@@ -23,11 +23,12 @@
 
 StorageCapabilityImpl::StorageCapabilityImpl()
 {
-#ifndef __NuttX__
+/* #ifndef __NuttX__
   storageFilePath = "data.json";
 #else
   storageFilePath = "/tmp/data.json";
-#endif
+#endif */
+  storageFilePath = "/opt/ros2/cyberdog/data/token.json";
   loadKeyValuesFromFile();
 }