{Feature} Properly support random access for encoded audio signals.

Summary:
This diff adds a "proper" solution to deal with random accessing Audio with Opus encoding.

The core idea is, every time when `readRecordByIndex()` is called for Opus Audio stream, the index is compared with the previously-read index:
1. If new_index == prev_index, no need to pre-roll. Audio Player will simply return the cached data.
2. If new_index == prev_index + 1, no need to pre-roll. Just do normal decoding.
3. Else, pre-roll is needed. 100ms is pre-fetched before this index.

Reviewed By: kongchen1992

Differential Revision: D83426928

fbshipit-source-id: 93e312e381978fef58a9c38b0fc4251d0811e9bc

Author: YLouWashU

core/data_provider/RecordReaderInterface.cpp

@@ -231,6 +231,9 @@ RecordReaderInterface::RecordReaderInterface(
   }
   fileTags_ = reader_->getTags();
   vrsMetadata_ = getMetadata();
+
+  // Initialize Opus detection cache
+  initializeOpusDetection();
 }
 
 std::set<vrs::StreamId> RecordReaderInterface::getStreamIds() const {
@@ -327,15 +330,37 @@ const vrs::IndexRecord::RecordInfo* RecordReaderInterface::readRecordByIndex(
     const int index) {
   std::lock_guard<std::mutex> lockGuard(*readerMutex_);
 
+  // Check bounds first to prevent invalid pre-roll attempts
   if (index < 0 || index >= reader_->getRecordCount(streamId, vrs::Record::Type::DATA)) {
     return nullptr;
   }
+
+  // Handle Opus audio pre-roll before reading the target record
+  if (getSensorDataType(streamId) == SensorDataType::Audio && isOpusAudioStream(streamId) &&
+      needsOpusPreroll(streamId, index)) {
+    performOpusPreroll(streamId, index);
+  }
   const vrs::IndexRecord::RecordInfo* recordInfo =
       reader_->getRecord(streamId, vrs::Record::Type::DATA, static_cast<uint32_t>(index));
   checkAndThrow(
       recordInfo, fmt::format("getRecord failed for {}, index {}", streamId.getName(), index));
   const int errorCode = reader_->readRecord(*recordInfo);
   streamIdToLastReadRecord_[streamId] = recordInfo;
+
+  // Update tracking for audio streams after successful read
+  if (getSensorDataType(streamId) == SensorDataType::Audio) {
+    audioStreamLastReadIndex_[streamId] = index;
+
+    // Update Opus detection cache after first successful read
+    auto audioPlayerIt = audioPlayers_.find(streamId);
+    if (audioPlayerIt != audioPlayers_.end()) {
+      vrs::AudioFormat detectedFormat = audioPlayerIt->second->getDetectedAudioFormat();
+      if (detectedFormat != vrs::AudioFormat::UNDEFINED) {
+        updateOpusDetection(streamId, detectedFormat == vrs::AudioFormat::OPUS);
+      }
+    }
+  }
+
   if (errorCode != 0) {
     XR_LOGE(
         "Fail to read record {} from streamId {} with code {}",
@@ -684,4 +709,62 @@ OnDeviceHandPoseData RecordReaderInterface::getLastCachedHandPoseData(
   return handPoseData;
 }
 
+bool RecordReaderInterface::needsOpusPreroll(const vrs::StreamId& streamId, int targetIndex) {
+  auto lastIndexIt = audioStreamLastReadIndex_.find(streamId);
+
+  if (lastIndexIt == audioStreamLastReadIndex_.end()) {
+    // First read for this stream - need pre-roll if not starting from index 0
+    return targetIndex > 0;
+  }
+
+  int lastIndex = lastIndexIt->second;
+  // Need pre-roll if not sequential (not lastIndex + 1) and not same index
+  return (targetIndex != lastIndex + 1 && targetIndex != lastIndex);
+}
+
+void RecordReaderInterface::performOpusPreroll(const vrs::StreamId& streamId, int targetIndex) {
+  // Reset Opus decoder first
+  resetOpusDecoder(streamId);
+
+  // Read previous frames for pre-roll (but don't update lastReadIndex yet)
+  int prerollStart = std::max(0, targetIndex - kAudioDecodingPrerollLength);
+  for (int prerollIndex = prerollStart; prerollIndex < targetIndex; ++prerollIndex) {
+    const vrs::IndexRecord::RecordInfo* prerollRecord =
+        reader_->getRecord(streamId, vrs::Record::Type::DATA, static_cast<uint32_t>(prerollIndex));
+    if (prerollRecord) {
+      reader_->readRecord(*prerollRecord);
+      // Audio data is processed by the player but we don't cache or return it
+    }
+  }
+}
+
+bool RecordReaderInterface::isOpusAudioStream(const vrs::StreamId& streamId) {
+  // Use cached Opus detection result
+  auto it = audioStreamIsOpus_.find(streamId);
+  return it != audioStreamIsOpus_.end() && it->second;
+}
+
+void RecordReaderInterface::initializeOpusDetection() {
+  // Initialize all audio streams as undetermined (will be detected lazily on first read)
+  for (const auto& [streamId, audioPlayer] : audioPlayers_) {
+    // Initialize as undetermined - will be detected during first actual read
+    // when the AudioPlayer's onAudioRead() method can access the content block format
+    audioStreamIsOpus_[streamId] = false;
+  }
+}
+
+void RecordReaderInterface::updateOpusDetection(const vrs::StreamId& streamId, bool isOpus) {
+  audioStreamIsOpus_[streamId] = isOpus;
+}
+
+void RecordReaderInterface::resetOpusDecoder(const vrs::StreamId& streamId) {
+  if (audioPlayers_.find(streamId) != audioPlayers_.end()) {
+    audioPlayers_[streamId]->resetOpusDecoder();
+  } else {
+    fmt::print(
+        "Warning: streamId {} not found in audioPlayers_, Opus decoder cannot be reset\n",
+        streamId.getNumericName());
+  }
+}
+
 } // namespace projectaria::tools::data_provider

core/data_provider/RecordReaderInterface.h

@@ -105,7 +105,36 @@ class RecordReaderInterface {
 
   [[nodiscard]] std::optional<MetadataTimeSyncMode> getTimeSyncMode() const;
 
+  // Explicit function to reset AudioPlayer's Opus decoder. This is needed to support random access
+  // for Audio data
+  void resetOpusDecoder(const vrs::StreamId& streamId);
+
  private:
+  // Parameter to control the "pre-roll" length for encoded audio signal.
+  // 5 samples stands for approx 100ms, which is enough for pre-roll
+  static constexpr int kAudioDecodingPrerollLength = 5;
+
+  // Track last read index per audio stream for Opus pre-roll detection
+  std::map<vrs::StreamId, int> audioStreamLastReadIndex_;
+
+  // Cache which audio streams use Opus encoding (determined during initialization)
+  std::map<vrs::StreamId, bool> audioStreamIsOpus_;
+
+  // Helper method to determine if audio stream needs Opus pre-roll
+  bool needsOpusPreroll(const vrs::StreamId& streamId, int targetIndex);
+
+  // Helper method to perform Opus pre-roll
+  void performOpusPreroll(const vrs::StreamId& streamId, int targetIndex);
+
+  // Check if audio stream uses Opus encoding (cached lookup)
+  bool isOpusAudioStream(const vrs::StreamId& streamId);
+
+  // Initialize Opus detection cache during construction
+  void initializeOpusDetection();
+
+  // Update Opus detection cache for a specific stream
+  void updateOpusDetection(const vrs::StreamId& streamId, bool isOpus);
+
   std::shared_ptr<vrs::MultiRecordFileReader> reader_;
 
   calibration::DeviceVersion deviceVersion_;

core/data_provider/VrsDataProvider.cpp

@@ -526,6 +526,12 @@ SensorData VrsDataProvider::getSensorDataByTimeNs(
     const TimeDomain& timeDomain,
     const TimeQueryOptions& timeQueryOptions) {
   const int index = getIndexByTimeNs(streamId, timeNs, timeDomain, timeQueryOptions);
+
+  // Check for out-of-bounds index and return NotValid SensorData
+  if (index < 0) {
+    return SensorData(streamId, std::monostate{}, SensorDataType::NotValid, -1, {});
+  }
+
   return getSensorDataByIndex(streamId, index);
 }
 

core/data_provider/players/AudioPlayer.cpp

@@ -62,6 +62,11 @@ bool AudioPlayer::onAudioRead(
     const vrs::ContentBlock& cb) {
   const auto& audioSpec = cb.audio();
 
+  // Detect and store audio format on first read
+  if (detectedAudioFormat_ == vrs::AudioFormat::UNDEFINED) {
+    detectedAudioFormat_ = audioSpec.getAudioFormat();
+  }
+
   // Read Opus-encoded audio data with decoding
   if (audioSpec.getAudioFormat() == vrs::AudioFormat::OPUS &&
       audioSpec.getSampleFormat() == vrs::AudioSampleFormat::S16_LE) {
@@ -129,12 +134,10 @@ bool AudioPlayer::readAndDecodeAudioData(const vrs::CurrentRecord& r, const vrs:
   }
 
   if (needsReset) {
-    opus_multistream_decoder_destroy(opusDecoder_);
-    opusDecoder_ = nullptr;
-    lastDecodedTimestamp_ = -1.0; // Reset timestamp tracking
+    resetOpusDecoder();
   }
 
-  // Create decoder only if needed (reuse existing decoder when possible)
+  // Initialize decoder if not yet initialized
   if (opusDecoder_ == nullptr) {
     // Separate mono and coupled channels
     uint32_t totalCoupledAudioChannel = kStereoMultiplier * audioSpec.getStereoPairCount();
@@ -224,6 +227,7 @@ bool AudioPlayer::readAndDecodeAudioData(const vrs::CurrentRecord& r, const vrs:
     lastDecodedTimestamp_ = currentTimestamp;
 
     callback_(data_, dataRecord_, configRecord_, verbose_);
+
     return true;
   } else {
     // Error: result contains error code
@@ -232,4 +236,13 @@ bool AudioPlayer::readAndDecodeAudioData(const vrs::CurrentRecord& r, const vrs:
   }
 }
 
+void AudioPlayer::resetOpusDecoder() {
+  // Destroy existing decoder
+  if (opusDecoder_ != nullptr) {
+    opus_multistream_decoder_destroy(opusDecoder_);
+    opusDecoder_ = nullptr;
+    lastDecodedTimestamp_ = -1.0; // Reset timestamp tracking
+  }
+}
+
 } // namespace projectaria::tools::data_provider

core/data_provider/players/AudioPlayer.h

@@ -105,6 +105,15 @@ class AudioPlayer : public vrs::RecordFormatStreamPlayer {
     verbose_ = verbose;
   }
 
+  // Get the detected audio format (available after first read)
+  vrs::AudioFormat getDetectedAudioFormat() const {
+    return detectedAudioFormat_;
+  }
+
+  // A function to explicitly reset the Opus decoder. Note that this DOES NOT initialize the
+  // decoder!
+  void resetOpusDecoder();
+
  protected:
   bool onDataLayoutRead(const vrs::CurrentRecord& r, size_t blockIndex, vrs::DataLayout& dl)
       override;
@@ -133,6 +142,9 @@ class AudioPlayer : public vrs::RecordFormatStreamPlayer {
   OpusMSDecoder* opusDecoder_ = nullptr;
   vrs::AudioContentBlockSpec lastDecoderSpec_; // Track spec compatibility
   double lastDecodedTimestamp_ = -1.0; // Track last decoded timestamp for random access detection
+
+  // Track detected audio format (set on first audio read)
+  vrs::AudioFormat detectedAudioFormat_ = vrs::AudioFormat::UNDEFINED;
 };
 
 } // namespace projectaria::tools::data_provider