Fix: task export, data upload status, overlap

Author: ashish7515

cvat/apps/dataset_manager/task.py

@@ -13,7 +13,7 @@
 from pydub import AudioSegment
 import numpy as np
 import os
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from copy import deepcopy
 from enum import Enum
 from tempfile import TemporaryDirectory
@@ -1359,6 +1359,30 @@ def export_audino_task(task_id, dst_file, format_name, server_url=None, save_ima
     task_jobs = task.db_jobs
     audio_files_mapping = sorted(os.listdir(os.path.join(task.db_task.data.get_data_dirname(), "raw")), key=str.lower)
 
+    job_duration_without_overlap: list[float] = []
+    for audio_idx in range(len(task.db_task.audio_total_duration)):
+        duration_ms = task.db_task.audio_total_duration[audio_idx]
+        frame_count = task.db_task.total_frames_count[audio_idx]
+        if duration_ms > 0 and frame_count > 0:
+            frames_per_ms = frame_count / duration_ms
+            overlap_ms = task.db_task.overlap / frames_per_ms if frames_per_ms > 0 else 0
+            effective_segment_duration = task.db_task.segment_duration - overlap_ms
+            job_duration_without_overlap.append(effective_segment_duration / 1000.0)
+        else:
+            job_duration_without_overlap.append(0.0)
+
+    job_start_time_mapping: dict[int, float] = {}
+    audio_job_counters: dict[int, int] = defaultdict(int)
+    for job in task_jobs:
+        job_id = job.id
+        audio_index = job.segment.audio_file_index
+        if audio_index is not None and audio_index < len(job_duration_without_overlap):
+            duration_without_overlap = job_duration_without_overlap[audio_index]
+            job_start_time_mapping[job_id] = audio_job_counters[audio_index] * duration_without_overlap
+            audio_job_counters[audio_index] += 1
+        else:
+            job_start_time_mapping[job_id] = 0.0
+
     final_task_data = []
     final_annotation_chunk_paths = []
     with TemporaryDirectory(dir=temp_dir_base) as temp_dir:
@@ -1373,6 +1397,16 @@ def export_audino_task(task_id, dst_file, format_name, server_url=None, save_ima
             # Convert the data into a format
             final_data = convert_annotation_data_format(job, final_data, format_name)
 
+            # Modify start and end for each annotation to make time stamps absolute
+            for annotation in final_data:
+                job_id = annotation.get("job_id")
+                if job_id is not None:
+                    job_time_offset = job_start_time_mapping.get(job_id)
+                    if "start" in annotation:
+                        annotation["start"] += job_time_offset
+                    if "end" in annotation:
+                        annotation["end"] += job_time_offset
+
             final_task_data.append(final_data)
             final_annotation_chunk_paths.append(annotation_audio_chunk_file_paths)
 

cvat/apps/engine/background_operations.py

@@ -793,54 +793,38 @@ def create_gt_job_task(task_id, validated_data):
                             job_segments[index] = used_segments
 
 
-                        def save_concatenated_gt_audio_chunks(time_stamps, upload_dir):
+                        def save_concatenated_gt_audio_chunks(time_stamps: list, output_directory: str):
                             """
-                            Concatenates audio chunks from multiple WAV files in the same directory based on time_stamps into a single WAV file, saving it in the same directory.
-
-                            Args:
-                                time_stamps: A list where odd indices contain durations for each job (in milliseconds).
-                                upload_dir: The directory containing the input WAV files (0.wav, 1.wav, etc.) and where the output will be saved.
+                            Efficiently concatenates MP3 audio chunks into a single MP3, minimizing RAM usage
+                            by appending incrementally. Assumes input files are named '{job_index}.mp3'.
                             """
+                            number_of_jobs = len(time_stamps) // 2
+                            output_file_path = os.path.join(output_directory, f"{number_of_jobs}.mp3")
 
-                            num_jobs = len(time_stamps) // 2
-                            output_path = os.path.join(upload_dir, f"{num_jobs}.wav")
-
-                            # Check if the file with .mp3 exist or .wav
-                            if not os.path.exists(output_path):
-                                output_path = os.path.splitext(output_path)[0] + '.mp3'
-
-                            if os.path.exists(output_path):
-                                os.remove(output_path)
-
-                            concatenated_audio = AudioSegment.empty()
-
-                            for job_index in range(num_jobs):
-                                input_wav_path = os.path.join(upload_dir, f"{job_index}.wav")
-
-                                # Check if the file with .mp3 exist or .wav
-                                if not os.path.exists(input_wav_path):
-                                    input_wav_path = os.path.splitext(input_wav_path)[0] + '.mp3'
+                            if os.path.exists(output_file_path):
+                                os.remove(output_file_path)
 
-                                if not os.path.exists(input_wav_path):
-                                    print(f"Warning: Input file {input_wav_path} not found, skipping job {job_index}.")
+                            output_file_created = False
+                            for job_index in range(number_of_jobs):
+                                input_file_path = os.path.join(output_directory, f"{job_index}.mp3")
+                                if not os.path.exists(input_file_path):
                                     continue
 
                                 duration_ms = time_stamps[job_index * 2 + 1]
-
                                 try:
-                                    audio_segment = AudioSegment.from_file(input_wav_path)
-                                    audio_segment = audio_segment[:duration_ms]
-                                    concatenated_audio += audio_segment
+                                    audio_segment = AudioSegment.from_file(input_file_path)[:duration_ms]
+                                    if not output_file_created:
+                                        audio_segment.export(output_file_path, format="mp3")
+                                        output_file_created = True
+                                    else:
+                                        existing_audio = AudioSegment.from_file(output_file_path)
+                                        combined_audio = existing_audio + audio_segment
+                                        combined_audio.export(output_file_path, format="mp3")
                                 except Exception as e:
-                                    print(f"Error processing {input_wav_path}: {e}")
+                                    print(f"Error processing {input_file_path}: {e}")
 
-                            if len(concatenated_audio) > 0:
-                                try:
-                                    concatenated_audio.export(output_path, format="mp3")
-                                except Exception as e:
-                                    print(f"Error writing {output_path}: {e}")
-                            else:
-                                print("Warning: No valid audio data to concatenate.")
+                            if not output_file_created:
+                                print("Warning: No valid audio segments found for concatenation.")
 
                         save_concatenated_gt_audio_chunks(time_stamps, task.data.get_compressed_cache_dirname())
                     else:

cvat/apps/engine/media_extractors.py

@@ -508,17 +508,6 @@ def _has_frame(self, i):
 
         return False
 
-    def get_total_frames(self):
-        total_frame = 0
-        with self._get_av_container() as container:
-            stream = container.streams.audio[0]
-            stream.thread_type = 'AUTO'
-            for packet in container.demux(stream):
-                for _ in packet.decode():
-                    total_frame += 1
-
-        return total_frame
-
     def get_file_encoding(self, file_path):
 
         with open(file_path, 'rb') as f:
@@ -539,8 +528,8 @@ def __iter__(self):
                     if self._has_frame(frame_num - 1):
                         yield (image, self._source_path[0], image.pts)
 
-    def get_progress(self, pos):
-        duration = self._get_duration()
+    def get_progress(self, pos, audio_idx=0):
+        duration = self._get_duration(audio_idx)
         return pos / duration if duration else None
 
     def _get_av_container(self, file_path_idx = 0):
@@ -553,8 +542,8 @@ def _get_av_container(self, file_path_idx = 0):
         else:
             return av.open(self._source_path[file_path_idx])
 
-    def _get_duration(self):
-        with self._get_av_container() as container:
+    def _get_duration(self, audio_idx=0):
+        with self._get_av_container(audio_idx) as container:
             stream = container.streams.audio[0]
             duration = None
             if stream.duration:

cvat/apps/engine/task.py

@@ -1070,7 +1070,7 @@ def get_audio_duration_and_frame_count(file_path):
             db_task.audio_total_duration.append(duration_milliseconds)
             db_task.total_frames_count.append(total_frame)
 
-        db_task.overlap = db_task.overlap if db_task.overlap is not None else 0 # we want to hardcode overlap for audio
+        db_task.overlap = db_task.overlap if db_task.overlap else 191 # we want to hardcode overlap for audio (191 frames approx 5 sec)
 
         # Default chunk size = entire frames
         db_data.chunk_size = db_task.segment_size #db_task.data.size
@@ -1243,11 +1243,11 @@ def generate_chunks_with_overlap(
                             chunk.append((image, file_path, image.pts))
 
                             if frame_num == frame_count:
-                                yield chunk_idx, chunk
+                                yield chunk_idx, idx, chunk
                                 chunk_idx += 1
                                 chunk = []
                             elif len(chunk) == segment_size:
-                                yield chunk_idx, chunk[:segment_size]
+                                yield chunk_idx, idx, chunk[:segment_size]
                                 chunk_idx += 1
                                 chunk = chunk[segment_size - overlap:]
 
@@ -1264,7 +1264,7 @@ def generate_chunks_with_overlap(
             db_task.overlap,
             db_task.total_jobs_count
         )
-        generator = ((idx, list(chunk_data)) for idx, chunk_data in generator)
+        generator = ((idx, audio_idx, list(chunk_data)) for idx, audio_idx, chunk_data in generator)
 
         def save_chunks(
                 executor: concurrent.futures.ThreadPoolExecutor,
@@ -1296,7 +1296,7 @@ def save_chunks(
             # (path, frame, size)
             return list((i[0][1], i[0][2], i[1]) for i in zip(chunk_data, image_sizes))
 
-        def process_results(img_meta: list[tuple[str, int, tuple[int, int]]]):
+        def process_results(audio_idx: int, img_meta: list[tuple[str, int, tuple[int, int]]]):
             nonlocal db_images, db_data, video_path, video_size
 
             if db_task.mode == 'annotation':
@@ -1312,7 +1312,7 @@ def process_results(img_meta: list[tuple[str, int, tuple[int, int]]]):
                 video_size = img_meta[0][2]
                 video_path = img_meta[0][0]
 
-            progress = extractor.get_progress(img_meta[-1][1])
+            progress = extractor.get_progress(img_meta[-1][1], audio_idx)
             update_progress(progress)
 
         futures = queue.Queue(maxsize=settings.CVAT_CONCURRENT_CHUNK_PROCESSING)
@@ -1322,14 +1322,14 @@ def process_results(img_meta: list[tuple[str, int, tuple[int, int]]]):
             for frames in db_task.total_frames_count:
                 db_data.size += frames
 
-            for chunk_idx, chunk_data in generator:
+            for chunk_idx, audio_idx, chunk_data in generator:
                 slogger.glob.debug("Chunk {} with {} frames".format(chunk_idx, len(chunk_data)))
                 if futures.full():
-                    process_results(futures.get().result())
+                    process_results(audio_idx, futures.get().result())
                 futures.put(executor.submit(save_chunks, executor, chunk_idx, chunk_data))
 
             while not futures.empty():
-                process_results(futures.get().result())
+                process_results(audio_idx, futures.get().result())
 
     if db_task.mode == 'annotation':
         models.Image.objects.bulk_create(db_images)

cvat/apps/quality_control/quality_reports.py

@@ -2208,7 +2208,7 @@ def _match_annotations(self, ds_annotations, gt_annotations):
         gt_annotations = [
             gt_ann
             for gt_ann in gt_annotations
-            if job_start_time <= gt_ann["points"][0] and gt_ann["points"][3] <= job_end_time
+            if (job_start_time - 1.5) <= gt_ann["points"][0] and gt_ann["points"][3] <= (job_end_time + 1.5)
         ]
 
         ds_annotations = [