added support for s3 buckets with azure containers

ai-services/background-music-api/main.py

@@ -3,10 +3,10 @@
 from yt_dlp.utils import DownloadError
 from yt_dlp.extractor import get_info_extractor
 from moviepy.editor import VideoFileClip, AudioFileClip, concatenate_audioclips
+from utils.storage_factory import get_storage_provider
 from spleeter.separator import Separator
 import shutil
 from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
-from azure.storage.blob import BlobServiceClient
 from config import (
     storage_account_key,
     connection_string,
@@ -102,36 +102,26 @@ def utils_add_bg_music(file_path, video_link):
 
 
 def upload_audio_to_azure_blob(file_path, export_type, export):
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
     if export == False:
         AudioSegment.from_wav(file_path + "final.wav").export(
             file_path + "final.flac", format="flac"
         )
-        full_path_audio = file_path + "final.flac"
-        blob_client_audio = blob_service_client.get_blob_client(
-            container=container_name, blob=file_path.split("/")[-1] + ".flac"
-        )
+        local_file_to_upload = file_path + "final.flac"
+        remote_file_name = file_path.split("/")[-1] + ".flac"
     else:
-        full_path_audio = file_path.replace(".flac", "") + "." + export_type
-        blob_client_audio = blob_service_client.get_blob_client(
-            container=container_name,
-            blob=file_path.split("/")[-1].replace(".flac", "") + "." + export_type,
-        )
-    with open(full_path_audio, "rb") as data:
-        try:
-            if not blob_client_audio.exists():
-                blob_client_audio.upload_blob(data)
-                print("Audio uploaded successfully!")
-                print(blob_client_audio.url)
-            else:
-                blob_client_audio.delete_blob()
-                print("Old Audio deleted successfully!")
-                blob_client_audio.upload_blob(data)
-                print("New audio uploaded successfully!")
-        except Exception as e:
-            print("This audio can't be uploaded")
-    return blob_client_audio.url
+        local_file_to_upload = file_path.replace(".flac", "") + "." + export_type
+        remote_file_name = file_path.split("/")[-1].replace(".flac", "") + "." + export_type
 
+    storage = get_storage_provider()
+
+    try:
+        url = storage.upload(local_file_to_upload, remote_file_name)
+        print("Audio uploaded successfully!")
+        print(url)
+        return url
+    except Exception as e:
+        print("This audio can't be uploaded")
+        return None
 
 if __name__ == "__main__":
     os.mkdir("temporary_audio_storage")
@@ -143,16 +133,12 @@ class BGMusicRequest(BaseModel):
 
 
 def download_from_azure_blob(file_path):
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
-    encoded_file_path = file_path.split("/")[-1]
-    encoded_url_path = urllib.parse.unquote(encoded_file_path)
-    blob_client = blob_service_client.get_blob_client(
-        container=container_name, blob=encoded_url_path
-    )
-    with open(file=file_path.split("/")[-1], mode="wb") as sample_blob:
-        download_stream = blob_client.download_blob()
-        sample_blob.write(download_stream.readall())
+    remote_file_path = urllib.parse.unquote(file_path.split("/")[-1])
+    local_destination_path = file_path.split("/")[-1]
+
+    storage = get_storage_provider()
 
+    storage.download(remote_file_path, local_destination_path)
 
 @app.post("/add_background_music")
 async def add_background_music(audio_request: BGMusicRequest):

backend/organization/tasks.py

@@ -10,9 +10,8 @@
     get_org_report_projects_email,
 )
 from users.models import User
-from azure.storage.blob import BlobServiceClient
-from datetime import datetime, timedelta
-from config import storage_account_key, connection_string, reports_container_name
+from utils.storage_factory import get_storage_provider
+from datetime import datetime, timedelta, timezone
 
 
 @shared_task()
@@ -41,28 +40,13 @@ def send_email_with_projects_report(org_id, user_id):
 
 @shared_task(name="delete_reports")
 def delete_reports():
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
-    container_client = blob_service_client.get_container_client(reports_container_name)
+    storage = get_storage_provider(reports_container=True)
+
+    one_week_ago_date = (datetime.now(timezone.utc) - timedelta(days=7)).date()
 
-    current_date = datetime.now()
+    objects = storage.list_objects()
 
-    # Calculate one week ago
-    one_week_ago = current_date - timedelta(days=7)
-
-    # Convert the specific date to UTC format
-    specific_date_utc = datetime.strptime(one_week_ago, "%Y-%m-%d").replace(
-        tzinfo=datetime.timezone.utc
-    )
-
-    # List all blobs in the container
-    blobs = container_client.list_blobs()
-
-    for blob in blobs:
-        properties = blob.get_blob_properties()
-        last_modified = properties["last_modified"].astimezone(datetime.timezone.utc)
-
-        # Check if the blob was created on the specific date
-        if last_modified.date() == specific_date_utc.date():
-            blob_client = container_client.get_blob_client(blob.name)
-            blob_client.delete_blob()
-            print(f"Deleted: {blob.name}")
+    for obj in objects:
+        if obj.last_modified.date() == one_week_ago_date:
+            storage.delete(obj.name)
+            print(f"Deleted: {obj.name}")

backend/organization/utils.py

@@ -27,7 +27,7 @@
 from voiceover.models import VoiceOver
 from video.models import Video
 from task.models import Task
-from azure.storage.blob import BlobServiceClient
+from utils.storage_factory import get_storage_provider
 from config import storage_account_key, connection_string, reports_container_name
 from django.conf import settings
 import logging
@@ -37,27 +37,20 @@
 
 
 def send_mail_with_report(subject, body, user, csv_file_paths):
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
+    storage = get_storage_provider(reports_container=True)
     report_urls = []
 
     for file_path in csv_file_paths:
-        blob_client = blob_service_client.get_blob_client(
-            container=reports_container_name, blob=file_path
-        )
-        with open(file_path, "rb") as data:
-            try:
-                if not blob_client.exists():
-                    blob_client.upload_blob(data)
-                    logging.info("Report uploaded successfully!")
-                    logging.info(blob_client.url)
-                else:
-                    blob_client.delete_blob()
-                    logging.info("Old Report deleted successfully!")
-                    blob_client.upload_blob(data)
-                    logging.info("New Report uploaded successfully!")
-            except Exception as e:
-                logging.info("This report can't be uploaded")
-        report_urls.append(blob_client.url)
+        local_file = file_path
+        remote_file = file_path
+
+        try:
+            url = storage.upload(local_file, remote_file)
+            logging.info("Report uploaded successfully!")
+            logging.info(url)
+            report_urls.append(url)
+        except Exception as e:
+            logging.info("This report can't be uploaded")
 
     if len(report_urls) == 1:
         try:

backend/project/utils.py

@@ -21,12 +21,12 @@
 import logging
 import os
 from translation.metadata import TRANSLATION_LANGUAGE_CHOICES
+from utils.storage_factory import get_storage_provider
 from voiceover.metadata import VOICEOVER_LANGUAGE_CHOICES
 from transcript.models import Transcript
 from translation.models import Translation
 from voiceover.models import VoiceOver
 from video.models import Video
-from azure.storage.blob import BlobServiceClient
 from config import storage_account_key, connection_string, reports_container_name
 from django.conf import settings
 
@@ -227,27 +227,21 @@ def get_reports_for_users(pk, start, end):
 
 
 def send_mail_with_report(subject, body, user, csv_file_paths):
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
+    storage = get_storage_provider(reports_container=True)
     report_urls = []
 
     for file_path in csv_file_paths:
-        blob_client = blob_service_client.get_blob_client(
-            container=reports_container_name, blob=file_path
-        )
-        with open(file_path, "rb") as data:
-            try:
-                if not blob_client.exists():
-                    blob_client.upload_blob(data)
-                    logging.info("Report uploaded successfully!")
-                    logging.info(blob_client.url)
-                else:
-                    blob_client.delete_blob()
-                    logging.info("Old Report deleted successfully!")
-                    blob_client.upload_blob(data)
-                    logging.info("New Report uploaded successfully!")
-            except Exception as e:
-                logging.info("This report can't be uploaded")
-        report_urls.append(blob_client.url)
+        local_file = file_path
+        remote_file = file_path
+
+        try:
+            url = storage.upload(local_file, remote_file)
+            logging.info("Report uploaded successfully!")
+            logging.info(url)
+            report_urls.append(url)
+        except Exception as e:
+            logging.info("This report can't be uploaded")
+
 
     if len(report_urls) == 1:
         try:

backend/transcript/tasks.py

@@ -3,8 +3,6 @@
 from backend.celery import celery_app
 import json
 import logging
-from azure.storage.blob import BlobServiceClient
-import logging
 from config import (
     storage_account_key,
     connection_string,

backend/transcript/utils/ytt_align.py

@@ -5,7 +5,7 @@
 import subprocess
 import json
 from config import align_json_url
-from azure.storage.blob import BlobServiceClient
+from utils.storage_factory import get_storage_provider
 import logging
 from config import (
     storage_account_key,
@@ -54,27 +54,27 @@ def align_json_api(transcript_obj):
 
 
 def download_ytt_from_azure(file_name):
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
-    blob_client = blob_service_client.get_blob_client(
-        container=container_name, blob=file_name
-    )
-    with open(file=file_name, mode="wb") as sample_blob:
-        download_stream = blob_client.download_blob()
-        sample_blob.write(download_stream.readall())
+    storage = get_storage_provider()
+
+    remote_object_name = file_name
+    local_destination_path = file_name
+
+    storage.download(remote_object_name, local_destination_path)
 
 
 def upload_ytt_to_azure(transcript_obj, file_name):
-    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
-    blob_client_json = blob_service_client.get_blob_client(
-        container=container_name, blob=file_name
-    )
-    with open(file_name, "rb") as data:
-        if not blob_client_json.exists():
-            blob_client_json.upload_blob(data)
-            logging.info(blob_client_json.url)
-            transcript_obj.payload["ytt_azure_url"] = blob_client_json.url
-            transcript_obj.save()
-        else:
-            blob_client_json.delete_blob()
-            blob_client_json.upload_blob(data)
-            logging.info(blob_client_json.url)
+    storage = get_storage_provider()
+
+    local_file = file_name
+    remote_file = file_name
+
+    file_exists = storage.exists(remote_file)
+
+    url = storage.upload(local_file, remote_file)
+
+    if not file_exists:
+        logging.info(url)
+        transcript_obj.payload["ytt_azure_url"] = url
+        transcript_obj.save()
+    else:
+        logging.info(url)