feat: Add object storage support for PDF and Image extraction

app/api/pdf_extract.py

@@ -5,21 +5,28 @@
 
 from app.domain import pdf_ocr, fileUpload
 from app.model.TextSchema import TextExtractDocument
+from app.services import storage
 
 router = APIRouter()
 
 @router.post("/extract_pdf", response_model=List[TextExtractDocument])
 async def extract_pdf(
-    input_file: UploadFile = File(..., description="PDF file to process"),
+    input_file: Optional[UploadFile] = File(None, description="PDF file to process (required if source='upload')"),
+    source: str = Form("upload", description="Source of the file: 'upload' or 'object_storage'"),
+    client_id: Optional[str] = Form(None, description="Client ID for object storage (required if source='object_storage')"),
+    object_key: Optional[str] = Form(None, description="Object key (path) in the bucket (required if source='object_storage')"),
     lang: str = Form("eng+por", description="Language code (eng, por, eng+por). Use 'auto' to enable OSD."),
     mode: str = Form("fast", description="OCR mode: 'fast' or 'accurate'."),
     auto_detect: bool = Form(False, description="Enable Orientation and Script Detection (OSD)."),
     force_processing: bool = Form(False, description="Force processing even if page count > 10.")
 ):
     """
-    Extract text from uploaded PDF document.
+    Extract text from uploaded PDF document or from object storage.
 
-    - **input_file**: PDF file to process.
+    - **input_file**: PDF file to process (required if source='upload').
+    - **source**: Source of the file: 'upload' or 'object_storage'. Defaults to 'upload'.
+    - **client_id**: Client ID for object storage (required if source='object_storage').
+    - **object_key**: Object key (path) in the bucket (required if source='object_storage').
     - **lang**: Language(s) to use for OCR. Defaults to 'eng+por'.
     - **mode**: processing mode. 'fast' is quicker, 'accurate' performs preprocessing.
     - **auto_detect**: Explicitly enable OSD (Orientation and Script Detection).
@@ -36,15 +43,61 @@ async def extract_pdf(
         lang = "eng+por"
 
     start_time = time.time()
-    print(f"Processing PDF: {input_file.filename}")
+    filename = "unknown"
+    temp_file = None
 
-    # Validate file type
-    fileUpload.validate_pdf_file(input_file)
+    try:
+        if source == "upload":
+            if not input_file:
+                raise HTTPException(status_code=400, detail="input_file is required when source is 'upload'.")
 
-    # Save file using tempfile
-    temp_file = fileUpload._save_file_to_server(input_file)
+            # Reject if object storage params are present to avoid ambiguity
+            if client_id or object_key:
+                 raise HTTPException(status_code=400, detail="Ambiguous request: cannot provide both upload file and object storage parameters.")
+
+            filename = input_file.filename
+            print(f"Processing PDF upload: {filename}")
+
+            # Validate file type
+            fileUpload.validate_pdf_file(input_file)
+
+            # Save file using tempfile
+            temp_file = fileUpload._save_file_to_server(input_file)
+
+        elif source == "object_storage":
+            if input_file:
+                 raise HTTPException(status_code=400, detail="Ambiguous request: cannot provide both upload file and object storage parameters.")
+
+            if not client_id or not object_key:
+                raise HTTPException(status_code=400, detail="client_id and object_key are required when source is 'object_storage'.")
+
+            filename = object_key
+            print(f"Processing PDF from storage: {client_id}/{object_key}")
+
+            # Download file
+            try:
+                temp_file = storage.download_file_from_storage(client_id, object_key)
+            except ValueError as e:
+                raise HTTPException(status_code=400, detail=str(e))
+            except Exception as e:
+                # Log error
+                print(f"Error downloading file: {e}")
+                raise HTTPException(status_code=500, detail="Error retrieving file from storage.")
+
+            # Validate PDF header (simple check)
+            try:
+                with open(temp_file, "rb") as f:
+                    header = f.read(5)
+                if header != b"%PDF-":
+                    raise HTTPException(status_code=400, detail="Downloaded file is not a valid PDF.")
+            except HTTPException:
+                raise
+            except Exception as e:
+                 raise HTTPException(status_code=400, detail=f"Error validating file: {e}")
+
+        else:
+             raise HTTPException(status_code=400, detail="Invalid source. Use 'upload' or 'object_storage'.")
 
-    try:
         # Process PDF
         text = await pdf_ocr.process_pdf(
             file_path=temp_file,
@@ -53,15 +106,21 @@ async def extract_pdf(
             auto=auto_detect,
             force_processing=force_processing
         )
+
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        # Log unexpected errors
+        print(f"Error processing PDF: {e}")
+        raise HTTPException(status_code=500, detail=f"Error processing PDF: {str(e)}")
     finally:
-        if os.path.exists(temp_file):
+        if temp_file and os.path.exists(temp_file):
             os.remove(temp_file)
 
     time_taken = str(round((time.time() - start_time), 2))
 
-    # Return list with single document result (consistent with image endpoint structure)
     return [TextExtractDocument(
-        file_name=input_file.filename or "unknown",
+        file_name=filename or "unknown",
         text=text,
         time_taken=time_taken
     )]

app/api/text_extract.py

@@ -5,20 +5,27 @@
 
 from app.domain import ocr, fileUpload
 from app.model.TextSchema import TextExtractDocument
+from app.services import storage
 
 router = APIRouter()
 
 @router.post("/extract_text", response_model=List[TextExtractDocument])
 async def extract_text(
-    input_images: List[UploadFile] = File(...),
+    input_images: Optional[List[UploadFile]] = File(None, description="List of image files to process (required if source='upload')"),
+    source: str = Form("upload", description="Source of the file: 'upload' or 'object_storage'"),
+    client_id: Optional[str] = Form(None, description="Client ID for object storage (required if source='object_storage')"),
+    object_keys: Optional[List[str]] = Form(None, description="List of object keys (paths) in the bucket (required if source='object_storage')"),
     lang: str = Form("eng+por", description="Language code (eng, por, eng+por). Use 'auto' to enable OSD."),
     mode: str = Form("fast", description="OCR mode: 'fast' or 'accurate'."),
     auto_detect: bool = Form(False, description="Enable Orientation and Script Detection (OSD).")
 ):
     """
-    Extract text from uploaded images using Tesseract OCR.
+    Extract text from uploaded images or from object storage using Tesseract OCR.
 
-    - **input_images**: List of image files to process.
+    - **input_images**: List of image files to process (required if source='upload').
+    - **source**: Source of the file: 'upload' or 'object_storage'. Defaults to 'upload'.
+    - **client_id**: Client ID for object storage (required if source='object_storage').
+    - **object_keys**: List of object keys (paths) in the bucket (required if source='object_storage').
     - **lang**: Language(s) to use for OCR. Defaults to 'eng+por'. set to 'auto' to force OSD.
     - **mode**: processing mode. 'fast' is quicker, 'accurate' performs preprocessing (rescaling, etc.).
     - **auto_detect**: Explicitly enable OSD (Orientation and Script Detection).
@@ -35,34 +42,82 @@ async def extract_text(
 
     results = []
 
-    for img in input_images:
-        start_time = time.time()
-        print(f"Processing image: {img.filename}")
-
-        # Validate file type
-        fileUpload.validate_image_file(img)
-
-        # Save file using tempfile
-        temp_file = fileUpload._save_file_to_server(img)
-
-        try:
-            # Process OCR
-            text = await ocr.read_image(
-                img_path=temp_file,
-                lang=lang,
-                mode=mode,
-                auto=auto_detect
-            )
-        finally:
-            if os.path.exists(temp_file):
-                os.remove(temp_file)
-
-        time_taken = str(round((time.time() - start_time), 2))
-
-        results.append(TextExtractDocument(
-            file_name=img.filename or "unknown",
-            text=text,
-            time_taken=time_taken
-        ))
+    if source == "upload":
+        if not input_images:
+             raise HTTPException(status_code=400, detail="input_images is required when source is 'upload'.")
+
+        if client_id or object_keys:
+             raise HTTPException(status_code=400, detail="Ambiguous request: cannot provide both upload files and object storage parameters.")
+
+        for img in input_images:
+            start_time = time.time()
+            print(f"Processing image: {img.filename}")
+
+            # Validate file type
+            fileUpload.validate_image_file(img)
+
+            # Save file using tempfile
+            temp_file = fileUpload._save_file_to_server(img)
+
+            try:
+                # Process OCR
+                text = await ocr.read_image(
+                    img_path=temp_file,
+                    lang=lang,
+                    mode=mode,
+                    auto=auto_detect
+                )
+            finally:
+                if os.path.exists(temp_file):
+                    os.remove(temp_file)
+
+            time_taken = str(round((time.time() - start_time), 2))
+
+            results.append(TextExtractDocument(
+                file_name=img.filename or "unknown",
+                text=text,
+                time_taken=time_taken
+            ))
+
+    elif source == "object_storage":
+        if input_images:
+             raise HTTPException(status_code=400, detail="Ambiguous request: cannot provide both upload files and object storage parameters.")
+
+        if not client_id or not object_keys:
+             raise HTTPException(status_code=400, detail="client_id and object_keys are required when source is 'object_storage'.")
+
+        for key in object_keys:
+            start_time = time.time()
+            print(f"Processing image from storage: {client_id}/{key}")
+
+            temp_file = None
+            try:
+                try:
+                    temp_file = storage.download_file_from_storage(client_id, key)
+                except ValueError as e:
+                    raise HTTPException(status_code=400, detail=str(e))
+                except Exception as e:
+                     print(f"Error downloading file {key}: {e}")
+                     raise HTTPException(status_code=500, detail=f"Error retrieving file {key} from storage.")
+
+                # Process OCR
+                text = await ocr.read_image(
+                    img_path=temp_file,
+                    lang=lang,
+                    mode=mode,
+                    auto=auto_detect
+                )
+
+                time_taken = str(round((time.time() - start_time), 2))
+                results.append(TextExtractDocument(
+                    file_name=key,
+                    text=text,
+                    time_taken=time_taken
+                ))
+            finally:
+                if temp_file and os.path.exists(temp_file):
+                    os.remove(temp_file)
+    else:
+        raise HTTPException(status_code=400, detail="Invalid source. Use 'upload' or 'object_storage'.")
 
     return results

app/core/tenants.py

@@ -0,0 +1,40 @@
+import json
+import os
+from typing import Dict, Optional
+from pydantic import BaseModel
+
+class TenantConfig(BaseModel):
+    bucket_name: str
+    endpoint_url: Optional[str] = None
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    region_name: str
+
+# Cache for tenant configuration
+_TENANTS_CACHE: Dict[str, TenantConfig] = {}
+
+def load_tenants(config_path: str = "tenants.json"):
+    """
+    Load tenant configurations from a JSON file.
+    """
+    global _TENANTS_CACHE
+    if not os.path.exists(config_path):
+        return
+
+    try:
+        with open(config_path, "r") as f:
+            data = json.load(f)
+            for client_id, config in data.items():
+                _TENANTS_CACHE[client_id] = TenantConfig(**config)
+    except Exception as e:
+        # In a real app, logging here would be good
+        print(f"Error loading tenants configuration: {e}")
+
+def get_tenant_config(client_id: str) -> Optional[TenantConfig]:
+    """
+    Retrieve configuration for a specific tenant.
+    Initializes cache if empty.
+    """
+    if not _TENANTS_CACHE:
+        load_tenants()
+    return _TENANTS_CACHE.get(client_id)

app/services/storage.py

@@ -0,0 +1,55 @@
+import boto3
+import tempfile
+import os
+import logging
+from botocore.exceptions import ClientError
+from app.core.tenants import get_tenant_config
+
+log = logging.getLogger("uvicorn")
+
+def download_file_from_storage(client_id: str, object_key: str) -> str:
+    """
+    Downloads a file from object storage to a temporary file.
+
+    Args:
+        client_id (str): The tenant identifier.
+        object_key (str): The path to the file in the bucket.
+
+    Returns:
+        str: The path to the downloaded temporary file.
+
+    Raises:
+        ValueError: If tenant configuration is missing.
+        ClientError: If S3 interaction fails (e.g. file not found).
+    """
+    config = get_tenant_config(client_id)
+    if not config:
+        raise ValueError(f"Configuration for client {client_id} not found.")
+
+    try:
+        # Determine file extension if possible
+        suffix = os.path.splitext(object_key)[1]
+
+        # Create a temporary file (not automatically deleted on close, we delete it later)
+        # Using delete=False so we can return the path and use it.
+        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp_file:
+            log.info(f"Downloading {object_key} for client {client_id} from {config.endpoint_url or 'AWS S3'}...")
+
+            s3_client = boto3.client(
+                "s3",
+                endpoint_url=config.endpoint_url,
+                aws_access_key_id=config.aws_access_key_id,
+                aws_secret_access_key=config.aws_secret_access_key,
+                region_name=config.region_name
+            )
+
+            s3_client.download_fileobj(config.bucket_name, object_key, tmp_file)
+            log.info(f"Downloaded to {tmp_file.name}")
+            return tmp_file.name
+
+    except ClientError as e:
+        log.error(f"S3 ClientError downloading {object_key}: {e}")
+        raise e
+    except Exception as e:
+        log.error(f"Unexpected error downloading {object_key}: {e}")
+        raise e

pyproject.toml

@@ -5,6 +5,7 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
+    "boto3>=1.42.50",
     "fastapi>=0.129.0",
     "filetype>=1.2.0",
     "httpx>=0.28.1",

tenants.json

@@ -0,0 +1,9 @@
+{
+  "client_a": {
+    "bucket_name": "bucket-a",
+    "endpoint_url": "https://s3.example.com",
+    "aws_access_key_id": "mock_key",
+    "aws_secret_access_key": "mock_secret",
+    "region_name": "us-east-1"
+  }
+}