diff --git a/Dockerfile b/Dockerfile index cd331c3..222183d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,12 @@ FROM python:3.12-slim-bookworm WORKDIR /app -# Install runtime dependencies (tesseract and languages) +# Install runtime dependencies (tesseract, languages, and poppler) RUN apt-get update && apt-get install -y --no-install-recommends \ tesseract-ocr \ tesseract-ocr-por \ tesseract-ocr-eng \ + poppler-utils \ && rm -rf /var/lib/apt/lists/* # Copy virtual environment from builder diff --git a/README.md b/README.md index c1de010..a429bdd 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## 🇺🇸 English ### Overview -This project provides a robust API for Optical Character Recognition (OCR) capable of extracting text from images. It supports multiple languages (English and Portuguese), mixed-language documents, and offers configurable processing modes for speed or accuracy. +This project provides a robust API for Optical Character Recognition (OCR) capable of extracting text from images and PDF documents. It supports multiple languages (English and Portuguese), mixed-language documents, and offers configurable processing modes for speed or accuracy. ### Features - **Multi-language Support**: English (`eng`), Portuguese (`por`), and mixed (`eng+por`). @@ -20,6 +20,7 @@ This project provides a robust API for Optical Character Recognition (OCR) capab - `accurate`: Enhanced preprocessing (rescaling, contrast adjustment) for better results on difficult images. - **Auto-Detection**: Orientation and Script Detection (OSD) to handle rotated images or unknown scripts. - **Strict Validation**: Validates file types via magic bytes (MIME type verification) to ensure security. +- **PDF Support**: Extract text from PDF documents (hybrid approach: converts pages to images for robust OCR). - **Dockerized**: Optimized multi-stage Docker build for easy deployment. ### Installation & Running @@ -35,7 +36,7 @@ docker run -p 8080:8080 api_ocr The API will be available at `http://localhost:8080`. #### Local Development -Prerequisites: Python 3.12+, `uv` (dependency manager), and Tesseract OCR installed on your system. +Prerequisites: Python 3.12+, `uv` (dependency manager), `poppler-utils` (for PDF processing), and Tesseract OCR installed on your system. ```bash # Install dependencies @@ -47,7 +48,7 @@ uv run uvicorn app.main:app --reload --port 8080 ### API Usage -#### Extract Text +#### 1. Extract Text from Images **Endpoint**: `POST /extract_text` **Parameters (Form Data):** @@ -67,12 +68,31 @@ curl -X 'POST' \ -F 'mode=accurate' ``` +#### 2. Extract Text from PDF +**Endpoint**: `POST /extract_pdf` + +**Parameters (Form Data):** +- `input_file`: Single PDF file. +- `lang`, `mode`, `auto_detect`: Same as image endpoint. +- `force_processing`: Boolean (`true`/`false`). By default, PDFs with > 10 pages are rejected to save resources. Set this to `true` to override the limit. + +**Example (cURL):** +```bash +curl -X 'POST' \ + 'http://localhost:8080/extract_pdf' \ + -H 'accept: application/json' \ + -H 'Content-Type: multipart/form-data' \ + -F 'input_file=@document.pdf;type=application/pdf' \ + -F 'force_processing=true' +``` + ### Development Guide #### Architecture - **FastAPI**: Handles HTTP requests and validation. - **Tesseract OCR**: The core OCR engine. - **Pillow (PIL)**: Used for image preprocessing in `accurate` mode. +- **pdf2image / Poppler**: Converts PDF pages to images for reliable OCR. - **Filetype**: Validates image magic bytes securely. - **AsyncIO**: OCR tasks run in thread pools to prevent blocking the event loop. @@ -87,7 +107,7 @@ uv run pytest ## 🇧🇷 Português ### Visão Geral -Este projeto fornece uma API robusta para Reconhecimento Óptico de Caracteres (OCR) capaz de extrair texto de imagens. Suporta múltiplos idiomas (Inglês e Português), documentos com idiomas mistos e oferece modos de processamento configuráveis para velocidade ou precisão. +Este projeto fornece uma API robusta para Reconhecimento Óptico de Caracteres (OCR) capaz de extrair texto de imagens e documentos PDF. Suporta múltiplos idiomas (Inglês e Português), documentos com idiomas mistos e oferece modos de processamento configuráveis para velocidade ou precisão. ### Funcionalidades - **Suporte Multi-idioma**: Inglês (`eng`), Português (`por`) e misto (`eng+por`). @@ -96,6 +116,7 @@ Este projeto fornece uma API robusta para Reconhecimento Óptico de Caracteres ( - `accurate` (Preciso): Pré-processamento aprimorado (redimensionamento, ajuste de contraste) para melhores resultados em imagens difíceis. - **Detecção Automática**: Detecção de Orientação e Script (OSD) para lidar com imagens rotacionadas ou scripts desconhecidos. - **Validação Rigorosa**: Valida tipos de arquivos via *magic bytes* (verificação de tipo MIME) para garantir segurança. +- **Suporte a PDF**: Extrai texto de documentos PDF (abordagem híbrida: converte páginas em imagens para OCR robusto). - **Dockerizado**: Build Docker multi-estágio otimizado para fácil implantação. ### Instalação e Execução @@ -111,7 +132,7 @@ docker run -p 8080:8080 api_ocr A API estará disponível em `http://localhost:8080`. #### Desenvolvimento Local -Pré-requisitos: Python 3.12+, `uv` (gerenciador de dependências) e Tesseract OCR instalado no sistema. +Pré-requisitos: Python 3.12+, `uv` (gerenciador de dependências), `poppler-utils` (para processamento de PDF) e Tesseract OCR instalado no sistema. ```bash # Instalar dependências @@ -123,7 +144,7 @@ uv run uvicorn app.main:app --reload --port 8080 ### Uso da API -#### Extrair Texto +#### 1. Extrair Texto de Imagens **Endpoint**: `POST /extract_text` **Parâmetros (Form Data):** @@ -143,12 +164,31 @@ curl -X 'POST' \ -F 'mode=accurate' ``` +#### 2. Extrair Texto de PDF +**Endpoint**: `POST /extract_pdf` + +**Parâmetros (Form Data):** +- `input_file`: Arquivo PDF único. +- `lang`, `mode`, `auto_detect`: Iguais ao endpoint de imagem. +- `force_processing`: Booleano (`true`/`false`). Por padrão, PDFs com mais de 10 páginas são rejeitados para economizar recursos. Defina como `true` para ignorar o limite. + +**Exemplo (cURL):** +```bash +curl -X 'POST' \ + 'http://localhost:8080/extract_pdf' \ + -H 'accept: application/json' \ + -H 'Content-Type: multipart/form-data' \ + -F 'input_file=@documento.pdf;type=application/pdf' \ + -F 'force_processing=true' +``` + ### Guia de Desenvolvimento #### Arquitetura - **FastAPI**: Gerencia requisições HTTP e validação. - **Tesseract OCR**: O motor principal de OCR. - **Pillow (PIL)**: Usado para pré-processamento de imagens no modo `accurate`. +- **pdf2image / Poppler**: Converte páginas de PDF em imagens para OCR confiável. - **Filetype**: Valida *magic bytes* de imagens de forma segura. - **AsyncIO**: Tarefas de OCR rodam em *thread pools* para não bloquear o *event loop*. diff --git a/app/api/pdf_extract.py b/app/api/pdf_extract.py new file mode 100644 index 0000000..6b20b45 --- /dev/null +++ b/app/api/pdf_extract.py @@ -0,0 +1,67 @@ +from fastapi import APIRouter, UploadFile, File, Form, HTTPException +from typing import List, Optional +import time +import os + +from app.domain import pdf_ocr, fileUpload +from app.model.TextSchema import TextExtractDocument + +router = APIRouter() + +@router.post("/extract_pdf", response_model=List[TextExtractDocument]) +async def extract_pdf( + input_file: UploadFile = File(..., description="PDF file to process"), + lang: str = Form("eng+por", description="Language code (eng, por, eng+por). Use 'auto' to enable OSD."), + mode: str = Form("fast", description="OCR mode: 'fast' or 'accurate'."), + auto_detect: bool = Form(False, description="Enable Orientation and Script Detection (OSD)."), + force_processing: bool = Form(False, description="Force processing even if page count > 10.") +): + """ + Extract text from uploaded PDF document. + + - **input_file**: PDF file to process. + - **lang**: Language(s) to use for OCR. Defaults to 'eng+por'. + - **mode**: processing mode. 'fast' is quicker, 'accurate' performs preprocessing. + - **auto_detect**: Explicitly enable OSD (Orientation and Script Detection). + - **force_processing**: Set to True to bypass the 10-page limit safeguard. + """ + + # Validate mode + if mode not in ["fast", "accurate"]: + raise HTTPException(status_code=400, detail="Invalid mode. Use 'fast' or 'accurate'.") + + # Handle 'auto' lang + if lang == "auto": + auto_detect = True + lang = "eng+por" + + start_time = time.time() + print(f"Processing PDF: {input_file.filename}") + + # Validate file type + fileUpload.validate_pdf_file(input_file) + + # Save file using tempfile + temp_file = fileUpload._save_file_to_server(input_file) + + try: + # Process PDF + text = await pdf_ocr.process_pdf( + file_path=temp_file, + lang=lang, + mode=mode, + auto=auto_detect, + force_processing=force_processing + ) + finally: + if os.path.exists(temp_file): + os.remove(temp_file) + + time_taken = str(round((time.time() - start_time), 2)) + + # Return list with single document result (consistent with image endpoint structure) + return [TextExtractDocument( + file_name=input_file.filename or "unknown", + text=text, + time_taken=time_taken + )] diff --git a/app/domain/fileUpload.py b/app/domain/fileUpload.py index abd5c83..6b25a74 100644 --- a/app/domain/fileUpload.py +++ b/app/domain/fileUpload.py @@ -36,3 +36,40 @@ def validate_image_file(file: UploadFile): if isinstance(e, HTTPException): raise e raise HTTPException(status_code=400, detail=f"Error validating file: {str(e)}") + +def validate_pdf_file(file: UploadFile): + """ + Validates if the uploaded file is a valid PDF using magic bytes. + Raises HTTPException(400) if invalid. + """ + try: + # Read the first 2048 bytes to detect file type + header = file.file.read(2048) + file.file.seek(0) # Reset file pointer + + if not header: + raise HTTPException(status_code=400, detail="Empty file uploaded.") + + kind = filetype.guess(header) + + # PDF magic bytes might not always be detected by filetype if it's not strictly following standard or is a fragment, + # but filetype lib is generally good. + # However, specifically check for PDF mime type. + + if kind is None: + # Fallback check for %PDF- + if header.startswith(b"%PDF-"): + return True + raise HTTPException(status_code=400, detail="Could not determine file type. Please upload a valid PDF.") + + if kind.mime != "application/pdf": + # Fallback check for %PDF- + if header.startswith(b"%PDF-"): + return True + raise HTTPException(status_code=400, detail=f"Invalid file type: {kind.mime}. Only PDF files are allowed.") + + return True + except Exception as e: + if isinstance(e, HTTPException): + raise e + raise HTTPException(status_code=400, detail=f"Error validating file: {str(e)}") diff --git a/app/domain/pdf_ocr.py b/app/domain/pdf_ocr.py new file mode 100644 index 0000000..2756064 --- /dev/null +++ b/app/domain/pdf_ocr.py @@ -0,0 +1,73 @@ +import asyncio +import tempfile +import os +from pdf2image import convert_from_path, pdfinfo_from_path +from app.domain import ocr +from fastapi import HTTPException + +async def process_pdf( + file_path: str, + lang: str = 'eng+por', + mode: str = 'fast', + auto: bool = False, + force_processing: bool = False +) -> str: + """ + Converts PDF to images and extracts text from each page. + + Args: + file_path (str): Path to the PDF file. + lang (str): Language code. + mode (str): 'fast' or 'accurate'. + auto (bool): Enable OSD. + force_processing (bool): If True, ignores the 10-page limit. + + Returns: + str: Concatenated text from all processed pages. + """ + try: + # Check page count + # pdfinfo_from_path runs 'pdfinfo' command which is part of poppler-utils + info = await asyncio.to_thread(pdfinfo_from_path, file_path) + page_count = info.get("Pages", 0) + + if page_count > 10 and not force_processing: + raise HTTPException( + status_code=400, + detail=f"PDF has {page_count} pages. Limit is 10. Set 'force_processing' to True to override." + ) + + # Determine DPI based on mode + dpi = 300 if mode == 'accurate' else 200 + + # Convert PDF to images + # We run this in a thread because it's CPU bound + def convert(): + return convert_from_path(file_path, dpi=dpi) + + images = await asyncio.to_thread(convert) + + extracted_text = [] + + for i, image in enumerate(images): + # Save image to temp file to reuse existing OCR logic + # Use delete=False so we can close it before OCR reads it (Windows compat, though running on Linux) + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_img: + image.save(temp_img.name) + temp_img_path = temp_img.name + + try: + # Reuse the existing OCR logic which handles preprocessing based on mode + page_text = await ocr.read_image(temp_img_path, lang=lang, mode=mode, auto=auto) + extracted_text.append(f"--- Page {i+1} ---\n{page_text}") + finally: + if os.path.exists(temp_img_path): + os.remove(temp_img_path) + + return "\n\n".join(extracted_text) + + except HTTPException as he: + raise he + except Exception as e: + # Log the error here if logging was set up + raise HTTPException(status_code=500, detail=f"Error processing PDF: {str(e)}") diff --git a/app/main.py b/app/main.py index a8eb1ac..60ae7ea 100644 --- a/app/main.py +++ b/app/main.py @@ -5,7 +5,7 @@ from fastapi import FastAPI import sentry_sdk -from app.api import text_extract # updated +from app.api import text_extract, pdf_extract # updated sentry_sdk.init( dsn="https://5c74c0bf64424183a3d8fea7a803a9b0@o4505535984828416.ingest.sentry.io/4505535986335744", traces_sample_rate=1.0, @@ -32,6 +32,7 @@ def create_application() -> FastAPI: lifespan=lifespan ) application.include_router(text_extract.router) + application.include_router(pdf_extract.router) return application diff --git a/pyproject.toml b/pyproject.toml index f3015ce..70f9430 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "fastapi>=0.129.0", "filetype>=1.2.0", "httpx>=0.28.1", + "pdf2image>=1.17.0", "pillow>=12.1.1", "pydantic-settings>=2.13.0", "pytesseract>=0.3.13", diff --git a/tests/test_file_validation.py b/tests/test_file_validation.py index 6be0b84..498e76e 100644 --- a/tests/test_file_validation.py +++ b/tests/test_file_validation.py @@ -1,5 +1,5 @@ from fastapi import UploadFile -from app.domain.fileUpload import validate_image_file +from app.domain.fileUpload import validate_image_file, validate_pdf_file from io import BytesIO import pytest from fastapi import HTTPException @@ -24,3 +24,16 @@ def test_validate_empty_file(): validate_image_file(file) assert exc.value.status_code == 400 assert "Empty file uploaded" in exc.value.detail + +def test_validate_valid_pdf(): + # Create a dummy PDF file (signature: %PDF-) + pdf_header = b'%PDF-1.4\n' + file = UploadFile(filename="test.pdf", file=BytesIO(pdf_header)) + assert validate_pdf_file(file) is True + +def test_validate_invalid_pdf(): + # Create a text file + file = UploadFile(filename="test.txt", file=BytesIO(b"Hello World")) + with pytest.raises(HTTPException) as exc: + validate_pdf_file(file) + assert exc.value.status_code == 400 diff --git a/tests/test_pdf_extract.py b/tests/test_pdf_extract.py new file mode 100644 index 0000000..2e63511 --- /dev/null +++ b/tests/test_pdf_extract.py @@ -0,0 +1,85 @@ +import pytest +from unittest.mock import patch, AsyncMock, MagicMock +from fastapi.testclient import TestClient +from app.main import app + +client = TestClient(app) + +@pytest.fixture +def mock_ocr_read_image(): + with patch("app.domain.ocr.read_image", new_callable=AsyncMock) as mock: + mock.return_value = "PDF Page Text" + yield mock + +@pytest.fixture +def mock_pdf_tools(): + # Mock both convert_from_path and pdfinfo_from_path + with patch("app.domain.pdf_ocr.convert_from_path") as mock_convert, \ + patch("app.domain.pdf_ocr.pdfinfo_from_path") as mock_info: + + # Mock convert_from_path to return a list of dummy images + mock_image = MagicMock() + mock_convert.return_value = [mock_image, mock_image] # 2 pages + + # Mock pdfinfo to return page count + mock_info.return_value = {"Pages": 2} + + yield mock_convert, mock_info + +def test_extract_pdf_success(mock_ocr_read_image, mock_pdf_tools): + mock_convert, mock_info = mock_pdf_tools + + # Valid PDF header + pdf_content = b'%PDF-1.4\n' + files = {"input_file": ("test.pdf", pdf_content, "application/pdf")} + + response = client.post("/extract_pdf", files=files) + + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert "--- Page 1 ---" in data[0]["text"] + assert "--- Page 2 ---" in data[0]["text"] + assert "PDF Page Text" in data[0]["text"] + +def test_extract_pdf_page_limit_exceeded(mock_ocr_read_image, mock_pdf_tools): + mock_convert, mock_info = mock_pdf_tools + + # Simulate 15 pages + mock_info.return_value = {"Pages": 15} + + pdf_content = b'%PDF-1.4\n' + files = {"input_file": ("large.pdf", pdf_content, "application/pdf")} + + response = client.post("/extract_pdf", files=files) + + assert response.status_code == 400 + assert "Limit is 10" in response.json()["detail"] + +def test_extract_pdf_force_processing(mock_ocr_read_image, mock_pdf_tools): + mock_convert, mock_info = mock_pdf_tools + + # Simulate 15 pages + mock_info.return_value = {"Pages": 15} + # Mock convert to return 15 images (dummy) + mock_convert.return_value = [MagicMock()] * 15 + + pdf_content = b'%PDF-1.4\n' + files = {"input_file": ("large.pdf", pdf_content, "application/pdf")} + data = {"force_processing": "true"} + + response = client.post("/extract_pdf", files=files, data=data) + + assert response.status_code == 200 + # Should process successfully + assert len(response.json()) == 1 + +def test_extract_pdf_invalid_file(mock_ocr_read_image, mock_pdf_tools): + # Text file content + txt_content = b"Not a PDF" + files = {"input_file": ("fake.pdf", txt_content, "application/pdf")} + + response = client.post("/extract_pdf", files=files) + + assert response.status_code == 400 + assert "Invalid file type" in response.json()["detail"] or "Could not determine" in response.json()["detail"] diff --git a/uv.lock b/uv.lock index 8585a62..797763f 100644 --- a/uv.lock +++ b/uv.lock @@ -41,6 +41,7 @@ dependencies = [ { name = "fastapi" }, { name = "filetype" }, { name = "httpx" }, + { name = "pdf2image" }, { name = "pillow" }, { name = "pydantic-settings" }, { name = "pytesseract" }, @@ -64,6 +65,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.129.0" }, { name = "filetype", specifier = ">=1.2.0" }, { name = "httpx", specifier = ">=0.28.1" }, + { name = "pdf2image", specifier = ">=1.17.0" }, { name = "pillow", specifier = ">=12.1.1" }, { name = "pydantic-settings", specifier = ">=2.13.0" }, { name = "pytesseract", specifier = ">=0.3.13" }, @@ -321,6 +323,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] +[[package]] +name = "pdf2image" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/d8/b280f01045555dc257b8153c00dee3bc75830f91a744cd5f84ef3a0a64b1/pdf2image-1.17.0.tar.gz", hash = "sha256:eaa959bc116b420dd7ec415fcae49b98100dda3dd18cd2fdfa86d09f112f6d57", size = 12811, upload-time = "2024-01-07T20:33:01.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/33/61766ae033518957f877ab246f87ca30a85b778ebaad65b7f74fa7e52988/pdf2image-1.17.0-py3-none-any.whl", hash = "sha256:ecdd58d7afb810dffe21ef2b1bbc057ef434dabbac6c33778a38a3f7744a27e2", size = 11618, upload-time = "2024-01-07T20:32:59.957Z" }, +] + [[package]] name = "pillow" version = "12.1.1"