Added Docling Service

Utilities/DOCLING_SERVICES/README.md

@@ -0,0 +1,112 @@
+# DOCLING_SERVICES
+
+## Overview
+
+DOCLING_SERVICES is a FastAPI-based application designed to process PDF files and convert them into Markdown format. It leverages the Docling library for efficient document conversion and provides a simple API for seamless integration.
+
+## Features
+
+- **PDF to Markdown Conversion:** Easily convert PDF documents to Markdown.
+- **Concurrent Processing:** Handles multiple PDF processing tasks concurrently.
+- **Configurable Settings:** Customize host, port, and other parameters via `config.json` or environment variables.
+- **Logging:** Comprehensive logging for monitoring and debugging.
+
+## Setup
+
+### Prerequisites
+
+- Python 3.8 or higher
+- Git
+
+**Configure Environment (Optional)**
+
+   Customize settings by editing `config.json` or setting environment variables. Default configurations are provided in `config.json`.
+
+## Running the Service
+
+### Using the Setup Script
+
+Execute the setup and run the service:
+
+1. **Clone the Repository**
+    ```bash
+    git clone <repository-url>
+    cd DOCLING_SERVICES
+    ```
+
+2. **Install Dependencies**
+    ```bash
+    ./setup.sh
+    ```
+
+### Manually Starting the Service
+
+Ensure the virtual environment is activated, then run:
+```bash
+python -m app.main
+```
+
+The service will start on the host and port specified in `config.json` (default is `0.0.0.0:8000`).
+
+## Usage
+
+### API Endpoint
+
+- **POST** `/process-pdf-markdown/`
+
+    Upload a PDF file to convert it into Markdown format.
+
+### Example `curl` Command
+
+```bash
+curl -X POST "http://0.0.0.0:8000/process-pdf-markdown/" \
+  -H "accept: application/json" \
+  -H "Content-Type: multipart/form-data" \
+  -F "file=@/path/to/your/document.pdf"
+```
+
+### Response Format
+
+Upon successful processing, the API returns a JSON response:
+
+```json
+{
+  "status": "success",
+  "chunks": [
+    {
+      "chunkText": "Markdown content of page 1",
+      "filename": "document.pdf",
+      "page_number": 1
+    },
+    {
+      "chunkText": "Markdown content of page 2",
+      "filename": "document.pdf",
+      "page_number": 2
+    }
+    // ... more pages
+  ]
+}
+```
+
+### Error Handling
+
+If an error occurs during processing, the API responds with an appropriate HTTP status code and error message.
+
+
+
+## Logging
+
+Logs are stored in the `logs/` directory by default. You can monitor the `markdown_service.log` file for detailed insights into the application's operations.
+
+## Configuration
+
+Settings can be adjusted in the `config.json` file or via environment variables. Key configurations include:
+
+- `HOST`: Server host (default: `0.0.0.0`)
+- `MARKDOWN_SERVICE_PORT`: Server port (default: `8000`)
+- `PDF_THREAD_POOL_SIZE`: Number of worker threads for PDF processing (default: `3`)
+- `LOGS_DIR`: Directory for log files (default: `logs`)
+- `MARKDOWN_LOG_FILE`: Log file name (default: `markdown_service.log`)
+
+## License
+Kore.ai

Utilities/DOCLING_SERVICES/app/__init__.py

@@ -0,0 +1 @@
+# This file can be empty 

Utilities/DOCLING_SERVICES/app/config.py

@@ -0,0 +1,33 @@
+import os
+import json
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Get the base directory of the service
+BASE_DIR = Path(__file__).resolve().parent.parent
+ENV_FILE = BASE_DIR / 'config' / '.env'
+load_dotenv(ENV_FILE)
+
+# Try to load configuration from JSON file if it exists
+config_data = {}
+config_file_path = BASE_DIR / 'config.json'
+if config_file_path.exists():
+    with open(config_file_path, 'r') as config_file:
+        config_data = json.load(config_file)
+
+# Configuration values with environment variable fallbacks
+HOST = os.getenv('HOST', config_data.get('HOST', '0.0.0.0'))
+MARKDOWN_SERVICE_PORT = int(os.getenv('MARKDOWN_SERVICE_PORT', config_data.get('MARKDOWN_SERVICE_PORT', 8000)))
+PDF_THREAD_POOL_SIZE = int(os.getenv('PDF_THREAD_POOL_SIZE', config_data.get('PDF_THREAD_POOL_SIZE', 3)))
+
+# Logging configuration
+LOGS_DIR = os.path.join(BASE_DIR, 'logs')  # Relative to service directory
+MARKDOWN_LOG_FILE = os.path.join(LOGS_DIR, 'markdown_service.log')
+
+# Ensure logs directory exists
+os.makedirs(LOGS_DIR, exist_ok=True)
+
+# Processing configuration
+MAX_RETRIES = int(os.getenv('MAX_RETRIES', config_data.get('MAX_RETRIES', 3)))
+RETRY_DELAY = int(os.getenv('RETRY_DELAY', config_data.get('RETRY_DELAY', 5000)))
+REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', config_data.get('REQUEST_TIMEOUT', 300000))) 

Utilities/DOCLING_SERVICES/app/main.py

@@ -0,0 +1,35 @@
+from fastapi import FastAPI
+import uvicorn
+from app.routes.pdf_processing import router as pdf_router
+from app.utils.logger import setup_logger
+import app.config as config
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="PDF Markdown Service",
+    description="Service for converting PDF files to markdown format",
+    version="1.0.0"
+)
+
+# Setup logger
+logger = setup_logger('markdown-service')
+
+# Include routes
+app.include_router(pdf_router)
+
+@app.on_event("startup")
+async def startup_event():
+    logger.info(f"Starting PDF Markdown Service on {config.HOST}:{config.MARKDOWN_SERVICE_PORT}")
+    logger.info(f"Thread pool size: {config.PDF_THREAD_POOL_SIZE}")
+
+if __name__ == "__main__":
+    try:
+        uvicorn.run(
+            app,
+            host=config.HOST,
+            port=config.MARKDOWN_SERVICE_PORT,
+            log_config=None  # Use our custom logging config
+        )
+    except Exception as e:
+        logger.error(f"Failed to start server: {str(e)}")
+        raise 

Utilities/DOCLING_SERVICES/app/models/processing.py

@@ -0,0 +1,72 @@
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import DocumentConverter, PdfFormatOption
+from docling.datamodel.pipeline_options import PdfPipelineOptions
+from docling_core.types.doc import ImageRefMode
+from pathlib import Path
+import time
+import logging
+from typing import List, Dict
+import app.config as config
+import os
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s | %(levelname)s | %(message)s',
+    handlers=[
+        logging.FileHandler(config.MARKDOWN_LOG_FILE),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+def ensure_logs_directory():
+    """Ensure that the logs directory exists."""
+    logs_dir = 'logs'
+    if not os.path.exists(logs_dir):
+        os.makedirs(logs_dir)
+
+def process_uploaded_file_sync(file_path: str) -> List[Dict]:
+    """Synchronous version of process_uploaded_file"""
+    # Set up pipeline options
+    pipeline_options = PdfPipelineOptions()
+    pipeline_options.do_ocr = False
+    pipeline_options.do_table_structure = True
+    pipeline_options.table_structure_options.do_cell_matching = True
+    pipeline_options.generate_page_images = False
+    pipeline_options.generate_picture_images = False
+
+    # Create converter instance
+    doc_converter = DocumentConverter(
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
+        }
+    )
+
+    chunks = []
+    input_doc_path = Path(file_path)
+
+    try:
+        start_time = time.time()
+        # Convert document
+        logger.info(f"Started processing Document {file_path}")
+        conv_result = doc_converter.convert(input_doc_path)
+        end_time = time.time() - start_time
+        logger.info(f"Document {file_path} converted in {end_time:.2f} seconds.")
+
+        # Process each page
+        for page_no, page in conv_result.document.pages.items():
+            page_content = conv_result.document.export_to_markdown(
+                image_mode=ImageRefMode.REFERENCED,
+                page_no=page_no
+            )
+            chunk_obj = {
+                'chunkText': page_content,
+                'filename': str(input_doc_path.name),
+                'page_number': page_no
+            }
+            chunks.append(chunk_obj)
+
+        return chunks
+    except Exception as e:
+        logger.error(f"Error processing document {file_path}: {str(e)}")
+        raise Exception(f"Error processing document: {str(e)}") 

Utilities/DOCLING_SERVICES/app/routes/pdf_processing.py

@@ -0,0 +1,36 @@
+from fastapi import APIRouter, UploadFile, HTTPException
+from fastapi.responses import JSONResponse
+import time
+import logging
+from app.services.file_processing import save_and_process_file
+from datetime import datetime
+
+router = APIRouter()
+
+logger = logging.getLogger(__name__)
+
+@router.post("/process-pdf-markdown/")
+async def process_pdf_markdown(file: UploadFile) -> JSONResponse:
+    if not file:
+        raise HTTPException(status_code=400, detail="No file provided")
+
+    if not file.filename.lower().endswith('.pdf'):
+        raise HTTPException(status_code=400, detail="File must be a PDF")
+
+    start_time = time.time()
+    filename = file.filename
+    logger.info(f"[MARKDOWN] Starting processing for file: {filename}")
+    logger.info(f"[MARKDOWN] Start time: {datetime.fromtimestamp(start_time).isoformat()}")
+
+    try:
+        chunks = await save_and_process_file(file)
+        processing_time = time.time() - start_time
+        logger.info(f"[MARKDOWN] Completed processing for file: {filename}")
+        logger.info(f"[MARKDOWN] Processing time: {processing_time:.2f} seconds")
+        logger.info(f"[MARKDOWN] End time: {datetime.now().isoformat()}")
+
+        return JSONResponse(content={"status": "success", "chunks": chunks}, status_code=200)
+
+    except Exception as e:
+        logger.error(f"[MARKDOWN] Error processing {filename}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Error processing file: {str(e)}") 

Utilities/DOCLING_SERVICES/app/services/file_processing.py

@@ -0,0 +1,25 @@
+import os
+import tempfile
+import asyncio
+from fastapi import UploadFile
+from concurrent.futures import ProcessPoolExecutor
+from typing import List, Dict
+from app.models.processing import process_uploaded_file_sync
+import app.config as config
+import shutil
+
+# Create a process pool for concurrent CPU-bound processing
+process_pool = ProcessPoolExecutor(max_workers=config.PDF_THREAD_POOL_SIZE)
+
+async def save_and_process_file(file: UploadFile) -> List[Dict]:
+    """Save the uploaded file and process it."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_file_path = os.path.join(temp_dir, file.filename)
+        save_uploaded_file(file, temp_file_path)
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(process_pool, process_uploaded_file_sync, temp_file_path)
+
+def save_uploaded_file(file: UploadFile, destination: str) -> None:
+    """Save the uploaded file to the specified destination."""
+    with open(destination, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer) 

Utilities/DOCLING_SERVICES/app/utils/ensure_logs_dir.py

@@ -0,0 +1,7 @@
+import os
+
+def ensure_logs_directory():
+    """Ensure that the logs directory exists."""
+    logs_dir = 'logs'
+    if not os.path.exists(logs_dir):
+        os.makedirs(logs_dir) 

Utilities/DOCLING_SERVICES/app/utils/logger.py

@@ -0,0 +1,25 @@
+import logging
+import os
+from app.config import LOGS_DIR, MARKDOWN_LOG_FILE
+
+def setup_logger(name):
+    """Set up logger with file and console handlers."""
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    # Create formatters and handlers
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+
+    # File handler
+    file_handler = logging.FileHandler(MARKDOWN_LOG_FILE)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Console handler
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+
+    return logger 

Utilities/DOCLING_SERVICES/config.json

@@ -0,0 +1,11 @@
+{
+    "HOST": "0.0.0.0",
+    "MARKDOWN_SERVICE_PORT": 8000,
+    "PDF_THREAD_POOL_SIZE": 3,
+    "PDF_PROCESS_POOL_SIZE": 3,
+    "MAX_RETRIES": 5,
+    "RETRY_DELAY": 5000,
+    "REQUEST_TIMEOUT": 300000,
+    "LOGS_DIR": "logs",
+    "MARKDOWN_LOG_FILE": "markdown.log"
+} 

Utilities/DOCLING_SERVICES/requirements.txt

@@ -0,0 +1,9 @@
+docling==2.21.0
+docling-core==2.18.0
+docling-ibm-models==3.3.1
+docling-parse==3.3.0
+easyocr==1.7.2
+fastapi==0.115.8
+python-dotenv==1.0.1
+python-multipart==0.0.20
+uvicorn==0.34.0