feat: Use LlamaParse to parse the private files

.changeset/gold-mugs-perform.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add using LlamaParse for private file uploader

templates/components/loaders/python/__init__.py

@@ -1,11 +1,9 @@
-import os
-import yaml
-import importlib
 import logging
-from typing import Dict
+
+import yaml
+from app.engine.loaders.db import DBLoaderConfig, get_db_documents
 from app.engine.loaders.file import FileLoaderConfig, get_file_documents
 from app.engine.loaders.web import WebLoaderConfig, get_web_documents
-from app.engine.loaders.db import DBLoaderConfig, get_db_documents
 
 logger = logging.getLogger(__name__)
 

templates/components/loaders/python/file.py

@@ -62,7 +62,8 @@ def get_file_documents(config: FileLoaderConfig):
         )
         return reader.load_data()
     except Exception as e:
-        import sys, traceback
+        import sys
+        import traceback
 
         # Catch the error if the data dir is empty
         # and return as empty document list

templates/types/streaming/fastapi/app/api/routers/upload.py

@@ -1,9 +1,10 @@
 import logging
 from typing import List
+
+from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
-from fastapi import HTTPException
-from fastapi import APIRouter
-from app.api.controllers.file import FileController
+
+from app.api.services.file import PrivateFileService
 
 file_upload_router = r = APIRouter()
 
@@ -18,7 +19,7 @@ class FileUploadRequest(BaseModel):
 def upload_file(request: FileUploadRequest) -> List[str]:
     try:
         logger.info("Processing file")
-        return FileController.process_file(request.base64)
+        return PrivateFileService.process_file(request.base64)
     except Exception as e:
         logger.error(f"Error processing file: {e}", exc_info=True)
         raise HTTPException(status_code=500, detail="Error processing file")

templates/types/streaming/fastapi/app/api/controllers/file.py → templates/types/streaming/fastapi/app/api/services/file.py

@@ -1,73 +1,90 @@
-import os
 import base64
 import mimetypes
-from uuid import uuid4
+import os
 from pathlib import Path
-from typing import List, Dict
+from typing import Dict, List
+from uuid import uuid4
+
+from app.engine.index import get_index
 from llama_index.core import VectorStoreIndex
-from llama_index.readers.file import FlatReader
+from llama_index.core.ingestion import IngestionPipeline
 from llama_index.core.readers.file.base import (
     _try_loading_included_file_formats as get_file_loaders_map,
+)
+from llama_index.core.readers.file.base import (
     default_file_metadata_func,
 )
 from llama_index.core.schema import Document
-from llama_index.core.ingestion import IngestionPipeline
-from app.engine.index import get_index
-
+from llama_index.readers.file import FlatReader
 
 
 def file_metadata_func(*args, **kwargs) -> Dict:
     default_meta = default_file_metadata_func(*args, **kwargs)
     default_meta["private"] = "true"
     return default_meta
 
-def file_loaders_map():
+
+def get_llamaparse_parser():
+    from app.engine.loaders import load_configs
+    from app.engine.loaders.file import FileLoaderConfig, llama_parse_parser
+
+    config = load_configs()
+    file_loader_config = FileLoaderConfig(**config["file"])
+    if file_loader_config.use_llama_parse:
+        return llama_parse_parser()
+    else:
+        return None
+
+
+def default_file_loaders_map():
     default_loaders = get_file_loaders_map()
     default_loaders[".txt"] = FlatReader
     return default_loaders
 
 
-
-class FileController:
-
-    PRIVATE_STORE_PATH="output/uploaded"
+class PrivateFileService:
+    PRIVATE_STORE_PATH = "output/uploaded"
 
     @staticmethod
     def preprocess_base64_file(base64_content: str) -> tuple:
         header, data = base64_content.split(",", 1)
         mime_type = header.split(";")[0].split(":", 1)[1]
         extension = mimetypes.guess_extension(mime_type)
         # File data as bytes
-        data = base64.b64decode(data)
-        return data, extension
+        return base64.b64decode(data), extension
 
     @staticmethod
     def store_and_parse_file(file_data, extension) -> List[Document]:
         # Store file to the private directory
-        os.makedirs(FileController.PRIVATE_STORE_PATH, exist_ok=True)
+        os.makedirs(PrivateFileService.PRIVATE_STORE_PATH, exist_ok=True)
 
         # random file name
         file_name = f"{uuid4().hex}{extension}"
-        file_path = Path(os.path.join(FileController.PRIVATE_STORE_PATH, file_name))
+        file_path = Path(os.path.join(PrivateFileService.PRIVATE_STORE_PATH, file_name))
 
         # write file
         with open(file_path, "wb") as f:
             f.write(file_data)
 
         # Load file to documents
-        reader_cls = file_loaders_map().get(extension)
-        if reader_cls is None:
-            raise ValueError(f"File extension {extension} is not supported")
-        documents = reader_cls().load_data(file_path)
+        # If LlamaParse is enabled, use it to parse the file
+        # Otherwise, use the default file loaders
+        reader = get_llamaparse_parser()
+        if reader is None:
+            reader_cls = default_file_loaders_map().get(extension)
+            if reader_cls is None:
+                raise ValueError(f"File extension {extension} is not supported")
+            reader = reader_cls()
+        documents = reader.load_data(file_path)
         # Add custom metadata
         for doc in documents:
             doc.metadata["private"] = "true"
         return documents
 
     @staticmethod
     def process_file(base64_content: str) -> List[str]:
-        file_data, extension = FileController.preprocess_base64_file(base64_content)
-        documents = FileController.store_and_parse_file(file_data, extension)
+        file_data, extension = PrivateFileService.preprocess_base64_file(base64_content)
+        documents = PrivateFileService.store_and_parse_file(file_data, extension)
 
         # Only process nodes, no store the index
         pipeline = IngestionPipeline()