fix: outdated code snippets for loaders module in doc (#1310)

Author: WHALEEYE

.gitignore

@@ -428,3 +428,6 @@ temp_files/
 #Benchmark
 benchmark/gaia/Dataset
 benchmark/gaia/results.jsonl
+
+# Secret files for docker
+.container/.env

camel/loaders/__init__.py

@@ -13,14 +13,16 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 from .apify_reader import Apify
-from .base_io import File
+from .base_io import File, create_file, create_file_from_raw_bytes
 from .chunkr_reader import ChunkrReader
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
 from .unstructured_io import UnstructuredIO
 
 __all__ = [
     'File',
+    'create_file',
+    'create_file_from_raw_bytes',
     'UnstructuredIO',
     'JinaURLReader',
     'Firecrawl',

camel/loaders/base_io.py

@@ -22,6 +22,47 @@
 from camel.utils import dependencies_required
 
 
+def create_file(file: BytesIO, filename: str) -> "File":
+    r"""Reads an uploaded file and returns a File object.
+
+    Args:
+        file (BytesIO): A BytesIO object representing the contents of the
+            file.
+        filename (str): The name of the file.
+
+    Returns:
+        File: A File object.
+    """
+    ext_to_cls = {
+        "docx": DocxFile,
+        "pdf": PdfFile,
+        "txt": TxtFile,
+        "json": JsonFile,
+        "html": HtmlFile,
+    }
+
+    ext = filename.split(".")[-1].lower()
+    if ext not in ext_to_cls:
+        raise NotImplementedError(f"File type {ext} not supported")
+
+    out_file = ext_to_cls[ext].from_bytes(file, filename)
+    return out_file
+
+
+def create_file_from_raw_bytes(raw_bytes: bytes, filename: str) -> "File":
+    r"""Reads raw bytes and returns a File object.
+
+    Args:
+        raw_bytes (bytes): The raw bytes content of the file.
+        filename (str): The name of the file.
+
+    Returns:
+        File: A File object.
+    """
+    file = BytesIO(raw_bytes)
+    return create_file(file, filename)
+
+
 class File(ABC):
     r"""Represents an uploaded file comprised of Documents.
 
@@ -79,47 +120,6 @@ def from_raw_bytes(cls, raw_bytes: bytes, filename: str) -> "File":
         file = BytesIO(raw_bytes)
         return cls.from_bytes(file, filename)
 
-    @staticmethod
-    def create_file(file: BytesIO, filename: str) -> "File":
-        r"""Reads an uploaded file and returns a File object.
-
-        Args:
-            file (BytesIO): A BytesIO object representing the contents of the
-                file.
-            filename (str): The name of the file.
-
-        Returns:
-            File: A File object.
-        """
-        ext_to_cls = {
-            "docx": DocxFile,
-            "pdf": PdfFile,
-            "txt": TxtFile,
-            "json": JsonFile,
-            "html": HtmlFile,
-        }
-
-        ext = filename.split(".")[-1].lower()
-        if ext not in ext_to_cls:
-            raise NotImplementedError(f"File type {ext} not supported")
-
-        out_file = ext_to_cls[ext].from_bytes(file, filename)
-        return out_file
-
-    @staticmethod
-    def create_file_from_raw_bytes(raw_bytes: bytes, filename: str) -> "File":
-        r"""Reads raw bytes and returns a File object.
-
-        Args:
-            raw_bytes (bytes): The raw bytes content of the file.
-            filename (str): The name of the file.
-
-        Returns:
-            File: A File object.
-        """
-        file = BytesIO(raw_bytes)
-        return File.create_file(file, filename)
-
     def __repr__(self) -> str:
         return (
             f"File(name={self.name}, id={self.file_id}, "

camel/storages/object_storages/amazon_s3.py

@@ -17,7 +17,7 @@
 from typing import Optional, Tuple
 from warnings import warn
 
-from camel.loaders import File
+from camel.loaders import File, create_file_from_raw_bytes
 from camel.storages.object_storages.base import BaseObjectStorage
 
 
@@ -156,7 +156,7 @@ def _get_file(self, file_key: str, filename: str) -> File:
             Bucket=self._bucket_name, Key=file_key
         )
         raw_bytes = response["Body"].read()
-        return File.create_file_from_raw_bytes(raw_bytes, filename)
+        return create_file_from_raw_bytes(raw_bytes, filename)
 
     def _upload_file(
         self, local_file_path: Path, remote_file_key: str

camel/storages/object_storages/azure_blob.py

@@ -16,7 +16,7 @@
 from typing import Optional, Tuple
 from warnings import warn
 
-from camel.loaders import File
+from camel.loaders import File, create_file_from_raw_bytes
 from camel.storages.object_storages.base import BaseObjectStorage
 
 
@@ -123,7 +123,7 @@ def _get_file(self, file_key: str, filename: str) -> File:
             File: The object from the container.
         """
         raw_bytes = self._client.download_blob(file_key).readall()
-        file = File.create_file_from_raw_bytes(raw_bytes, filename)
+        file = create_file_from_raw_bytes(raw_bytes, filename)
         return file
 
     def _upload_file(

camel/storages/object_storages/google_cloud.py

@@ -15,7 +15,7 @@
 from typing import Tuple
 from warnings import warn
 
-from camel.loaders import File
+from camel.loaders import File, create_file_from_raw_bytes
 from camel.storages.object_storages.base import BaseObjectStorage
 
 
@@ -111,7 +111,7 @@ def _get_file(self, file_key: str, filename: str) -> File:
             File: The object from the S3 bucket.
         """
         raw_bytes = self._client.get_blob(file_key).download_as_bytes()
-        return File.create_file_from_raw_bytes(raw_bytes, filename)
+        return create_file_from_raw_bytes(raw_bytes, filename)
 
     def _upload_file(
         self, local_file_path: Path, remote_file_key: str

docs/key_modules/loaders.md

@@ -33,15 +33,14 @@ This module is designed to read files of various formats, extract their contents
 
 ```python
 from io import BytesIO
-from camel.loaders import read_file
+from camel.loaders import create_file_from_raw_bytes
 
 # Read a pdf file from disk
 with open("test.pdf", "rb") as file:
-    file_content = BytesIO(file.read())
-    file_content.name = "test.pdf"
+    file_content = file.read()
 
-# Use the read_file function to create an object based on the file extension
-file_obj = read_file(file_content)
+# Use the create_file function to create an object based on the file extension
+file_obj = create_file_from_raw_bytes(file_content, "test.pdf")
 
 # Once you have the File object, you can access its content
 print(file_obj.docs[0]["page_content"])
@@ -53,6 +52,9 @@ To get started with the `Unstructured IO` module, you first need to import the m
 
 Utilize `parse_file_or_url` to load and parse unstructured data from a file or URL
 ```python
+from camel.loaders import UnstructuredIO
+
+uio = UnstructuredIO()
 # Set example url
 example_url = (
     "https://www.cnn.com/2023/01/30/sport/empire-state-building-green-"
@@ -93,7 +95,7 @@ print(cleaned_text)
 Utilize `extract_data_from_text` to do text extraction operation
 ```python
 # Set example email to extract
-example_email_text = ("Contact me at [email protected].")
+example_email_text = "Contact me at [email protected]."
 
 extracted_text = uio.extract_data_from_text(text=example_email_text,
 extract_type="extract_email_address")
@@ -139,6 +141,8 @@ This is a basic guide to get you started with the `Unstructured IO` module. For
 
 Initialize the client, set up the required actors and parameters.
 ```python
+from camel.loaders import Apify
+
 apify = Apify()
 
 run_input = {