expose more things

Author: zzstoatzz

examples/refresh_vectorstore/tpuf_namespace.py

@@ -47,6 +47,22 @@
             repo="PrefectHQ/prefect",
             include_globs=["flows/"],
         ),
+        GitHubRepoLoader(
+            repo="PrefectHQ/prefect",
+            include_globs=["src/prefect/*.py"],
+        ),
+        GitHubRepoLoader(
+            repo="PrefectHQ/prefect-background-task-examples",
+            include_globs=["**/*.py", "**/*.md"],
+        ),
+        GitHubRepoLoader(
+            repo="zzstoatzz/prefect-pack",
+            include_globs=["**/*.py", "**/*.md"],
+        ),
+        GitHubRepoLoader(
+            repo="zzstoatzz/prefect-monorepo",
+            include_globs=["**/*.py", "**/*.md", "**/*.yaml"],
+        ),
     ],
     "controlflow": [
         SitemapLoader(
@@ -97,7 +113,7 @@ def refresh_tpuf_namespace(
         for doc in future.result()  # type: ignore
     ]
 
-    print(f"Loaded {len(documents)} documents from the Prefect community.")
+    print(f"Gathered {len(documents)} documents from the Prefect community.")
 
     with TurboPuffer(namespace=namespace) as tpuf:
         if reset:

src/raggy/loaders/github.py

@@ -194,6 +194,8 @@ class GitHubRepoLoader(Loader):
     repo: str = Field(...)
     include_globs: list[str] | None = Field(default=None)
     exclude_globs: list[str] | None = Field(default=None)
+    chunk_size: int = Field(default=500)
+    overlap: float = Field(default=0.1)
 
     @field_validator("repo")
     def validate_repo(cls, v: str) -> str:
@@ -243,7 +245,10 @@ async def load(self) -> list[Document]:
                             Document(
                                 text=await read_file_with_chardet(Path(tmp_dir) / file),
                                 metadata=metadata,
-                            )
+                            ),
+                            chunk_size=self.chunk_size,
+                            overlap=self.overlap,
                         )
                     )
                 return documents
+