feat: Add form filling use case for Python (#403)

---------
Co-authored-by: Marcus Schiesser <[email protected]>

Author: leehuwuj

.changeset/large-parents-exercise.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add form filling use case (Python)

helpers/tools.ts

@@ -267,6 +267,22 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Form Filling",
+    name: "form_filling",
+    supportedFrameworks: ["fastapi"],
+    type: ToolType.LOCAL,
+    dependencies: [
+      {
+        name: "pandas",
+        version: "^2.2.3",
+      },
+      {
+        name: "tabulate",
+        version: "^0.9.0",
+      },
+    ],
+  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {

helpers/types.ts

@@ -48,7 +48,7 @@ export type TemplateDataSource = {
 };
 export type TemplateDataSourceType = "file" | "web" | "db";
 export type TemplateObservability = "none" | "traceloop" | "llamatrace";
-export type TemplateAgents = "financial_report" | "blog";
+export type TemplateAgents = "financial_report" | "blog" | "form_filling";
 // Config for both file and folder
 export type FileSourceConfig =
   | {

questions/questions.ts

@@ -177,6 +177,34 @@ export const askProQuestions = async (program: QuestionArgs) => {
     program.observability = observability;
   }
 
+  // Ask agents
+  if (program.template === "multiagent" && !program.agents) {
+    const { agents } = await prompts(
+      {
+        type: "select",
+        name: "agents",
+        message: "Which agents would you like to use?",
+        choices: [
+          {
+            title: "Financial report (generate a financial report)",
+            value: "financial_report",
+          },
+          {
+            title: "Form filling (fill missing value in a CSV file)",
+            value: "form_filling",
+          },
+          {
+            title: "Blog writer (Write a blog post)",
+            value: "blog_writer",
+          },
+        ],
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    program.agents = agents;
+  }
+
   if (!program.modelConfig) {
     const modelConfig = await askModelConfig({
       openAiKey: program.openAiKey,

questions/simple.ts

@@ -10,6 +10,7 @@ type AppType =
   | "rag"
   | "code_artifact"
   | "financial_report_agent"
+  | "form_filling"
   | "extractor"
   | "data_scientist";
 
@@ -35,8 +36,12 @@ export const askSimpleQuestions = async (
           title: "Financial Report Generator (using Workflows)",
           value: "financial_report_agent",
         },
+        {
+          title: "Form Filler (using Workflows)",
+          value: "form_filling",
+        },
         { title: "Code Artifact Agent", value: "code_artifact" },
-        { title: "Structured extraction", value: "extractor" },
+        { title: "Information Extractor", value: "extractor" },
       ],
     },
     questionHandlers,
@@ -47,19 +52,22 @@ export const askSimpleQuestions = async (
   let useLlamaCloud = false;
 
   if (appType !== "extractor") {
-    const { language: newLanguage } = await prompts(
-      {
-        type: "select",
-        name: "language",
-        message: "What language do you want to use?",
-        choices: [
-          { title: "Python (FastAPI)", value: "fastapi" },
-          { title: "Typescript (NextJS)", value: "nextjs" },
-        ],
-      },
-      questionHandlers,
-    );
-    language = newLanguage;
+    // TODO: Add TS support for form filling use case
+    if (appType !== "form_filling") {
+      const { language: newLanguage } = await prompts(
+        {
+          type: "select",
+          name: "language",
+          message: "What language do you want to use?",
+          choices: [
+            { title: "Python (FastAPI)", value: "fastapi" },
+            { title: "Typescript (NextJS)", value: "nextjs" },
+          ],
+        },
+        questionHandlers,
+      );
+      language = newLanguage;
+    }
 
     const { useLlamaCloud: newUseLlamaCloud } = await prompts(
       {
@@ -152,6 +160,14 @@ const convertAnswers = async (
       frontend: true,
       modelConfig: MODEL_GPT4o,
     },
+    form_filling: {
+      template: "multiagent",
+      agents: "form_filling",
+      tools: getTools(["form_filling"]),
+      dataSources: EXAMPLE_10K_SEC_FILES,
+      frontend: true,
+      modelConfig: MODEL_GPT4o,
+    },
     extractor: {
       template: "extractor",
       tools: [],

templates/components/agents/python/blog/app/agents/publisher.py

@@ -11,11 +11,11 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]:
     tools = []
     # Get configured tools from the tools.yaml file
     configured_tools = ToolFactory.from_env(map_result=True)
-    if "document_generator" in configured_tools.keys():
-        tools.extend(configured_tools["document_generator"])
+    if "generate_document" in configured_tools.keys():
+        tools.append(configured_tools["generate_document"])
         prompt_instructions = dedent("""
             Normally, reply the blog post content to the user directly. 
-            But if user requested to generate a file, use the document_generator tool to generate the file and reply the link to the file.
+            But if user requested to generate a file, use the generate_document tool to generate the file and reply the link to the file.
         """)
         description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format."
     else:

templates/components/agents/python/blog/app/agents/researcher.py

@@ -42,11 +42,15 @@ def _get_research_tools(**kwargs) -> QueryEngineTool:
     query_engine_tool = _create_query_engine_tool(**kwargs)
     if query_engine_tool is not None:
         tools.append(query_engine_tool)
-    researcher_tool_names = ["duckduckgo", "wikipedia.WikipediaToolSpec"]
+    researcher_tool_names = [
+        "duckduckgo_search",
+        "duckduckgo_image_search",
+        "wikipedia.WikipediaToolSpec",
+    ]
     configured_tools = ToolFactory.from_env(map_result=True)
     for tool_name, tool in configured_tools.items():
         if tool_name in researcher_tool_names:
-            tools.extend(tool)
+            tools.append(tool)
     return tools
 
 

templates/components/agents/python/financial_report/app/agents/analyst.py

@@ -22,8 +22,8 @@ def _get_analyst_params() -> Tuple[List[type[FunctionTool]], str, str]:
     description = "Expert in analyzing financial data"
     configured_tools = ToolFactory.from_env(map_result=True)
     # Check if the interpreter tool is configured
-    if "interpreter" in configured_tools.keys():
-        tools.extend(configured_tools["interpreter"])
+    if "interpret" in configured_tools.keys():
+        tools.append(configured_tools["interpret"])
         prompt_instructions += dedent("""
             You are able to visualize the financial data using code interpreter tool.
             It's very useful to create and include visualizations to the report (make sure you include the right code and data for the visualization).

templates/components/agents/python/financial_report/app/agents/reporter.py

@@ -24,8 +24,8 @@ def _get_reporter_params(
         """
     )
     configured_tools = ToolFactory.from_env(map_result=True)
-    if "document_generator" in configured_tools:  # type: ignore
-        tools.extend(configured_tools["document_generator"])  # type: ignore
+    if "generate_document" in configured_tools:  # type: ignore
+        tools.append(configured_tools["generate_document"])  # type: ignore
         prompt_instructions += (
             "\nYou are also able to generate a file document (PDF/HTML) of the report."
         )

templates/components/agents/python/form_filling/README-template.md

@@ -0,0 +1,59 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, run the development server:
+
+```shell
+poetry run python main.py
+```
+
+## Use Case: Filling Financial CSV Template
+
+To reproduce the use case, start the [frontend](../frontend/README.md) and follow these steps in the frontend:
+
+1. Upload the Apple and Tesla financial reports from the [data](./data) directory. Just send an empty message.
+2. Upload the CSV file [sec_10k_template.csv](./sec_10k_template.csv) and send the message "Fill the missing cells in the CSV file".
+
+The agent will fill the missing cells by retrieving the information from the uploaded financial reports and return a new CSV file with the filled cells.
+
+### API endpoints
+
+The example provides one streaming API endpoint `/api/chat`.
+You can test the endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "What can you do?" }] }'
+```
+
+You can start editing the API by modifying `app/api/routers/chat.py` or `app/agents/form_filling.py`. The API auto-updates as you save the files.
+
+Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+
+The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+
+```
+ENVIRONMENT=prod poetry run python main.py
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!