From 5be66d71e3bc84c012b73ab8555554826bfe5faa Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 25 Sep 2024 15:43:21 +0700 Subject: [PATCH 01/77] add artifact generator agent --- .../multiagent/fastapi/app/agents/multi.py | 5 +- .../app/examples/artifact_generator.py | 19 +++ .../fastapi/app/examples/choreography.py | 12 +- .../fastapi/app/examples/orchestrator.py | 8 +- .../fastapi/app/examples/workflow.py | 33 +++- .../multiagent/fastapi/app/tools/artifact.py | 154 ++++++++++++++++++ .../types/multiagent/fastapi/pyproject.toml | 2 + 7 files changed, 219 insertions(+), 14 deletions(-) create mode 100644 templates/types/multiagent/fastapi/app/examples/artifact_generator.py create mode 100644 templates/types/multiagent/fastapi/app/tools/artifact.py diff --git a/templates/types/multiagent/fastapi/app/agents/multi.py b/templates/types/multiagent/fastapi/app/agents/multi.py index 9a04a3da8..d20f372a0 100644 --- a/templates/types/multiagent/fastapi/app/agents/multi.py +++ b/templates/types/multiagent/fastapi/app/agents/multi.py @@ -8,7 +8,7 @@ ) from llama_index.core.tools.types import ToolMetadata, ToolOutput from llama_index.core.tools.utils import create_schema_from_function -from llama_index.core.workflow import Context, Workflow +from llama_index.core.workflow import Context, StopEvent, Workflow class AgentCallTool(ContextAwareTool): @@ -35,7 +35,8 @@ async def acall(self, ctx: Context, input: str) -> ToolOutput: handler = self.agent.run(input=input) # bubble all events while running the agent to the calling agent async for ev in handler.stream_events(): - ctx.write_event_to_stream(ev) + if type(ev) is not StopEvent: + ctx.write_event_to_stream(ev) ret: AgentRunResult = await handler response = ret.response.message.content return ToolOutput( diff --git a/templates/types/multiagent/fastapi/app/examples/artifact_generator.py b/templates/types/multiagent/fastapi/app/examples/artifact_generator.py new file mode 100644 index 000000000..6486ac4a4 --- /dev/null +++ b/templates/types/multiagent/fastapi/app/examples/artifact_generator.py @@ -0,0 +1,19 @@ +from typing import List + +from app.agents.single import FunctionCallingAgent +from app.tools.artifact import ArtifactGenerator +from llama_index.core.chat_engine.types import ChatMessage +from llama_index.core.tools import FunctionTool + + +def create_artifact_generator(chat_history: List[ChatMessage]): + artifact_tool = FunctionTool.from_defaults(ArtifactGenerator.generate_artifact) + + return FunctionCallingAgent( + name="ArtifactGenerator", + tools=[artifact_tool], + role="expert in generating artifacts (pdf, html)", + system_prompt="You are generator that help generate artifacts (pdf, html) from a given content.", + chat_history=chat_history, + verbose=True, + ) diff --git a/templates/types/multiagent/fastapi/app/examples/choreography.py b/templates/types/multiagent/fastapi/app/examples/choreography.py index aa7c197d6..5e10fb298 100644 --- a/templates/types/multiagent/fastapi/app/examples/choreography.py +++ b/templates/types/multiagent/fastapi/app/examples/choreography.py @@ -1,25 +1,31 @@ from typing import List, Optional -from app.agents.single import FunctionCallingAgent + from app.agents.multi import AgentCallingAgent +from app.agents.single import FunctionCallingAgent +from app.examples.artifact_generator import create_artifact_generator from app.examples.researcher import create_researcher from llama_index.core.chat_engine.types import ChatMessage def create_choreography(chat_history: Optional[List[ChatMessage]] = None): researcher = create_researcher(chat_history) + artifact_generator = create_artifact_generator(chat_history) reviewer = FunctionCallingAgent( name="reviewer", role="expert in reviewing blog posts", system_prompt="You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. If the post is good, you can say 'The post is good.'", chat_history=chat_history, + verbose=True, ) return AgentCallingAgent( name="writer", - agents=[researcher, reviewer], + agents=[researcher, reviewer, artifact_generator], role="expert in writing blog posts", system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. - You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post.""", + You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. + Finally, always request the artifact generator to create an artifact (pdf, html) that user can download and use for publishing the blog post.""", # TODO: add chat_history support to AgentCallingAgent # chat_history=chat_history, + verbose=True, ) diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/types/multiagent/fastapi/app/examples/orchestrator.py index 9f9151241..411398fd2 100644 --- a/templates/types/multiagent/fastapi/app/examples/orchestrator.py +++ b/templates/types/multiagent/fastapi/app/examples/orchestrator.py @@ -1,8 +1,9 @@ from typing import List, Optional -from app.agents.single import FunctionCallingAgent + from app.agents.multi import AgentOrchestrator +from app.agents.single import FunctionCallingAgent +from app.examples.artifact_generator import create_artifact_generator from app.examples.researcher import create_researcher - from llama_index.core.chat_engine.types import ChatMessage @@ -21,7 +22,8 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.""", chat_history=chat_history, ) + artifact_generator = create_artifact_generator(chat_history) return AgentOrchestrator( - agents=[writer, reviewer, researcher], + agents=[writer, reviewer, researcher, artifact_generator], refine_plan=False, ) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index c92f96ab9..79dd78a8d 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -1,6 +1,7 @@ from typing import AsyncGenerator, List, Optional from app.agents.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent +from app.examples.artifact_generator import create_artifact_generator from app.examples.researcher import create_researcher from llama_index.core.chat_engine.types import ChatMessage from llama_index.core.workflow import ( @@ -17,6 +18,9 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): researcher = create_researcher( chat_history=chat_history, ) + artifact_generator = create_artifact_generator( + chat_history=chat_history, + ) writer = FunctionCallingAgent( name="writer", role="expert in writing blog posts", @@ -30,7 +34,12 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): chat_history=chat_history, ) workflow = BlogPostWorkflow(timeout=360) - workflow.add_workflows(researcher=researcher, writer=writer, reviewer=reviewer) + workflow.add_workflows( + researcher=researcher, + writer=writer, + reviewer=reviewer, + artifact_generator=artifact_generator, + ) return workflow @@ -40,13 +49,16 @@ class ResearchEvent(Event): class WriteEvent(Event): input: str - is_good: bool = False class ReviewEvent(Event): input: str +class GenerateArtifactEvent(Event): + input: str + + class BlogPostWorkflow(Workflow): @step() async def start(self, ctx: Context, ev: StartEvent) -> ResearchEvent: @@ -80,7 +92,7 @@ async def write( msg=f"Too many attempts ({MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.", ) ) - if ev.is_good or too_many_attempts: + if too_many_attempts: # too many attempts or the blog post is good - stream final response if requested result = await self.run_agent( ctx, writer, ev.input, streaming=ctx.data["streaming"] @@ -93,7 +105,7 @@ async def write( @step() async def review( self, ctx: Context, ev: ReviewEvent, reviewer: FunctionCallingAgent - ) -> WriteEvent: + ) -> WriteEvent | GenerateArtifactEvent: result: AgentRunResult = await self.run_agent(ctx, reviewer, ev.input) review = result.response.message.content old_content = ctx.data["result"].response.message.content @@ -105,9 +117,8 @@ async def review( ) ) if post_is_good: - return WriteEvent( + return GenerateArtifactEvent( input=f"You're blog post is ready for publication. Please respond with just the blog post. Blog post: ```{old_content}```", - is_good=True, ) else: return WriteEvent( @@ -123,6 +134,16 @@ async def review( ```""" ) + @step() + async def generate_artifact( + self, + ctx: Context, + ev: GenerateArtifactEvent, + artifact_generator: FunctionCallingAgent, + ) -> StopEvent: + result: AgentRunResult = await self.run_agent(ctx, artifact_generator, ev.input) + return StopEvent(result=result) + async def run_agent( self, ctx: Context, diff --git a/templates/types/multiagent/fastapi/app/tools/artifact.py b/templates/types/multiagent/fastapi/app/tools/artifact.py new file mode 100644 index 000000000..d8392744d --- /dev/null +++ b/templates/types/multiagent/fastapi/app/tools/artifact.py @@ -0,0 +1,154 @@ +import os +from enum import Enum +from io import BytesIO + +OUTPUT_DIR = "output/tools" + + +class ArtifactType(Enum): + PDF = "pdf" + HTML = "html" + + +class ArtifactGenerator: + @classmethod + def _generate_pdf(cls, original_content: str) -> BytesIO: + """ + Generate a PDF from the original content (markdown). + """ + try: + import markdown + from reportlab.lib.pagesizes import letter + from reportlab.lib.styles import getSampleStyleSheet + from reportlab.platypus import Paragraph, SimpleDocTemplate + except ImportError: + raise ImportError( + "Failed to import required modules. Please install reportlab and markdown." + ) + + # Convert markdown to HTML + html = markdown.markdown(original_content) + + buffer = BytesIO() + + doc = SimpleDocTemplate(buffer, pagesize=letter) + + # Create a list to store the flowables (content elements) + elements = [] + styles = getSampleStyleSheet() + # TODO: Make the format nicer + for paragraph in html.split("

"): + if paragraph: + clean_text = paragraph.replace("

", "").strip() + elements.append(Paragraph(clean_text, styles["Normal"])) + + # Build the PDF document + doc.build(elements) + + # Reset the buffer position to the beginning + buffer.seek(0) + + return buffer + + @classmethod + def _generate_html(cls, original_content: str) -> str: + """ + Generate an HTML from the original content (markdown). + """ + try: + import markdown + except ImportError: + raise ImportError( + "Failed to import required modules. Please install markdown." + ) + + # Convert markdown to HTML + html_content = markdown.markdown(original_content) + + # Create a complete HTML document with basic styling + html_document = f""" + + + + + + Generated HTML Document + + + + {html_content} + + + """ + + return html_document + + @classmethod + def _write_to_file(cls, content: BytesIO, file_path: str): + """ + Write the content to a file. + """ + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as file: + file.write(content.getvalue()) + + @classmethod + def generate_artifact( + cls, original_content: str, artifact_type: str, file_name: str + ) -> str: + """ + Generate an artifact from the original content and write it to a file. + Parameters: + original_content: str (markdown style) + artifact_type: str (pdf or html). Use pdf for report, html for blog post. + file_name: str (name of the artifact file), don't need to include the file extension. It will be added automatically based on the artifact type. + Returns: + str (URL to the artifact file): the url that already available to the file server. No need to change the path anymore. + """ + try: + artifact_type = ArtifactType(artifact_type.lower()) + except ValueError: + raise ValueError( + f"Invalid artifact type: {artifact_type}. Must be 'pdf' or 'html'." + ) + + if artifact_type == ArtifactType.PDF: + content = cls._generate_pdf(original_content) + file_extension = "pdf" + elif artifact_type == ArtifactType.HTML: + content = BytesIO(cls._generate_html(original_content).encode("utf-8")) + file_extension = "html" + else: + raise ValueError(f"Unexpected artifact type: {artifact_type}") + + file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") + cls._write_to_file(content, file_path) + file_url = f"{os.getenv('FILESERVER_URL_PREFIX')}/{file_path}" + return file_url diff --git a/templates/types/multiagent/fastapi/pyproject.toml b/templates/types/multiagent/fastapi/pyproject.toml index 5c779f27c..7971b9adf 100644 --- a/templates/types/multiagent/fastapi/pyproject.toml +++ b/templates/types/multiagent/fastapi/pyproject.toml @@ -18,6 +18,8 @@ python-dotenv = "^1.0.0" uvicorn = { extras = ["standard"], version = "^0.23.2" } cachetools = "^5.3.3" aiostream = "^0.5.2" +markdown = "^3.7" +reportlab = "^4.2.2" [tool.poetry.dependencies.docx2txt] version = "^0.8" From c3f2e05c8f04ff83ea88ece3170a080897997e62 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 25 Sep 2024 15:58:48 +0700 Subject: [PATCH 02/77] enhance workflow --- .../multiagent/fastapi/app/examples/artifact_generator.py | 2 +- templates/types/multiagent/fastapi/app/examples/workflow.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/examples/artifact_generator.py b/templates/types/multiagent/fastapi/app/examples/artifact_generator.py index 6486ac4a4..0448f6ed0 100644 --- a/templates/types/multiagent/fastapi/app/examples/artifact_generator.py +++ b/templates/types/multiagent/fastapi/app/examples/artifact_generator.py @@ -13,7 +13,7 @@ def create_artifact_generator(chat_history: List[ChatMessage]): name="ArtifactGenerator", tools=[artifact_tool], role="expert in generating artifacts (pdf, html)", - system_prompt="You are generator that help generate artifacts (pdf, html) from a given content.", + system_prompt="You are generator that help generate artifacts (pdf, html) from a given content. Please always respond the content again along with the generated artifact.", chat_history=chat_history, verbose=True, ) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index 79dd78a8d..bd4a3fc7f 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -66,6 +66,7 @@ async def start(self, ctx: Context, ev: StartEvent) -> ResearchEvent: ctx.data["streaming"] = getattr(ev, "streaming", False) # start the workflow with researching about a topic ctx.data["task"] = ev.input + ctx.data["user_input"] = ev.input return ResearchEvent(input=f"Research for this task: {ev.input}") @step() @@ -117,8 +118,9 @@ async def review( ) ) if post_is_good: + user_input = ctx.data["user_input"] return GenerateArtifactEvent( - input=f"You're blog post is ready for publication. Please respond with just the blog post. Blog post: ```{old_content}```", + input=f"Please generate an artifact for this content: ```{old_content}```. The user input is: ```{user_input}```", ) else: return WriteEvent( From 17c34f1d8ddb146146e7feddd1e97f114ee77e16 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 25 Sep 2024 16:12:31 +0700 Subject: [PATCH 03/77] rename to publisher --- .../fastapi/app/examples/choreography.py | 8 +++---- .../fastapi/app/examples/orchestrator.py | 6 ++--- .../{artifact_generator.py => publisher.py} | 4 ++-- .../fastapi/app/examples/workflow.py | 22 +++++++++---------- .../chat/chat-message/chat-agent-events.tsx | 1 + 5 files changed, 21 insertions(+), 20 deletions(-) rename templates/types/multiagent/fastapi/app/examples/{artifact_generator.py => publisher.py} (87%) diff --git a/templates/types/multiagent/fastapi/app/examples/choreography.py b/templates/types/multiagent/fastapi/app/examples/choreography.py index 5e10fb298..7ac48b8d5 100644 --- a/templates/types/multiagent/fastapi/app/examples/choreography.py +++ b/templates/types/multiagent/fastapi/app/examples/choreography.py @@ -2,14 +2,14 @@ from app.agents.multi import AgentCallingAgent from app.agents.single import FunctionCallingAgent -from app.examples.artifact_generator import create_artifact_generator +from app.examples.publisher import create_publisher from app.examples.researcher import create_researcher from llama_index.core.chat_engine.types import ChatMessage def create_choreography(chat_history: Optional[List[ChatMessage]] = None): researcher = create_researcher(chat_history) - artifact_generator = create_artifact_generator(chat_history) + publisher = create_publisher(chat_history) reviewer = FunctionCallingAgent( name="reviewer", role="expert in reviewing blog posts", @@ -19,12 +19,12 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): ) return AgentCallingAgent( name="writer", - agents=[researcher, reviewer, artifact_generator], + agents=[researcher, reviewer, publisher], role="expert in writing blog posts", system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. - Finally, always request the artifact generator to create an artifact (pdf, html) that user can download and use for publishing the blog post.""", + Finally, always request the publisher to create an artifact (pdf, html) and publish the blog post.""", # TODO: add chat_history support to AgentCallingAgent # chat_history=chat_history, verbose=True, diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/types/multiagent/fastapi/app/examples/orchestrator.py index 411398fd2..5247a2946 100644 --- a/templates/types/multiagent/fastapi/app/examples/orchestrator.py +++ b/templates/types/multiagent/fastapi/app/examples/orchestrator.py @@ -2,7 +2,7 @@ from app.agents.multi import AgentOrchestrator from app.agents.single import FunctionCallingAgent -from app.examples.artifact_generator import create_artifact_generator +from app.examples.publisher import create_publisher from app.examples.researcher import create_researcher from llama_index.core.chat_engine.types import ChatMessage @@ -22,8 +22,8 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.""", chat_history=chat_history, ) - artifact_generator = create_artifact_generator(chat_history) + publisher = create_publisher(chat_history) return AgentOrchestrator( - agents=[writer, reviewer, researcher, artifact_generator], + agents=[writer, reviewer, researcher, publisher], refine_plan=False, ) diff --git a/templates/types/multiagent/fastapi/app/examples/artifact_generator.py b/templates/types/multiagent/fastapi/app/examples/publisher.py similarity index 87% rename from templates/types/multiagent/fastapi/app/examples/artifact_generator.py rename to templates/types/multiagent/fastapi/app/examples/publisher.py index 0448f6ed0..9b03f168d 100644 --- a/templates/types/multiagent/fastapi/app/examples/artifact_generator.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -6,11 +6,11 @@ from llama_index.core.tools import FunctionTool -def create_artifact_generator(chat_history: List[ChatMessage]): +def create_publisher(chat_history: List[ChatMessage]): artifact_tool = FunctionTool.from_defaults(ArtifactGenerator.generate_artifact) return FunctionCallingAgent( - name="ArtifactGenerator", + name="publisher", tools=[artifact_tool], role="expert in generating artifacts (pdf, html)", system_prompt="You are generator that help generate artifacts (pdf, html) from a given content. Please always respond the content again along with the generated artifact.", diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index bd4a3fc7f..af9b6848a 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, List, Optional from app.agents.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent -from app.examples.artifact_generator import create_artifact_generator +from app.examples.publisher import create_publisher from app.examples.researcher import create_researcher from llama_index.core.chat_engine.types import ChatMessage from llama_index.core.workflow import ( @@ -18,7 +18,7 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): researcher = create_researcher( chat_history=chat_history, ) - artifact_generator = create_artifact_generator( + publisher = create_publisher( chat_history=chat_history, ) writer = FunctionCallingAgent( @@ -38,7 +38,7 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): researcher=researcher, writer=writer, reviewer=reviewer, - artifact_generator=artifact_generator, + publisher=publisher, ) return workflow @@ -55,7 +55,7 @@ class ReviewEvent(Event): input: str -class GenerateArtifactEvent(Event): +class PublishEvent(Event): input: str @@ -106,7 +106,7 @@ async def write( @step() async def review( self, ctx: Context, ev: ReviewEvent, reviewer: FunctionCallingAgent - ) -> WriteEvent | GenerateArtifactEvent: + ) -> WriteEvent | PublishEvent: result: AgentRunResult = await self.run_agent(ctx, reviewer, ev.input) review = result.response.message.content old_content = ctx.data["result"].response.message.content @@ -119,8 +119,8 @@ async def review( ) if post_is_good: user_input = ctx.data["user_input"] - return GenerateArtifactEvent( - input=f"Please generate an artifact for this content: ```{old_content}```. The user input is: ```{user_input}```", + return PublishEvent( + input=f"Please publish this content: ```{old_content}```. The user request was: ```{user_input}```", ) else: return WriteEvent( @@ -137,13 +137,13 @@ async def review( ) @step() - async def generate_artifact( + async def publish( self, ctx: Context, - ev: GenerateArtifactEvent, - artifact_generator: FunctionCallingAgent, + ev: PublishEvent, + publisher: FunctionCallingAgent, ) -> StopEvent: - result: AgentRunResult = await self.run_agent(ctx, artifact_generator, ev.input) + result: AgentRunResult = await self.run_agent(ctx, publisher, ev.input) return StopEvent(result=result) async def run_agent( diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-agent-events.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-agent-events.tsx index 618dd0645..8fea31dfa 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-agent-events.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-agent-events.tsx @@ -17,6 +17,7 @@ const AgentIcons: Record = { researcher: icons.ScanSearch, writer: icons.PenLine, reviewer: icons.MessageCircle, + publisher: icons.BookCheck, }; type MergedEvent = { From 4691c3cee2604a2e6b62f754ff181059227527e5 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 25 Sep 2024 16:14:50 +0700 Subject: [PATCH 04/77] add changeset --- .changeset/flat-singers-share.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/flat-singers-share.md diff --git a/.changeset/flat-singers-share.md b/.changeset/flat-singers-share.md new file mode 100644 index 000000000..da5304c49 --- /dev/null +++ b/.changeset/flat-singers-share.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +Add publisher agent that generates artifact for the content From ab2be638eb377a59e2497fb8922442825845f0ba Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 25 Sep 2024 16:31:48 +0700 Subject: [PATCH 05/77] enhance code --- .../multiagent/fastapi/app/examples/publisher.py | 1 - .../multiagent/fastapi/app/examples/workflow.py | 13 +++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index 9b03f168d..baafca4e2 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -15,5 +15,4 @@ def create_publisher(chat_history: List[ChatMessage]): role="expert in generating artifacts (pdf, html)", system_prompt="You are generator that help generate artifacts (pdf, html) from a given content. Please always respond the content again along with the generated artifact.", chat_history=chat_history, - verbose=True, ) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index af9b6848a..42595bdd6 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -143,8 +143,17 @@ async def publish( ev: PublishEvent, publisher: FunctionCallingAgent, ) -> StopEvent: - result: AgentRunResult = await self.run_agent(ctx, publisher, ev.input) - return StopEvent(result=result) + try: + result: AgentRunResult = await self.run_agent(ctx, publisher, ev.input) + return StopEvent(result=result) + except Exception as e: + ctx.write_event_to_stream( + AgentRunEvent( + name=publisher.name, + msg=f"Error publishing: {e}", + ) + ) + return StopEvent(result=result) async def run_agent( self, From 7ec56c92eb8e5783430e78f8a3ef91ec3f55f849 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 07:42:10 +0700 Subject: [PATCH 06/77] refactor code --- .../fastapi/app/examples/publisher.py | 7 +- .../multiagent/fastapi/app/tools/artifact.py | 201 ++++++++++-------- .../types/multiagent/fastapi/pyproject.toml | 2 +- 3 files changed, 117 insertions(+), 93 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index baafca4e2..f7341a830 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -12,7 +12,10 @@ def create_publisher(chat_history: List[ChatMessage]): return FunctionCallingAgent( name="publisher", tools=[artifact_tool], - role="expert in generating artifacts (pdf, html)", - system_prompt="You are generator that help generate artifacts (pdf, html) from a given content. Please always respond the content again along with the generated artifact.", + role="expert in publishing, need to specify the type of artifact (pdf, html, or markdown)", + system_prompt="""You are a publisher that help publish the blog post. + For a normal request, you should choose the type of artifact either pdf or html or just reply to the user. + """, chat_history=chat_history, + verbose=True, ) diff --git a/templates/types/multiagent/fastapi/app/tools/artifact.py b/templates/types/multiagent/fastapi/app/tools/artifact.py index d8392744d..b3fc15efb 100644 --- a/templates/types/multiagent/fastapi/app/tools/artifact.py +++ b/templates/types/multiagent/fastapi/app/tools/artifact.py @@ -1,4 +1,5 @@ import os +import re from enum import Enum from io import BytesIO @@ -10,127 +11,118 @@ class ArtifactType(Enum): HTML = "html" +HTML_FILE_TEMPLATE = """ + + + + + + + + + {html_content} + + +""" + + class ArtifactGenerator: @classmethod - def _generate_pdf(cls, original_content: str) -> BytesIO: + def _generate_html_content(cls, original_content: str) -> str: """ - Generate a PDF from the original content (markdown). + Generate HTML content from the original markdown content. """ try: import markdown - from reportlab.lib.pagesizes import letter - from reportlab.lib.styles import getSampleStyleSheet - from reportlab.platypus import Paragraph, SimpleDocTemplate except ImportError: raise ImportError( - "Failed to import required modules. Please install reportlab and markdown." + "Failed to import required modules. Please install markdown." ) # Convert markdown to HTML - html = markdown.markdown(original_content) - - buffer = BytesIO() - - doc = SimpleDocTemplate(buffer, pagesize=letter) - - # Create a list to store the flowables (content elements) - elements = [] - styles = getSampleStyleSheet() - # TODO: Make the format nicer - for paragraph in html.split("

"): - if paragraph: - clean_text = paragraph.replace("

", "").strip() - elements.append(Paragraph(clean_text, styles["Normal"])) - - # Build the PDF document - doc.build(elements) - - # Reset the buffer position to the beginning - buffer.seek(0) - - return buffer + html_content = markdown.markdown(original_content) + return html_content @classmethod - def _generate_html(cls, original_content: str) -> str: + def _generate_pdf(cls, html_content: str) -> BytesIO: """ - Generate an HTML from the original content (markdown). + Generate a PDF from the HTML content. """ try: - import markdown + from xhtml2pdf import pisa except ImportError: raise ImportError( - "Failed to import required modules. Please install markdown." + "Failed to import required modules. Please install xhtml2pdf." ) - # Convert markdown to HTML - html_content = markdown.markdown(original_content) - - # Create a complete HTML document with basic styling - html_document = f""" - - - - - - Generated HTML Document - - - - {html_content} - - - """ - - return html_document + buffer = BytesIO() + pdf = pisa.pisaDocument( + BytesIO(html_content.encode("UTF-8")), + buffer, + encoding="UTF-8", + path=".", + link_callback=None, + debug=0, + default_css=None, + xhtml=False, + xml_output=None, + ident=0, + show_error_as_pdf=False, + quiet=True, + capacity=100 * 1024 * 1024, + raise_exception=True, + ) + if pdf.err: + raise ValueError("PDF generation failed") + buffer.seek(0) + return buffer @classmethod - def _write_to_file(cls, content: BytesIO, file_path: str): + def _generate_html(cls, html_content: str) -> str: """ - Write the content to a file. + Generate a complete HTML document with the given HTML content. """ - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, "wb") as file: - file.write(content.getvalue()) + return HTML_FILE_TEMPLATE.format(html_content=html_content) @classmethod def generate_artifact( cls, original_content: str, artifact_type: str, file_name: str ) -> str: """ - Generate an artifact from the original content and write it to a file. + To generate artifact as PDF or HTML file. Parameters: original_content: str (markdown style) - artifact_type: str (pdf or html). Use pdf for report, html for blog post. - file_name: str (name of the artifact file), don't need to include the file extension. It will be added automatically based on the artifact type. + artifact_type: str (pdf or html) specify the type of the file format based on the use case + file_name: str (name of the artifact file) must be a valid file name, no extensions needed Returns: - str (URL to the artifact file): the url that already available to the file server. No need to change the path anymore. + str (URL to the artifact file): A file URL ready to serve. """ try: artifact_type = ArtifactType(artifact_type.lower()) @@ -138,17 +130,46 @@ def generate_artifact( raise ValueError( f"Invalid artifact type: {artifact_type}. Must be 'pdf' or 'html'." ) + # Always generate html content first + html_content = cls._generate_html_content(original_content) + # Based on the type of artifact, generate the corresponding file if artifact_type == ArtifactType.PDF: - content = cls._generate_pdf(original_content) + content = cls._generate_pdf(cls._generate_html(html_content)) file_extension = "pdf" elif artifact_type == ArtifactType.HTML: - content = BytesIO(cls._generate_html(original_content).encode("utf-8")) + content = BytesIO(cls._generate_html(html_content).encode("utf-8")) file_extension = "html" else: raise ValueError(f"Unexpected artifact type: {artifact_type}") + file_name = cls._validate_file_name(file_name) file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") + cls._write_to_file(content, file_path) + file_url = f"{os.getenv('FILESERVER_URL_PREFIX')}/{file_path}" return file_url + + @staticmethod + def _write_to_file(content: BytesIO, file_path: str): + """ + Write the content to a file. + """ + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as file: + file.write(content.getvalue()) + + @staticmethod + def _validate_file_name(file_name: str) -> str: + """ + Validate the file name. + """ + # Don't allow directory traversal + if os.path.isabs(file_name): + raise ValueError("File name is not allowed.") + # Don't allow special characters + if re.match(r"^[a-zA-Z0-9_.-]+$", file_name): + return file_name + else: + raise ValueError("File name is not allowed to contain special characters.") diff --git a/templates/types/multiagent/fastapi/pyproject.toml b/templates/types/multiagent/fastapi/pyproject.toml index 7971b9adf..dc6bd7c3d 100644 --- a/templates/types/multiagent/fastapi/pyproject.toml +++ b/templates/types/multiagent/fastapi/pyproject.toml @@ -19,7 +19,7 @@ uvicorn = { extras = ["standard"], version = "^0.23.2" } cachetools = "^5.3.3" aiostream = "^0.5.2" markdown = "^3.7" -reportlab = "^4.2.2" +xhtml2pdf = "^0.2.16" [tool.poetry.dependencies.docx2txt] version = "^0.8" From 6b8109ffc56a822a1c66c69730623e7a56e6cc99 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 07:57:05 +0700 Subject: [PATCH 07/77] add write file error handling --- templates/types/multiagent/fastapi/app/tools/artifact.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/tools/artifact.py b/templates/types/multiagent/fastapi/app/tools/artifact.py index b3fc15efb..9960c7a1a 100644 --- a/templates/types/multiagent/fastapi/app/tools/artifact.py +++ b/templates/types/multiagent/fastapi/app/tools/artifact.py @@ -156,9 +156,12 @@ def _write_to_file(content: BytesIO, file_path: str): """ Write the content to a file. """ - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, "wb") as file: - file.write(content.getvalue()) + try: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as file: + file.write(content.getvalue()) + except Exception as e: + raise e @staticmethod def _validate_file_name(file_name: str) -> str: From 13ad7147d54a9e4d07c369e34a6d82af9d1b42ca Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 08:22:33 +0700 Subject: [PATCH 08/77] keep old workflow --- .../fastapi/app/examples/workflow.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index 42595bdd6..1d49792ca 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -49,6 +49,7 @@ class ResearchEvent(Event): class WriteEvent(Event): input: str + is_good: bool = False class ReviewEvent(Event): @@ -82,7 +83,7 @@ async def research( @step() async def write( self, ctx: Context, ev: WriteEvent, writer: FunctionCallingAgent - ) -> ReviewEvent | StopEvent: + ) -> ReviewEvent | PublishEvent: MAX_ATTEMPTS = 2 ctx.data["attempts"] = ctx.data.get("attempts", 0) + 1 too_many_attempts = ctx.data["attempts"] > MAX_ATTEMPTS @@ -93,12 +94,11 @@ async def write( msg=f"Too many attempts ({MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.", ) ) - if too_many_attempts: + if ev.is_good or too_many_attempts: # too many attempts or the blog post is good - stream final response if requested - result = await self.run_agent( - ctx, writer, ev.input, streaming=ctx.data["streaming"] + return PublishEvent( + input=f"Please publish this content: ```{ev.input}```. The user request was: ```{ctx.data['user_input']}```", ) - return StopEvent(result=result) result: AgentRunResult = await self.run_agent(ctx, writer, ev.input) ctx.data["result"] = result return ReviewEvent(input=result.response.message.content) @@ -106,7 +106,7 @@ async def write( @step() async def review( self, ctx: Context, ev: ReviewEvent, reviewer: FunctionCallingAgent - ) -> WriteEvent | PublishEvent: + ) -> WriteEvent: result: AgentRunResult = await self.run_agent(ctx, reviewer, ev.input) review = result.response.message.content old_content = ctx.data["result"].response.message.content @@ -118,9 +118,9 @@ async def review( ) ) if post_is_good: - user_input = ctx.data["user_input"] - return PublishEvent( - input=f"Please publish this content: ```{old_content}```. The user request was: ```{user_input}```", + return WriteEvent( + input=f"You're blog post is ready for publication. Please respond with just the blog post. Blog post: ```{old_content}```", + is_good=True, ) else: return WriteEvent( From f0d58cac1e60e308b66abfa04f419ec1605ea686 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 09:58:35 +0700 Subject: [PATCH 09/77] add artifact generator tool --- helpers/tools.ts | 24 +++ .../engines/python/agent/tools/artifact.py | 183 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 templates/components/engines/python/agent/tools/artifact.py diff --git a/helpers/tools.ts b/helpers/tools.ts index a635e3fd1..5a0b46aa8 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -110,6 +110,30 @@ For better results, you can specify the region parameter to get results from a s }, ], }, + { + display: "Artifact generator", + name: "artifact", + // TODO: add support for Typescript templates + supportedFrameworks: ["fastapi"], + dependencies: [ + { + name: "xhtml2pdf", + version: "^0.2.16", + }, + { + name: "markdown", + version: "^3.7", + }, + ], + type: ToolType.LOCAL, + envVars: [ + { + name: TOOL_SYSTEM_PROMPT_ENV_VAR, + description: "System prompt for artifact tool.", + value: `If user request for a report or a post, use artifact tool to create a file and reply with the link to the file.`, + }, + ], + }, { display: "Code Interpreter", name: "interpreter", diff --git a/templates/components/engines/python/agent/tools/artifact.py b/templates/components/engines/python/agent/tools/artifact.py new file mode 100644 index 000000000..c4b996c0d --- /dev/null +++ b/templates/components/engines/python/agent/tools/artifact.py @@ -0,0 +1,183 @@ +import os +import re +from enum import Enum +from io import BytesIO + +from llama_index.core.tools.function_tool import FunctionTool + +OUTPUT_DIR = "output/tools" + + +class ArtifactType(Enum): + PDF = "pdf" + HTML = "html" + + +HTML_FILE_TEMPLATE = """ + + + + + + + + + {html_content} + + +""" + + +class ArtifactGenerator: + @classmethod + def _generate_html_content(cls, original_content: str) -> str: + """ + Generate HTML content from the original markdown content. + """ + try: + import markdown + except ImportError: + raise ImportError( + "Failed to import required modules. Please install markdown." + ) + + # Convert markdown to HTML + html_content = markdown.markdown(original_content) + return html_content + + @classmethod + def _generate_pdf(cls, html_content: str) -> BytesIO: + """ + Generate a PDF from the HTML content. + """ + try: + from xhtml2pdf import pisa + except ImportError: + raise ImportError( + "Failed to import required modules. Please install xhtml2pdf." + ) + + buffer = BytesIO() + pdf = pisa.pisaDocument( + BytesIO(html_content.encode("UTF-8")), + buffer, + encoding="UTF-8", + path=".", + link_callback=None, + debug=0, + default_css=None, + xhtml=False, + xml_output=None, + ident=0, + show_error_as_pdf=False, + quiet=True, + capacity=100 * 1024 * 1024, + raise_exception=True, + ) + if pdf.err: + raise ValueError("PDF generation failed") + buffer.seek(0) + return buffer + + @classmethod + def _generate_html(cls, html_content: str) -> str: + """ + Generate a complete HTML document with the given HTML content. + """ + return HTML_FILE_TEMPLATE.format(html_content=html_content) + + @classmethod + def generate_artifact( + cls, original_content: str, artifact_type: str, file_name: str + ) -> str: + """ + To generate artifact as PDF or HTML file. + Parameters: + original_content: str (markdown style) + artifact_type: str (pdf or html) specify the type of the file format based on the use case + file_name: str (name of the artifact file) must be a valid file name, no extensions needed + Returns: + str (URL to the artifact file): A file URL ready to serve. + """ + try: + artifact_type = ArtifactType(artifact_type.lower()) + except ValueError: + raise ValueError( + f"Invalid artifact type: {artifact_type}. Must be 'pdf' or 'html'." + ) + # Always generate html content first + html_content = cls._generate_html_content(original_content) + + # Based on the type of artifact, generate the corresponding file + if artifact_type == ArtifactType.PDF: + content = cls._generate_pdf(cls._generate_html(html_content)) + file_extension = "pdf" + elif artifact_type == ArtifactType.HTML: + content = BytesIO(cls._generate_html(html_content).encode("utf-8")) + file_extension = "html" + else: + raise ValueError(f"Unexpected artifact type: {artifact_type}") + + file_name = cls._validate_file_name(file_name) + file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") + + cls._write_to_file(content, file_path) + + file_url = f"{os.getenv('FILESERVER_URL_PREFIX')}/{file_path}" + return file_url + + @staticmethod + def _write_to_file(content: BytesIO, file_path: str): + """ + Write the content to a file. + """ + try: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as file: + file.write(content.getvalue()) + except Exception as e: + raise e + + @staticmethod + def _validate_file_name(file_name: str) -> str: + """ + Validate the file name. + """ + # Don't allow directory traversal + if os.path.isabs(file_name): + raise ValueError("File name is not allowed.") + # Don't allow special characters + if re.match(r"^[a-zA-Z0-9_.-]+$", file_name): + return file_name + else: + raise ValueError("File name is not allowed to contain special characters.") + +def get_tools(**kwargs): + return [FunctionTool.from_defaults(ArtifactGenerator.generate_artifact)] From 72e57eecc7ca915981ffd25795db848e24f98375 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 10:00:28 +0700 Subject: [PATCH 10/77] add changeset and format code --- .changeset/gorgeous-penguins-shout.md | 5 +++++ templates/components/engines/python/agent/tools/artifact.py | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/gorgeous-penguins-shout.md diff --git a/.changeset/gorgeous-penguins-shout.md b/.changeset/gorgeous-penguins-shout.md new file mode 100644 index 000000000..b4689266f --- /dev/null +++ b/.changeset/gorgeous-penguins-shout.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +Add artifact generator tool diff --git a/templates/components/engines/python/agent/tools/artifact.py b/templates/components/engines/python/agent/tools/artifact.py index c4b996c0d..37e59e45c 100644 --- a/templates/components/engines/python/agent/tools/artifact.py +++ b/templates/components/engines/python/agent/tools/artifact.py @@ -179,5 +179,6 @@ def _validate_file_name(file_name: str) -> str: else: raise ValueError("File name is not allowed to contain special characters.") + def get_tools(**kwargs): return [FunctionTool.from_defaults(ArtifactGenerator.generate_artifact)] From d28e4223dad0c6eeae046fed5a0885a4745156fc Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 11:25:28 +0700 Subject: [PATCH 11/77] improve pdf looking --- .../engines/python/agent/tools/artifact.py | 137 ++++++++++++------ 1 file changed, 91 insertions(+), 46 deletions(-) diff --git a/templates/components/engines/python/agent/tools/artifact.py b/templates/components/engines/python/agent/tools/artifact.py index 37e59e45c..2a9bb638f 100644 --- a/templates/components/engines/python/agent/tools/artifact.py +++ b/templates/components/engines/python/agent/tools/artifact.py @@ -1,3 +1,4 @@ +import logging import os import re from enum import Enum @@ -13,43 +14,85 @@ class ArtifactType(Enum): HTML = "html" -HTML_FILE_TEMPLATE = """ +COMMON_STYLES = """ +body { + font-family: Arial, sans-serif; + line-height: 1.3; + color: #333; +} +h1, h2, h3, h4, h5, h6 { + margin-top: 1em; + margin-bottom: 0.5em; +} +p { + margin-bottom: 0.7em; +} +code { + background-color: #f4f4f4; + padding: 2px 4px; + border-radius: 4px; +} +pre { + background-color: #f4f4f4; + padding: 10px; + border-radius: 4px; + overflow-x: auto; +} +table { + border-collapse: collapse; + width: 100%; + margin-bottom: 1em; +} +th, td { + border: 1px solid #ddd; + padding: 8px; + text-align: left; +} +th { + background-color: #f2f2f2; + font-weight: bold; +} +""" + +HTML_SPECIFIC_STYLES = """ +body { + max-width: 800px; + margin: 0 auto; + padding: 20px; +} +""" + +PDF_SPECIFIC_STYLES = """ +@page { + size: letter; + margin: 2cm; +} +body { + font-size: 11pt; +} +h1 { font-size: 18pt; } +h2 { font-size: 16pt; } +h3 { font-size: 14pt; } +h4, h5, h6 { font-size: 12pt; } +pre, code { + font-family: Courier, monospace; + font-size: 0.9em; +} +""" + +HTML_TEMPLATE = """ - {html_content} + {content} """ @@ -68,8 +111,10 @@ def _generate_html_content(cls, original_content: str) -> str: "Failed to import required modules. Please install markdown." ) - # Convert markdown to HTML - html_content = markdown.markdown(original_content) + # Convert markdown to HTML with fenced code and table extensions + html_content = markdown.markdown( + original_content, extensions=["fenced_code", "tables"] + ) return html_content @classmethod @@ -84,25 +129,21 @@ def _generate_pdf(cls, html_content: str) -> BytesIO: "Failed to import required modules. Please install xhtml2pdf." ) + pdf_html = HTML_TEMPLATE.format( + common_styles=COMMON_STYLES, + specific_styles=PDF_SPECIFIC_STYLES, + content=html_content, + ) + buffer = BytesIO() pdf = pisa.pisaDocument( - BytesIO(html_content.encode("UTF-8")), - buffer, - encoding="UTF-8", - path=".", - link_callback=None, - debug=0, - default_css=None, - xhtml=False, - xml_output=None, - ident=0, - show_error_as_pdf=False, - quiet=True, - capacity=100 * 1024 * 1024, - raise_exception=True, + BytesIO(pdf_html.encode("UTF-8")), buffer, encoding="UTF-8" ) + if pdf.err: + logging.error(f"PDF generation failed: {pdf.err}") raise ValueError("PDF generation failed") + buffer.seek(0) return buffer @@ -111,7 +152,11 @@ def _generate_html(cls, html_content: str) -> str: """ Generate a complete HTML document with the given HTML content. """ - return HTML_FILE_TEMPLATE.format(html_content=html_content) + return HTML_TEMPLATE.format( + common_styles=COMMON_STYLES, + specific_styles=HTML_SPECIFIC_STYLES, + content=html_content, + ) @classmethod def generate_artifact( @@ -137,7 +182,7 @@ def generate_artifact( # Based on the type of artifact, generate the corresponding file if artifact_type == ArtifactType.PDF: - content = cls._generate_pdf(cls._generate_html(html_content)) + content = cls._generate_pdf(html_content) file_extension = "pdf" elif artifact_type == ArtifactType.HTML: content = BytesIO(cls._generate_html(html_content).encode("utf-8")) From 540e850f078e8a0e996d0b1ae3c1a3278c0c6f57 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 12:37:19 +0700 Subject: [PATCH 12/77] add duckduckgo image search --- .../engines/python/agent/tools/duckduckgo.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/templates/components/engines/python/agent/tools/duckduckgo.py b/templates/components/engines/python/agent/tools/duckduckgo.py index b63612a77..ec0f63326 100644 --- a/templates/components/engines/python/agent/tools/duckduckgo.py +++ b/templates/components/engines/python/agent/tools/duckduckgo.py @@ -32,5 +32,37 @@ def duckduckgo_search( return results +def duckduckgo_image_search( + query: str, + region: str = "wt-wt", + max_results: int = 10, +): + """ + Use this function to search for images in DuckDuckGo. + Args: + query (str): The query to search in DuckDuckGo. + region Optional(str): The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc... + max_results Optional(int): The maximum number of results to be returned. Default is 10. + """ + try: + from duckduckgo_search import DDGS + except ImportError: + raise ImportError( + "duckduckgo_search package is required to use this function." + "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`" + ) + params = { + "keywords": query, + "region": region, + "max_results": max_results, + } + with DDGS() as ddg: + results = list(ddg.images(**params)) + return results + + def get_tools(**kwargs): - return [FunctionTool.from_defaults(duckduckgo_search)] + return [ + FunctionTool.from_defaults(duckduckgo_search), + FunctionTool.from_defaults(duckduckgo_image_search), + ] From ea1f11244e6a8807f11a456f3f9b0435241d9518 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 15:20:14 +0700 Subject: [PATCH 13/77] add duckduckgo tool and improve prompting --- .../fastapi/app/examples/orchestrator.py | 11 +- .../fastapi/app/examples/researcher.py | 12 +- .../multiagent/fastapi/app/tools/artifact.py | 172 ++++++++++++------ .../fastapi/app/tools/duckduckgo.py | 68 +++++++ 4 files changed, 197 insertions(+), 66 deletions(-) create mode 100644 templates/types/multiagent/fastapi/app/tools/duckduckgo.py diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/types/multiagent/fastapi/app/examples/orchestrator.py index 5247a2946..69739744a 100644 --- a/templates/types/multiagent/fastapi/app/examples/orchestrator.py +++ b/templates/types/multiagent/fastapi/app/examples/orchestrator.py @@ -11,14 +11,19 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): researcher = create_researcher(chat_history) writer = FunctionCallingAgent( name="writer", - role="expert in writing blog posts", - system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Don't make up any information yourself. If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". If you have all the information needed, write the blog post.""", + role="expert in writing blog posts, need information and images to write a post", + system_prompt="""You are an expert in writing blog posts. + You are given a task to write a blog post. Don't make up any information yourself. + If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". + If you need to use images, reply "I need images about the topic to write the blog post". Don't use any dummy images made up by you. + If you have all the information needed, write the blog post.""", chat_history=chat_history, ) reviewer = FunctionCallingAgent( name="reviewer", - role="expert in reviewing blog posts", + role="expert in reviewing blog posts, need a written blog post to review", system_prompt="""You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix the issues found yourself. You must output a final blog post. + A post must include at lease one valid image, if not, reply "I need images about the topic to write the blog post". An image URL start with example or your website is not valid. Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.""", chat_history=chat_history, ) diff --git a/templates/types/multiagent/fastapi/app/examples/researcher.py b/templates/types/multiagent/fastapi/app/examples/researcher.py index ed6819d4d..6c4d9d594 100644 --- a/templates/types/multiagent/fastapi/app/examples/researcher.py +++ b/templates/types/multiagent/fastapi/app/examples/researcher.py @@ -1,10 +1,11 @@ import os from typing import List -from llama_index.core.tools import QueryEngineTool, ToolMetadata + from app.agents.single import FunctionCallingAgent from app.engine.index import get_index - +from app.tools.duckduckgo import get_tools as get_duckduckgo_tools from llama_index.core.chat_engine.types import ChatMessage +from llama_index.core.tools import QueryEngineTool, ToolMetadata def get_query_engine_tool() -> QueryEngineTool: @@ -30,10 +31,11 @@ def get_query_engine_tool() -> QueryEngineTool: def create_researcher(chat_history: List[ChatMessage]): + duckduckgo_search_tools = get_duckduckgo_tools() return FunctionCallingAgent( name="researcher", - tools=[get_query_engine_tool()], - role="expert in retrieving any unknown content", - system_prompt="You are a researcher agent. You are given a researching task. You must use your tools to complete the research.", + tools=[get_query_engine_tool(), *duckduckgo_search_tools], + role="expert in retrieving any unknown content or searching for images from the internet", + system_prompt="You are a researcher agent. You are given a researching task. You must use tools to retrieve information from the knowledge base and search for needed images from the internet for the post.", chat_history=chat_history, ) diff --git a/templates/types/multiagent/fastapi/app/tools/artifact.py b/templates/types/multiagent/fastapi/app/tools/artifact.py index 9960c7a1a..9c4673770 100644 --- a/templates/types/multiagent/fastapi/app/tools/artifact.py +++ b/templates/types/multiagent/fastapi/app/tools/artifact.py @@ -1,8 +1,11 @@ +import logging import os import re from enum import Enum from io import BytesIO +from llama_index.core.tools.function_tool import FunctionTool + OUTPUT_DIR = "output/tools" @@ -11,43 +14,92 @@ class ArtifactType(Enum): HTML = "html" -HTML_FILE_TEMPLATE = """ +COMMON_STYLES = """ +body { + font-family: Arial, sans-serif; + line-height: 1.3; + color: #333; +} +h1, h2, h3, h4, h5, h6 { + margin-top: 1em; + margin-bottom: 0.5em; +} +p { + margin-bottom: 0.7em; +} +code { + background-color: #f4f4f4; + padding: 2px 4px; + border-radius: 4px; +} +pre { + background-color: #f4f4f4; + padding: 10px; + border-radius: 4px; + overflow-x: auto; +} +table { + border-collapse: collapse; + width: 100%; + margin-bottom: 1em; +} +th, td { + border: 1px solid #ddd; + padding: 8px; + text-align: left; +} +th { + background-color: #f2f2f2; + font-weight: bold; +} +img { + max-width: 90%; + height: auto; + display: block; + margin: 1em auto; + border-radius: 10px; +} +""" + +HTML_SPECIFIC_STYLES = """ +body { + max-width: 800px; + margin: 0 auto; + padding: 20px; +} +""" + +PDF_SPECIFIC_STYLES = """ +@page { + size: letter; + margin: 2cm; +} +body { + font-size: 11pt; +} +h1 { font-size: 18pt; } +h2 { font-size: 16pt; } +h3 { font-size: 14pt; } +h4, h5, h6 { font-size: 12pt; } +pre, code { + font-family: Courier, monospace; + font-size: 0.9em; +} +""" + +HTML_TEMPLATE = """ - {html_content} + {content} """ @@ -66,10 +118,23 @@ def _generate_html_content(cls, original_content: str) -> str: "Failed to import required modules. Please install markdown." ) - # Convert markdown to HTML - html_content = markdown.markdown(original_content) + # Convert markdown to HTML with fenced code and table extensions + html_content = markdown.markdown( + original_content, extensions=["fenced_code", "tables"] + ) return html_content + @classmethod + def _generate_html_document(cls, html_content: str) -> str: + """ + Generate a complete HTML document with the given HTML content. + """ + return HTML_TEMPLATE.format( + common_styles=COMMON_STYLES, + specific_styles=HTML_SPECIFIC_STYLES, + content=html_content, + ) + @classmethod def _generate_pdf(cls, html_content: str) -> BytesIO: """ @@ -82,35 +147,24 @@ def _generate_pdf(cls, html_content: str) -> BytesIO: "Failed to import required modules. Please install xhtml2pdf." ) + pdf_html = HTML_TEMPLATE.format( + common_styles=COMMON_STYLES, + specific_styles=PDF_SPECIFIC_STYLES, + content=html_content, + ) + buffer = BytesIO() pdf = pisa.pisaDocument( - BytesIO(html_content.encode("UTF-8")), - buffer, - encoding="UTF-8", - path=".", - link_callback=None, - debug=0, - default_css=None, - xhtml=False, - xml_output=None, - ident=0, - show_error_as_pdf=False, - quiet=True, - capacity=100 * 1024 * 1024, - raise_exception=True, + BytesIO(pdf_html.encode("UTF-8")), buffer, encoding="UTF-8" ) + if pdf.err: + logging.error(f"PDF generation failed: {pdf.err}") raise ValueError("PDF generation failed") + buffer.seek(0) return buffer - @classmethod - def _generate_html(cls, html_content: str) -> str: - """ - Generate a complete HTML document with the given HTML content. - """ - return HTML_FILE_TEMPLATE.format(html_content=html_content) - @classmethod def generate_artifact( cls, original_content: str, artifact_type: str, file_name: str @@ -119,10 +173,11 @@ def generate_artifact( To generate artifact as PDF or HTML file. Parameters: original_content: str (markdown style) - artifact_type: str (pdf or html) specify the type of the file format based on the use case + artifact_type: str (pdf or html or image) specify the type of the file format based on the use case file_name: str (name of the artifact file) must be a valid file name, no extensions needed Returns: - str (URL to the artifact file): A file URL ready to serve. + str (URL to the artifact file): A file URL ready to serve (pdf or html). + list[str] (URLs to the artifact files): A list of image URLs ready to serve (png). """ try: artifact_type = ArtifactType(artifact_type.lower()) @@ -133,15 +188,12 @@ def generate_artifact( # Always generate html content first html_content = cls._generate_html_content(original_content) - # Based on the type of artifact, generate the corresponding file if artifact_type == ArtifactType.PDF: - content = cls._generate_pdf(cls._generate_html(html_content)) + content = cls._generate_pdf(html_content) file_extension = "pdf" elif artifact_type == ArtifactType.HTML: - content = BytesIO(cls._generate_html(html_content).encode("utf-8")) + content = BytesIO(cls._generate_html_document(html_content).encode("utf-8")) file_extension = "html" - else: - raise ValueError(f"Unexpected artifact type: {artifact_type}") file_name = cls._validate_file_name(file_name) file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") @@ -176,3 +228,7 @@ def _validate_file_name(file_name: str) -> str: return file_name else: raise ValueError("File name is not allowed to contain special characters.") + + +def get_tools(**kwargs): + return [FunctionTool.from_defaults(ArtifactGenerator.generate_artifact)] diff --git a/templates/types/multiagent/fastapi/app/tools/duckduckgo.py b/templates/types/multiagent/fastapi/app/tools/duckduckgo.py new file mode 100644 index 000000000..ec0f63326 --- /dev/null +++ b/templates/types/multiagent/fastapi/app/tools/duckduckgo.py @@ -0,0 +1,68 @@ +from llama_index.core.tools.function_tool import FunctionTool + + +def duckduckgo_search( + query: str, + region: str = "wt-wt", + max_results: int = 10, +): + """ + Use this function to search for any query in DuckDuckGo. + Args: + query (str): The query to search in DuckDuckGo. + region Optional(str): The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc... + max_results Optional(int): The maximum number of results to be returned. Default is 10. + """ + try: + from duckduckgo_search import DDGS + except ImportError: + raise ImportError( + "duckduckgo_search package is required to use this function." + "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`" + ) + + params = { + "keywords": query, + "region": region, + "max_results": max_results, + } + results = [] + with DDGS() as ddg: + results = list(ddg.text(**params)) + return results + + +def duckduckgo_image_search( + query: str, + region: str = "wt-wt", + max_results: int = 10, +): + """ + Use this function to search for images in DuckDuckGo. + Args: + query (str): The query to search in DuckDuckGo. + region Optional(str): The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc... + max_results Optional(int): The maximum number of results to be returned. Default is 10. + """ + try: + from duckduckgo_search import DDGS + except ImportError: + raise ImportError( + "duckduckgo_search package is required to use this function." + "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`" + ) + params = { + "keywords": query, + "region": region, + "max_results": max_results, + } + with DDGS() as ddg: + results = list(ddg.images(**params)) + return results + + +def get_tools(**kwargs): + return [ + FunctionTool.from_defaults(duckduckgo_search), + FunctionTool.from_defaults(duckduckgo_image_search), + ] From 9af221c775ada37f1029d0750e9550081a41b3fd Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 15:27:23 +0700 Subject: [PATCH 14/77] add missing duckduckgo package --- templates/types/multiagent/fastapi/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/types/multiagent/fastapi/pyproject.toml b/templates/types/multiagent/fastapi/pyproject.toml index dc6bd7c3d..397acd278 100644 --- a/templates/types/multiagent/fastapi/pyproject.toml +++ b/templates/types/multiagent/fastapi/pyproject.toml @@ -20,6 +20,7 @@ cachetools = "^5.3.3" aiostream = "^0.5.2" markdown = "^3.7" xhtml2pdf = "^0.2.16" +duckduckgo-search = "^6.2.13" [tool.poetry.dependencies.docx2txt] version = "^0.8" From fd5d86ea8c6c3f995606730e491c165a01fdf480 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 15:33:57 +0700 Subject: [PATCH 15/77] improve publisher prompt --- templates/types/multiagent/fastapi/app/examples/publisher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index f7341a830..a27b7e6e2 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -14,7 +14,7 @@ def create_publisher(chat_history: List[ChatMessage]): tools=[artifact_tool], role="expert in publishing, need to specify the type of artifact (pdf, html, or markdown)", system_prompt="""You are a publisher that help publish the blog post. - For a normal request, you should choose the type of artifact either pdf or html or just reply to the user. + For a normal request, you should choose the type of artifact either pdf or html or just reply to the user the markdown content directly with out generating any artifact file. """, chat_history=chat_history, verbose=True, From c2df7a5fdcf2bc42436bc5ef74fa8dcb2325b43b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 22:18:30 +0700 Subject: [PATCH 16/77] Refine the planner by include chat history --- .../multiagent/fastapi/app/agents/planner.py | 31 ++++++++++-- .../fastapi/app/api/routers/chat.py | 2 +- .../fastapi/app/api/routers/models.py | 49 +++++++++++++++++-- .../fastapi/app/examples/orchestrator.py | 1 + 4 files changed, 74 insertions(+), 9 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/agents/planner.py b/templates/types/multiagent/fastapi/app/agents/planner.py index 07306f2ba..71152c1dc 100644 --- a/templates/types/multiagent/fastapi/app/agents/planner.py +++ b/templates/types/multiagent/fastapi/app/agents/planner.py @@ -11,6 +11,7 @@ SubTask, ) from llama_index.core.bridge.pydantic import ValidationError +from llama_index.core.chat_engine.types import ChatMessage from llama_index.core.llms.function_calling import FunctionCallingLLM from llama_index.core.prompts import PromptTemplate from llama_index.core.settings import Settings @@ -24,6 +25,18 @@ step, ) +INITIAL_PLANNER_PROMPT = """\ +Think step-by-step. Given a conversation, set of tools and a user request. Your responsibility is to create a plan to complete the task. +The plan must adapt with the user request and the conversation. It's fine to just start with needed tasks first and asking user for the next step approval. + +The tools available are: +{tools_str} + +Conversation: {chat_history} + +Overall Task: {task} +""" + class ExecutePlanEvent(Event): pass @@ -62,14 +75,21 @@ def __init__( tools: List[BaseTool] | None = None, timeout: float = 360.0, refine_plan: bool = False, + chat_history: Optional[List[ChatMessage]] = None, **kwargs: Any, ) -> None: super().__init__(*args, timeout=timeout, **kwargs) self.name = name self.refine_plan = refine_plan + self.chat_history = chat_history self.tools = tools or [] - self.planner = Planner(llm=llm, tools=self.tools, verbose=self._verbose) + self.planner = Planner( + llm=llm, + tools=self.tools, + initial_plan_prompt=INITIAL_PLANNER_PROMPT, + verbose=self._verbose, + ) # The executor is keeping the memory of all tool calls and decides to call the right tool for the task self.executor = FunctionCallingAgent( name="executor", @@ -89,7 +109,9 @@ async def create_plan( ctx.data["streaming"] = getattr(ev, "streaming", False) ctx.data["task"] = ev.input - plan_id, plan = await self.planner.create_plan(input=ev.input) + plan_id, plan = await self.planner.create_plan( + input=ev.input, chat_history=self.chat_history + ) ctx.data["act_plan_id"] = plan_id # inform about the new plan @@ -213,7 +235,9 @@ def __init__( plan_refine_prompt = PromptTemplate(plan_refine_prompt) self.plan_refine_prompt = plan_refine_prompt - async def create_plan(self, input: str) -> Tuple[str, Plan]: + async def create_plan( + self, input: str, chat_history: Optional[List[ChatMessage]] = None + ) -> Tuple[str, Plan]: tools = self.tools tools_str = "" for tool in tools: @@ -225,6 +249,7 @@ async def create_plan(self, input: str) -> Tuple[str, Plan]: self.initial_plan_prompt, tools_str=tools_str, task=input, + chat_history=chat_history, ) except (ValueError, ValidationError): if self.verbose: diff --git a/templates/types/multiagent/fastapi/app/api/routers/chat.py b/templates/types/multiagent/fastapi/app/api/routers/chat.py index 2b7a5636f..b9776775d 100644 --- a/templates/types/multiagent/fastapi/app/api/routers/chat.py +++ b/templates/types/multiagent/fastapi/app/api/routers/chat.py @@ -20,7 +20,7 @@ async def chat( ): try: last_message_content = data.get_last_message_content() - messages = data.get_history_messages() + messages = data.get_history_messages(include_agent_messages=True) # TODO: generate filters based on doc_ids # for now just use all documents # doc_ids = data.get_chat_document_ids() diff --git a/templates/types/multiagent/fastapi/app/api/routers/models.py b/templates/types/multiagent/fastapi/app/api/routers/models.py index 29648608f..f70f99afd 100644 --- a/templates/types/multiagent/fastapi/app/api/routers/models.py +++ b/templates/types/multiagent/fastapi/app/api/routers/models.py @@ -2,13 +2,12 @@ import os from typing import Any, Dict, List, Literal, Optional +from app.config import DATA_DIR from llama_index.core.llms import ChatMessage, MessageRole from llama_index.core.schema import NodeWithScore from pydantic import BaseModel, Field, validator from pydantic.alias_generators import to_camel -from app.config import DATA_DIR - logger = logging.getLogger("uvicorn") @@ -50,9 +49,14 @@ class Config: alias_generator = to_camel +class AgentAnnotation(BaseModel): + agent: str + text: str + + class Annotation(BaseModel): type: str - data: AnnotationFileData | List[str] + data: AnnotationFileData | List[str] | AgentAnnotation def to_content(self) -> str | None: if self.type == "document_file": @@ -119,14 +123,49 @@ def get_last_message_content(self) -> str: break return message_content - def get_history_messages(self) -> List[ChatMessage]: + def _get_agent_messages(self, max_messages: int = 5) -> List[str]: + """ + Construct agent messages from the annotations in the chat messages + """ + agent_messages = [] + for message in self.messages: + if ( + message.role == MessageRole.ASSISTANT + and message.annotations is not None + ): + for annotation in message.annotations: + if annotation.type == "agent" and isinstance( + annotation.data, AgentAnnotation + ): + text = annotation.data.text + if not text.startswith("Finished task"): + agent_messages.append( + f"\nAgent: {annotation.data.agent}\nsaid: {text}\n" + ) + if len(agent_messages) >= max_messages: + break + return agent_messages + + def get_history_messages( + self, include_agent_messages: bool = False + ) -> List[ChatMessage]: """ Get the history messages """ - return [ + chat_messages = [ ChatMessage(role=message.role, content=message.content) for message in self.messages[:-1] ] + if include_agent_messages: + agent_messages = self._get_agent_messages(max_messages=5) + if len(agent_messages) > 0: + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="Previous agent events: \n" + "\n".join(agent_messages), + ) + chat_messages.append(message) + + return chat_messages def is_last_message_from_user(self) -> bool: return self.messages[-1].role == MessageRole.USER diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/types/multiagent/fastapi/app/examples/orchestrator.py index 69739744a..85ff262e7 100644 --- a/templates/types/multiagent/fastapi/app/examples/orchestrator.py +++ b/templates/types/multiagent/fastapi/app/examples/orchestrator.py @@ -31,4 +31,5 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): return AgentOrchestrator( agents=[writer, reviewer, researcher, publisher], refine_plan=False, + chat_history=chat_history, ) From 10ec398123b468467af992f82f99e30d1d1fc705 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 22:24:24 +0700 Subject: [PATCH 17/77] change role to description --- templates/types/multiagent/fastapi/app/agents/multi.py | 2 +- .../types/multiagent/fastapi/app/examples/choreography.py | 4 ++-- .../types/multiagent/fastapi/app/examples/orchestrator.py | 4 ++-- templates/types/multiagent/fastapi/app/examples/publisher.py | 4 ++-- templates/types/multiagent/fastapi/app/examples/researcher.py | 2 +- templates/types/multiagent/fastapi/app/examples/workflow.py | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/agents/multi.py b/templates/types/multiagent/fastapi/app/agents/multi.py index d20f372a0..6370a7611 100644 --- a/templates/types/multiagent/fastapi/app/agents/multi.py +++ b/templates/types/multiagent/fastapi/app/agents/multi.py @@ -25,7 +25,7 @@ async def schema_call(input: str) -> str: name=name, description=( f"Use this tool to delegate a sub task to the {agent.name} agent." - + (f" The agent is an {agent.role}." if agent.role else "") + + (f" The agent is an {agent.description}." if agent.description else "") ), fn_schema=fn_schema, ) diff --git a/templates/types/multiagent/fastapi/app/examples/choreography.py b/templates/types/multiagent/fastapi/app/examples/choreography.py index 7ac48b8d5..27a73abcd 100644 --- a/templates/types/multiagent/fastapi/app/examples/choreography.py +++ b/templates/types/multiagent/fastapi/app/examples/choreography.py @@ -12,7 +12,7 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): publisher = create_publisher(chat_history) reviewer = FunctionCallingAgent( name="reviewer", - role="expert in reviewing blog posts", + description="expert in reviewing blog posts, need a written post to review", system_prompt="You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. If the post is good, you can say 'The post is good.'", chat_history=chat_history, verbose=True, @@ -20,7 +20,7 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): return AgentCallingAgent( name="writer", agents=[researcher, reviewer, publisher], - role="expert in writing blog posts", + description="expert in writing blog posts, need provided information and images to write a blog post", system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/types/multiagent/fastapi/app/examples/orchestrator.py index 85ff262e7..7f4a1839d 100644 --- a/templates/types/multiagent/fastapi/app/examples/orchestrator.py +++ b/templates/types/multiagent/fastapi/app/examples/orchestrator.py @@ -11,7 +11,7 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): researcher = create_researcher(chat_history) writer = FunctionCallingAgent( name="writer", - role="expert in writing blog posts, need information and images to write a post", + description="expert in writing blog posts, need information and images to write a post", system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Don't make up any information yourself. If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". @@ -21,7 +21,7 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): ) reviewer = FunctionCallingAgent( name="reviewer", - role="expert in reviewing blog posts, need a written blog post to review", + description="expert in reviewing blog posts, need a written blog post to review", system_prompt="""You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix the issues found yourself. You must output a final blog post. A post must include at lease one valid image, if not, reply "I need images about the topic to write the blog post". An image URL start with example or your website is not valid. Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.""", diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index a27b7e6e2..79bccf7f5 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -12,9 +12,9 @@ def create_publisher(chat_history: List[ChatMessage]): return FunctionCallingAgent( name="publisher", tools=[artifact_tool], - role="expert in publishing, need to specify the type of artifact (pdf, html, or markdown)", + description="expert in publishing, need to specify the type of artifact use a file (pdf, html) or just reply the content directly", system_prompt="""You are a publisher that help publish the blog post. - For a normal request, you should choose the type of artifact either pdf or html or just reply to the user the markdown content directly with out generating any artifact file. + For a normal request, you should choose the type of artifact either pdf or html or just reply to the user directly without generating any artifact file. """, chat_history=chat_history, verbose=True, diff --git a/templates/types/multiagent/fastapi/app/examples/researcher.py b/templates/types/multiagent/fastapi/app/examples/researcher.py index 6c4d9d594..5fbdf484c 100644 --- a/templates/types/multiagent/fastapi/app/examples/researcher.py +++ b/templates/types/multiagent/fastapi/app/examples/researcher.py @@ -35,7 +35,7 @@ def create_researcher(chat_history: List[ChatMessage]): return FunctionCallingAgent( name="researcher", tools=[get_query_engine_tool(), *duckduckgo_search_tools], - role="expert in retrieving any unknown content or searching for images from the internet", + description="expert in retrieving any unknown content or searching for images from the internet", system_prompt="You are a researcher agent. You are given a researching task. You must use tools to retrieve information from the knowledge base and search for needed images from the internet for the post.", chat_history=chat_history, ) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index 1d49792ca..dd9a6b853 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -23,13 +23,13 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): ) writer = FunctionCallingAgent( name="writer", - role="expert in writing blog posts", + description="expert in writing blog posts, need information and images to write a post", system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Don't make up any information yourself.""", chat_history=chat_history, ) reviewer = FunctionCallingAgent( name="reviewer", - role="expert in reviewing blog posts", + description="expert in reviewing blog posts, need a written blog post to review", system_prompt="You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review.", chat_history=chat_history, ) From c217f047b34a708fe74e2c1c9005a35ec8388859 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 26 Sep 2024 22:37:08 +0700 Subject: [PATCH 18/77] fix missing change --- templates/types/multiagent/fastapi/app/agents/single.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/agents/single.py b/templates/types/multiagent/fastapi/app/agents/single.py index b47662f8d..754cc5ac4 100644 --- a/templates/types/multiagent/fastapi/app/agents/single.py +++ b/templates/types/multiagent/fastapi/app/agents/single.py @@ -5,10 +5,8 @@ from llama_index.core.llms.function_calling import FunctionCallingLLM from llama_index.core.memory import ChatMemoryBuffer from llama_index.core.settings import Settings -from llama_index.core.tools import ToolOutput, ToolSelection +from llama_index.core.tools import FunctionTool, ToolOutput, ToolSelection from llama_index.core.tools.types import BaseTool -from llama_index.core.tools import FunctionTool - from llama_index.core.workflow import ( Context, Event, @@ -64,13 +62,13 @@ def __init__( timeout: float = 360.0, name: str, write_events: bool = True, - role: Optional[str] = None, + description: Optional[str] = None, **kwargs: Any, ) -> None: super().__init__(*args, verbose=verbose, timeout=timeout, **kwargs) self.tools = tools or [] self.name = name - self.role = role + self.description = description self.write_events = write_events if llm is None: From 818a0bb7bbb6a9f1d78c061204a992911dffbf7c Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 14:20:53 +0700 Subject: [PATCH 19/77] add document generator tool --- helpers/tools.ts | 10 +- helpers/typescript.ts | 15 ++ .../agent/tools/document_generator.ts | 240 ++++++++++++++++++ .../typescript/agent/tools/duckduckgo.ts | 95 ++++++- .../engines/typescript/agent/tools/index.ts | 7 + 5 files changed, 355 insertions(+), 12 deletions(-) create mode 100644 templates/components/engines/typescript/agent/tools/document_generator.ts diff --git a/helpers/tools.ts b/helpers/tools.ts index 5a0b46aa8..0d758f23e 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -111,10 +111,10 @@ For better results, you can specify the region parameter to get results from a s ], }, { - display: "Artifact generator", - name: "artifact", + display: "Document generator", + name: "document_generator", // TODO: add support for Typescript templates - supportedFrameworks: ["fastapi"], + supportedFrameworks: ["fastapi", "nextjs", "express"], dependencies: [ { name: "xhtml2pdf", @@ -129,8 +129,8 @@ For better results, you can specify the region parameter to get results from a s envVars: [ { name: TOOL_SYSTEM_PROMPT_ENV_VAR, - description: "System prompt for artifact tool.", - value: `If user request for a report or a post, use artifact tool to create a file and reply with the link to the file.`, + description: "System prompt for document generator tool.", + value: `If user request for a report or a post, use document generator tool to create a file and reply with the link to the file.`, }, ], }, diff --git a/helpers/typescript.ts b/helpers/typescript.ts index ffebae4a3..0688a80f6 100644 --- a/helpers/typescript.ts +++ b/helpers/typescript.ts @@ -209,6 +209,7 @@ export const installTSTemplate = async ({ ui, observability, vectorDb, + tools, }); if (postInstallAction === "runApp" || postInstallAction === "dependencies") { @@ -230,6 +231,7 @@ async function updatePackageJson({ ui, observability, vectorDb, + tools, }: Pick< InstallTemplateArgs, | "root" @@ -239,6 +241,7 @@ async function updatePackageJson({ | "ui" | "observability" | "vectorDb" + | "tools" > & { relativeEngineDestPath: string; }): Promise { @@ -325,6 +328,18 @@ async function updatePackageJson({ }; } + if (tools && tools.length > 0) { + tools.forEach((tool) => { + if (tool.name === "document_generator") { + packageJson.dependencies = { + ...packageJson.dependencies, + puppeteer: "^23.4.1", + marked: "^14.1.2", + }; + } + }); + } + await fs.writeFile( packageJsonFile, JSON.stringify(packageJson, null, 2) + os.EOL, diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document_generator.ts new file mode 100644 index 000000000..9c9ac950a --- /dev/null +++ b/templates/components/engines/typescript/agent/tools/document_generator.ts @@ -0,0 +1,240 @@ +import { JSONSchemaType } from "ajv"; +import fs from "fs"; +import { BaseTool, ToolMetadata } from "llamaindex"; +import { marked } from "marked"; +import path from "path"; +import puppeteer from "puppeteer"; + +const OUTPUT_DIR = "output/tools"; + +enum DocumentType { + HTML = "html", + PDF = "pdf", +} + +type DocumentParameter = { + originalContent: string; + documentType: string; + fileName: string; +}; + +const DEFAULT_METADATA: ToolMetadata> = { + name: "document_generator", + description: + "Generate document as PDF or HTML file from markdown content. Return a file url to the document", + parameters: { + type: "object", + properties: { + originalContent: { + type: "string", + description: "The original markdown content to convert.", + }, + documentType: { + type: "string", + description: "The type of document to generate (pdf or html).", + }, + fileName: { + type: "string", + description: "The name of the document file (without extension).", + }, + }, + required: ["originalContent", "documentType", "fileName"], + }, +}; + +const COMMON_STYLES = ` + body { + font-family: Arial, sans-serif; + line-height: 1.3; + color: #333; + } + h1, h2, h3, h4, h5, h6 { + margin-top: 1em; + margin-bottom: 0.5em; + } + p { + margin-bottom: 0.7em; + } + code { + background-color: #f4f4f4; + padding: 2px 4px; + border-radius: 4px; + } + pre { + background-color: #f4f4f4; + padding: 10px; + border-radius: 4px; + overflow-x: auto; + } + table { + border-collapse: collapse; + width: 100%; + margin-bottom: 1em; + } + th, td { + border: 1px solid #ddd; + padding: 8px; + text-align: left; + } + th { + background-color: #f2f2f2; + font-weight: bold; + } + img { + max-width: 90%; + height: auto; + display: block; + margin: 1em auto; + border-radius: 10px; + } +`; + +const HTML_SPECIFIC_STYLES = ` + body { + max-width: 800px; + margin: 0 auto; + padding: 20px; + } +`; + +const HTML_TEMPLATE = ` + + + + + + + + + {{content}} + + +`; + +const PDF_SPECIFIC_STYLES = ` + @page { + size: letter; + margin: 2cm; + } + body { + font-size: 11pt; + } + h1 { font-size: 18pt; } + h2 { font-size: 16pt; } + h3 { font-size: 14pt; } + h4, h5, h6 { font-size: 12pt; } + pre, code { + font-family: Courier, monospace; + font-size: 0.9em; + } +`; + +export interface DocumentGeneratorParams { + metadata?: ToolMetadata>; +} + +export class DocumentGenerator implements BaseTool { + metadata: ToolMetadata>; + + constructor(params: DocumentGeneratorParams) { + this.metadata = params.metadata ?? DEFAULT_METADATA; + } + + private static generateHtmlContent(originalContent: string): string { + return marked(originalContent); + } + + private static generateHtmlDocument(htmlContent: string): string { + return HTML_TEMPLATE.replace("{{content}}", htmlContent); + } + + private static validateFileName(fileName: string): string { + if (path.isAbsolute(fileName)) { + throw new Error("File name is not allowed."); + } + if (/^[a-zA-Z0-9_.-]+$/.test(fileName)) { + return fileName; + } else { + throw new Error( + "File name is not allowed to contain special characters.", + ); + } + } + + private static writeToFile(content: string | Buffer, filePath: string): void { + try { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + if (typeof content === "string") { + fs.writeFileSync(filePath, content, "utf8"); + } else { + fs.writeFileSync(filePath, content); + } + } catch (error) { + throw error; + } + } + + private static generatePdfDocument(htmlContent: string): string { + return HTML_TEMPLATE.replace("{{content}}", htmlContent).replace( + HTML_SPECIFIC_STYLES, + PDF_SPECIFIC_STYLES, + ); + } + + private static async generatePdf(htmlContent: string): Promise { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.setContent(htmlContent, { waitUntil: "networkidle0" }); + const pdf = await page.pdf({ format: "A4" }); + await browser.close(); + return pdf; + } + + async call(input: DocumentParameter): Promise { + const { originalContent, documentType, fileName } = input; + + let fileContent: string | Buffer; + let fileExtension: string; + + // Generate the HTML from the original content (markdown) + const htmlContent = DocumentGenerator.generateHtmlContent(originalContent); + + try { + if (documentType.toLowerCase() === DocumentType.HTML) { + const htmlDocument = + DocumentGenerator.generateHtmlDocument(htmlContent); + fileContent = DocumentGenerator.generateHtmlDocument(htmlDocument); + fileExtension = "html"; + } else if (documentType.toLowerCase() === DocumentType.PDF) { + const pdfDocument = DocumentGenerator.generatePdfDocument(htmlContent); + fileContent = await DocumentGenerator.generatePdf(pdfDocument); + fileExtension = "pdf"; + } else { + throw new Error( + `Invalid document type: ${documentType}. Must be 'pdf' or 'html'.`, + ); + } + } catch (error) { + console.error("Error generating document:", error); + throw new Error("Failed to generate document"); + } + + const validatedFileName = DocumentGenerator.validateFileName(fileName); + const filePath = path.join( + OUTPUT_DIR, + `${validatedFileName}.${fileExtension}`, + ); + + DocumentGenerator.writeToFile(fileContent, filePath); + + const fileUrl = `${process.env.FILESERVER_URL_PREFIX}/${filePath}`; + return fileUrl; + } +} + +export function getTools(): BaseTool[] { + return [new DocumentGenerator({})]; +} diff --git a/templates/components/engines/typescript/agent/tools/duckduckgo.ts b/templates/components/engines/typescript/agent/tools/duckduckgo.ts index 19423e353..5100bdfdb 100644 --- a/templates/components/engines/typescript/agent/tools/duckduckgo.ts +++ b/templates/components/engines/typescript/agent/tools/duckduckgo.ts @@ -1,19 +1,23 @@ import { JSONSchemaType } from "ajv"; -import { search } from "duck-duck-scrape"; +import { ImageSearchOptions, search, searchImages } from "duck-duck-scrape"; import { BaseTool, ToolMetadata } from "llamaindex"; export type DuckDuckGoParameter = { query: string; region?: string; + maxResults?: number; }; export type DuckDuckGoToolParams = { metadata?: ToolMetadata>; }; -const DEFAULT_META_DATA: ToolMetadata> = { - name: "duckduckgo", - description: "Use this function to search for any query in DuckDuckGo.", +const DEFAULT_SEARCH_METADATA: ToolMetadata< + JSONSchemaType +> = { + name: "duckduckgo_search", + description: + "Use this function to search for information in the internet using DuckDuckGo.", parameters: { type: "object", properties: { @@ -27,6 +31,41 @@ const DEFAULT_META_DATA: ToolMetadata> = { "Optional, The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...", nullable: true, }, + maxResults: { + type: "number", + description: + "Optional, The maximum number of results to be returned. Default is 10.", + nullable: true, + }, + }, + required: ["query"], + }, +}; + +const DEFAULT_IMAGE_SEARCH_METADATA: ToolMetadata< + JSONSchemaType +> = { + name: "duckduckgo_image_search", + description: "Use this function to search for images in DuckDuckGo.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The query to search in DuckDuckGo.", + }, + region: { + type: "string", + description: + "Optional, The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...", + nullable: true, + }, + maxResults: { + type: "number", + description: + "Optional, The maximum number of results to be returned. Default is 10.", + nullable: true, + }, }, required: ["query"], }, @@ -38,19 +77,28 @@ type DuckDuckGoSearchResult = { url: string; }; +type DuckDuckGoImageResult = { + image: string; + title: string; + source: string; + url: string; +}; + export class DuckDuckGoSearchTool implements BaseTool { metadata: ToolMetadata>; constructor(params: DuckDuckGoToolParams) { - this.metadata = params.metadata ?? DEFAULT_META_DATA; + this.metadata = params.metadata ?? DEFAULT_SEARCH_METADATA; } async call(input: DuckDuckGoParameter) { - const { query, region } = input; + const { query, region, maxResults = 10 } = input; const options = region ? { region } : {}; + // Temporarily sleep to reduce overloading the DuckDuckGo + await new Promise((resolve) => setTimeout(resolve, 1000)); const searchResults = await search(query, options); - return searchResults.results.map((result) => { + return searchResults.results.slice(0, maxResults).map((result) => { return { title: result.title, description: result.description, @@ -59,3 +107,36 @@ export class DuckDuckGoSearchTool implements BaseTool { }); } } + +export class DuckDuckGoImageSearchTool + implements BaseTool +{ + metadata: ToolMetadata>; + + constructor(params: DuckDuckGoToolParams) { + this.metadata = params.metadata ?? DEFAULT_IMAGE_SEARCH_METADATA; + } + + async call(input: DuckDuckGoParameter) { + const { query, region, maxResults = 5 } = input; + const options: Partial = region + ? { locale: region } + : {}; + // Temporarily sleep to reduce overloading the DuckDuckGo + await new Promise((resolve) => setTimeout(resolve, 1000)); + const imageResults = await searchImages(query, options); + + return imageResults.results.slice(0, maxResults).map((result) => { + return { + image: result.image, + title: result.title, + source: result.source, + url: result.url, + } as DuckDuckGoImageResult; + }); + } +} + +export function getTools() { + return [new DuckDuckGoSearchTool({}), new DuckDuckGoImageSearchTool({})]; +} diff --git a/templates/components/engines/typescript/agent/tools/index.ts b/templates/components/engines/typescript/agent/tools/index.ts index c442a315d..7fe850fb9 100644 --- a/templates/components/engines/typescript/agent/tools/index.ts +++ b/templates/components/engines/typescript/agent/tools/index.ts @@ -1,5 +1,9 @@ import { BaseToolWithCall } from "llamaindex"; import { ToolsFactory } from "llamaindex/tools/ToolsFactory"; +import { + DocumentGenerator, + DocumentGeneratorParams, +} from "./document_generator"; import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo"; import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen"; import { InterpreterTool, InterpreterToolParams } from "./interpreter"; @@ -43,6 +47,9 @@ const toolFactory: Record = { img_gen: async (config: unknown) => { return [new ImgGeneratorTool(config as ImgGeneratorToolParams)]; }, + document_generator: async (config: unknown) => { + return [new DocumentGenerator(config as DocumentGeneratorParams)]; + }, }; async function createLocalTools( From 4298256980e894c5ed7f6f73fce6080fd5d5214a Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 14:23:12 +0700 Subject: [PATCH 20/77] rename artifact to document generator tool --- helpers/tools.ts | 2 +- .../{artifact.py => document_generator.py} | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) rename templates/components/engines/python/agent/tools/{artifact.py => document_generator.py} (90%) diff --git a/helpers/tools.ts b/helpers/tools.ts index 0d758f23e..875405804 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -72,7 +72,7 @@ export const supportedTools: Tool[] = [ name: TOOL_SYSTEM_PROMPT_ENV_VAR, description: "System prompt for DuckDuckGo search tool.", value: `You are a DuckDuckGo search agent. -You can use the duckduckgo search tool to get information from the web to answer user questions. +You can use the duckduckgo search tool to get information or images from the web to answer user questions. For better results, you can specify the region parameter to get results from a specific region but it's optional.`, }, ], diff --git a/templates/components/engines/python/agent/tools/artifact.py b/templates/components/engines/python/agent/tools/document_generator.py similarity index 90% rename from templates/components/engines/python/agent/tools/artifact.py rename to templates/components/engines/python/agent/tools/document_generator.py index 2a9bb638f..212542ee3 100644 --- a/templates/components/engines/python/agent/tools/artifact.py +++ b/templates/components/engines/python/agent/tools/document_generator.py @@ -9,7 +9,7 @@ OUTPUT_DIR = "output/tools" -class ArtifactType(Enum): +class DocumentType(Enum): PDF = "pdf" HTML = "html" @@ -98,7 +98,7 @@ class ArtifactType(Enum): """ -class ArtifactGenerator: +class DocumentGenerator: @classmethod def _generate_html_content(cls, original_content: str) -> str: """ @@ -159,36 +159,36 @@ def _generate_html(cls, html_content: str) -> str: ) @classmethod - def generate_artifact( - cls, original_content: str, artifact_type: str, file_name: str + def generate_document( + cls, original_content: str, document_type: str, file_name: str ) -> str: """ To generate artifact as PDF or HTML file. Parameters: original_content: str (markdown style) - artifact_type: str (pdf or html) specify the type of the file format based on the use case + document_type: str (pdf or html) specify the type of the file format based on the use case file_name: str (name of the artifact file) must be a valid file name, no extensions needed Returns: str (URL to the artifact file): A file URL ready to serve. """ try: - artifact_type = ArtifactType(artifact_type.lower()) + document_type = DocumentType(document_type.lower()) except ValueError: raise ValueError( - f"Invalid artifact type: {artifact_type}. Must be 'pdf' or 'html'." + f"Invalid document type: {document_type}. Must be 'pdf' or 'html'." ) # Always generate html content first html_content = cls._generate_html_content(original_content) # Based on the type of artifact, generate the corresponding file - if artifact_type == ArtifactType.PDF: + if document_type == DocumentType.PDF: content = cls._generate_pdf(html_content) file_extension = "pdf" - elif artifact_type == ArtifactType.HTML: + elif document_type == DocumentType.HTML: content = BytesIO(cls._generate_html(html_content).encode("utf-8")) file_extension = "html" else: - raise ValueError(f"Unexpected artifact type: {artifact_type}") + raise ValueError(f"Unexpected document type: {document_type}") file_name = cls._validate_file_name(file_name) file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") @@ -226,4 +226,4 @@ def _validate_file_name(file_name: str) -> str: def get_tools(**kwargs): - return [FunctionTool.from_defaults(ArtifactGenerator.generate_artifact)] + return [FunctionTool.from_defaults(DocumentGenerator.generate_document)] From 2af3c41143482f73a4458caf0289d898fd0cd6d8 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 14:42:31 +0700 Subject: [PATCH 21/77] add tools for multiagent --- questions.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/questions.ts b/questions.ts index 3619447cd..69d7f49ae 100644 --- a/questions.ts +++ b/questions.ts @@ -18,6 +18,7 @@ import { getAvailableLlamapackOptions } from "./helpers/llama-pack"; import { askModelConfig } from "./helpers/providers"; import { getProjectOptions } from "./helpers/repo"; import { + ToolType, supportedTools, toolRequiresConfig, toolsRequireConfig, @@ -365,6 +366,22 @@ export const askQuestions = async ( } } + // TODO: Remove this once we support selecting tools for multiagent template + if (program.template === "multiagent") { + program.tools = [ + { + name: "document_generator", + display: "Document Generator", + type: ToolType.LOCAL, + }, + { + name: "duckduckgo", + display: "DuckDuckGo", + type: ToolType.LOCAL, + }, + ]; + } + if (program.template === "community") { const projectOptions = await getProjectOptions( COMMUNITY_OWNER, From 4169525d57bd77454330f12dc23f5e2c975e5375 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 14:49:59 +0700 Subject: [PATCH 22/77] update python to change artifact to document generator --- helpers/python.ts | 9 + .../fastapi/app/examples/publisher.py | 12 +- .../multiagent/fastapi/app/tools/artifact.py | 234 ------------------ .../fastapi/app/tools/duckduckgo.py | 68 ----- 4 files changed, 16 insertions(+), 307 deletions(-) delete mode 100644 templates/types/multiagent/fastapi/app/tools/artifact.py delete mode 100644 templates/types/multiagent/fastapi/app/tools/duckduckgo.py diff --git a/helpers/python.ts b/helpers/python.ts index f5dac282b..06587dafe 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -402,6 +402,15 @@ export const installPythonTemplate = async ({ }); } + // Copy tools for multiagent template + // TODO: Remove this once we support selecting tools for multiagent template + if (template === "multiagent") { + // templates / components / engines / python / agent / tools; + await copy("**", path.join(root, "app", "tools"), { + cwd: path.join(compPath, "engines", "python", "agent", "tools"), + }); + } + if (template === "streaming") { // For the streaming template only: // Select and copy engine code based on data sources and tools diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index 79bccf7f5..140f8b852 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -1,20 +1,22 @@ from typing import List from app.agents.single import FunctionCallingAgent -from app.tools.artifact import ArtifactGenerator +from app.tools.document_generator import DocumentGenerator from llama_index.core.chat_engine.types import ChatMessage from llama_index.core.tools import FunctionTool def create_publisher(chat_history: List[ChatMessage]): - artifact_tool = FunctionTool.from_defaults(ArtifactGenerator.generate_artifact) + document_generator_tool = FunctionTool.from_defaults( + DocumentGenerator.generate_document + ) return FunctionCallingAgent( name="publisher", - tools=[artifact_tool], - description="expert in publishing, need to specify the type of artifact use a file (pdf, html) or just reply the content directly", + tools=[document_generator_tool], + description="expert in publishing, need to specify the type of document use a file (pdf, html) or just reply the content directly", system_prompt="""You are a publisher that help publish the blog post. - For a normal request, you should choose the type of artifact either pdf or html or just reply to the user directly without generating any artifact file. + For a normal request, you should choose the type of document either pdf or html or just reply to the user directly without generating any document file. """, chat_history=chat_history, verbose=True, diff --git a/templates/types/multiagent/fastapi/app/tools/artifact.py b/templates/types/multiagent/fastapi/app/tools/artifact.py deleted file mode 100644 index 9c4673770..000000000 --- a/templates/types/multiagent/fastapi/app/tools/artifact.py +++ /dev/null @@ -1,234 +0,0 @@ -import logging -import os -import re -from enum import Enum -from io import BytesIO - -from llama_index.core.tools.function_tool import FunctionTool - -OUTPUT_DIR = "output/tools" - - -class ArtifactType(Enum): - PDF = "pdf" - HTML = "html" - - -COMMON_STYLES = """ -body { - font-family: Arial, sans-serif; - line-height: 1.3; - color: #333; -} -h1, h2, h3, h4, h5, h6 { - margin-top: 1em; - margin-bottom: 0.5em; -} -p { - margin-bottom: 0.7em; -} -code { - background-color: #f4f4f4; - padding: 2px 4px; - border-radius: 4px; -} -pre { - background-color: #f4f4f4; - padding: 10px; - border-radius: 4px; - overflow-x: auto; -} -table { - border-collapse: collapse; - width: 100%; - margin-bottom: 1em; -} -th, td { - border: 1px solid #ddd; - padding: 8px; - text-align: left; -} -th { - background-color: #f2f2f2; - font-weight: bold; -} -img { - max-width: 90%; - height: auto; - display: block; - margin: 1em auto; - border-radius: 10px; -} -""" - -HTML_SPECIFIC_STYLES = """ -body { - max-width: 800px; - margin: 0 auto; - padding: 20px; -} -""" - -PDF_SPECIFIC_STYLES = """ -@page { - size: letter; - margin: 2cm; -} -body { - font-size: 11pt; -} -h1 { font-size: 18pt; } -h2 { font-size: 16pt; } -h3 { font-size: 14pt; } -h4, h5, h6 { font-size: 12pt; } -pre, code { - font-family: Courier, monospace; - font-size: 0.9em; -} -""" - -HTML_TEMPLATE = """ - - - - - - - - - {content} - - -""" - - -class ArtifactGenerator: - @classmethod - def _generate_html_content(cls, original_content: str) -> str: - """ - Generate HTML content from the original markdown content. - """ - try: - import markdown - except ImportError: - raise ImportError( - "Failed to import required modules. Please install markdown." - ) - - # Convert markdown to HTML with fenced code and table extensions - html_content = markdown.markdown( - original_content, extensions=["fenced_code", "tables"] - ) - return html_content - - @classmethod - def _generate_html_document(cls, html_content: str) -> str: - """ - Generate a complete HTML document with the given HTML content. - """ - return HTML_TEMPLATE.format( - common_styles=COMMON_STYLES, - specific_styles=HTML_SPECIFIC_STYLES, - content=html_content, - ) - - @classmethod - def _generate_pdf(cls, html_content: str) -> BytesIO: - """ - Generate a PDF from the HTML content. - """ - try: - from xhtml2pdf import pisa - except ImportError: - raise ImportError( - "Failed to import required modules. Please install xhtml2pdf." - ) - - pdf_html = HTML_TEMPLATE.format( - common_styles=COMMON_STYLES, - specific_styles=PDF_SPECIFIC_STYLES, - content=html_content, - ) - - buffer = BytesIO() - pdf = pisa.pisaDocument( - BytesIO(pdf_html.encode("UTF-8")), buffer, encoding="UTF-8" - ) - - if pdf.err: - logging.error(f"PDF generation failed: {pdf.err}") - raise ValueError("PDF generation failed") - - buffer.seek(0) - return buffer - - @classmethod - def generate_artifact( - cls, original_content: str, artifact_type: str, file_name: str - ) -> str: - """ - To generate artifact as PDF or HTML file. - Parameters: - original_content: str (markdown style) - artifact_type: str (pdf or html or image) specify the type of the file format based on the use case - file_name: str (name of the artifact file) must be a valid file name, no extensions needed - Returns: - str (URL to the artifact file): A file URL ready to serve (pdf or html). - list[str] (URLs to the artifact files): A list of image URLs ready to serve (png). - """ - try: - artifact_type = ArtifactType(artifact_type.lower()) - except ValueError: - raise ValueError( - f"Invalid artifact type: {artifact_type}. Must be 'pdf' or 'html'." - ) - # Always generate html content first - html_content = cls._generate_html_content(original_content) - - if artifact_type == ArtifactType.PDF: - content = cls._generate_pdf(html_content) - file_extension = "pdf" - elif artifact_type == ArtifactType.HTML: - content = BytesIO(cls._generate_html_document(html_content).encode("utf-8")) - file_extension = "html" - - file_name = cls._validate_file_name(file_name) - file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") - - cls._write_to_file(content, file_path) - - file_url = f"{os.getenv('FILESERVER_URL_PREFIX')}/{file_path}" - return file_url - - @staticmethod - def _write_to_file(content: BytesIO, file_path: str): - """ - Write the content to a file. - """ - try: - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, "wb") as file: - file.write(content.getvalue()) - except Exception as e: - raise e - - @staticmethod - def _validate_file_name(file_name: str) -> str: - """ - Validate the file name. - """ - # Don't allow directory traversal - if os.path.isabs(file_name): - raise ValueError("File name is not allowed.") - # Don't allow special characters - if re.match(r"^[a-zA-Z0-9_.-]+$", file_name): - return file_name - else: - raise ValueError("File name is not allowed to contain special characters.") - - -def get_tools(**kwargs): - return [FunctionTool.from_defaults(ArtifactGenerator.generate_artifact)] diff --git a/templates/types/multiagent/fastapi/app/tools/duckduckgo.py b/templates/types/multiagent/fastapi/app/tools/duckduckgo.py deleted file mode 100644 index ec0f63326..000000000 --- a/templates/types/multiagent/fastapi/app/tools/duckduckgo.py +++ /dev/null @@ -1,68 +0,0 @@ -from llama_index.core.tools.function_tool import FunctionTool - - -def duckduckgo_search( - query: str, - region: str = "wt-wt", - max_results: int = 10, -): - """ - Use this function to search for any query in DuckDuckGo. - Args: - query (str): The query to search in DuckDuckGo. - region Optional(str): The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc... - max_results Optional(int): The maximum number of results to be returned. Default is 10. - """ - try: - from duckduckgo_search import DDGS - except ImportError: - raise ImportError( - "duckduckgo_search package is required to use this function." - "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`" - ) - - params = { - "keywords": query, - "region": region, - "max_results": max_results, - } - results = [] - with DDGS() as ddg: - results = list(ddg.text(**params)) - return results - - -def duckduckgo_image_search( - query: str, - region: str = "wt-wt", - max_results: int = 10, -): - """ - Use this function to search for images in DuckDuckGo. - Args: - query (str): The query to search in DuckDuckGo. - region Optional(str): The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc... - max_results Optional(int): The maximum number of results to be returned. Default is 10. - """ - try: - from duckduckgo_search import DDGS - except ImportError: - raise ImportError( - "duckduckgo_search package is required to use this function." - "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`" - ) - params = { - "keywords": query, - "region": region, - "max_results": max_results, - } - with DDGS() as ddg: - results = list(ddg.images(**params)) - return results - - -def get_tools(**kwargs): - return [ - FunctionTool.from_defaults(duckduckgo_search), - FunctionTool.from_defaults(duckduckgo_image_search), - ] From 73c50b4e75e29b80d48e1f6335d1e232d1f6579f Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 14:51:09 +0700 Subject: [PATCH 23/77] update python to change artifact to document generator --- .../engines/python/agent/tools/document_generator.py | 8 ++++---- .../types/multiagent/fastapi/app/examples/choreography.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/components/engines/python/agent/tools/document_generator.py b/templates/components/engines/python/agent/tools/document_generator.py index 212542ee3..5609f1467 100644 --- a/templates/components/engines/python/agent/tools/document_generator.py +++ b/templates/components/engines/python/agent/tools/document_generator.py @@ -163,13 +163,13 @@ def generate_document( cls, original_content: str, document_type: str, file_name: str ) -> str: """ - To generate artifact as PDF or HTML file. + To generate document as PDF or HTML file. Parameters: original_content: str (markdown style) document_type: str (pdf or html) specify the type of the file format based on the use case - file_name: str (name of the artifact file) must be a valid file name, no extensions needed + file_name: str (name of the document file) must be a valid file name, no extensions needed Returns: - str (URL to the artifact file): A file URL ready to serve. + str (URL to the document file): A file URL ready to serve. """ try: document_type = DocumentType(document_type.lower()) @@ -180,7 +180,7 @@ def generate_document( # Always generate html content first html_content = cls._generate_html_content(original_content) - # Based on the type of artifact, generate the corresponding file + # Based on the type of document, generate the corresponding file if document_type == DocumentType.PDF: content = cls._generate_pdf(html_content) file_extension = "pdf" diff --git a/templates/types/multiagent/fastapi/app/examples/choreography.py b/templates/types/multiagent/fastapi/app/examples/choreography.py index 27a73abcd..f99f19ed0 100644 --- a/templates/types/multiagent/fastapi/app/examples/choreography.py +++ b/templates/types/multiagent/fastapi/app/examples/choreography.py @@ -24,7 +24,7 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. - Finally, always request the publisher to create an artifact (pdf, html) and publish the blog post.""", + Finally, always request the publisher to create an document (pdf, html) and publish the blog post.""", # TODO: add chat_history support to AgentCallingAgent # chat_history=chat_history, verbose=True, From 251e8422f65c9c41777c04c87ee171e07649b1de Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 15:00:18 +0700 Subject: [PATCH 24/77] fix wordings --- .../types/multiagent/fastapi/app/api/routers/models.py | 3 ++- .../types/multiagent/fastapi/app/examples/choreography.py | 6 ++---- .../types/multiagent/fastapi/app/examples/orchestrator.py | 2 +- .../types/multiagent/fastapi/app/examples/publisher.py | 1 - templates/types/multiagent/fastapi/app/examples/workflow.py | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/api/routers/models.py b/templates/types/multiagent/fastapi/app/api/routers/models.py index f70f99afd..b0fa4ecc5 100644 --- a/templates/types/multiagent/fastapi/app/api/routers/models.py +++ b/templates/types/multiagent/fastapi/app/api/routers/models.py @@ -125,7 +125,7 @@ def get_last_message_content(self) -> str: def _get_agent_messages(self, max_messages: int = 5) -> List[str]: """ - Construct agent messages from the annotations in the chat messages + Construct agent messages from the agent events in the annotations of the chat messages """ agent_messages = [] for message in self.messages: @@ -138,6 +138,7 @@ def _get_agent_messages(self, max_messages: int = 5) -> List[str]: annotation.data, AgentAnnotation ): text = annotation.data.text + # TODO: we should not filter the message by its text, but by its type - we need to send the event type in the AgentAnnotation if not text.startswith("Finished task"): agent_messages.append( f"\nAgent: {annotation.data.agent}\nsaid: {text}\n" diff --git a/templates/types/multiagent/fastapi/app/examples/choreography.py b/templates/types/multiagent/fastapi/app/examples/choreography.py index f99f19ed0..23f0efdad 100644 --- a/templates/types/multiagent/fastapi/app/examples/choreography.py +++ b/templates/types/multiagent/fastapi/app/examples/choreography.py @@ -12,20 +12,18 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): publisher = create_publisher(chat_history) reviewer = FunctionCallingAgent( name="reviewer", - description="expert in reviewing blog posts, need a written post to review", + description="expert in reviewing blog posts, needs a written post to review", system_prompt="You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. If the post is good, you can say 'The post is good.'", chat_history=chat_history, - verbose=True, ) return AgentCallingAgent( name="writer", agents=[researcher, reviewer, publisher], - description="expert in writing blog posts, need provided information and images to write a blog post", + description="expert in writing blog posts, needs researched information and images to write a blog post", system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. Finally, always request the publisher to create an document (pdf, html) and publish the blog post.""", # TODO: add chat_history support to AgentCallingAgent # chat_history=chat_history, - verbose=True, ) diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/types/multiagent/fastapi/app/examples/orchestrator.py index 7f4a1839d..6c0404559 100644 --- a/templates/types/multiagent/fastapi/app/examples/orchestrator.py +++ b/templates/types/multiagent/fastapi/app/examples/orchestrator.py @@ -21,7 +21,7 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): ) reviewer = FunctionCallingAgent( name="reviewer", - description="expert in reviewing blog posts, need a written blog post to review", + description="expert in reviewing blog posts, needs a written blog post to review", system_prompt="""You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix the issues found yourself. You must output a final blog post. A post must include at lease one valid image, if not, reply "I need images about the topic to write the blog post". An image URL start with example or your website is not valid. Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.""", diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index 140f8b852..d8998376a 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -19,5 +19,4 @@ def create_publisher(chat_history: List[ChatMessage]): For a normal request, you should choose the type of document either pdf or html or just reply to the user directly without generating any document file. """, chat_history=chat_history, - verbose=True, ) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index dd9a6b853..e0aa98d70 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -29,7 +29,7 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): ) reviewer = FunctionCallingAgent( name="reviewer", - description="expert in reviewing blog posts, need a written blog post to review", + description="expert in reviewing blog posts, needs a written blog post to review", system_prompt="You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review.", chat_history=chat_history, ) From 1e37edb51b571179a934efbeacc7d9f8402fcc5b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 15:01:31 +0700 Subject: [PATCH 25/77] fix linting --- templates/types/multiagent/fastapi/app/agents/multi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/types/multiagent/fastapi/app/agents/multi.py b/templates/types/multiagent/fastapi/app/agents/multi.py index 6370a7611..115038502 100644 --- a/templates/types/multiagent/fastapi/app/agents/multi.py +++ b/templates/types/multiagent/fastapi/app/agents/multi.py @@ -25,7 +25,11 @@ async def schema_call(input: str) -> str: name=name, description=( f"Use this tool to delegate a sub task to the {agent.name} agent." - + (f" The agent is an {agent.description}." if agent.description else "") + + ( + f" The agent is an {agent.description}." + if agent.description + else "" + ) ), fn_schema=fn_schema, ) From 73a3967c705bd5074f84c1f692a66ea433a69b91 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 15:06:11 +0700 Subject: [PATCH 26/77] fix doubled html --- .../engines/typescript/agent/tools/document_generator.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document_generator.ts index 9c9ac950a..0c0e9ef2c 100644 --- a/templates/components/engines/typescript/agent/tools/document_generator.ts +++ b/templates/components/engines/typescript/agent/tools/document_generator.ts @@ -204,9 +204,7 @@ export class DocumentGenerator implements BaseTool { try { if (documentType.toLowerCase() === DocumentType.HTML) { - const htmlDocument = - DocumentGenerator.generateHtmlDocument(htmlContent); - fileContent = DocumentGenerator.generateHtmlDocument(htmlDocument); + fileContent = DocumentGenerator.generateHtmlDocument(htmlContent); fileExtension = "html"; } else if (documentType.toLowerCase() === DocumentType.PDF) { const pdfDocument = DocumentGenerator.generatePdfDocument(htmlContent); From f89e377b855821e89767273015a1d45755aa6250 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Fri, 27 Sep 2024 15:06:43 +0700 Subject: [PATCH 27/77] Update helpers/tools.ts --- helpers/tools.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/helpers/tools.ts b/helpers/tools.ts index 875405804..0b4996bd5 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -113,7 +113,6 @@ For better results, you can specify the region parameter to get results from a s { display: "Document generator", name: "document_generator", - // TODO: add support for Typescript templates supportedFrameworks: ["fastapi", "nextjs", "express"], dependencies: [ { From 8fda798727bf0a383b873483853c259444d430c7 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 15:15:33 +0700 Subject: [PATCH 28/77] enhance error handling --- .../agent/tools/document_generator.ts | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document_generator.ts index 0c0e9ef2c..4c2dc4fb3 100644 --- a/templates/components/engines/typescript/agent/tools/document_generator.ts +++ b/templates/components/engines/typescript/agent/tools/document_generator.ts @@ -165,15 +165,11 @@ export class DocumentGenerator implements BaseTool { } private static writeToFile(content: string | Buffer, filePath: string): void { - try { - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - if (typeof content === "string") { - fs.writeFileSync(filePath, content, "utf8"); - } else { - fs.writeFileSync(filePath, content); - } - } catch (error) { - throw error; + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + if (typeof content === "string") { + fs.writeFileSync(filePath, content, "utf8"); + } else { + fs.writeFileSync(filePath, content); } } @@ -187,10 +183,16 @@ export class DocumentGenerator implements BaseTool { private static async generatePdf(htmlContent: string): Promise { const browser = await puppeteer.launch(); const page = await browser.newPage(); - await page.setContent(htmlContent, { waitUntil: "networkidle0" }); - const pdf = await page.pdf({ format: "A4" }); - await browser.close(); - return pdf; + try { + await page.setContent(htmlContent, { waitUntil: "networkidle0" }); + const pdf = await page.pdf({ format: "A4" }); + return pdf; + } catch (error) { + console.error("Error generating PDF:", error); + throw new Error("Failed to generate PDF"); + } finally { + await browser.close(); + } } async call(input: DocumentParameter): Promise { From 2fcbe2ba0b126dbb96973405acf87f4339383619 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 15:18:50 +0700 Subject: [PATCH 29/77] remove redundant code --- templates/types/multiagent/fastapi/app/agents/single.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/types/multiagent/fastapi/app/agents/single.py b/templates/types/multiagent/fastapi/app/agents/single.py index 754cc5ac4..05ae77b9f 100644 --- a/templates/types/multiagent/fastapi/app/agents/single.py +++ b/templates/types/multiagent/fastapi/app/agents/single.py @@ -62,13 +62,11 @@ def __init__( timeout: float = 360.0, name: str, write_events: bool = True, - description: Optional[str] = None, **kwargs: Any, ) -> None: super().__init__(*args, verbose=verbose, timeout=timeout, **kwargs) self.tools = tools or [] self.name = name - self.description = description self.write_events = write_events if llm is None: From 9bd02f03140a854f971acf476a3e9f9d4ee5f328 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 20:06:07 +0700 Subject: [PATCH 30/77] update python multiagent template --- helpers/python.ts | 2 +- questions.ts | 33 ++++------------ .../engines/python/agent/tools/__init__.py | 27 +++++++++---- .../multiagent/fastapi/app/agents/single.py | 2 + .../fastapi/app/examples/publisher.py | 33 ++++++++++------ .../fastapi/app/examples/researcher.py | 38 ++++++++++++++++--- .../fastapi/app/examples/workflow.py | 28 +++++++++++--- 7 files changed, 109 insertions(+), 54 deletions(-) diff --git a/helpers/python.ts b/helpers/python.ts index 06587dafe..bad1949ad 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -406,7 +406,7 @@ export const installPythonTemplate = async ({ // TODO: Remove this once we support selecting tools for multiagent template if (template === "multiagent") { // templates / components / engines / python / agent / tools; - await copy("**", path.join(root, "app", "tools"), { + await copy("**", path.join(root, "app", "engine", "tools"), { cwd: path.join(compPath, "engines", "python", "agent", "tools"), }); } diff --git a/questions.ts b/questions.ts index 69d7f49ae..81061f83f 100644 --- a/questions.ts +++ b/questions.ts @@ -18,7 +18,6 @@ import { getAvailableLlamapackOptions } from "./helpers/llama-pack"; import { askModelConfig } from "./helpers/providers"; import { getProjectOptions } from "./helpers/repo"; import { - ToolType, supportedTools, toolRequiresConfig, toolsRequireConfig, @@ -142,12 +141,10 @@ export const getDataSourceChoices = ( }); } if (selectedDataSource === undefined || selectedDataSource.length === 0) { - if (template !== "multiagent") { - choices.push({ - title: "No datasource", - value: "none", - }); - } + choices.push({ + title: "No datasource", + value: "none", + }); choices.push({ title: process.platform !== "linux" @@ -366,22 +363,6 @@ export const askQuestions = async ( } } - // TODO: Remove this once we support selecting tools for multiagent template - if (program.template === "multiagent") { - program.tools = [ - { - name: "document_generator", - display: "Document Generator", - type: ToolType.LOCAL, - }, - { - name: "duckduckgo", - display: "DuckDuckGo", - type: ToolType.LOCAL, - }, - ]; - } - if (program.template === "community") { const projectOptions = await getProjectOptions( COMMUNITY_OWNER, @@ -751,8 +732,10 @@ export const askQuestions = async ( } } - if (!program.tools && program.template === "streaming") { - // TODO: allow to select tools also for multi-agent framework + if ( + !program.tools && + (program.template === "streaming" || program.template === "multiagent") + ) { if (ciInfo.isCI) { program.tools = getPrefOrDefault("tools"); } else { diff --git a/templates/components/engines/python/agent/tools/__init__.py b/templates/components/engines/python/agent/tools/__init__.py index f24d988db..6b2184328 100644 --- a/templates/components/engines/python/agent/tools/__init__.py +++ b/templates/components/engines/python/agent/tools/__init__.py @@ -1,8 +1,9 @@ +import importlib import os + import yaml -import importlib -from llama_index.core.tools.tool_spec.base import BaseToolSpec from llama_index.core.tools.function_tool import FunctionTool +from llama_index.core.tools.tool_spec.base import BaseToolSpec class ToolType: @@ -40,14 +41,26 @@ def load_tools(tool_type: str, tool_name: str, config: dict) -> list[FunctionToo raise ValueError(f"Failed to load tool {tool_name}: {e}") @staticmethod - def from_env() -> list[FunctionTool]: - tools = [] + def from_env( + map_result: bool = False, + ) -> list[FunctionTool] | dict[str, FunctionTool]: + """ + Load tools from the configured file. + Params: + - use_map: if True, return map of tool name and the tool itself + """ + if map_result: + tools = {} + else: + tools = [] if os.path.exists("config/tools.yaml"): with open("config/tools.yaml", "r") as f: tool_configs = yaml.safe_load(f) for tool_type, config_entries in tool_configs.items(): for tool_name, config in config_entries.items(): - tools.extend( - ToolFactory.load_tools(tool_type, tool_name, config) - ) + tool = ToolFactory.load_tools(tool_type, tool_name, config) + if map_result: + tools[tool_name] = tool + else: + tools.extend(tool) return tools diff --git a/templates/types/multiagent/fastapi/app/agents/single.py b/templates/types/multiagent/fastapi/app/agents/single.py index 05ae77b9f..a598bdf65 100644 --- a/templates/types/multiagent/fastapi/app/agents/single.py +++ b/templates/types/multiagent/fastapi/app/agents/single.py @@ -62,12 +62,14 @@ def __init__( timeout: float = 360.0, name: str, write_events: bool = True, + description: str | None = None, **kwargs: Any, ) -> None: super().__init__(*args, verbose=verbose, timeout=timeout, **kwargs) self.tools = tools or [] self.name = name self.write_events = write_events + self.description = description if llm is None: llm = Settings.llm diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/types/multiagent/fastapi/app/examples/publisher.py index d8998376a..a7abd0df8 100644 --- a/templates/types/multiagent/fastapi/app/examples/publisher.py +++ b/templates/types/multiagent/fastapi/app/examples/publisher.py @@ -1,22 +1,33 @@ -from typing import List +from typing import List, Tuple from app.agents.single import FunctionCallingAgent -from app.tools.document_generator import DocumentGenerator +from app.engine.tools import ToolFactory from llama_index.core.chat_engine.types import ChatMessage from llama_index.core.tools import FunctionTool -def create_publisher(chat_history: List[ChatMessage]): - document_generator_tool = FunctionTool.from_defaults( - DocumentGenerator.generate_document - ) +def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]: + tools = [] + # Get configured tools from the tools.yaml file + configured_tools = ToolFactory.from_env(map_result=True) + if "document_generator" in configured_tools.keys(): + tools.extend(configured_tools["document_generator"]) + prompt_instructions = "You have access to a document generator tool that can create PDF or HTML document for the content. Based on the user request, please specify the type of document to generate or just reply to the user directly without generating any document file." + description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format." + else: + prompt_instructions = "You don't have a tool to generate document. Please reply the content directly." + description = "Expert in publishing the blog post" + return tools, prompt_instructions, description + +def create_publisher(chat_history: List[ChatMessage]): + tools, instructions, description = get_publisher_tools() + system_prompt = f"""You are a publisher that help publish the blog post. + {instructions}""" return FunctionCallingAgent( name="publisher", - tools=[document_generator_tool], - description="expert in publishing, need to specify the type of document use a file (pdf, html) or just reply the content directly", - system_prompt="""You are a publisher that help publish the blog post. - For a normal request, you should choose the type of document either pdf or html or just reply to the user directly without generating any document file. - """, + tools=tools, + description=description, + system_prompt=system_prompt, chat_history=chat_history, ) diff --git a/templates/types/multiagent/fastapi/app/examples/researcher.py b/templates/types/multiagent/fastapi/app/examples/researcher.py index 5fbdf484c..ed19218b4 100644 --- a/templates/types/multiagent/fastapi/app/examples/researcher.py +++ b/templates/types/multiagent/fastapi/app/examples/researcher.py @@ -3,12 +3,12 @@ from app.agents.single import FunctionCallingAgent from app.engine.index import get_index -from app.tools.duckduckgo import get_tools as get_duckduckgo_tools +from app.engine.tools import ToolFactory from llama_index.core.chat_engine.types import ChatMessage from llama_index.core.tools import QueryEngineTool, ToolMetadata -def get_query_engine_tool() -> QueryEngineTool: +def _create_query_engine_tool() -> QueryEngineTool: """ Provide an agent worker that can be used to query the index. """ @@ -30,12 +30,40 @@ def get_query_engine_tool() -> QueryEngineTool: ) +def _get_research_tools() -> QueryEngineTool: + """ + Researcher take responsibility for retrieving information. + Try init wikipedia or duckduckgo tool if available. + """ + researcher_tool_names = ["duckduckgo", "wikipedia.WikipediaToolSpec"] + # Always include the query engine tool + tools = [_create_query_engine_tool()] + configured_tools = ToolFactory.from_env(map_result=True) + print(configured_tools) + for tool_name, tool in configured_tools.items(): + if tool_name in researcher_tool_names: + tools.extend(tool) + return tools + + def create_researcher(chat_history: List[ChatMessage]): - duckduckgo_search_tools = get_duckduckgo_tools() + """ + Researcher is an agent that take responsibility for using tools to complete a given task. + """ + tools = _get_research_tools() return FunctionCallingAgent( name="researcher", - tools=[get_query_engine_tool(), *duckduckgo_search_tools], + tools=tools, description="expert in retrieving any unknown content or searching for images from the internet", - system_prompt="You are a researcher agent. You are given a researching task. You must use tools to retrieve information from the knowledge base and search for needed images from the internet for the post.", + system_prompt="""You are a researcher agent. +You are given a researching task. You must use tools to retrieve information needed for the task. +It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. +If you don't found any related information, please return "I didn't find any information." +Example: +Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." +-> +Your real task: Looking for information in english about the history of the internet +This is not your task: Create blog post, create PDF, write in English +""", chat_history=chat_history, ) diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/types/multiagent/fastapi/app/examples/workflow.py index e0aa98d70..ac62250ae 100644 --- a/templates/types/multiagent/fastapi/app/examples/workflow.py +++ b/templates/types/multiagent/fastapi/app/examples/workflow.py @@ -23,14 +23,32 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): ) writer = FunctionCallingAgent( name="writer", - description="expert in writing blog posts, need information and images to write a post", - system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Don't make up any information yourself.""", + description="expert in writing blog posts, need information and images to write a post.", + system_prompt="""You are an expert in writing blog posts. +You are given a task to write a blog post. Don't make up any information yourself. +It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. +Example: +Task: "Here is the information i found about the history of internet: +Create a blog post about the history of the internet, write in English and publish in PDF format." +-> Your task: Use the research content {...} to write a blog post in English. +-> This is not your task: Create PDF +""", chat_history=chat_history, ) reviewer = FunctionCallingAgent( name="reviewer", - description="expert in reviewing blog posts, needs a written blog post to review", - system_prompt="You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review.", + description="expert in reviewing blog posts, needs a written blog post to review.", + system_prompt="""You are an expert in reviewing blog posts. +You are given a task to review a blog post. +Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. +Furthermore, proofread the post for grammar and spelling errors. +Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. +It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. +Example: +Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." +-> Your task: Review is the main content of the post is about the history of the internet, is it written in English. +-> This is not your task: Create blog post, create PDF, write in English. +""", chat_history=chat_history, ) workflow = BlogPostWorkflow(timeout=360) @@ -153,7 +171,7 @@ async def publish( msg=f"Error publishing: {e}", ) ) - return StopEvent(result=result) + return StopEvent(result=None) async def run_agent( self, From aff5beb93d15bb7099fb36df8a2a2f9fdd02af24 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 20:22:54 +0700 Subject: [PATCH 31/77] reuse python streaming for multiagent --- helpers/python.ts | 27 +- .../multiagent/python}/README-template.md | 0 .../multiagent/python}/app/agents/multi.py | 0 .../multiagent/python}/app/agents/planner.py | 0 .../multiagent/python}/app/agents/single.py | 0 .../python}/app/examples/choreography.py | 0 .../python}/app/examples/factory.py | 0 .../python}/app/examples/orchestrator.py | 0 .../python}/app/examples/publisher.py | 0 .../python}/app/examples/researcher.py | 0 .../python}/app/examples/workflow.py | 0 .../multiagent/fastapi/app/api/__init__.py | 0 .../fastapi/app/api/routers/__init__.py | 0 .../fastapi/app/api/routers/chat.py | 39 --- .../fastapi/app/api/routers/chat_config.py | 48 ---- .../fastapi/app/api/routers/models.py | 267 ------------------ .../fastapi/app/api/routers/upload.py | 29 -- .../app/api/routers/vercel_response.py | 121 -------- .../types/multiagent/fastapi/app/config.py | 1 - .../multiagent/fastapi/app/observability.py | 2 - .../types/multiagent/fastapi/app/utils.py | 8 - templates/types/multiagent/fastapi/gitignore | 4 - templates/types/multiagent/fastapi/main.py | 72 ----- .../types/multiagent/fastapi/pyproject.toml | 30 -- 24 files changed, 15 insertions(+), 633 deletions(-) rename templates/{types/multiagent/fastapi => components/multiagent/python}/README-template.md (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/agents/multi.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/agents/planner.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/agents/single.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/examples/choreography.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/examples/factory.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/examples/orchestrator.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/examples/publisher.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/examples/researcher.py (100%) rename templates/{types/multiagent/fastapi => components/multiagent/python}/app/examples/workflow.py (100%) delete mode 100644 templates/types/multiagent/fastapi/app/api/__init__.py delete mode 100644 templates/types/multiagent/fastapi/app/api/routers/__init__.py delete mode 100644 templates/types/multiagent/fastapi/app/api/routers/chat.py delete mode 100644 templates/types/multiagent/fastapi/app/api/routers/chat_config.py delete mode 100644 templates/types/multiagent/fastapi/app/api/routers/models.py delete mode 100644 templates/types/multiagent/fastapi/app/api/routers/upload.py delete mode 100644 templates/types/multiagent/fastapi/app/api/routers/vercel_response.py delete mode 100644 templates/types/multiagent/fastapi/app/config.py delete mode 100644 templates/types/multiagent/fastapi/app/observability.py delete mode 100644 templates/types/multiagent/fastapi/app/utils.py delete mode 100644 templates/types/multiagent/fastapi/gitignore delete mode 100644 templates/types/multiagent/fastapi/main.py delete mode 100644 templates/types/multiagent/fastapi/pyproject.toml diff --git a/helpers/python.ts b/helpers/python.ts index bad1949ad..2c89aa6c4 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -364,7 +364,12 @@ export const installPythonTemplate = async ({ | "modelConfig" >) => { console.log("\nInitializing Python project with template:", template, "\n"); - const templatePath = path.join(templatesDir, "types", template, framework); + let templatePath; + if (template === "extractor") { + templatePath = path.join(templatesDir, "types", "extractor", framework); + } else { + templatePath = path.join(templatesDir, "types", "streaming", framework); + } await copy("**", root, { parents: true, cwd: templatePath, @@ -402,17 +407,7 @@ export const installPythonTemplate = async ({ }); } - // Copy tools for multiagent template - // TODO: Remove this once we support selecting tools for multiagent template - if (template === "multiagent") { - // templates / components / engines / python / agent / tools; - await copy("**", path.join(root, "app", "engine", "tools"), { - cwd: path.join(compPath, "engines", "python", "agent", "tools"), - }); - } - - if (template === "streaming") { - // For the streaming template only: + if (template === "streaming" || template === "multiagent") { // Select and copy engine code based on data sources and tools let engine; if (dataSources.length > 0 && (!tools || tools.length === 0)) { @@ -427,6 +422,14 @@ export const installPythonTemplate = async ({ }); } + if (template === "multiagent") { + // Copy multi-agent code + await copy("**", path.join(root), { + parents: true, + cwd: path.join(compPath, "multiagent", "python"), + }); + } + console.log("Adding additional dependencies"); const addOnDependencies = getAdditionalDependencies( diff --git a/templates/types/multiagent/fastapi/README-template.md b/templates/components/multiagent/python/README-template.md similarity index 100% rename from templates/types/multiagent/fastapi/README-template.md rename to templates/components/multiagent/python/README-template.md diff --git a/templates/types/multiagent/fastapi/app/agents/multi.py b/templates/components/multiagent/python/app/agents/multi.py similarity index 100% rename from templates/types/multiagent/fastapi/app/agents/multi.py rename to templates/components/multiagent/python/app/agents/multi.py diff --git a/templates/types/multiagent/fastapi/app/agents/planner.py b/templates/components/multiagent/python/app/agents/planner.py similarity index 100% rename from templates/types/multiagent/fastapi/app/agents/planner.py rename to templates/components/multiagent/python/app/agents/planner.py diff --git a/templates/types/multiagent/fastapi/app/agents/single.py b/templates/components/multiagent/python/app/agents/single.py similarity index 100% rename from templates/types/multiagent/fastapi/app/agents/single.py rename to templates/components/multiagent/python/app/agents/single.py diff --git a/templates/types/multiagent/fastapi/app/examples/choreography.py b/templates/components/multiagent/python/app/examples/choreography.py similarity index 100% rename from templates/types/multiagent/fastapi/app/examples/choreography.py rename to templates/components/multiagent/python/app/examples/choreography.py diff --git a/templates/types/multiagent/fastapi/app/examples/factory.py b/templates/components/multiagent/python/app/examples/factory.py similarity index 100% rename from templates/types/multiagent/fastapi/app/examples/factory.py rename to templates/components/multiagent/python/app/examples/factory.py diff --git a/templates/types/multiagent/fastapi/app/examples/orchestrator.py b/templates/components/multiagent/python/app/examples/orchestrator.py similarity index 100% rename from templates/types/multiagent/fastapi/app/examples/orchestrator.py rename to templates/components/multiagent/python/app/examples/orchestrator.py diff --git a/templates/types/multiagent/fastapi/app/examples/publisher.py b/templates/components/multiagent/python/app/examples/publisher.py similarity index 100% rename from templates/types/multiagent/fastapi/app/examples/publisher.py rename to templates/components/multiagent/python/app/examples/publisher.py diff --git a/templates/types/multiagent/fastapi/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py similarity index 100% rename from templates/types/multiagent/fastapi/app/examples/researcher.py rename to templates/components/multiagent/python/app/examples/researcher.py diff --git a/templates/types/multiagent/fastapi/app/examples/workflow.py b/templates/components/multiagent/python/app/examples/workflow.py similarity index 100% rename from templates/types/multiagent/fastapi/app/examples/workflow.py rename to templates/components/multiagent/python/app/examples/workflow.py diff --git a/templates/types/multiagent/fastapi/app/api/__init__.py b/templates/types/multiagent/fastapi/app/api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/types/multiagent/fastapi/app/api/routers/__init__.py b/templates/types/multiagent/fastapi/app/api/routers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/types/multiagent/fastapi/app/api/routers/chat.py b/templates/types/multiagent/fastapi/app/api/routers/chat.py deleted file mode 100644 index b9776775d..000000000 --- a/templates/types/multiagent/fastapi/app/api/routers/chat.py +++ /dev/null @@ -1,39 +0,0 @@ -import logging - -from app.api.routers.models import ( - ChatData, -) -from app.api.routers.vercel_response import VercelStreamResponse -from app.examples.factory import create_agent -from fastapi import APIRouter, HTTPException, Request, status -from llama_index.core.workflow import Workflow - -chat_router = r = APIRouter() - -logger = logging.getLogger("uvicorn") - - -@r.post("") -async def chat( - request: Request, - data: ChatData, -): - try: - last_message_content = data.get_last_message_content() - messages = data.get_history_messages(include_agent_messages=True) - # TODO: generate filters based on doc_ids - # for now just use all documents - # doc_ids = data.get_chat_document_ids() - # TODO: use params - # params = data.data or {} - - agent: Workflow = create_agent(chat_history=messages) - handler = agent.run(input=last_message_content, streaming=True) - - return VercelStreamResponse(request, handler, agent.stream_events, data) - except Exception as e: - logger.exception("Error in agent", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error in agent: {e}", - ) from e diff --git a/templates/types/multiagent/fastapi/app/api/routers/chat_config.py b/templates/types/multiagent/fastapi/app/api/routers/chat_config.py deleted file mode 100644 index 8d926e50b..000000000 --- a/templates/types/multiagent/fastapi/app/api/routers/chat_config.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging -import os - -from fastapi import APIRouter - -from app.api.routers.models import ChatConfig - - -config_router = r = APIRouter() - -logger = logging.getLogger("uvicorn") - - -@r.get("") -async def chat_config() -> ChatConfig: - starter_questions = None - conversation_starters = os.getenv("CONVERSATION_STARTERS") - if conversation_starters and conversation_starters.strip(): - starter_questions = conversation_starters.strip().split("\n") - return ChatConfig(starter_questions=starter_questions) - - -try: - from app.engine.service import LLamaCloudFileService - - logger.info("LlamaCloud is configured. Adding /config/llamacloud route.") - - @r.get("/llamacloud") - async def chat_llama_cloud_config(): - projects = LLamaCloudFileService.get_all_projects_with_pipelines() - pipeline = os.getenv("LLAMA_CLOUD_INDEX_NAME") - project = os.getenv("LLAMA_CLOUD_PROJECT_NAME") - pipeline_config = None - if pipeline and project: - pipeline_config = { - "pipeline": pipeline, - "project": project, - } - return { - "projects": projects, - "pipeline": pipeline_config, - } - -except ImportError: - logger.debug( - "LlamaCloud is not configured. Skipping adding /config/llamacloud route." - ) - pass diff --git a/templates/types/multiagent/fastapi/app/api/routers/models.py b/templates/types/multiagent/fastapi/app/api/routers/models.py deleted file mode 100644 index b0fa4ecc5..000000000 --- a/templates/types/multiagent/fastapi/app/api/routers/models.py +++ /dev/null @@ -1,267 +0,0 @@ -import logging -import os -from typing import Any, Dict, List, Literal, Optional - -from app.config import DATA_DIR -from llama_index.core.llms import ChatMessage, MessageRole -from llama_index.core.schema import NodeWithScore -from pydantic import BaseModel, Field, validator -from pydantic.alias_generators import to_camel - -logger = logging.getLogger("uvicorn") - - -class FileContent(BaseModel): - type: Literal["text", "ref"] - # If the file is pure text then the value is be a string - # otherwise, it's a list of document IDs - value: str | List[str] - - -class File(BaseModel): - id: str - content: FileContent - filename: str - filesize: int - filetype: str - - -class AnnotationFileData(BaseModel): - files: List[File] = Field( - default=[], - description="List of files", - ) - - class Config: - json_schema_extra = { - "example": { - "csvFiles": [ - { - "content": "Name, Age\nAlice, 25\nBob, 30", - "filename": "example.csv", - "filesize": 123, - "id": "123", - "type": "text/csv", - } - ] - } - } - alias_generator = to_camel - - -class AgentAnnotation(BaseModel): - agent: str - text: str - - -class Annotation(BaseModel): - type: str - data: AnnotationFileData | List[str] | AgentAnnotation - - def to_content(self) -> str | None: - if self.type == "document_file": - # We only support generating context content for CSV files for now - csv_files = [file for file in self.data.files if file.filetype == "csv"] - if len(csv_files) > 0: - return "Use data from following CSV raw content\n" + "\n".join( - [f"```csv\n{csv_file.content.value}\n```" for csv_file in csv_files] - ) - else: - logger.warning( - f"The annotation {self.type} is not supported for generating context content" - ) - return None - - -class Message(BaseModel): - role: MessageRole - content: str - annotations: List[Annotation] | None = None - - -class ChatData(BaseModel): - messages: List[Message] - data: Any = None - - class Config: - json_schema_extra = { - "example": { - "messages": [ - { - "role": "user", - "content": "What standards for letters exist?", - } - ] - } - } - - @validator("messages") - def messages_must_not_be_empty(cls, v): - if len(v) == 0: - raise ValueError("Messages must not be empty") - return v - - def get_last_message_content(self) -> str: - """ - Get the content of the last message along with the data content if available. - Fallback to use data content from previous messages - """ - if len(self.messages) == 0: - raise ValueError("There is not any message in the chat") - last_message = self.messages[-1] - message_content = last_message.content - for message in reversed(self.messages): - if message.role == MessageRole.USER and message.annotations is not None: - annotation_contents = filter( - None, - [annotation.to_content() for annotation in message.annotations], - ) - if not annotation_contents: - continue - annotation_text = "\n".join(annotation_contents) - message_content = f"{message_content}\n{annotation_text}" - break - return message_content - - def _get_agent_messages(self, max_messages: int = 5) -> List[str]: - """ - Construct agent messages from the agent events in the annotations of the chat messages - """ - agent_messages = [] - for message in self.messages: - if ( - message.role == MessageRole.ASSISTANT - and message.annotations is not None - ): - for annotation in message.annotations: - if annotation.type == "agent" and isinstance( - annotation.data, AgentAnnotation - ): - text = annotation.data.text - # TODO: we should not filter the message by its text, but by its type - we need to send the event type in the AgentAnnotation - if not text.startswith("Finished task"): - agent_messages.append( - f"\nAgent: {annotation.data.agent}\nsaid: {text}\n" - ) - if len(agent_messages) >= max_messages: - break - return agent_messages - - def get_history_messages( - self, include_agent_messages: bool = False - ) -> List[ChatMessage]: - """ - Get the history messages - """ - chat_messages = [ - ChatMessage(role=message.role, content=message.content) - for message in self.messages[:-1] - ] - if include_agent_messages: - agent_messages = self._get_agent_messages(max_messages=5) - if len(agent_messages) > 0: - message = ChatMessage( - role=MessageRole.ASSISTANT, - content="Previous agent events: \n" + "\n".join(agent_messages), - ) - chat_messages.append(message) - - return chat_messages - - def is_last_message_from_user(self) -> bool: - return self.messages[-1].role == MessageRole.USER - - def get_chat_document_ids(self) -> List[str]: - """ - Get the document IDs from the chat messages - """ - document_ids: List[str] = [] - for message in self.messages: - if message.role == MessageRole.USER and message.annotations is not None: - for annotation in message.annotations: - if ( - annotation.type == "document_file" - and annotation.data.files is not None - ): - for fi in annotation.data.files: - if fi.content.type == "ref": - document_ids += fi.content.value - return list(set(document_ids)) - - -class SourceNodes(BaseModel): - id: str - metadata: Dict[str, Any] - score: Optional[float] - text: str - url: Optional[str] - - @classmethod - def from_source_node(cls, source_node: NodeWithScore): - metadata = source_node.node.metadata - url = cls.get_url_from_metadata(metadata) - - return cls( - id=source_node.node.node_id, - metadata=metadata, - score=source_node.score, - text=source_node.node.text, # type: ignore - url=url, - ) - - @classmethod - def get_url_from_metadata(cls, metadata: Dict[str, Any]) -> str: - url_prefix = os.getenv("FILESERVER_URL_PREFIX") - if not url_prefix: - logger.warning( - "Warning: FILESERVER_URL_PREFIX not set in environment variables. Can't use file server" - ) - file_name = metadata.get("file_name") - - if file_name and url_prefix: - # file_name exists and file server is configured - pipeline_id = metadata.get("pipeline_id") - if pipeline_id: - # file is from LlamaCloud - file_name = f"{pipeline_id}${file_name}" - return f"{url_prefix}/output/llamacloud/{file_name}" - is_private = metadata.get("private", "false") == "true" - if is_private: - # file is a private upload - return f"{url_prefix}/output/uploaded/{file_name}" - # file is from calling the 'generate' script - # Get the relative path of file_path to data_dir - file_path = metadata.get("file_path") - data_dir = os.path.abspath(DATA_DIR) - if file_path and data_dir: - relative_path = os.path.relpath(file_path, data_dir) - return f"{url_prefix}/data/{relative_path}" - # fallback to URL in metadata (e.g. for websites) - return metadata.get("URL") - - @classmethod - def from_source_nodes(cls, source_nodes: List[NodeWithScore]): - return [cls.from_source_node(node) for node in source_nodes] - - -class Result(BaseModel): - result: Message - nodes: List[SourceNodes] - - -class ChatConfig(BaseModel): - starter_questions: Optional[List[str]] = Field( - default=None, - description="List of starter questions", - serialization_alias="starterQuestions", - ) - - class Config: - json_schema_extra = { - "example": { - "starterQuestions": [ - "What standards for letters exist?", - "What are the requirements for a letter to be considered a letter?", - ] - } - } diff --git a/templates/types/multiagent/fastapi/app/api/routers/upload.py b/templates/types/multiagent/fastapi/app/api/routers/upload.py deleted file mode 100644 index ccc03004b..000000000 --- a/templates/types/multiagent/fastapi/app/api/routers/upload.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging -from typing import List, Any - -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel - -from app.api.services.file import PrivateFileService - -file_upload_router = r = APIRouter() - -logger = logging.getLogger("uvicorn") - - -class FileUploadRequest(BaseModel): - base64: str - filename: str - params: Any = None - - -@r.post("") -def upload_file(request: FileUploadRequest) -> List[str]: - try: - logger.info("Processing file") - return PrivateFileService.process_file( - request.filename, request.base64, request.params - ) - except Exception as e: - logger.error(f"Error processing file: {e}", exc_info=True) - raise HTTPException(status_code=500, detail="Error processing file") diff --git a/templates/types/multiagent/fastapi/app/api/routers/vercel_response.py b/templates/types/multiagent/fastapi/app/api/routers/vercel_response.py deleted file mode 100644 index 29bcf852b..000000000 --- a/templates/types/multiagent/fastapi/app/api/routers/vercel_response.py +++ /dev/null @@ -1,121 +0,0 @@ -import json -import logging -from asyncio import Task -from typing import AsyncGenerator, List - -from aiostream import stream -from app.agents.single import AgentRunEvent, AgentRunResult -from app.api.routers.models import ChatData, Message -from app.api.services.suggestion import NextQuestionSuggestion -from fastapi import Request -from fastapi.responses import StreamingResponse - -logger = logging.getLogger("uvicorn") - - -class VercelStreamResponse(StreamingResponse): - """ - Class to convert the response from the chat engine to the streaming format expected by Vercel - """ - - TEXT_PREFIX = "0:" - DATA_PREFIX = "8:" - - @classmethod - def convert_text(cls, token: str): - # Escape newlines and double quotes to avoid breaking the stream - token = json.dumps(token) - return f"{cls.TEXT_PREFIX}{token}\n" - - @classmethod - def convert_data(cls, data: dict): - data_str = json.dumps(data) - return f"{cls.DATA_PREFIX}[{data_str}]\n" - - def __init__( - self, - request: Request, - task: Task[AgentRunResult | AsyncGenerator], - events: AsyncGenerator[AgentRunEvent, None], - chat_data: ChatData, - verbose: bool = True, - ): - content = VercelStreamResponse.content_generator( - request, task, events, chat_data, verbose - ) - super().__init__(content=content) - - @classmethod - async def content_generator( - cls, - request: Request, - task: Task[AgentRunResult | AsyncGenerator], - events: AsyncGenerator[AgentRunEvent, None], - chat_data: ChatData, - verbose: bool = True, - ): - # Yield the text response - async def _chat_response_generator(): - result = await task - final_response = "" - - if isinstance(result, AgentRunResult): - for token in result.response.message.content: - final_response += token - yield cls.convert_text(token) - - if isinstance(result, AsyncGenerator): - async for token in result: - final_response += token.delta - yield cls.convert_text(token.delta) - - # Generate next questions if next question prompt is configured - question_data = await cls._generate_next_questions( - chat_data.messages, final_response - ) - if question_data: - yield cls.convert_data(question_data) - - # TODO: stream sources - - # Yield the events from the event handler - async def _event_generator(): - async for event in events(): - event_response = cls._event_to_response(event) - if verbose: - logger.debug(event_response) - if event_response is not None: - yield cls.convert_data(event_response) - - combine = stream.merge(_chat_response_generator(), _event_generator()) - - is_stream_started = False - async with combine.stream() as streamer: - if not is_stream_started: - is_stream_started = True - # Stream a blank message to start the stream - yield cls.convert_text("") - - async for output in streamer: - yield output - if await request.is_disconnected(): - break - - @staticmethod - def _event_to_response(event: AgentRunEvent) -> dict: - return { - "type": "agent", - "data": {"agent": event.name, "text": event.msg}, - } - - @staticmethod - async def _generate_next_questions(chat_history: List[Message], response: str): - questions = await NextQuestionSuggestion.suggest_next_questions( - chat_history, response - ) - if questions: - return { - "type": "suggested_questions", - "data": questions, - } - return None diff --git a/templates/types/multiagent/fastapi/app/config.py b/templates/types/multiagent/fastapi/app/config.py deleted file mode 100644 index 29fa8d9a2..000000000 --- a/templates/types/multiagent/fastapi/app/config.py +++ /dev/null @@ -1 +0,0 @@ -DATA_DIR = "data" diff --git a/templates/types/multiagent/fastapi/app/observability.py b/templates/types/multiagent/fastapi/app/observability.py deleted file mode 100644 index 28019c37e..000000000 --- a/templates/types/multiagent/fastapi/app/observability.py +++ /dev/null @@ -1,2 +0,0 @@ -def init_observability(): - pass diff --git a/templates/types/multiagent/fastapi/app/utils.py b/templates/types/multiagent/fastapi/app/utils.py deleted file mode 100644 index ac43ccbb3..000000000 --- a/templates/types/multiagent/fastapi/app/utils.py +++ /dev/null @@ -1,8 +0,0 @@ -import os - - -def load_from_env(var: str, throw_error: bool = True) -> str: - res = os.getenv(var) - if res is None and throw_error: - raise ValueError(f"Missing environment variable: {var}") - return res diff --git a/templates/types/multiagent/fastapi/gitignore b/templates/types/multiagent/fastapi/gitignore deleted file mode 100644 index ae22d348e..000000000 --- a/templates/types/multiagent/fastapi/gitignore +++ /dev/null @@ -1,4 +0,0 @@ -__pycache__ -storage -.env -output diff --git a/templates/types/multiagent/fastapi/main.py b/templates/types/multiagent/fastapi/main.py deleted file mode 100644 index 11395a079..000000000 --- a/templates/types/multiagent/fastapi/main.py +++ /dev/null @@ -1,72 +0,0 @@ -# flake8: noqa: E402 -import os -from dotenv import load_dotenv - -from app.config import DATA_DIR - -load_dotenv() - -import logging - -import uvicorn -from app.api.routers.chat import chat_router -from app.api.routers.chat_config import config_router -from app.api.routers.upload import file_upload_router -from app.observability import init_observability -from app.settings import init_settings -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import RedirectResponse -from fastapi.staticfiles import StaticFiles - -app = FastAPI() - -init_settings() -init_observability() - - -environment = os.getenv("ENVIRONMENT", "dev") # Default to 'development' if not set -logger = logging.getLogger("uvicorn") - -if environment == "dev": - logger.warning("Running in development mode - allowing CORS for all origins") - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - - # Redirect to documentation page when accessing base URL - @app.get("/") - async def redirect_to_docs(): - return RedirectResponse(url="/docs") - - -def mount_static_files(directory, path): - if os.path.exists(directory): - logger.info(f"Mounting static files '{directory}' at '{path}'") - app.mount( - path, - StaticFiles(directory=directory, check_dir=False), - name=f"{directory}-static", - ) - - -# Mount the data files to serve the file viewer -mount_static_files(DATA_DIR, "/api/files/data") -# Mount the output files from tools -mount_static_files("output", "/api/files/output") - -app.include_router(chat_router, prefix="/api/chat") -app.include_router(config_router, prefix="/api/chat/config") -app.include_router(file_upload_router, prefix="/api/chat/upload") - - -if __name__ == "__main__": - app_host = os.getenv("APP_HOST", "0.0.0.0") - app_port = int(os.getenv("APP_PORT", "8000")) - reload = True if environment == "dev" else False - - uvicorn.run(app="main:app", host=app_host, port=app_port, reload=reload) diff --git a/templates/types/multiagent/fastapi/pyproject.toml b/templates/types/multiagent/fastapi/pyproject.toml deleted file mode 100644 index 397acd278..000000000 --- a/templates/types/multiagent/fastapi/pyproject.toml +++ /dev/null @@ -1,30 +0,0 @@ -[tool] -[tool.poetry] -name = "app" -version = "0.1.0" -description = "" -authors = ["Marcus Schiesser "] -readme = "README.md" - -[tool.poetry.scripts] -generate = "app.engine.generate:generate_datasource" - -[tool.poetry.dependencies] -python = ">=3.11,<3.13" -llama-index-agent-openai = ">=0.3.0,<0.4.0" -llama-index = "0.11.11" -fastapi = "^0.112.2" -python-dotenv = "^1.0.0" -uvicorn = { extras = ["standard"], version = "^0.23.2" } -cachetools = "^5.3.3" -aiostream = "^0.5.2" -markdown = "^3.7" -xhtml2pdf = "^0.2.16" -duckduckgo-search = "^6.2.13" - -[tool.poetry.dependencies.docx2txt] -version = "^0.8" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" From ce3a5f3dd0f1227ec5f0b7f498ddba01fa3fa966 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 20:42:14 +0700 Subject: [PATCH 32/77] refactor vercel response to for reusing --- .../{examples/factory.py => engine/engine.py} | 14 +- .../streaming/fastapi/app/api/routers/chat.py | 38 +++- .../app/api/routers/vercel_response.py | 180 +++++++++++++----- 3 files changed, 169 insertions(+), 63 deletions(-) rename templates/components/multiagent/python/app/{examples/factory.py => engine/engine.py} (81%) diff --git a/templates/components/multiagent/python/app/examples/factory.py b/templates/components/multiagent/python/app/engine/engine.py similarity index 81% rename from templates/components/multiagent/python/app/examples/factory.py rename to templates/components/multiagent/python/app/engine/engine.py index 2a376a327..e9563975d 100644 --- a/templates/components/multiagent/python/app/examples/factory.py +++ b/templates/components/multiagent/python/app/engine/engine.py @@ -1,20 +1,20 @@ import logging +import os from typing import List, Optional + from app.examples.choreography import create_choreography from app.examples.orchestrator import create_orchestrator from app.examples.workflow import create_workflow - - -from llama_index.core.workflow import Workflow from llama_index.core.chat_engine.types import ChatMessage - - -import os +from llama_index.core.workflow import Workflow logger = logging.getLogger("uvicorn") -def create_agent(chat_history: Optional[List[ChatMessage]] = None) -> Workflow: +def get_chat_engine( + chat_history: Optional[List[ChatMessage]] = None, **kwargs +) -> Workflow: + # TODO: the EXAMPLE_TYPE could be passed as a chat config parameter? agent_type = os.getenv("EXAMPLE_TYPE", "").lower() match agent_type: case "choreography": diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 39894361a..9b7533ccf 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -2,6 +2,8 @@ from typing import List from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status +from llama_index.core.agent import AgentRunner +from llama_index.core.chat_engine import CondensePlusContextChatEngine from llama_index.core.chat_engine.types import BaseChatEngine, NodeWithScore from llama_index.core.llms import MessageRole @@ -12,7 +14,10 @@ Result, SourceNodes, ) -from app.api.routers.vercel_response import VercelStreamResponse +from app.api.routers.vercel_response import ( + ChatEngineVercelStreamResponse, + WorkflowVercelStreamResponse, +) from app.engine import get_chat_engine from app.engine.query_filter import generate_filters @@ -40,12 +45,35 @@ async def chat( ) event_handler = EventCallbackHandler() chat_engine = get_chat_engine( - filters=filters, params=params, event_handlers=[event_handler] + filters=filters, + params=params, + event_handlers=[event_handler], + chat_history=messages, ) - response = await chat_engine.astream_chat(last_message_content, messages) - process_response_nodes(response.source_nodes, background_tasks) - return VercelStreamResponse(request, event_handler, response, data) + if isinstance(chat_engine, CondensePlusContextChatEngine) or isinstance( + chat_engine, AgentRunner + ): + event_handler = EventCallbackHandler() + chat_engine.callback_manager.handlers.append(event_handler) # type: ignore + + response = await chat_engine.astream_chat(last_message_content, messages) + process_response_nodes(response.source_nodes, background_tasks) + + return ChatEngineVercelStreamResponse( + request=request, + chat_data=data, + event_handler=event_handler, + response=response, + ) + else: + event_handler = chat_engine.run(input=last_message_content, streaming=True) + return WorkflowVercelStreamResponse( + request=request, + chat_data=data, + event_handler=event_handler, + events=chat_engine.stream_events(), + ) except Exception as e: logger.exception("Error in chat engine", exc_info=True) raise HTTPException( diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 924c60ce5..df90de9ed 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -1,24 +1,59 @@ import json -from typing import List +import logging +from abc import ABC, abstractmethod +from typing import AsyncGenerator, List from aiostream import stream from fastapi import Request from fastapi.responses import StreamingResponse from llama_index.core.chat_engine.types import StreamingAgentChatResponse +from app.agents.single import AgentRunEvent, AgentRunResult from app.api.routers.events import EventCallbackHandler from app.api.routers.models import ChatData, Message, SourceNodes from app.api.services.suggestion import NextQuestionSuggestion +logger = logging.getLogger("uvicorn") -class VercelStreamResponse(StreamingResponse): + +class BaseVercelStreamResponse(StreamingResponse, ABC): """ - Class to convert the response from the chat engine to the streaming format expected by Vercel + Base class to convert the response from the chat engine to the streaming format expected by Vercel """ TEXT_PREFIX = "0:" DATA_PREFIX = "8:" + def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs): + self.request = request + + stream = self._create_stream(request, chat_data, *args, **kwargs) + content = self.content_generator(stream) + + super().__init__(content=content) + + @abstractmethod + def _create_stream(self, request: Request, chat_data: ChatData, *args, **kwargs): + """ + Create the stream that will be used to generate the response. + """ + raise NotImplementedError("Subclasses must implement _create_stream") + + async def content_generator(self, stream): + is_stream_started = False + + async with stream.stream() as streamer: + async for output in streamer: + if not is_stream_started: + is_stream_started = True + # Stream a blank message to start the stream + yield self.convert_text("") + + yield output + + if await self.request.is_disconnected(): + break + @classmethod def convert_text(cls, token: str): # Escape newlines and double quotes to avoid breaking the stream @@ -30,54 +65,51 @@ def convert_data(cls, data: dict): data_str = json.dumps(data) return f"{cls.DATA_PREFIX}[{data_str}]\n" - def __init__( - self, - request: Request, - event_handler: EventCallbackHandler, - response: StreamingAgentChatResponse, - chat_data: ChatData, - ): - content = VercelStreamResponse.content_generator( - request, event_handler, response, chat_data + @staticmethod + async def _generate_next_questions(chat_history: List[Message], response: str): + questions = await NextQuestionSuggestion.suggest_next_questions( + chat_history, response ) - super().__init__(content=content) + if questions: + return { + "type": "suggested_questions", + "data": questions, + } + return None - @classmethod - async def content_generator( - cls, + +class ChatEngineVercelStreamResponse(BaseVercelStreamResponse): + """ + Class to convert the response from the chat engine to the streaming format expected by Vercel + """ + + def _create_stream( + self, request: Request, + chat_data: ChatData, event_handler: EventCallbackHandler, response: StreamingAgentChatResponse, - chat_data: ChatData, ): # Yield the text response async def _chat_response_generator(): final_response = "" async for token in response.async_response_gen(): final_response += token - yield cls.convert_text(token) + yield self.convert_text(token) # Generate next questions if next question prompt is configured - question_data = await cls._generate_next_questions( + question_data = await self._generate_next_questions( chat_data.messages, final_response ) if question_data: - yield cls.convert_data(question_data) + yield self.convert_data(question_data) # the text_generator is the leading stream, once it's finished, also finish the event stream event_handler.is_done = True # Yield the source nodes - yield cls.convert_data( - { - "type": "sources", - "data": { - "nodes": [ - SourceNodes.from_source_node(node).model_dump() - for node in response.source_nodes - ] - }, - } + yield self.convert_data( + self._source_nodes_to_response(response.source_nodes) ) # Yield the events from the event handler @@ -85,30 +117,76 @@ async def _event_generator(): async for event in event_handler.async_event_gen(): event_response = event.to_response() if event_response is not None: - yield cls.convert_data(event_response) + yield self.convert_data(event_response) combine = stream.merge(_chat_response_generator(), _event_generator()) - is_stream_started = False - async with combine.stream() as streamer: - async for output in streamer: - if not is_stream_started: - is_stream_started = True - # Stream a blank message to start the stream - yield cls.convert_text("") + return combine - yield output + @staticmethod + def _source_nodes_to_response(source_nodes: List): + return { + "type": "sources", + "data": { + "nodes": [ + SourceNodes.from_source_node(node).model_dump() + for node in source_nodes + ] + }, + } + + +class WorkflowVercelStreamResponse(BaseVercelStreamResponse): + """ + Class to convert the response from the chat engine to the streaming format expected by Vercel + """ - if await request.is_disconnected(): - break + def _create_stream( + self, + request: Request, + chat_data: ChatData, + event_handler: AgentRunResult | AsyncGenerator, + events: AsyncGenerator[AgentRunEvent, None], + verbose: bool = True, + ): + # Yield the text response + async def _chat_response_generator(): + result = await event_handler + final_response = "" + + if isinstance(result, AgentRunResult): + for token in result.response.message.content: + final_response += token + yield self.convert_text(token) + + if isinstance(result, AsyncGenerator): + async for token in result: + final_response += token.delta + yield self.convert_text(token.delta) + + # Generate next questions if next question prompt is configured + question_data = await self._generate_next_questions( + chat_data.messages, final_response + ) + if question_data: + yield self.convert_data(question_data) + + # TODO: stream sources + + # Yield the events from the event handler + async def _event_generator(): + async for event in events: + event_response = self._event_to_response(event) + if verbose: + logger.debug(event_response) + if event_response is not None: + yield self.convert_data(event_response) + + combine = stream.merge(_chat_response_generator(), _event_generator()) + return combine @staticmethod - async def _generate_next_questions(chat_history: List[Message], response: str): - questions = await NextQuestionSuggestion.suggest_next_questions( - chat_history, response - ) - if questions: - return { - "type": "suggested_questions", - "data": questions, - } - return None + def _event_to_response(event: AgentRunEvent) -> dict: + return { + "type": "agent", + "data": {"agent": event.name, "text": event.msg}, + } From a84e88535f2486f269adae1a4795cc447f32f97b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Fri, 27 Sep 2024 21:12:30 +0700 Subject: [PATCH 33/77] add tools usage for ts --- .../multiagent/typescript/workflow/agents.ts | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 7abaa6d0a..4bf892cea 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -1,5 +1,8 @@ -import { ChatMessage, QueryEngineTool } from "llamaindex"; +import fs from "fs/promises"; +import { BaseToolWithCall, ChatMessage, QueryEngineTool } from "llamaindex"; +import path from "path"; import { getDataSource } from "../engine"; +import { createTools } from "../engine/tools/index"; import { FunctionCallingAgent } from "./single-agent"; const getQueryEngineTool = async () => { @@ -22,10 +25,37 @@ const getQueryEngineTool = async () => { }); }; +const getAvailableTools = async () => { + const configFile = path.join("config", "tools.json"); + let toolConfig: any; + const tools: BaseToolWithCall[] = []; + try { + toolConfig = JSON.parse(await fs.readFile(configFile, "utf8")); + } catch (e) { + console.info(`Could not read ${configFile} file. Using no tools.`); + } + if (toolConfig) { + tools.push(...(await createTools(toolConfig))); + } + + return tools; +}; + export const createResearcher = async (chatHistory: ChatMessage[]) => { + const tools = await getAvailableTools(); + const researcherTools = [await getQueryEngineTool()]; + // Add wikipedia, duckduckgo if they are available + for (const tool of tools) { + if ( + tool.metadata.name === "wikipedia.WikipediaToolSpec" || + tool.metadata.name === "duckduckgo_search" + ) { + researcherTools.push(tool as any); + } + } return new FunctionCallingAgent({ name: "researcher", - tools: [await getQueryEngineTool()], + tools: researcherTools, systemPrompt: "You are a researcher agent. You are given a researching task. You must use your tools to complete the research.", chatHistory, @@ -49,3 +79,20 @@ export const createReviewer = (chatHistory: ChatMessage[]) => { chatHistory, }); }; + +export const createPublisher = async (chatHistory: ChatMessage[]) => { + const tools = await getAvailableTools(); + const publisherTools: BaseToolWithCall[] = []; + for (const tool of tools) { + if (tool.metadata.name === "document_generator") { + publisherTools.push(tool); + } + } + return new FunctionCallingAgent({ + name: "publisher", + tools: publisherTools, + systemPrompt: + "You are an expert in publishing blog posts. You are given a task to publish a blog post.", + chatHistory, + }); +}; From 48614d4f629966ff3f692c5fa0d0610b36170750 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 10:54:27 +0700 Subject: [PATCH 34/77] add publisher for TS --- .../typescript/agent/tools/duckduckgo.ts | 4 +- .../engines/typescript/agent/tools/index.ts | 11 +++- .../multiagent/typescript/workflow/agents.ts | 59 +++++++++++++------ .../multiagent/typescript/workflow/factory.ts | 32 ++++++---- 4 files changed, 76 insertions(+), 30 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/duckduckgo.ts b/templates/components/engines/typescript/agent/tools/duckduckgo.ts index 5100bdfdb..80ca7de0a 100644 --- a/templates/components/engines/typescript/agent/tools/duckduckgo.ts +++ b/templates/components/engines/typescript/agent/tools/duckduckgo.ts @@ -46,7 +46,8 @@ const DEFAULT_IMAGE_SEARCH_METADATA: ToolMetadata< JSONSchemaType > = { name: "duckduckgo_image_search", - description: "Use this function to search for images in DuckDuckGo.", + description: + "Use this function to search for images in internet using DuckDuckGo.", parameters: { type: "object", properties: { @@ -96,6 +97,7 @@ export class DuckDuckGoSearchTool implements BaseTool { const options = region ? { region } : {}; // Temporarily sleep to reduce overloading the DuckDuckGo await new Promise((resolve) => setTimeout(resolve, 1000)); + const searchResults = await search(query, options); return searchResults.results.slice(0, maxResults).map((result) => { diff --git a/templates/components/engines/typescript/agent/tools/index.ts b/templates/components/engines/typescript/agent/tools/index.ts index 7fe850fb9..511791ebe 100644 --- a/templates/components/engines/typescript/agent/tools/index.ts +++ b/templates/components/engines/typescript/agent/tools/index.ts @@ -4,7 +4,11 @@ import { DocumentGenerator, DocumentGeneratorParams, } from "./document_generator"; -import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo"; +import { + DuckDuckGoImageSearchTool, + DuckDuckGoSearchTool, + DuckDuckGoToolParams, +} from "./duckduckgo"; import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen"; import { InterpreterTool, InterpreterToolParams } from "./interpreter"; import { OpenAPIActionTool } from "./openapi-action"; @@ -42,7 +46,10 @@ const toolFactory: Record = { return await openAPIActionTool.toToolFunctions(); }, duckduckgo: async (config: unknown) => { - return [new DuckDuckGoSearchTool(config as DuckDuckGoToolParams)]; + return [ + new DuckDuckGoSearchTool(config as DuckDuckGoToolParams), + new DuckDuckGoImageSearchTool(config as DuckDuckGoToolParams), + ]; }, img_gen: async (config: unknown) => { return [new ImgGeneratorTool(config as ImgGeneratorToolParams)]; diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 4bf892cea..3263ecbd1 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -5,12 +5,10 @@ import { getDataSource } from "../engine"; import { createTools } from "../engine/tools/index"; import { FunctionCallingAgent } from "./single-agent"; -const getQueryEngineTool = async () => { +const getQueryEngineTool = async (): Promise => { const index = await getDataSource(); if (!index) { - throw new Error( - "StorageContext is empty - call 'npm run generate' to generate the storage first.", - ); + return null; } const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined; @@ -42,22 +40,33 @@ const getAvailableTools = async () => { }; export const createResearcher = async (chatHistory: ChatMessage[]) => { - const tools = await getAvailableTools(); - const researcherTools = [await getQueryEngineTool()]; + const tools: BaseToolWithCall[] = []; + const queryEngineTool = await getQueryEngineTool(); + if (queryEngineTool) { + tools.push(queryEngineTool); + } + const availableTools = await getAvailableTools(); // Add wikipedia, duckduckgo if they are available - for (const tool of tools) { + for (const tool of availableTools) { if ( tool.metadata.name === "wikipedia.WikipediaToolSpec" || tool.metadata.name === "duckduckgo_search" ) { - researcherTools.push(tool as any); + tools.push(tool); } } return new FunctionCallingAgent({ name: "researcher", - tools: researcherTools, - systemPrompt: - "You are a researcher agent. You are given a researching task. You must use your tools to complete the research.", + tools: tools, + systemPrompt: `You are a researcher agent. +You are given a researching task. You must use tools to retrieve information needed for the task. +It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. +If you don't found any related information, please return "I didn't find any information." +Example: +Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." +-> +Your task: Looking for information/images in English about the history of the Internet +This is not your task: Create blog post, create PDF, write in English`, chatHistory, }); }; @@ -65,8 +74,14 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { export const createWriter = (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "writer", - systemPrompt: - "You are an expert in writing blog posts. You are given a task to write a blog post. Don't make up any information yourself.", + systemPrompt: `You are an expert in writing blog posts. +You are given a task to write a blog post. Don't make up any information yourself. +It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. +Example: +Task: "Here is the information i found about the history of internet: +Create a blog post about the history of the internet, write in English and publish in PDF format." +-> Your task: Use the research content {...} to write a blog post in English. +-> This is not your task: Create PDF`, chatHistory, }); }; @@ -74,8 +89,16 @@ export const createWriter = (chatHistory: ChatMessage[]) => { export const createReviewer = (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "reviewer", - systemPrompt: - "You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review.", + systemPrompt: `You are an expert in reviewing blog posts. +You are given a task to review a blog post. +Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. +Furthermore, proofread the post for grammar and spelling errors. +Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. +It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. +Example: +Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." +-> Your task: Review is the main content of the post is about the history of the internet, is it written in English. +-> This is not your task: Create blog post, create PDF, write in English.`, chatHistory, }); }; @@ -83,16 +106,18 @@ export const createReviewer = (chatHistory: ChatMessage[]) => { export const createPublisher = async (chatHistory: ChatMessage[]) => { const tools = await getAvailableTools(); const publisherTools: BaseToolWithCall[] = []; + let systemPrompt = + "You are an expert in publishing blog posts. You are given a task to publish a blog post."; for (const tool of tools) { if (tool.metadata.name === "document_generator") { publisherTools.push(tool); + systemPrompt = `${systemPrompt}.If user request for a file, use the document_generator tool to generate the file and reply the link to the file.`; } } return new FunctionCallingAgent({ name: "publisher", tools: publisherTools, - systemPrompt: - "You are an expert in publishing blog posts. You are given a task to publish a blog post.", + systemPrompt: systemPrompt, chatHistory, }); }; diff --git a/templates/components/multiagent/typescript/workflow/factory.ts b/templates/components/multiagent/typescript/workflow/factory.ts index 8853b08b2..8ea5ffea2 100644 --- a/templates/components/multiagent/typescript/workflow/factory.ts +++ b/templates/components/multiagent/typescript/workflow/factory.ts @@ -6,7 +6,12 @@ import { WorkflowEvent, } from "@llamaindex/core/workflow"; import { ChatMessage, ChatResponseChunk } from "llamaindex"; -import { createResearcher, createReviewer, createWriter } from "./agents"; +import { + createPublisher, + createResearcher, + createReviewer, + createWriter, +} from "./agents"; import { AgentInput, AgentRunEvent } from "./type"; const TIMEOUT = 360 * 1000; @@ -18,6 +23,7 @@ class WriteEvent extends WorkflowEvent<{ isGood: boolean; }> {} class ReviewEvent extends WorkflowEvent<{ input: string }> {} +class PublishEvent extends WorkflowEvent<{ input: string }> {} export const createWorkflow = (chatHistory: ChatMessage[]) => { const runAgent = async ( @@ -66,14 +72,9 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { } if (ev.data.isGood || tooManyAttempts) { - // The text is ready for publication, we just use the writer to stream the output - const writer = createWriter(chatHistory); - const content = context.get("result"); - - return (await runAgent(context, writer, { - message: `You're blog post is ready for publication. Please respond with just the blog post. Blog post: \`\`\`${content}\`\`\``, - streaming: true, - })) as unknown as StopEvent>; + return new PublishEvent({ + input: "Please help me to publish the blog post.", + }); } const writer = createWriter(chatHistory); @@ -123,11 +124,22 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { }); }; + const publish = async (context: Context, ev: PublishEvent) => { + const publisher = await createPublisher(chatHistory); + const result = context.get("result"); + + return (await runAgent(context, publisher, { + message: `Please publish this blog post: ${result}`, + streaming: true, + })) as unknown as StopEvent>; + }; + const workflow = new Workflow({ timeout: TIMEOUT, validate: true }); workflow.addStep(StartEvent, start, { outputs: ResearchEvent }); workflow.addStep(ResearchEvent, research, { outputs: WriteEvent }); - workflow.addStep(WriteEvent, write, { outputs: [ReviewEvent, StopEvent] }); + workflow.addStep(WriteEvent, write, { outputs: [ReviewEvent, PublishEvent] }); workflow.addStep(ReviewEvent, review, { outputs: WriteEvent }); + workflow.addStep(PublishEvent, publish, { outputs: StopEvent }); return workflow; }; From ca82a226d7578a83a8f3395301b8ce40f6801435 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 10:59:40 +0700 Subject: [PATCH 35/77] only use query engine if index is not None --- .../multiagent/python/app/examples/researcher.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py index ed19218b4..2714d102d 100644 --- a/templates/components/multiagent/python/app/examples/researcher.py +++ b/templates/components/multiagent/python/app/examples/researcher.py @@ -14,7 +14,7 @@ def _create_query_engine_tool() -> QueryEngineTool: """ index = get_index() if index is None: - raise ValueError("Index not found. Please create an index first.") + return None top_k = int(os.getenv("TOP_K", 0)) query_engine = index.as_query_engine( **({"similarity_top_k": top_k} if top_k != 0 else {}) @@ -35,11 +35,12 @@ def _get_research_tools() -> QueryEngineTool: Researcher take responsibility for retrieving information. Try init wikipedia or duckduckgo tool if available. """ + tools = [] + query_engine_tool = _create_query_engine_tool() + if query_engine_tool is not None: + tools.append(query_engine_tool) researcher_tool_names = ["duckduckgo", "wikipedia.WikipediaToolSpec"] - # Always include the query engine tool - tools = [_create_query_engine_tool()] configured_tools = ToolFactory.from_env(map_result=True) - print(configured_tools) for tool_name, tool in configured_tools.items(): if tool_name in researcher_tool_names: tools.extend(tool) From ab94dc0082c29be0eb89dfc50e97ad1803c8e963 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 11:54:55 +0700 Subject: [PATCH 36/77] fix chat issue --- templates/components/engines/python/agent/engine.py | 2 +- templates/components/engines/python/chat/engine.py | 2 +- .../types/streaming/fastapi/app/api/routers/chat.py | 7 ++++--- .../streaming/fastapi/app/api/routers/vercel_response.py | 9 ++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/templates/components/engines/python/agent/engine.py b/templates/components/engines/python/agent/engine.py index 22a30d0e9..c71d37047 100644 --- a/templates/components/engines/python/agent/engine.py +++ b/templates/components/engines/python/agent/engine.py @@ -8,7 +8,7 @@ from llama_index.core.tools.query_engine import QueryEngineTool -def get_chat_engine(filters=None, params=None, event_handlers=None): +def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs): system_prompt = os.getenv("SYSTEM_PROMPT") top_k = int(os.getenv("TOP_K", 0)) tools = [] diff --git a/templates/components/engines/python/chat/engine.py b/templates/components/engines/python/chat/engine.py index cb7e00827..83c73cc65 100644 --- a/templates/components/engines/python/chat/engine.py +++ b/templates/components/engines/python/chat/engine.py @@ -9,7 +9,7 @@ from llama_index.core.settings import Settings -def get_chat_engine(filters=None, params=None, event_handlers=None): +def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs): system_prompt = os.getenv("SYSTEM_PROMPT") citation_prompt = os.getenv("SYSTEM_CITATION_PROMPT", None) top_k = int(os.getenv("TOP_K", 0)) diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 9b7533ccf..4cbe9f685 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -1,10 +1,10 @@ import logging from typing import List -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status from llama_index.core.agent import AgentRunner from llama_index.core.chat_engine import CondensePlusContextChatEngine -from llama_index.core.chat_engine.types import BaseChatEngine, NodeWithScore +from llama_index.core.chat_engine.types import NodeWithScore from llama_index.core.llms import MessageRole from app.api.routers.events import EventCallbackHandler @@ -86,11 +86,12 @@ async def chat( @r.post("/request") async def chat_request( data: ChatData, - chat_engine: BaseChatEngine = Depends(get_chat_engine), ) -> Result: last_message_content = data.get_last_message_content() messages = data.get_history_messages() + chat_engine = get_chat_engine(filters=None, params=None) + response = await chat_engine.achat(last_message_content, messages) return Result( result=Message(role=MessageRole.ASSISTANT, content=response.response), diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index df90de9ed..293bd4c0c 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -8,7 +8,6 @@ from fastapi.responses import StreamingResponse from llama_index.core.chat_engine.types import StreamingAgentChatResponse -from app.agents.single import AgentRunEvent, AgentRunResult from app.api.routers.events import EventCallbackHandler from app.api.routers.models import ChatData, Message, SourceNodes from app.api.services.suggestion import NextQuestionSuggestion @@ -144,8 +143,8 @@ def _create_stream( self, request: Request, chat_data: ChatData, - event_handler: AgentRunResult | AsyncGenerator, - events: AsyncGenerator[AgentRunEvent, None], + event_handler: "AgentRunResult" | AsyncGenerator, + events: AsyncGenerator["AgentRunEvent", None], verbose: bool = True, ): # Yield the text response @@ -153,7 +152,7 @@ async def _chat_response_generator(): result = await event_handler final_response = "" - if isinstance(result, AgentRunResult): + if isinstance(result, "AgentRunResult"): for token in result.response.message.content: final_response += token yield self.convert_text(token) @@ -185,7 +184,7 @@ async def _event_generator(): return combine @staticmethod - def _event_to_response(event: AgentRunEvent) -> dict: + def _event_to_response(event: "AgentRunEvent") -> dict: return { "type": "agent", "data": {"agent": event.name, "text": event.msg}, From f9f19b42e845fa414ab54c2f84e396a5d6449ef8 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 11:58:01 +0700 Subject: [PATCH 37/77] ignore typing F821 --- .../streaming/fastapi/app/api/routers/vercel_response.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 293bd4c0c..588765816 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -143,8 +143,8 @@ def _create_stream( self, request: Request, chat_data: ChatData, - event_handler: "AgentRunResult" | AsyncGenerator, - events: AsyncGenerator["AgentRunEvent", None], + event_handler: "AgentRunResult" | AsyncGenerator, # noqa: F821 + events: AsyncGenerator["AgentRunEvent", None], # noqa: F821 verbose: bool = True, ): # Yield the text response @@ -184,7 +184,7 @@ async def _event_generator(): return combine @staticmethod - def _event_to_response(event: "AgentRunEvent") -> dict: + def _event_to_response(event: "AgentRunEvent") -> dict: # noqa: F821 return { "type": "agent", "data": {"agent": event.name, "text": event.msg}, From e96c861ede95eb5d1efe526981ff4ba1b8be05e1 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 12:11:31 +0700 Subject: [PATCH 38/77] fix: always add tool dependencies for TS --- helpers/typescript.ts | 12 ------------ templates/types/streaming/express/package.json | 4 +++- templates/types/streaming/nextjs/package.json | 4 +++- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/helpers/typescript.ts b/helpers/typescript.ts index 0688a80f6..605e9802c 100644 --- a/helpers/typescript.ts +++ b/helpers/typescript.ts @@ -328,18 +328,6 @@ async function updatePackageJson({ }; } - if (tools && tools.length > 0) { - tools.forEach((tool) => { - if (tool.name === "document_generator") { - packageJson.dependencies = { - ...packageJson.dependencies, - puppeteer: "^23.4.1", - marked: "^14.1.2", - }; - } - }); - } - await fs.writeFile( packageJsonFile, JSON.stringify(packageJson, null, 2) + os.EOL, diff --git a/templates/types/streaming/express/package.json b/templates/types/streaming/express/package.json index 084f39c86..ccba195be 100644 --- a/templates/types/streaming/express/package.json +++ b/templates/types/streaming/express/package.json @@ -27,7 +27,9 @@ "@e2b/code-interpreter": "^0.0.5", "got": "^14.4.1", "@apidevtools/swagger-parser": "^10.1.0", - "formdata-node": "^6.0.3" + "formdata-node": "^6.0.3", + "puppeteer": "^23.4.1", + "marked": "^14.1.2" }, "devDependencies": { "@types/cors": "^2.8.16", diff --git a/templates/types/streaming/nextjs/package.json b/templates/types/streaming/nextjs/package.json index 775bee983..d985ff93f 100644 --- a/templates/types/streaming/nextjs/package.json +++ b/templates/types/streaming/nextjs/package.json @@ -42,7 +42,9 @@ "tailwind-merge": "^2.1.0", "tiktoken": "^1.0.15", "uuid": "^9.0.1", - "vaul": "^0.9.1" + "vaul": "^0.9.1", + "puppeteer": "^23.4.1", + "marked": "^14.1.2" }, "devDependencies": { "@types/node": "^20.10.3", From 461eb4ecd3e6f01000dd35e2a7191bf59c155448 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 12:55:31 +0700 Subject: [PATCH 39/77] fix: wrong engine template --- helpers/python.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/helpers/python.ts b/helpers/python.ts index 2c89aa6c4..c2f1a0a96 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -406,16 +406,27 @@ export const installPythonTemplate = async ({ cwd: path.join(compPath, "services", "python"), }); } - + // Copy engine code if (template === "streaming" || template === "multiagent") { // Select and copy engine code based on data sources and tools let engine; - if (dataSources.length > 0 && (!tools || tools.length === 0)) { - console.log("\nNo tools selected - use optimized context chat engine\n"); - engine = "chat"; - } else { + // Multiagent always uses agent engine + if (template === "multiagent") { engine = "agent"; + } else { + // For streaming, use chat engine by default + // Unless tools are selected, in which case use agent engine + if (dataSources.length > 0 && (!tools || tools.length === 0)) { + console.log( + "\nNo tools selected - use optimized context chat engine\n", + ); + engine = "chat"; + } else { + engine = "agent"; + } } + + // Copy engine code await copy("**", enginePath, { parents: true, cwd: path.join(compPath, "engines", "python", engine), From ababbe1b32dc11e3f118af64e394fb3b73dc072e Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 13:03:11 +0700 Subject: [PATCH 40/77] fix: ts type checking --- .../typescript/agent/tools/document_generator.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document_generator.ts index 4c2dc4fb3..93b79eec8 100644 --- a/templates/components/engines/typescript/agent/tools/document_generator.ts +++ b/templates/components/engines/typescript/agent/tools/document_generator.ts @@ -143,8 +143,10 @@ export class DocumentGenerator implements BaseTool { this.metadata = params.metadata ?? DEFAULT_METADATA; } - private static generateHtmlContent(originalContent: string): string { - return marked(originalContent); + private static async generateHtmlContent( + originalContent: string, + ): Promise { + return await marked(originalContent); } private static generateHtmlDocument(htmlContent: string): string { @@ -185,8 +187,8 @@ export class DocumentGenerator implements BaseTool { const page = await browser.newPage(); try { await page.setContent(htmlContent, { waitUntil: "networkidle0" }); - const pdf = await page.pdf({ format: "A4" }); - return pdf; + const pdfArray = await page.pdf({ format: "A4" }); + return Buffer.from(pdfArray); } catch (error) { console.error("Error generating PDF:", error); throw new Error("Failed to generate PDF"); @@ -202,7 +204,8 @@ export class DocumentGenerator implements BaseTool { let fileExtension: string; // Generate the HTML from the original content (markdown) - const htmlContent = DocumentGenerator.generateHtmlContent(originalContent); + const htmlContent = + await DocumentGenerator.generateHtmlContent(originalContent); try { if (documentType.toLowerCase() === DocumentType.HTML) { From eadca2a84d4c531af2b8d7aa05af96150742c84c Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 14:49:31 +0700 Subject: [PATCH 41/77] fix: e2e --- e2e/utils.ts | 8 +++++--- helpers/typescript.ts | 3 --- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/e2e/utils.ts b/e2e/utils.ts index 361ad7c6c..aec1e30d6 100644 --- a/e2e/utils.ts +++ b/e2e/utils.ts @@ -109,7 +109,9 @@ export async function runCreateLlama({ if (appType) { commandArgs.push(appType); } - if (!useLlamaParse) { + if (useLlamaParse) { + commandArgs.push("--use-llama-parse"); + } else { commandArgs.push("--no-llama-parse"); } @@ -141,10 +143,10 @@ export async function runCreateLlama({ externalPort, ); } else if (postInstallAction === "dependencies") { - await waitForProcess(appProcess, 1000 * 60); // wait 1 min for dependencies to be resolved + await waitForProcess(appProcess, 2000 * 60); // wait 2 min for dependencies to be resolved } else { // wait 10 seconds for create-llama to exit - await waitForProcess(appProcess, 1000 * 10); + await waitForProcess(appProcess, 1000 * 20); } return { diff --git a/helpers/typescript.ts b/helpers/typescript.ts index 605e9802c..ffebae4a3 100644 --- a/helpers/typescript.ts +++ b/helpers/typescript.ts @@ -209,7 +209,6 @@ export const installTSTemplate = async ({ ui, observability, vectorDb, - tools, }); if (postInstallAction === "runApp" || postInstallAction === "dependencies") { @@ -231,7 +230,6 @@ async function updatePackageJson({ ui, observability, vectorDb, - tools, }: Pick< InstallTemplateArgs, | "root" @@ -241,7 +239,6 @@ async function updatePackageJson({ | "ui" | "observability" | "vectorDb" - | "tools" > & { relativeEngineDestPath: string; }): Promise { From 498ad16f0496c879a5791aec3e404d62247a451d Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 16:33:02 +0700 Subject: [PATCH 42/77] remove pdf generator in TS --- .../agent/tools/document_generator.ts | 96 ++----------------- .../types/streaming/express/package.json | 1 - templates/types/streaming/nextjs/package.json | 1 - 3 files changed, 9 insertions(+), 89 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document_generator.ts index 93b79eec8..33f29a1da 100644 --- a/templates/components/engines/typescript/agent/tools/document_generator.ts +++ b/templates/components/engines/typescript/agent/tools/document_generator.ts @@ -3,25 +3,18 @@ import fs from "fs"; import { BaseTool, ToolMetadata } from "llamaindex"; import { marked } from "marked"; import path from "path"; -import puppeteer from "puppeteer"; const OUTPUT_DIR = "output/tools"; -enum DocumentType { - HTML = "html", - PDF = "pdf", -} - type DocumentParameter = { originalContent: string; - documentType: string; fileName: string; }; const DEFAULT_METADATA: ToolMetadata> = { name: "document_generator", description: - "Generate document as PDF or HTML file from markdown content. Return a file url to the document", + "Generate HTML document from markdown content. Return a file url to the document", parameters: { type: "object", properties: { @@ -29,16 +22,12 @@ const DEFAULT_METADATA: ToolMetadata> = { type: "string", description: "The original markdown content to convert.", }, - documentType: { - type: "string", - description: "The type of document to generate (pdf or html).", - }, fileName: { type: "string", description: "The name of the document file (without extension).", }, }, - required: ["originalContent", "documentType", "fileName"], + required: ["originalContent", "fileName"], }, }; @@ -114,24 +103,6 @@ const HTML_TEMPLATE = ` `; -const PDF_SPECIFIC_STYLES = ` - @page { - size: letter; - margin: 2cm; - } - body { - font-size: 11pt; - } - h1 { font-size: 18pt; } - h2 { font-size: 16pt; } - h3 { font-size: 14pt; } - h4, h5, h6 { font-size: 12pt; } - pre, code { - font-family: Courier, monospace; - font-size: 0.9em; - } -`; - export interface DocumentGeneratorParams { metadata?: ToolMetadata>; } @@ -143,10 +114,8 @@ export class DocumentGenerator implements BaseTool { this.metadata = params.metadata ?? DEFAULT_METADATA; } - private static async generateHtmlContent( - originalContent: string, - ): Promise { - return await marked(originalContent); + private static generateHtmlContent(originalContent: string): string { + return marked(originalContent); } private static generateHtmlDocument(htmlContent: string): string { @@ -175,61 +144,14 @@ export class DocumentGenerator implements BaseTool { } } - private static generatePdfDocument(htmlContent: string): string { - return HTML_TEMPLATE.replace("{{content}}", htmlContent).replace( - HTML_SPECIFIC_STYLES, - PDF_SPECIFIC_STYLES, - ); - } - - private static async generatePdf(htmlContent: string): Promise { - const browser = await puppeteer.launch(); - const page = await browser.newPage(); - try { - await page.setContent(htmlContent, { waitUntil: "networkidle0" }); - const pdfArray = await page.pdf({ format: "A4" }); - return Buffer.from(pdfArray); - } catch (error) { - console.error("Error generating PDF:", error); - throw new Error("Failed to generate PDF"); - } finally { - await browser.close(); - } - } - async call(input: DocumentParameter): Promise { - const { originalContent, documentType, fileName } = input; - - let fileContent: string | Buffer; - let fileExtension: string; - - // Generate the HTML from the original content (markdown) - const htmlContent = - await DocumentGenerator.generateHtmlContent(originalContent); - - try { - if (documentType.toLowerCase() === DocumentType.HTML) { - fileContent = DocumentGenerator.generateHtmlDocument(htmlContent); - fileExtension = "html"; - } else if (documentType.toLowerCase() === DocumentType.PDF) { - const pdfDocument = DocumentGenerator.generatePdfDocument(htmlContent); - fileContent = await DocumentGenerator.generatePdf(pdfDocument); - fileExtension = "pdf"; - } else { - throw new Error( - `Invalid document type: ${documentType}. Must be 'pdf' or 'html'.`, - ); - } - } catch (error) { - console.error("Error generating document:", error); - throw new Error("Failed to generate document"); - } + const { originalContent, fileName } = input; + + const htmlContent = DocumentGenerator.generateHtmlContent(originalContent); + const fileContent = DocumentGenerator.generateHtmlDocument(htmlContent); const validatedFileName = DocumentGenerator.validateFileName(fileName); - const filePath = path.join( - OUTPUT_DIR, - `${validatedFileName}.${fileExtension}`, - ); + const filePath = path.join(OUTPUT_DIR, `${validatedFileName}.html`); DocumentGenerator.writeToFile(fileContent, filePath); diff --git a/templates/types/streaming/express/package.json b/templates/types/streaming/express/package.json index ccba195be..39d23f857 100644 --- a/templates/types/streaming/express/package.json +++ b/templates/types/streaming/express/package.json @@ -28,7 +28,6 @@ "got": "^14.4.1", "@apidevtools/swagger-parser": "^10.1.0", "formdata-node": "^6.0.3", - "puppeteer": "^23.4.1", "marked": "^14.1.2" }, "devDependencies": { diff --git a/templates/types/streaming/nextjs/package.json b/templates/types/streaming/nextjs/package.json index d985ff93f..38ced8514 100644 --- a/templates/types/streaming/nextjs/package.json +++ b/templates/types/streaming/nextjs/package.json @@ -43,7 +43,6 @@ "tiktoken": "^1.0.15", "uuid": "^9.0.1", "vaul": "^0.9.1", - "puppeteer": "^23.4.1", "marked": "^14.1.2" }, "devDependencies": { From 6c654692ce8c1f3568fe3cc057cc550a5248ece4 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 16:37:21 +0700 Subject: [PATCH 43/77] fix ts typing --- .../engines/typescript/agent/tools/document_generator.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document_generator.ts index 33f29a1da..a7741ce53 100644 --- a/templates/components/engines/typescript/agent/tools/document_generator.ts +++ b/templates/components/engines/typescript/agent/tools/document_generator.ts @@ -114,8 +114,10 @@ export class DocumentGenerator implements BaseTool { this.metadata = params.metadata ?? DEFAULT_METADATA; } - private static generateHtmlContent(originalContent: string): string { - return marked(originalContent); + private static async generateHtmlContent( + originalContent: string, + ): Promise { + return await marked(originalContent); } private static generateHtmlDocument(htmlContent: string): string { @@ -147,7 +149,8 @@ export class DocumentGenerator implements BaseTool { async call(input: DocumentParameter): Promise { const { originalContent, fileName } = input; - const htmlContent = DocumentGenerator.generateHtmlContent(originalContent); + const htmlContent = + await DocumentGenerator.generateHtmlContent(originalContent); const fileContent = DocumentGenerator.generateHtmlDocument(htmlContent); const validatedFileName = DocumentGenerator.validateFileName(fileName); From 138b9c0c9a1a2081d77bc808e7ddf31bec023ca8 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 30 Sep 2024 17:25:25 +0700 Subject: [PATCH 44/77] always use agent template for multi-agent --- helpers/typescript.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/helpers/typescript.ts b/helpers/typescript.ts index ffebae4a3..90d2079e8 100644 --- a/helpers/typescript.ts +++ b/helpers/typescript.ts @@ -157,7 +157,10 @@ export const installTSTemplate = async ({ // Select and copy engine code based on data sources and tools let engine; tools = tools ?? []; - if (dataSources.length > 0 && tools.length === 0) { + // multiagent template always uses agent engine + if (template === "multiagent") { + engine = "agent"; + } else if (dataSources.length > 0 && tools.length === 0) { console.log("\nNo tools selected - use optimized context chat engine\n"); engine = "chat"; } else { From ee4fdc5ca0470edb9f2033ef5728a49f436c92f8 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 08:45:09 +0700 Subject: [PATCH 45/77] format code --- .../python/app/examples/choreography.py | 11 +-- .../python/app/examples/orchestrator.py | 21 +++--- .../python/app/examples/publisher.py | 12 +++- .../python/app/examples/researcher.py | 22 +++--- .../python/app/examples/workflow.py | 67 ++++++++++--------- .../multiagent/typescript/workflow/factory.ts | 5 +- 6 files changed, 81 insertions(+), 57 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/choreography.py b/templates/components/multiagent/python/app/examples/choreography.py index 23f0efdad..2ad180f47 100644 --- a/templates/components/multiagent/python/app/examples/choreography.py +++ b/templates/components/multiagent/python/app/examples/choreography.py @@ -1,3 +1,4 @@ +from textwrap import dedent from typing import List, Optional from app.agents.multi import AgentCallingAgent @@ -20,10 +21,12 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): name="writer", agents=[researcher, reviewer, publisher], description="expert in writing blog posts, needs researched information and images to write a blog post", - system_prompt="""You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. - After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. - You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. - Finally, always request the publisher to create an document (pdf, html) and publish the blog post.""", + system_prompt=dedent(""" + You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. + After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. + You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. + Finally, always request the publisher to create an document (pdf, html) and publish the blog post. + """), # TODO: add chat_history support to AgentCallingAgent # chat_history=chat_history, ) diff --git a/templates/components/multiagent/python/app/examples/orchestrator.py b/templates/components/multiagent/python/app/examples/orchestrator.py index 6c0404559..fb6a336d9 100644 --- a/templates/components/multiagent/python/app/examples/orchestrator.py +++ b/templates/components/multiagent/python/app/examples/orchestrator.py @@ -1,3 +1,4 @@ +from textwrap import dedent from typing import List, Optional from app.agents.multi import AgentOrchestrator @@ -12,19 +13,23 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): writer = FunctionCallingAgent( name="writer", description="expert in writing blog posts, need information and images to write a post", - system_prompt="""You are an expert in writing blog posts. - You are given a task to write a blog post. Don't make up any information yourself. - If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". - If you need to use images, reply "I need images about the topic to write the blog post". Don't use any dummy images made up by you. - If you have all the information needed, write the blog post.""", + system_prompt=dedent(""" + You are an expert in writing blog posts. + You are given a task to write a blog post. Don't make up any information yourself. + If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". + If you need to use images, reply "I need images about the topic to write the blog post". Don't use any dummy images made up by you. + If you have all the information needed, write the blog post. + """), chat_history=chat_history, ) reviewer = FunctionCallingAgent( name="reviewer", description="expert in reviewing blog posts, needs a written blog post to review", - system_prompt="""You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix the issues found yourself. You must output a final blog post. - A post must include at lease one valid image, if not, reply "I need images about the topic to write the blog post". An image URL start with example or your website is not valid. - Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.""", + system_prompt=dedent(""" + You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix the issues found yourself. You must output a final blog post. + A post must include at lease one valid image, if not, reply "I need images about the topic to write the blog post". An image URL start with example or your website is not valid. + Especially check for logical inconsistencies and proofread the post for grammar and spelling errors. + """), chat_history=chat_history, ) publisher = create_publisher(chat_history) diff --git a/templates/components/multiagent/python/app/examples/publisher.py b/templates/components/multiagent/python/app/examples/publisher.py index a7abd0df8..617e273bb 100644 --- a/templates/components/multiagent/python/app/examples/publisher.py +++ b/templates/components/multiagent/python/app/examples/publisher.py @@ -1,3 +1,4 @@ +from textwrap import dedent from typing import List, Tuple from app.agents.single import FunctionCallingAgent @@ -12,7 +13,10 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]: configured_tools = ToolFactory.from_env(map_result=True) if "document_generator" in configured_tools.keys(): tools.extend(configured_tools["document_generator"]) - prompt_instructions = "You have access to a document generator tool that can create PDF or HTML document for the content. Based on the user request, please specify the type of document to generate or just reply to the user directly without generating any document file." + prompt_instructions = dedent(""" + You have access to a document generator tool that can create PDF or HTML document for the content. + Based on the user request, please specify the type of document to generate or just reply to the user directly without generating any document file. + """) description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format." else: prompt_instructions = "You don't have a tool to generate document. Please reply the content directly." @@ -22,8 +26,10 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]: def create_publisher(chat_history: List[ChatMessage]): tools, instructions, description = get_publisher_tools() - system_prompt = f"""You are a publisher that help publish the blog post. - {instructions}""" + system_prompt = dedent(f""" + You are a publisher that help publish the blog post. + {instructions} + """) return FunctionCallingAgent( name="publisher", tools=tools, diff --git a/templates/components/multiagent/python/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py index 2714d102d..cc480ec80 100644 --- a/templates/components/multiagent/python/app/examples/researcher.py +++ b/templates/components/multiagent/python/app/examples/researcher.py @@ -1,4 +1,5 @@ import os +from textwrap import dedent from typing import List from app.agents.single import FunctionCallingAgent @@ -56,15 +57,16 @@ def create_researcher(chat_history: List[ChatMessage]): name="researcher", tools=tools, description="expert in retrieving any unknown content or searching for images from the internet", - system_prompt="""You are a researcher agent. -You are given a researching task. You must use tools to retrieve information needed for the task. -It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. -If you don't found any related information, please return "I didn't find any information." -Example: -Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." --> -Your real task: Looking for information in english about the history of the internet -This is not your task: Create blog post, create PDF, write in English -""", + system_prompt=dedent(""" + You are a researcher agent. + You are given a researching task. You must use tools to retrieve information needed for the task. + It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. + If you don't found any related information, please return "I didn't find any information." + Example: + Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." + -> + Your real task: Looking for information in english about the history of the internet + This is not your task: Create blog post, create PDF, write in English + """), chat_history=chat_history, ) diff --git a/templates/components/multiagent/python/app/examples/workflow.py b/templates/components/multiagent/python/app/examples/workflow.py index ac62250ae..d1de62b50 100644 --- a/templates/components/multiagent/python/app/examples/workflow.py +++ b/templates/components/multiagent/python/app/examples/workflow.py @@ -1,3 +1,4 @@ +from textwrap import dedent from typing import AsyncGenerator, List, Optional from app.agents.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent @@ -24,31 +25,33 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): writer = FunctionCallingAgent( name="writer", description="expert in writing blog posts, need information and images to write a post.", - system_prompt="""You are an expert in writing blog posts. -You are given a task to write a blog post. Don't make up any information yourself. -It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. -Example: -Task: "Here is the information i found about the history of internet: -Create a blog post about the history of the internet, write in English and publish in PDF format." --> Your task: Use the research content {...} to write a blog post in English. --> This is not your task: Create PDF -""", + system_prompt=dedent(""" + You are an expert in writing blog posts. + You are given a task to write a blog post. Don't make up any information yourself. + It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. + Example: + Task: "Here is the information i found about the history of internet: + Create a blog post about the history of the internet, write in English and publish in PDF format." + -> Your task: Use the research content {...} to write a blog post in English. + -> This is not your task: Create PDF + """), chat_history=chat_history, ) reviewer = FunctionCallingAgent( name="reviewer", description="expert in reviewing blog posts, needs a written blog post to review.", - system_prompt="""You are an expert in reviewing blog posts. -You are given a task to review a blog post. -Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. -Furthermore, proofread the post for grammar and spelling errors. -Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. -It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. -Example: -Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." --> Your task: Review is the main content of the post is about the history of the internet, is it written in English. --> This is not your task: Create blog post, create PDF, write in English. -""", + system_prompt=dedent(""" + You are an expert in reviewing blog posts. + You are given a task to review a blog post. + Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. + Furthermore, proofread the post for grammar and spelling errors. + Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. + It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. + Example: + Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." + -> Your task: Review is the main content of the post is about the history of the internet, is it written in English. + -> This is not your task: Create blog post, create PDF, write in English. + """), chat_history=chat_history, ) workflow = BlogPostWorkflow(timeout=360) @@ -142,16 +145,20 @@ async def review( ) else: return WriteEvent( - input=f"""Improve the writing of a given blog post by using a given review. -Blog post: -``` -{old_content} -``` - -Review: -``` -{review} -```""" + input=dedent( + f""" + Improve the writing of a given blog post by using a given review. + Blog post: + ``` + {old_content} + ``` + + Review: + ``` + {review} + ``` + """ + ), ) @step() diff --git a/templates/components/multiagent/typescript/workflow/factory.ts b/templates/components/multiagent/typescript/workflow/factory.ts index 8ea5ffea2..47290fe8b 100644 --- a/templates/components/multiagent/typescript/workflow/factory.ts +++ b/templates/components/multiagent/typescript/workflow/factory.ts @@ -128,10 +128,11 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { const publisher = await createPublisher(chatHistory); const result = context.get("result"); - return (await runAgent(context, publisher, { + const publishResult = await runAgent(context, publisher, { message: `Please publish this blog post: ${result}`, streaming: true, - })) as unknown as StopEvent>; + }); + return publishResult as StopEvent>; }; const workflow = new Workflow({ timeout: TIMEOUT, validate: true }); From 56e29a67b0b0d8cb3ab29d031d04e2e90e5107e2 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 09:29:09 +0700 Subject: [PATCH 46/77] fix python vercel stream generator --- .../fastapi/app/api/routers/vercel_response.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 588765816..01bf7c597 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -152,15 +152,18 @@ async def _chat_response_generator(): result = await event_handler final_response = "" - if isinstance(result, "AgentRunResult"): - for token in result.response.message.content: - final_response += token - yield self.convert_text(token) - if isinstance(result, AsyncGenerator): async for token in result: final_response += token.delta yield self.convert_text(token.delta) + else: + try: + for token in result.response.message.content: + final_response += token + yield self.convert_text(token) + except Exception as e: + logger.error(f"Error in chat response generator: {e}") + raise e # Generate next questions if next question prompt is configured question_data = await self._generate_next_questions( From ab9890a3e0425bad0f26b64e423203032ee0f187 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Tue, 1 Oct 2024 12:35:50 +0700 Subject: [PATCH 47/77] Update .changeset/flat-singers-share.md --- .changeset/flat-singers-share.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/flat-singers-share.md b/.changeset/flat-singers-share.md index da5304c49..fce3d6e92 100644 --- a/.changeset/flat-singers-share.md +++ b/.changeset/flat-singers-share.md @@ -2,4 +2,4 @@ "create-llama": patch --- -Add publisher agent that generates artifact for the content +Add publisher agent to multi-agents for generating documents (PDF and HTML) From 7df29e6a52fec1ecfc58ce3710f7b3e0a721f666 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Tue, 1 Oct 2024 12:36:52 +0700 Subject: [PATCH 48/77] Update .changeset/gorgeous-penguins-shout.md --- .changeset/gorgeous-penguins-shout.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/gorgeous-penguins-shout.md b/.changeset/gorgeous-penguins-shout.md index b4689266f..26dba1940 100644 --- a/.changeset/gorgeous-penguins-shout.md +++ b/.changeset/gorgeous-penguins-shout.md @@ -2,4 +2,4 @@ "create-llama": patch --- -Add artifact generator tool +Allow tool selection for multi-agents (Python and TS) From 42d0a05a551df32ee1961369682fefe28944be41 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Tue, 1 Oct 2024 13:12:52 +0700 Subject: [PATCH 49/77] rename doc generator --- .../tools/{document_generator.ts => document-generator.ts} | 0 templates/components/engines/typescript/agent/tools/index.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename templates/components/engines/typescript/agent/tools/{document_generator.ts => document-generator.ts} (100%) diff --git a/templates/components/engines/typescript/agent/tools/document_generator.ts b/templates/components/engines/typescript/agent/tools/document-generator.ts similarity index 100% rename from templates/components/engines/typescript/agent/tools/document_generator.ts rename to templates/components/engines/typescript/agent/tools/document-generator.ts diff --git a/templates/components/engines/typescript/agent/tools/index.ts b/templates/components/engines/typescript/agent/tools/index.ts index 511791ebe..8c337f4db 100644 --- a/templates/components/engines/typescript/agent/tools/index.ts +++ b/templates/components/engines/typescript/agent/tools/index.ts @@ -3,7 +3,7 @@ import { ToolsFactory } from "llamaindex/tools/ToolsFactory"; import { DocumentGenerator, DocumentGeneratorParams, -} from "./document_generator"; +} from "./document-generator"; import { DuckDuckGoImageSearchTool, DuckDuckGoSearchTool, From 45932cb9390522e907bf4c8dc8b7c9508c870d3d Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 13:44:57 +0700 Subject: [PATCH 50/77] separate chat api for multi-agent code --- .../multiagent/python/app/api/routers/chat.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 templates/components/multiagent/python/app/api/routers/chat.py diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py new file mode 100644 index 000000000..1dbe47569 --- /dev/null +++ b/templates/components/multiagent/python/app/api/routers/chat.py @@ -0,0 +1,49 @@ +import logging + +from app.api.routers.events import EventCallbackHandler +from app.api.routers.models import ( + ChatData, +) +from app.api.routers.vercel_response import ( + WorkflowVercelStreamResponse, +) +from app.engine import get_chat_engine +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status + +chat_router = r = APIRouter() + +logger = logging.getLogger("uvicorn") + + +# streaming endpoint - delete if not needed +@r.post("") +async def chat( + request: Request, + data: ChatData, + background_tasks: BackgroundTasks, +): + try: + last_message_content = data.get_last_message_content() + messages = data.get_history_messages() + + event_handler = EventCallbackHandler() + # The chat API supports passing private document filters and chat params + # but agent workflow does not support them yet + # ignore chat params and use all documents for now + # TODO: generate filters based on doc_ids + # TODO: use chat params + engine = get_chat_engine(chat_history=messages) + + event_handler = engine.run(input=last_message_content, streaming=True) + return WorkflowVercelStreamResponse( + request=request, + chat_data=data, + event_handler=event_handler, + events=engine.stream_events(), + ) + except Exception as e: + logger.exception("Error in chat engine", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error in chat engine: {e}", + ) from e From 7cedd55364e16b092e3dc7f373c79900ead3274b Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Tue, 1 Oct 2024 13:55:04 +0700 Subject: [PATCH 51/77] use saveDocument from file server and simplify tool factory for agents --- .../agent/tools/document-generator.ts | 36 ++------- .../multiagent/typescript/workflow/agents.ts | 74 +++---------------- .../multiagent/typescript/workflow/factory.ts | 4 +- .../multiagent/typescript/workflow/tools.ts | 52 +++++++++++++ 4 files changed, 72 insertions(+), 94 deletions(-) create mode 100644 templates/components/multiagent/typescript/workflow/tools.ts diff --git a/templates/components/engines/typescript/agent/tools/document-generator.ts b/templates/components/engines/typescript/agent/tools/document-generator.ts index a7741ce53..e2e6b6bfe 100644 --- a/templates/components/engines/typescript/agent/tools/document-generator.ts +++ b/templates/components/engines/typescript/agent/tools/document-generator.ts @@ -1,10 +1,10 @@ import { JSONSchemaType } from "ajv"; -import fs from "fs"; import { BaseTool, ToolMetadata } from "llamaindex"; import { marked } from "marked"; -import path from "path"; +import path from "node:path"; +import { saveDocument } from "../../llamaindex/documents/helper"; -const OUTPUT_DIR = "output/tools"; +const OUTPUT_DIR = "output/tool"; type DocumentParameter = { originalContent: string; @@ -124,28 +124,6 @@ export class DocumentGenerator implements BaseTool { return HTML_TEMPLATE.replace("{{content}}", htmlContent); } - private static validateFileName(fileName: string): string { - if (path.isAbsolute(fileName)) { - throw new Error("File name is not allowed."); - } - if (/^[a-zA-Z0-9_.-]+$/.test(fileName)) { - return fileName; - } else { - throw new Error( - "File name is not allowed to contain special characters.", - ); - } - } - - private static writeToFile(content: string | Buffer, filePath: string): void { - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - if (typeof content === "string") { - fs.writeFileSync(filePath, content, "utf8"); - } else { - fs.writeFileSync(filePath, content); - } - } - async call(input: DocumentParameter): Promise { const { originalContent, fileName } = input; @@ -153,13 +131,9 @@ export class DocumentGenerator implements BaseTool { await DocumentGenerator.generateHtmlContent(originalContent); const fileContent = DocumentGenerator.generateHtmlDocument(htmlContent); - const validatedFileName = DocumentGenerator.validateFileName(fileName); - const filePath = path.join(OUTPUT_DIR, `${validatedFileName}.html`); - - DocumentGenerator.writeToFile(fileContent, filePath); + const filePath = path.join(OUTPUT_DIR, `${fileName}.html`); - const fileUrl = `${process.env.FILESERVER_URL_PREFIX}/${filePath}`; - return fileUrl; + return await saveDocument(filePath, fileContent); } } diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 3263ecbd1..43a7fccd5 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -1,60 +1,14 @@ -import fs from "fs/promises"; -import { BaseToolWithCall, ChatMessage, QueryEngineTool } from "llamaindex"; -import path from "path"; -import { getDataSource } from "../engine"; -import { createTools } from "../engine/tools/index"; +import { ChatMessage } from "llamaindex"; import { FunctionCallingAgent } from "./single-agent"; - -const getQueryEngineTool = async (): Promise => { - const index = await getDataSource(); - if (!index) { - return null; - } - - const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined; - return new QueryEngineTool({ - queryEngine: index.asQueryEngine({ - similarityTopK: topK, - }), - metadata: { - name: "query_index", - description: `Use this tool to retrieve information about the text corpus from the index.`, - }, - }); -}; - -const getAvailableTools = async () => { - const configFile = path.join("config", "tools.json"); - let toolConfig: any; - const tools: BaseToolWithCall[] = []; - try { - toolConfig = JSON.parse(await fs.readFile(configFile, "utf8")); - } catch (e) { - console.info(`Could not read ${configFile} file. Using no tools.`); - } - if (toolConfig) { - tools.push(...(await createTools(toolConfig))); - } - - return tools; -}; +import { lookupTools } from "./tools"; export const createResearcher = async (chatHistory: ChatMessage[]) => { - const tools: BaseToolWithCall[] = []; - const queryEngineTool = await getQueryEngineTool(); - if (queryEngineTool) { - tools.push(queryEngineTool); - } - const availableTools = await getAvailableTools(); - // Add wikipedia, duckduckgo if they are available - for (const tool of availableTools) { - if ( - tool.metadata.name === "wikipedia.WikipediaToolSpec" || - tool.metadata.name === "duckduckgo_search" - ) { - tools.push(tool); - } - } + const tools = await lookupTools([ + "query_index", + "wikipedia.WikipediaToolSpec", + "duckduckgo_search", + ]); + return new FunctionCallingAgent({ name: "researcher", tools: tools, @@ -104,19 +58,15 @@ Task: "Create a blog post about the history of the internet, write in English an }; export const createPublisher = async (chatHistory: ChatMessage[]) => { - const tools = await getAvailableTools(); - const publisherTools: BaseToolWithCall[] = []; + const tools = await lookupTools(["document_generator"]); let systemPrompt = "You are an expert in publishing blog posts. You are given a task to publish a blog post."; - for (const tool of tools) { - if (tool.metadata.name === "document_generator") { - publisherTools.push(tool); - systemPrompt = `${systemPrompt}.If user request for a file, use the document_generator tool to generate the file and reply the link to the file.`; - } + if (tools.length > 0) { + systemPrompt = `${systemPrompt}. If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file.`; } return new FunctionCallingAgent({ name: "publisher", - tools: publisherTools, + tools: tools, systemPrompt: systemPrompt, chatHistory, }); diff --git a/templates/components/multiagent/typescript/workflow/factory.ts b/templates/components/multiagent/typescript/workflow/factory.ts index 47290fe8b..ac1bdb1a0 100644 --- a/templates/components/multiagent/typescript/workflow/factory.ts +++ b/templates/components/multiagent/typescript/workflow/factory.ts @@ -132,7 +132,9 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { message: `Please publish this blog post: ${result}`, streaming: true, }); - return publishResult as StopEvent>; + return publishResult as unknown as StopEvent< + AsyncGenerator + >; }; const workflow = new Workflow({ timeout: TIMEOUT, validate: true }); diff --git a/templates/components/multiagent/typescript/workflow/tools.ts b/templates/components/multiagent/typescript/workflow/tools.ts new file mode 100644 index 000000000..ac4e5fb98 --- /dev/null +++ b/templates/components/multiagent/typescript/workflow/tools.ts @@ -0,0 +1,52 @@ +import fs from "fs/promises"; +import { BaseToolWithCall, QueryEngineTool } from "llamaindex"; +import path from "path"; +import { getDataSource } from "../engine"; +import { createTools } from "../engine/tools/index"; + +const getQueryEngineTool = async (): Promise => { + const index = await getDataSource(); + if (!index) { + return null; + } + + const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined; + return new QueryEngineTool({ + queryEngine: index.asQueryEngine({ + similarityTopK: topK, + }), + metadata: { + name: "query_index", + description: `Use this tool to retrieve information about the text corpus from the index.`, + }, + }); +}; + +export const getAvailableTools = async () => { + const configFile = path.join("config", "tools.json"); + let toolConfig: any; + const tools: BaseToolWithCall[] = []; + try { + toolConfig = JSON.parse(await fs.readFile(configFile, "utf8")); + } catch (e) { + console.info(`Could not read ${configFile} file. Using no tools.`); + } + if (toolConfig) { + tools.push(...(await createTools(toolConfig))); + } + const queryEngineTool = await getQueryEngineTool(); + if (queryEngineTool) { + tools.push(queryEngineTool); + } + + return tools; +}; + +export const lookupTools = async ( + toolNames: string[], +): Promise => { + const availableTools = await getAvailableTools(); + return availableTools.filter((tool) => + toolNames.includes(tool.metadata.name), + ); +}; From 01410e2a9d36906ffeff1128925d21059e13735f Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 14:11:26 +0700 Subject: [PATCH 52/77] add resolve package test for document generator tool --- e2e/python/resolve_dependencies.spec.ts | 1 + e2e/utils.ts | 4 ++-- helpers/tools.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e/python/resolve_dependencies.spec.ts b/e2e/python/resolve_dependencies.spec.ts index 96864ac06..6d778a1f8 100644 --- a/e2e/python/resolve_dependencies.spec.ts +++ b/e2e/python/resolve_dependencies.spec.ts @@ -33,6 +33,7 @@ if ( const toolOptions = [ "wikipedia.WikipediaToolSpec", "google.GoogleSearchToolSpec", + "document_generator", ]; const dataSources = [ diff --git a/e2e/utils.ts b/e2e/utils.ts index aec1e30d6..956872cfc 100644 --- a/e2e/utils.ts +++ b/e2e/utils.ts @@ -143,10 +143,10 @@ export async function runCreateLlama({ externalPort, ); } else if (postInstallAction === "dependencies") { - await waitForProcess(appProcess, 2000 * 60); // wait 2 min for dependencies to be resolved + await waitForProcess(appProcess, 1000 * 60); // wait 1 min for dependencies to be resolved } else { // wait 10 seconds for create-llama to exit - await waitForProcess(appProcess, 1000 * 20); + await waitForProcess(appProcess, 1000 * 10); } return { diff --git a/helpers/tools.ts b/helpers/tools.ts index 0b4996bd5..ac8072325 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -117,7 +117,7 @@ For better results, you can specify the region parameter to get results from a s dependencies: [ { name: "xhtml2pdf", - version: "^0.2.16", + version: "^0.2.14", }, { name: "markdown", From 5248530b87c0e9c4d7c24df14b842dda96d0f53a Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 14:33:16 +0700 Subject: [PATCH 53/77] add duckduckgo image search and tunning the prompt --- .../components/multiagent/typescript/workflow/agents.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 43a7fccd5..79a1d8376 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -7,6 +7,7 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { "query_index", "wikipedia.WikipediaToolSpec", "duckduckgo_search", + "duckduckgo_image_search", ]); return new FunctionCallingAgent({ @@ -15,7 +16,7 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { systemPrompt: `You are a researcher agent. You are given a researching task. You must use tools to retrieve information needed for the task. It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. -If you don't found any related information, please return "I didn't find any information." +If you don't found any related information, please return "I didn't find any information.". Don't try to make up information yourself. Example: Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> @@ -29,7 +30,9 @@ export const createWriter = (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "writer", systemPrompt: `You are an expert in writing blog posts. -You are given a task to write a blog post. Don't make up any information yourself. +You are given a task to write a blog post from the research content provided by the researcher agent. Don't make up any information yourself. +If there is no research content provided, you must return "I don't have any research content to write about." +If the content is not valid (ex: broken link, broken image, etc.) don't use it. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. Example: Task: "Here is the information i found about the history of internet: @@ -60,7 +63,7 @@ Task: "Create a blog post about the history of the internet, write in English an export const createPublisher = async (chatHistory: ChatMessage[]) => { const tools = await lookupTools(["document_generator"]); let systemPrompt = - "You are an expert in publishing blog posts. You are given a task to publish a blog post."; + "You are an expert in publishing blog posts. You are given a task to publish a blog post. If the writer say that there was an error you should reply with the error and not publish the post."; if (tools.length > 0) { systemPrompt = `${systemPrompt}. If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file.`; } From 55409357a2cdf8a345697065e030a8df779d0330 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 15:04:45 +0700 Subject: [PATCH 54/77] revert change in streaming chat.py --- .../multiagent/python/app/api/routers/chat.py | 1 - .../streaming/fastapi/app/api/routers/chat.py | 47 ++++--------------- 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py index 1dbe47569..718382af9 100644 --- a/templates/components/multiagent/python/app/api/routers/chat.py +++ b/templates/components/multiagent/python/app/api/routers/chat.py @@ -15,7 +15,6 @@ logger = logging.getLogger("uvicorn") -# streaming endpoint - delete if not needed @r.post("") async def chat( request: Request, diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 4cbe9f685..94f124511 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -1,10 +1,8 @@ import logging from typing import List -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status -from llama_index.core.agent import AgentRunner -from llama_index.core.chat_engine import CondensePlusContextChatEngine -from llama_index.core.chat_engine.types import NodeWithScore +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status +from llama_index.core.chat_engine.types import BaseChatEngine, NodeWithScore from llama_index.core.llms import MessageRole from app.api.routers.events import EventCallbackHandler @@ -14,10 +12,7 @@ Result, SourceNodes, ) -from app.api.routers.vercel_response import ( - ChatEngineVercelStreamResponse, - WorkflowVercelStreamResponse, -) +from app.api.routers.vercel_response import VercelStreamResponse from app.engine import get_chat_engine from app.engine.query_filter import generate_filters @@ -45,35 +40,12 @@ async def chat( ) event_handler = EventCallbackHandler() chat_engine = get_chat_engine( - filters=filters, - params=params, - event_handlers=[event_handler], - chat_history=messages, + filters=filters, params=params, event_handlers=[event_handler] ) + response = await chat_engine.astream_chat(last_message_content, messages) + process_response_nodes(response.source_nodes, background_tasks) - if isinstance(chat_engine, CondensePlusContextChatEngine) or isinstance( - chat_engine, AgentRunner - ): - event_handler = EventCallbackHandler() - chat_engine.callback_manager.handlers.append(event_handler) # type: ignore - - response = await chat_engine.astream_chat(last_message_content, messages) - process_response_nodes(response.source_nodes, background_tasks) - - return ChatEngineVercelStreamResponse( - request=request, - chat_data=data, - event_handler=event_handler, - response=response, - ) - else: - event_handler = chat_engine.run(input=last_message_content, streaming=True) - return WorkflowVercelStreamResponse( - request=request, - chat_data=data, - event_handler=event_handler, - events=chat_engine.stream_events(), - ) + return VercelStreamResponse(request, event_handler, response, data) except Exception as e: logger.exception("Error in chat engine", exc_info=True) raise HTTPException( @@ -86,12 +58,11 @@ async def chat( @r.post("/request") async def chat_request( data: ChatData, + chat_engine: BaseChatEngine = Depends(get_chat_engine), ) -> Result: last_message_content = data.get_last_message_content() messages = data.get_history_messages() - chat_engine = get_chat_engine(filters=None, params=None) - response = await chat_engine.achat(last_message_content, messages) return Result( result=Message(role=MessageRole.ASSISTANT, content=response.response), @@ -110,4 +81,4 @@ def process_response_nodes( LLamaCloudFileService.download_files_from_nodes(nodes, background_tasks) except ImportError: logger.debug("LlamaCloud is not configured. Skipping post processing of nodes") - pass + pass \ No newline at end of file From 610f0d09f1b7fa9fc11550e7ace9a3abcc00810b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 15:04:55 +0700 Subject: [PATCH 55/77] fix wrong wikipedia tool name --- templates/components/multiagent/typescript/workflow/agents.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 79a1d8376..c1dba4801 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -5,7 +5,7 @@ import { lookupTools } from "./tools"; export const createResearcher = async (chatHistory: ChatMessage[]) => { const tools = await lookupTools([ "query_index", - "wikipedia.WikipediaToolSpec", + "wikipedia_tool", "duckduckgo_search", "duckduckgo_image_search", ]); From dd850c61cfe9129c1078590cf8f983318c3b97e6 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 15:07:41 +0700 Subject: [PATCH 56/77] fix linting --- templates/types/streaming/fastapi/app/api/routers/chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 94f124511..39894361a 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -81,4 +81,4 @@ def process_response_nodes( LLamaCloudFileService.download_files_from_nodes(nodes, background_tasks) except ImportError: logger.debug("LlamaCloud is not configured. Skipping post processing of nodes") - pass \ No newline at end of file + pass From 8f097e272515037677ac3850df6ba899db47a811 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 15:17:28 +0700 Subject: [PATCH 57/77] only use VercelStreamResponse --- .../multiagent/python/app/api/routers/chat.py | 6 +- .../python/app/api/routers/vercel_response.py | 128 ++++++++++++++++++ .../app/api/routers/vercel_response.py | 74 +--------- 3 files changed, 134 insertions(+), 74 deletions(-) create mode 100644 templates/components/multiagent/python/app/api/routers/vercel_response.py diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py index 718382af9..39ec50d7c 100644 --- a/templates/components/multiagent/python/app/api/routers/chat.py +++ b/templates/components/multiagent/python/app/api/routers/chat.py @@ -4,9 +4,7 @@ from app.api.routers.models import ( ChatData, ) -from app.api.routers.vercel_response import ( - WorkflowVercelStreamResponse, -) +from app.api.routers.vercel_response import VercelStreamResponse from app.engine import get_chat_engine from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status @@ -34,7 +32,7 @@ async def chat( engine = get_chat_engine(chat_history=messages) event_handler = engine.run(input=last_message_content, streaming=True) - return WorkflowVercelStreamResponse( + return VercelStreamResponse( request=request, chat_data=data, event_handler=event_handler, diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py new file mode 100644 index 000000000..6a4d66cca --- /dev/null +++ b/templates/components/multiagent/python/app/api/routers/vercel_response.py @@ -0,0 +1,128 @@ +import json +import logging +from abc import abstractmethod +from typing import AsyncGenerator, List + +from aiostream import stream +from app.api.routers.models import ChatData, Message +from app.api.services.suggestion import NextQuestionSuggestion +from fastapi import Request +from fastapi.responses import StreamingResponse + +logger = logging.getLogger("uvicorn") + + +class VercelStreamResponse(StreamingResponse): + """ + Class to convert the response from the chat engine to the streaming format expected by Vercel + """ + + TEXT_PREFIX = "0:" + DATA_PREFIX = "8:" + + def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs): + self.request = request + + stream = self._create_stream(request, chat_data, *args, **kwargs) + content = self.content_generator(stream) + + super().__init__(content=content) + + @abstractmethod + def _create_stream(self, request: Request, chat_data: ChatData, *args, **kwargs): + """ + Create the stream that will be used to generate the response. + """ + raise NotImplementedError("Subclasses must implement _create_stream") + + async def content_generator(self, stream): + is_stream_started = False + + async with stream.stream() as streamer: + async for output in streamer: + if not is_stream_started: + is_stream_started = True + # Stream a blank message to start the stream + yield self.convert_text("") + + yield output + + if await self.request.is_disconnected(): + break + + @classmethod + def convert_text(cls, token: str): + # Escape newlines and double quotes to avoid breaking the stream + token = json.dumps(token) + return f"{cls.TEXT_PREFIX}{token}\n" + + @classmethod + def convert_data(cls, data: dict): + data_str = json.dumps(data) + return f"{cls.DATA_PREFIX}[{data_str}]\n" + + @staticmethod + async def _generate_next_questions(chat_history: List[Message], response: str): + questions = await NextQuestionSuggestion.suggest_next_questions( + chat_history, response + ) + if questions: + return { + "type": "suggested_questions", + "data": questions, + } + return None + + def _create_stream( + self, + request: Request, + chat_data: ChatData, + event_handler: "AgentRunResult" | AsyncGenerator, # noqa: F821 + events: AsyncGenerator["AgentRunEvent", None], # noqa: F821 + verbose: bool = True, + ): + # Yield the text response + async def _chat_response_generator(): + result = await event_handler + final_response = "" + + if isinstance(result, AsyncGenerator): + async for token in result: + final_response += token.delta + yield self.convert_text(token.delta) + else: + try: + for token in result.response.message.content: + final_response += token + yield self.convert_text(token) + except Exception as e: + logger.error(f"Error in chat response generator: {e}") + raise e + + # Generate next questions if next question prompt is configured + question_data = await self._generate_next_questions( + chat_data.messages, final_response + ) + if question_data: + yield self.convert_data(question_data) + + # TODO: stream sources + + # Yield the events from the event handler + async def _event_generator(): + async for event in events: + event_response = self._event_to_response(event) + if verbose: + logger.debug(event_response) + if event_response is not None: + yield self.convert_data(event_response) + + combine = stream.merge(_chat_response_generator(), _event_generator()) + return combine + + @staticmethod + def _event_to_response(event: "AgentRunEvent") -> dict: # noqa: F821 + return { + "type": "agent", + "data": {"agent": event.name, "text": event.msg}, + } diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 01bf7c597..ee50f9995 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -1,7 +1,7 @@ import json import logging -from abc import ABC, abstractmethod -from typing import AsyncGenerator, List +from abc import abstractmethod +from typing import List from aiostream import stream from fastapi import Request @@ -15,9 +15,9 @@ logger = logging.getLogger("uvicorn") -class BaseVercelStreamResponse(StreamingResponse, ABC): +class VercelStreamResponse(StreamingResponse): """ - Base class to convert the response from the chat engine to the streaming format expected by Vercel + Class to convert the response from the chat engine to the streaming format expected by Vercel """ TEXT_PREFIX = "0:" @@ -76,12 +76,6 @@ async def _generate_next_questions(chat_history: List[Message], response: str): } return None - -class ChatEngineVercelStreamResponse(BaseVercelStreamResponse): - """ - Class to convert the response from the chat engine to the streaming format expected by Vercel - """ - def _create_stream( self, request: Request, @@ -132,63 +126,3 @@ def _source_nodes_to_response(source_nodes: List): ] }, } - - -class WorkflowVercelStreamResponse(BaseVercelStreamResponse): - """ - Class to convert the response from the chat engine to the streaming format expected by Vercel - """ - - def _create_stream( - self, - request: Request, - chat_data: ChatData, - event_handler: "AgentRunResult" | AsyncGenerator, # noqa: F821 - events: AsyncGenerator["AgentRunEvent", None], # noqa: F821 - verbose: bool = True, - ): - # Yield the text response - async def _chat_response_generator(): - result = await event_handler - final_response = "" - - if isinstance(result, AsyncGenerator): - async for token in result: - final_response += token.delta - yield self.convert_text(token.delta) - else: - try: - for token in result.response.message.content: - final_response += token - yield self.convert_text(token) - except Exception as e: - logger.error(f"Error in chat response generator: {e}") - raise e - - # Generate next questions if next question prompt is configured - question_data = await self._generate_next_questions( - chat_data.messages, final_response - ) - if question_data: - yield self.convert_data(question_data) - - # TODO: stream sources - - # Yield the events from the event handler - async def _event_generator(): - async for event in events: - event_response = self._event_to_response(event) - if verbose: - logger.debug(event_response) - if event_response is not None: - yield self.convert_data(event_response) - - combine = stream.merge(_chat_response_generator(), _event_generator()) - return combine - - @staticmethod - def _event_to_response(event: "AgentRunEvent") -> dict: # noqa: F821 - return { - "type": "agent", - "data": {"agent": event.name, "text": event.msg}, - } From d8dd59c08052f031015b2af6516ce3bc6aea0d58 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Tue, 1 Oct 2024 15:46:37 +0700 Subject: [PATCH 58/77] improve URL handling of doc gen in TS --- .../engines/typescript/agent/tools/document-generator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/components/engines/typescript/agent/tools/document-generator.ts b/templates/components/engines/typescript/agent/tools/document-generator.ts index e2e6b6bfe..aec343a33 100644 --- a/templates/components/engines/typescript/agent/tools/document-generator.ts +++ b/templates/components/engines/typescript/agent/tools/document-generator.ts @@ -133,7 +133,7 @@ export class DocumentGenerator implements BaseTool { const filePath = path.join(OUTPUT_DIR, `${fileName}.html`); - return await saveDocument(filePath, fileContent); + return `URL: ${await saveDocument(filePath, fileContent)}`; } } From bd64972d87efc15c41b7983ffeb23274c0808ea3 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 15:54:31 +0700 Subject: [PATCH 59/77] revised python chat api --- .../multiagent/python/app/api/routers/chat.py | 2 +- .../python/app/api/routers/vercel_response.py | 68 ++++------- .../streaming/fastapi/app/api/routers/chat.py | 14 ++- .../fastapi/app/api/routers/models.py | 46 ++++++- .../app/api/routers/vercel_response.py | 114 ++++++++---------- 5 files changed, 130 insertions(+), 114 deletions(-) diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py index 39ec50d7c..23135c809 100644 --- a/templates/components/multiagent/python/app/api/routers/chat.py +++ b/templates/components/multiagent/python/app/api/routers/chat.py @@ -21,7 +21,7 @@ async def chat( ): try: last_message_content = data.get_last_message_content() - messages = data.get_history_messages() + messages = data.get_history_messages(include_agent_messages=True) event_handler = EventCallbackHandler() # The chat API supports passing private document filters and chat params diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py index 6a4d66cca..fce55230e 100644 --- a/templates/components/multiagent/python/app/api/routers/vercel_response.py +++ b/templates/components/multiagent/python/app/api/routers/vercel_response.py @@ -1,13 +1,15 @@ import json import logging -from abc import abstractmethod from typing import AsyncGenerator, List from aiostream import stream +from app.agents.single import AgentRunEvent, AgentRunResult +from app.api.routers.events import EventCallbackHandler from app.api.routers.models import ChatData, Message from app.api.services.suggestion import NextQuestionSuggestion from fastapi import Request from fastapi.responses import StreamingResponse +from llama_index.core.chat_engine.types import StreamingAgentChatResponse logger = logging.getLogger("uvicorn") @@ -20,36 +22,6 @@ class VercelStreamResponse(StreamingResponse): TEXT_PREFIX = "0:" DATA_PREFIX = "8:" - def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs): - self.request = request - - stream = self._create_stream(request, chat_data, *args, **kwargs) - content = self.content_generator(stream) - - super().__init__(content=content) - - @abstractmethod - def _create_stream(self, request: Request, chat_data: ChatData, *args, **kwargs): - """ - Create the stream that will be used to generate the response. - """ - raise NotImplementedError("Subclasses must implement _create_stream") - - async def content_generator(self, stream): - is_stream_started = False - - async with stream.stream() as streamer: - async for output in streamer: - if not is_stream_started: - is_stream_started = True - # Stream a blank message to start the stream - yield self.convert_text("") - - yield output - - if await self.request.is_disconnected(): - break - @classmethod def convert_text(cls, token: str): # Escape newlines and double quotes to avoid breaking the stream @@ -61,6 +33,18 @@ def convert_data(cls, data: dict): data_str = json.dumps(data) return f"{cls.DATA_PREFIX}[{data_str}]\n" + def __init__( + self, + request: Request, + event_handler: EventCallbackHandler, + response: StreamingAgentChatResponse, + chat_data: ChatData, + ): + content = VercelStreamResponse.content_generator( + request, event_handler, response, chat_data + ) + super().__init__(content=content) + @staticmethod async def _generate_next_questions(chat_history: List[Message], response: str): questions = await NextQuestionSuggestion.suggest_next_questions( @@ -73,12 +57,13 @@ async def _generate_next_questions(chat_history: List[Message], response: str): } return None - def _create_stream( + @classmethod + def content_generator( self, request: Request, chat_data: ChatData, - event_handler: "AgentRunResult" | AsyncGenerator, # noqa: F821 - events: AsyncGenerator["AgentRunEvent", None], # noqa: F821 + event_handler: AgentRunResult | AsyncGenerator, + events: AsyncGenerator[AgentRunEvent, None], verbose: bool = True, ): # Yield the text response @@ -86,18 +71,15 @@ async def _chat_response_generator(): result = await event_handler final_response = "" + if isinstance(result, AgentRunResult): + for token in result.response.message.content: + final_response += token + yield self.convert_text(token) + if isinstance(result, AsyncGenerator): async for token in result: final_response += token.delta yield self.convert_text(token.delta) - else: - try: - for token in result.response.message.content: - final_response += token - yield self.convert_text(token) - except Exception as e: - logger.error(f"Error in chat response generator: {e}") - raise e # Generate next questions if next question prompt is configured question_data = await self._generate_next_questions( @@ -121,7 +103,7 @@ async def _event_generator(): return combine @staticmethod - def _event_to_response(event: "AgentRunEvent") -> dict: # noqa: F821 + def _event_to_response(event: AgentRunEvent) -> dict: return { "type": "agent", "data": {"agent": event.name, "text": event.msg}, diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 39894361a..48876efb6 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -1,8 +1,8 @@ import logging from typing import List -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status -from llama_index.core.chat_engine.types import BaseChatEngine, NodeWithScore +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status +from llama_index.core.chat_engine.types import NodeWithScore from llama_index.core.llms import MessageRole from app.api.routers.events import EventCallbackHandler @@ -58,11 +58,19 @@ async def chat( @r.post("/request") async def chat_request( data: ChatData, - chat_engine: BaseChatEngine = Depends(get_chat_engine), ) -> Result: last_message_content = data.get_last_message_content() messages = data.get_history_messages() + doc_ids = data.get_chat_document_ids() + filters = generate_filters(doc_ids) + params = data.data or {} + logger.info( + f"Creating chat engine with filters: {str(filters)}", + ) + + chat_engine = get_chat_engine(filters=filters, params=params) + response = await chat_engine.achat(last_message_content, messages) return Result( result=Message(role=MessageRole.ASSISTANT, content=response.response), diff --git a/templates/types/streaming/fastapi/app/api/routers/models.py b/templates/types/streaming/fastapi/app/api/routers/models.py index 29648608f..12568b4d0 100644 --- a/templates/types/streaming/fastapi/app/api/routers/models.py +++ b/templates/types/streaming/fastapi/app/api/routers/models.py @@ -50,9 +50,14 @@ class Config: alias_generator = to_camel +class AgentAnnotation(BaseModel): + agent: str + text: str + + class Annotation(BaseModel): type: str - data: AnnotationFileData | List[str] + data: AnnotationFileData | List[str] | AgentAnnotation def to_content(self) -> str | None: if self.type == "document_file": @@ -119,14 +124,49 @@ def get_last_message_content(self) -> str: break return message_content - def get_history_messages(self) -> List[ChatMessage]: + def _get_agent_messages(self, max_messages: int = 5) -> List[str]: + """ + Construct agent messages from the annotations in the chat messages + """ + agent_messages = [] + for message in self.messages: + if ( + message.role == MessageRole.ASSISTANT + and message.annotations is not None + ): + for annotation in message.annotations: + if annotation.type == "agent" and isinstance( + annotation.data, AgentAnnotation + ): + text = annotation.data.text + if not text.startswith("Finished task"): + agent_messages.append( + f"\nAgent: {annotation.data.agent}\nsaid: {text}\n" + ) + if len(agent_messages) >= max_messages: + break + return agent_messages + + def get_history_messages( + self, include_agent_messages: bool = False + ) -> List[ChatMessage]: """ Get the history messages """ - return [ + chat_messages = [ ChatMessage(role=message.role, content=message.content) for message in self.messages[:-1] ] + if include_agent_messages: + agent_messages = self._get_agent_messages(max_messages=5) + if len(agent_messages) > 0: + message = ChatMessage( + role=MessageRole.ASSISTANT, + content="Previous agent events: \n" + "\n".join(agent_messages), + ) + chat_messages.append(message) + + return chat_messages def is_last_message_from_user(self) -> bool: return self.messages[-1].role == MessageRole.USER diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index ee50f9995..924c60ce5 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -1,6 +1,4 @@ import json -import logging -from abc import abstractmethod from typing import List from aiostream import stream @@ -12,8 +10,6 @@ from app.api.routers.models import ChatData, Message, SourceNodes from app.api.services.suggestion import NextQuestionSuggestion -logger = logging.getLogger("uvicorn") - class VercelStreamResponse(StreamingResponse): """ @@ -23,36 +19,6 @@ class VercelStreamResponse(StreamingResponse): TEXT_PREFIX = "0:" DATA_PREFIX = "8:" - def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs): - self.request = request - - stream = self._create_stream(request, chat_data, *args, **kwargs) - content = self.content_generator(stream) - - super().__init__(content=content) - - @abstractmethod - def _create_stream(self, request: Request, chat_data: ChatData, *args, **kwargs): - """ - Create the stream that will be used to generate the response. - """ - raise NotImplementedError("Subclasses must implement _create_stream") - - async def content_generator(self, stream): - is_stream_started = False - - async with stream.stream() as streamer: - async for output in streamer: - if not is_stream_started: - is_stream_started = True - # Stream a blank message to start the stream - yield self.convert_text("") - - yield output - - if await self.request.is_disconnected(): - break - @classmethod def convert_text(cls, token: str): # Escape newlines and double quotes to avoid breaking the stream @@ -64,45 +30,54 @@ def convert_data(cls, data: dict): data_str = json.dumps(data) return f"{cls.DATA_PREFIX}[{data_str}]\n" - @staticmethod - async def _generate_next_questions(chat_history: List[Message], response: str): - questions = await NextQuestionSuggestion.suggest_next_questions( - chat_history, response - ) - if questions: - return { - "type": "suggested_questions", - "data": questions, - } - return None - - def _create_stream( + def __init__( self, request: Request, + event_handler: EventCallbackHandler, + response: StreamingAgentChatResponse, chat_data: ChatData, + ): + content = VercelStreamResponse.content_generator( + request, event_handler, response, chat_data + ) + super().__init__(content=content) + + @classmethod + async def content_generator( + cls, + request: Request, event_handler: EventCallbackHandler, response: StreamingAgentChatResponse, + chat_data: ChatData, ): # Yield the text response async def _chat_response_generator(): final_response = "" async for token in response.async_response_gen(): final_response += token - yield self.convert_text(token) + yield cls.convert_text(token) # Generate next questions if next question prompt is configured - question_data = await self._generate_next_questions( + question_data = await cls._generate_next_questions( chat_data.messages, final_response ) if question_data: - yield self.convert_data(question_data) + yield cls.convert_data(question_data) # the text_generator is the leading stream, once it's finished, also finish the event stream event_handler.is_done = True # Yield the source nodes - yield self.convert_data( - self._source_nodes_to_response(response.source_nodes) + yield cls.convert_data( + { + "type": "sources", + "data": { + "nodes": [ + SourceNodes.from_source_node(node).model_dump() + for node in response.source_nodes + ] + }, + } ) # Yield the events from the event handler @@ -110,19 +85,30 @@ async def _event_generator(): async for event in event_handler.async_event_gen(): event_response = event.to_response() if event_response is not None: - yield self.convert_data(event_response) + yield cls.convert_data(event_response) combine = stream.merge(_chat_response_generator(), _event_generator()) - return combine + is_stream_started = False + async with combine.stream() as streamer: + async for output in streamer: + if not is_stream_started: + is_stream_started = True + # Stream a blank message to start the stream + yield cls.convert_text("") + + yield output + + if await request.is_disconnected(): + break @staticmethod - def _source_nodes_to_response(source_nodes: List): - return { - "type": "sources", - "data": { - "nodes": [ - SourceNodes.from_source_node(node).model_dump() - for node in source_nodes - ] - }, - } + async def _generate_next_questions(chat_history: List[Message], response: str): + questions = await NextQuestionSuggestion.suggest_next_questions( + chat_history, response + ) + if questions: + return { + "type": "suggested_questions", + "data": questions, + } + return None From 6def1e3e7d8be5b75b2b9296bdd6948a7b238ec6 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 16:04:30 +0700 Subject: [PATCH 60/77] fix vercel stream for multiagent --- .../python/app/api/routers/vercel_response.py | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py index fce55230e..12082496e 100644 --- a/templates/components/multiagent/python/app/api/routers/vercel_response.py +++ b/templates/components/multiagent/python/app/api/routers/vercel_response.py @@ -1,64 +1,50 @@ import json import logging +from abc import ABC from typing import AsyncGenerator, List from aiostream import stream from app.agents.single import AgentRunEvent, AgentRunResult -from app.api.routers.events import EventCallbackHandler from app.api.routers.models import ChatData, Message from app.api.services.suggestion import NextQuestionSuggestion from fastapi import Request from fastapi.responses import StreamingResponse -from llama_index.core.chat_engine.types import StreamingAgentChatResponse logger = logging.getLogger("uvicorn") -class VercelStreamResponse(StreamingResponse): +class VercelStreamResponse(StreamingResponse, ABC): """ - Class to convert the response from the chat engine to the streaming format expected by Vercel + Base class to convert the response from the chat engine to the streaming format expected by Vercel """ TEXT_PREFIX = "0:" DATA_PREFIX = "8:" - @classmethod - def convert_text(cls, token: str): - # Escape newlines and double quotes to avoid breaking the stream - token = json.dumps(token) - return f"{cls.TEXT_PREFIX}{token}\n" + def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs): + self.request = request - @classmethod - def convert_data(cls, data: dict): - data_str = json.dumps(data) - return f"{cls.DATA_PREFIX}[{data_str}]\n" + stream = self._create_stream(request, chat_data, *args, **kwargs) + content = self.content_generator(stream) - def __init__( - self, - request: Request, - event_handler: EventCallbackHandler, - response: StreamingAgentChatResponse, - chat_data: ChatData, - ): - content = VercelStreamResponse.content_generator( - request, event_handler, response, chat_data - ) super().__init__(content=content) - @staticmethod - async def _generate_next_questions(chat_history: List[Message], response: str): - questions = await NextQuestionSuggestion.suggest_next_questions( - chat_history, response - ) - if questions: - return { - "type": "suggested_questions", - "data": questions, - } - return None + async def content_generator(self, stream): + is_stream_started = False - @classmethod - def content_generator( + async with stream.stream() as streamer: + async for output in streamer: + if not is_stream_started: + is_stream_started = True + # Stream a blank message to start the stream + yield self.convert_text("") + + yield output + + if await self.request.is_disconnected(): + break + + def _create_stream( self, request: Request, chat_data: ChatData, @@ -108,3 +94,26 @@ def _event_to_response(event: AgentRunEvent) -> dict: "type": "agent", "data": {"agent": event.name, "text": event.msg}, } + + @classmethod + def convert_text(cls, token: str): + # Escape newlines and double quotes to avoid breaking the stream + token = json.dumps(token) + return f"{cls.TEXT_PREFIX}{token}\n" + + @classmethod + def convert_data(cls, data: dict): + data_str = json.dumps(data) + return f"{cls.DATA_PREFIX}[{data_str}]\n" + + @staticmethod + async def _generate_next_questions(chat_history: List[Message], response: str): + questions = await NextQuestionSuggestion.suggest_next_questions( + chat_history, response + ) + if questions: + return { + "type": "suggested_questions", + "data": questions, + } + return None From efd16e3963d3c03b35f53cb4df953d13ba7c1f9b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 1 Oct 2024 20:07:32 +0700 Subject: [PATCH 61/77] add previous agent message to chat history and refine the prompt --- .../multiagent/typescript/workflow/agents.ts | 17 +++--- .../multiagent/typescript/workflow/factory.ts | 54 ++++++++++++++++--- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index c1dba4801..d4db8bdcd 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -13,10 +13,11 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "researcher", tools: tools, - systemPrompt: `You are a researcher agent. -You are given a researching task. You must use tools to retrieve information needed for the task. + systemPrompt: `You are a researcher agent. You are given a researching task. +If the conversation already included the information and there is no new request from the user, you should return the appropriate content to the writer. +Otherwise, you must use tools to retrieve information needed for the task. It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. -If you don't found any related information, please return "I didn't find any information.". Don't try to make up information yourself. +If you don't found any related information, please return "I didn't find any new information for {the topic}.". Don't try to make up information yourself. Example: Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> @@ -31,7 +32,9 @@ export const createWriter = (chatHistory: ChatMessage[]) => { name: "writer", systemPrompt: `You are an expert in writing blog posts. You are given a task to write a blog post from the research content provided by the researcher agent. Don't make up any information yourself. -If there is no research content provided, you must return "I don't have any research content to write about." +It's important to read the whole conversation history to write the blog post correctly. +If you received a review from the reviewer, update the post with the review and return the new post content. +If user request for an update with an new thing but there is no research content provided, you must return "I don't have any research content to write about." If the content is not valid (ex: broken link, broken image, etc.) don't use it. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. Example: @@ -47,7 +50,7 @@ export const createReviewer = (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "reviewer", systemPrompt: `You are an expert in reviewing blog posts. -You are given a task to review a blog post. +You are given a task to review a blog post. As a reviewer, it's important that your review is matching with the user request. Please focus on the user request to review the post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. @@ -62,8 +65,8 @@ Task: "Create a blog post about the history of the internet, write in English an export const createPublisher = async (chatHistory: ChatMessage[]) => { const tools = await lookupTools(["document_generator"]); - let systemPrompt = - "You are an expert in publishing blog posts. You are given a task to publish a blog post. If the writer say that there was an error you should reply with the error and not publish the post."; + let systemPrompt = `You are an expert in publishing blog posts. You are given a task to publish a blog post. +If the writer say that there was an error you should reply with the error and not publish the post.`; if (tools.length > 0) { systemPrompt = `${systemPrompt}. If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file.`; } diff --git a/templates/components/multiagent/typescript/workflow/factory.ts b/templates/components/multiagent/typescript/workflow/factory.ts index ac1bdb1a0..d265eb29e 100644 --- a/templates/components/multiagent/typescript/workflow/factory.ts +++ b/templates/components/multiagent/typescript/workflow/factory.ts @@ -25,7 +25,50 @@ class WriteEvent extends WorkflowEvent<{ class ReviewEvent extends WorkflowEvent<{ input: string }> {} class PublishEvent extends WorkflowEvent<{ input: string }> {} +const prepareChatHistory = (chatHistory: ChatMessage[]) => { + // By default, the chat history only contains the assistant and user messages + // all the agents messages are stored in annotation data which is not visible to the LLM + + const MAX_AGENT_MESSAGES = 10; + + // Construct a new agent message from agent messages + // Get annotations from assistant messages + const agentAnnotations = chatHistory + .filter((msg) => msg.role === "assistant") + .flatMap((msg) => msg.annotations || []) + .filter( + (annotation) => + annotation.type === "agent" && annotation.data.text !== "Finished task", + ) + .slice(-MAX_AGENT_MESSAGES); + + const agentMessages = agentAnnotations + .map( + (annotation) => + `\n<${annotation.data.agent}>\n${annotation.data.text}\n`, + ) + .join("\n"); + + const agentContent = agentMessages + ? "Here is the previous conversation of agents:\n" + agentMessages + : ""; + + if (agentContent) { + const agentMessage: ChatMessage = { + role: "assistant", + content: agentContent, + }; + return [ + ...chatHistory.slice(0, -1), + agentMessage, + chatHistory.slice(-1)[0], + ]; + } + return chatHistory; +}; + export const createWorkflow = (chatHistory: ChatMessage[]) => { + const chatHistoryWithAgentMessages = prepareChatHistory(chatHistory); const runAgent = async ( context: Context, agent: Workflow, @@ -48,7 +91,7 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { }; const research = async (context: Context, ev: ResearchEvent) => { - const researcher = await createResearcher(chatHistory); + const researcher = await createResearcher(chatHistoryWithAgentMessages); const researchRes = await runAgent(context, researcher, { message: ev.data.input, }); @@ -77,7 +120,7 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { }); } - const writer = createWriter(chatHistory); + const writer = createWriter(chatHistoryWithAgentMessages); const writeRes = await runAgent(context, writer, { message: ev.data.input, }); @@ -87,7 +130,7 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { }; const review = async (context: Context, ev: ReviewEvent) => { - const reviewer = createReviewer(chatHistory); + const reviewer = createReviewer(chatHistoryWithAgentMessages); const reviewRes = await reviewer.run( new StartEvent({ input: { message: ev.data.input } }), ); @@ -125,11 +168,10 @@ export const createWorkflow = (chatHistory: ChatMessage[]) => { }; const publish = async (context: Context, ev: PublishEvent) => { - const publisher = await createPublisher(chatHistory); - const result = context.get("result"); + const publisher = await createPublisher(chatHistoryWithAgentMessages); const publishResult = await runAgent(context, publisher, { - message: `Please publish this blog post: ${result}`, + message: `${ev.data.input}`, streaming: true, }); return publishResult as unknown as StopEvent< From 341f39a5cbd48323202c0a1a8e82cdeaa1aeca16 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 10:56:08 +0700 Subject: [PATCH 62/77] tunning publisher prompt for publish file or reply the content --- .../components/multiagent/python/app/examples/publisher.py | 3 ++- templates/components/multiagent/typescript/workflow/agents.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/publisher.py b/templates/components/multiagent/python/app/examples/publisher.py index 617e273bb..1775a7ad3 100644 --- a/templates/components/multiagent/python/app/examples/publisher.py +++ b/templates/components/multiagent/python/app/examples/publisher.py @@ -15,7 +15,8 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]: tools.extend(configured_tools["document_generator"]) prompt_instructions = dedent(""" You have access to a document generator tool that can create PDF or HTML document for the content. - Based on the user request, please specify the type of document to generate or just reply to the user directly without generating any document file. + If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file. + Otherwise, just return the content of the post. """) description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format." else: diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index d4db8bdcd..29ee8798d 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -68,7 +68,9 @@ export const createPublisher = async (chatHistory: ChatMessage[]) => { let systemPrompt = `You are an expert in publishing blog posts. You are given a task to publish a blog post. If the writer say that there was an error you should reply with the error and not publish the post.`; if (tools.length > 0) { - systemPrompt = `${systemPrompt}. If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file.`; + systemPrompt = `${systemPrompt}. +If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file. +Otherwise, just return the content of the post.`; } return new FunctionCallingAgent({ name: "publisher", From 74cda777e4862e80f1e6a35457b996d9c7652829 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 11:35:00 +0700 Subject: [PATCH 63/77] remove duckduckgo image search in TS --- .../typescript/agent/tools/duckduckgo.ts | 72 +------------------ .../engines/typescript/agent/tools/index.ts | 11 +-- .../multiagent/typescript/workflow/agents.ts | 19 +++-- 3 files changed, 18 insertions(+), 84 deletions(-) diff --git a/templates/components/engines/typescript/agent/tools/duckduckgo.ts b/templates/components/engines/typescript/agent/tools/duckduckgo.ts index 80ca7de0a..419ff90ac 100644 --- a/templates/components/engines/typescript/agent/tools/duckduckgo.ts +++ b/templates/components/engines/typescript/agent/tools/duckduckgo.ts @@ -1,5 +1,5 @@ import { JSONSchemaType } from "ajv"; -import { ImageSearchOptions, search, searchImages } from "duck-duck-scrape"; +import { search } from "duck-duck-scrape"; import { BaseTool, ToolMetadata } from "llamaindex"; export type DuckDuckGoParameter = { @@ -17,37 +17,7 @@ const DEFAULT_SEARCH_METADATA: ToolMetadata< > = { name: "duckduckgo_search", description: - "Use this function to search for information in the internet using DuckDuckGo.", - parameters: { - type: "object", - properties: { - query: { - type: "string", - description: "The query to search in DuckDuckGo.", - }, - region: { - type: "string", - description: - "Optional, The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...", - nullable: true, - }, - maxResults: { - type: "number", - description: - "Optional, The maximum number of results to be returned. Default is 10.", - nullable: true, - }, - }, - required: ["query"], - }, -}; - -const DEFAULT_IMAGE_SEARCH_METADATA: ToolMetadata< - JSONSchemaType -> = { - name: "duckduckgo_image_search", - description: - "Use this function to search for images in internet using DuckDuckGo.", + "Use this function to search for information (only text) in the internet using DuckDuckGo.", parameters: { type: "object", properties: { @@ -78,13 +48,6 @@ type DuckDuckGoSearchResult = { url: string; }; -type DuckDuckGoImageResult = { - image: string; - title: string; - source: string; - url: string; -}; - export class DuckDuckGoSearchTool implements BaseTool { metadata: ToolMetadata>; @@ -110,35 +73,6 @@ export class DuckDuckGoSearchTool implements BaseTool { } } -export class DuckDuckGoImageSearchTool - implements BaseTool -{ - metadata: ToolMetadata>; - - constructor(params: DuckDuckGoToolParams) { - this.metadata = params.metadata ?? DEFAULT_IMAGE_SEARCH_METADATA; - } - - async call(input: DuckDuckGoParameter) { - const { query, region, maxResults = 5 } = input; - const options: Partial = region - ? { locale: region } - : {}; - // Temporarily sleep to reduce overloading the DuckDuckGo - await new Promise((resolve) => setTimeout(resolve, 1000)); - const imageResults = await searchImages(query, options); - - return imageResults.results.slice(0, maxResults).map((result) => { - return { - image: result.image, - title: result.title, - source: result.source, - url: result.url, - } as DuckDuckGoImageResult; - }); - } -} - export function getTools() { - return [new DuckDuckGoSearchTool({}), new DuckDuckGoImageSearchTool({})]; + return [new DuckDuckGoSearchTool({})]; } diff --git a/templates/components/engines/typescript/agent/tools/index.ts b/templates/components/engines/typescript/agent/tools/index.ts index 8c337f4db..b29af0484 100644 --- a/templates/components/engines/typescript/agent/tools/index.ts +++ b/templates/components/engines/typescript/agent/tools/index.ts @@ -4,11 +4,7 @@ import { DocumentGenerator, DocumentGeneratorParams, } from "./document-generator"; -import { - DuckDuckGoImageSearchTool, - DuckDuckGoSearchTool, - DuckDuckGoToolParams, -} from "./duckduckgo"; +import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo"; import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen"; import { InterpreterTool, InterpreterToolParams } from "./interpreter"; import { OpenAPIActionTool } from "./openapi-action"; @@ -46,10 +42,7 @@ const toolFactory: Record = { return await openAPIActionTool.toToolFunctions(); }, duckduckgo: async (config: unknown) => { - return [ - new DuckDuckGoSearchTool(config as DuckDuckGoToolParams), - new DuckDuckGoImageSearchTool(config as DuckDuckGoToolParams), - ]; + return [new DuckDuckGoSearchTool(config as DuckDuckGoToolParams)]; }, img_gen: async (config: unknown) => { return [new ImgGeneratorTool(config as ImgGeneratorToolParams)]; diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 29ee8798d..a4f6cb5ce 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -7,22 +7,29 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { "query_index", "wikipedia_tool", "duckduckgo_search", - "duckduckgo_image_search", + "image_generator", ]); return new FunctionCallingAgent({ name: "researcher", tools: tools, systemPrompt: `You are a researcher agent. You are given a researching task. -If the conversation already included the information and there is no new request from the user, you should return the appropriate content to the writer. +If the conversation already included the information and there is no new request for a new information from the user, you should return the appropriate content to the writer. Otherwise, you must use tools to retrieve information needed for the task. -It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. -If you don't found any related information, please return "I didn't find any new information for {the topic}.". Don't try to make up information yourself. +It's normal that the task include some ambiguity which you must always think carefully about the context of the user request to understand what is the real request that need to retrieve information +If you called the tools but don't found any related information, please return "I didn't find any new information for {the topic}.". Don't try to make up information yourself. +If the request don't need for any new information because it was in the conversation history, please return "The task don't need any new information. Please reuse the old content in the conversation history.". Example: Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> -Your task: Looking for information/images in English about the history of the Internet -This is not your task: Create blog post, create PDF, write in English`, +Your task: Looking for information in English about the history of the Internet +This is not your task: Create blog post, looking for how to create a PDF + +Next request: "Publish the blog post in HTML format." +-> +Your task: Return the previous content of the post to the writer. Don't need to do any research. +This is not your task: looking for how to create a HTML file. +`, chatHistory, }); }; From 1f17805ecafee07accc5fc98131700acf896d5ef Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 11:44:38 +0700 Subject: [PATCH 64/77] fix wrong output dir --- templates/components/engines/typescript/agent/tools/img-gen.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/components/engines/typescript/agent/tools/img-gen.ts b/templates/components/engines/typescript/agent/tools/img-gen.ts index 05cc12aba..d24d5567b 100644 --- a/templates/components/engines/typescript/agent/tools/img-gen.ts +++ b/templates/components/engines/typescript/agent/tools/img-gen.ts @@ -37,7 +37,7 @@ const DEFAULT_META_DATA: ToolMetadata> = { export class ImgGeneratorTool implements BaseTool { readonly IMG_OUTPUT_FORMAT = "webp"; - readonly IMG_OUTPUT_DIR = "output/tool"; + readonly IMG_OUTPUT_DIR = "output/tools"; readonly IMG_GEN_API = "https://api.stability.ai/v2beta/stable-image/generate/core"; From 7ccdecb150c79d231fefd76e6ce6555d0cd12183 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 12:05:01 +0700 Subject: [PATCH 65/77] refine prompt image link --- .../components/multiagent/typescript/workflow/agents.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index a4f6cb5ce..838f39b8f 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -44,11 +44,13 @@ If you received a review from the reviewer, update the post with the review and If user request for an update with an new thing but there is no research content provided, you must return "I don't have any research content to write about." If the content is not valid (ex: broken link, broken image, etc.) don't use it. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. +If you updated the post for the reviewer, please firstly reply what did you change in the post and then return the new post content. Example: Task: "Here is the information i found about the history of internet: Create a blog post about the history of the internet, write in English and publish in PDF format." -> Your task: Use the research content {...} to write a blog post in English. --> This is not your task: Create PDF`, +-> This is not your task: Create PDF +Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid.`, chatHistory, }); }; @@ -62,6 +64,7 @@ Review the post for logical inconsistencies, ask critical questions, and provide Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. +Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid. Example: Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> Your task: Review is the main content of the post is about the history of the internet, is it written in English. From 65474af02cce6e84a02a769eac89f9471a726e26 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 12:23:05 +0700 Subject: [PATCH 66/77] update in consistent paths --- .../engines/python/agent/tools/img_gen.py | 9 +++++---- .../engines/python/agent/tools/interpreter.py | 14 +++++++------- .../typescript/agent/tools/document-generator.ts | 2 +- .../engines/typescript/agent/tools/interpreter.ts | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/templates/components/engines/python/agent/tools/img_gen.py b/templates/components/engines/python/agent/tools/img_gen.py index 966e95d0d..8c2ae7bc0 100644 --- a/templates/components/engines/python/agent/tools/img_gen.py +++ b/templates/components/engines/python/agent/tools/img_gen.py @@ -1,10 +1,11 @@ +import logging import os import uuid -import logging -import requests from typing import Optional -from pydantic import BaseModel, Field + +import requests from llama_index.core.tools import FunctionTool +from pydantic import BaseModel, Field logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ class ImageGeneratorToolOutput(BaseModel): class ImageGeneratorTool: _IMG_OUTPUT_FORMAT = "webp" - _IMG_OUTPUT_DIR = "output/tool" + _IMG_OUTPUT_DIR = "output/tools" _IMG_GEN_API = "https://api.stability.ai/v2beta/stable-image/generate/core" def __init__(self, api_key: str = None): diff --git a/templates/components/engines/python/agent/tools/interpreter.py b/templates/components/engines/python/agent/tools/interpreter.py index 8e701c58f..0f4c10b95 100644 --- a/templates/components/engines/python/agent/tools/interpreter.py +++ b/templates/components/engines/python/agent/tools/interpreter.py @@ -1,13 +1,13 @@ -import os -import logging import base64 +import logging +import os import uuid -from pydantic import BaseModel -from typing import List, Dict, Optional -from llama_index.core.tools import FunctionTool +from typing import Dict, List, Optional + from e2b_code_interpreter import CodeInterpreter from e2b_code_interpreter.models import Logs - +from llama_index.core.tools import FunctionTool +from pydantic import BaseModel logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ class E2BToolOutput(BaseModel): class E2BCodeInterpreter: - output_dir = "output/tool" + output_dir = "output/tools" def __init__(self, api_key: str = None): if api_key is None: diff --git a/templates/components/engines/typescript/agent/tools/document-generator.ts b/templates/components/engines/typescript/agent/tools/document-generator.ts index aec343a33..b630db2c5 100644 --- a/templates/components/engines/typescript/agent/tools/document-generator.ts +++ b/templates/components/engines/typescript/agent/tools/document-generator.ts @@ -4,7 +4,7 @@ import { marked } from "marked"; import path from "node:path"; import { saveDocument } from "../../llamaindex/documents/helper"; -const OUTPUT_DIR = "output/tool"; +const OUTPUT_DIR = "output/tools"; type DocumentParameter = { originalContent: string; diff --git a/templates/components/engines/typescript/agent/tools/interpreter.ts b/templates/components/engines/typescript/agent/tools/interpreter.ts index 6870e5487..24573c205 100644 --- a/templates/components/engines/typescript/agent/tools/interpreter.ts +++ b/templates/components/engines/typescript/agent/tools/interpreter.ts @@ -56,7 +56,7 @@ const DEFAULT_META_DATA: ToolMetadata> = { }; export class InterpreterTool implements BaseTool { - private readonly outputDir = "output/tool"; + private readonly outputDir = "output/tools"; private apiKey?: string; private fileServerURLPrefix?: string; metadata: ToolMetadata>; From 800d79e1ea4bc31f2b23b725350de3e5edf14bef Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 12:39:49 +0700 Subject: [PATCH 67/77] update refined prompt for python --- .../python/app/examples/researcher.py | 21 ++++++++++++------- .../python/app/examples/workflow.py | 11 ++++++++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py index cc480ec80..043be6880 100644 --- a/templates/components/multiagent/python/app/examples/researcher.py +++ b/templates/components/multiagent/python/app/examples/researcher.py @@ -58,15 +58,22 @@ def create_researcher(chat_history: List[ChatMessage]): tools=tools, description="expert in retrieving any unknown content or searching for images from the internet", system_prompt=dedent(""" - You are a researcher agent. - You are given a researching task. You must use tools to retrieve information needed for the task. - It's normal that the task include some ambiguity which you must identify what is the real request that need to retrieve information. - If you don't found any related information, please return "I didn't find any information." + You are a researcher agent. You are given a researching task. + If the conversation already included the information and there is no new request for a new information from the user, you should return the appropriate content to the writer. + Otherwise, you must use tools to retrieve information needed for the task. + It's normal that the task include some ambiguity which you must always think carefully about the context of the user request to understand what is the real request that need to retrieve information + If you called the tools but don't found any related information, please return "I didn't find any new information for {the topic}.". Don't try to make up information yourself. + If the request don't need for any new information because it was in the conversation history, please return "The task don't need any new information. Please reuse the old content in the conversation history.". Example: - Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." + Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> - Your real task: Looking for information in english about the history of the internet - This is not your task: Create blog post, create PDF, write in English + Your task: Looking for information in English about the history of the Internet + This is not your task: Create blog post, looking for how to create a PDF + + Next request: "Publish the blog post in HTML format." + -> + Your task: Return the previous content of the post to the writer. Don't need to do any research. + This is not your task: looking for how to create a HTML file. """), chat_history=chat_history, ) diff --git a/templates/components/multiagent/python/app/examples/workflow.py b/templates/components/multiagent/python/app/examples/workflow.py index d1de62b50..1ca9d6786 100644 --- a/templates/components/multiagent/python/app/examples/workflow.py +++ b/templates/components/multiagent/python/app/examples/workflow.py @@ -27,13 +27,19 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): description="expert in writing blog posts, need information and images to write a post.", system_prompt=dedent(""" You are an expert in writing blog posts. - You are given a task to write a blog post. Don't make up any information yourself. + You are given a task to write a blog post from the research content provided by the researcher agent. Don't make up any information yourself. + It's important to read the whole conversation history to write the blog post correctly. + If you received a review from the reviewer, update the post with the review and return the new post content. + If user request for an update with an new thing but there is no research content provided, you must return "I don't have any research content to write about." + If the content is not valid (ex: broken link, broken image, etc.) don't use it. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. + If you updated the post for the reviewer, please firstly reply what did you change in the post and then return the new post content. Example: Task: "Here is the information i found about the history of internet: Create a blog post about the history of the internet, write in English and publish in PDF format." -> Your task: Use the research content {...} to write a blog post in English. -> This is not your task: Create PDF + Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid. """), chat_history=chat_history, ) @@ -42,11 +48,12 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): description="expert in reviewing blog posts, needs a written blog post to review.", system_prompt=dedent(""" You are an expert in reviewing blog posts. - You are given a task to review a blog post. + You are given a task to review a blog post. As a reviewer, it's important that your review is matching with the user request. Please focus on the user request to review the post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. + Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid. Example: Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> Your task: Review is the main content of the post is about the history of the internet, is it written in English. From cdfa99ff969a0d4a5e1ebe00c05b8a28fe058174 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 12:58:42 +0700 Subject: [PATCH 68/77] refine python agent prompt --- .../multiagent/python/app/examples/publisher.py | 13 ++++--------- .../multiagent/python/app/examples/workflow.py | 4 ++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/publisher.py b/templates/components/multiagent/python/app/examples/publisher.py index 1775a7ad3..2a170d886 100644 --- a/templates/components/multiagent/python/app/examples/publisher.py +++ b/templates/components/multiagent/python/app/examples/publisher.py @@ -14,9 +14,8 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]: if "document_generator" in configured_tools.keys(): tools.extend(configured_tools["document_generator"]) prompt_instructions = dedent(""" - You have access to a document generator tool that can create PDF or HTML document for the content. - If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file. - Otherwise, just return the content of the post. + Normally, reply the blog post content to the user directly. + But if user requested to generate a file, use the document_generator tool to generate the file and reply the link to the file. """) description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format." else: @@ -26,15 +25,11 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]: def create_publisher(chat_history: List[ChatMessage]): - tools, instructions, description = get_publisher_tools() - system_prompt = dedent(f""" - You are a publisher that help publish the blog post. - {instructions} - """) + tools, prompt_instructions, description = get_publisher_tools() return FunctionCallingAgent( name="publisher", tools=tools, description=description, - system_prompt=system_prompt, + system_prompt=prompt_instructions, chat_history=chat_history, ) diff --git a/templates/components/multiagent/python/app/examples/workflow.py b/templates/components/multiagent/python/app/examples/workflow.py index 1ca9d6786..a0528cfe9 100644 --- a/templates/components/multiagent/python/app/examples/workflow.py +++ b/templates/components/multiagent/python/app/examples/workflow.py @@ -33,7 +33,7 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): If user request for an update with an new thing but there is no research content provided, you must return "I don't have any research content to write about." If the content is not valid (ex: broken link, broken image, etc.) don't use it. It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. - If you updated the post for the reviewer, please firstly reply what did you change in the post and then return the new post content. + If you updated the post for the reviewer, please firstly reply what did you change in the post and then return the new post content, don't include the review from reviewer. Example: Task: "Here is the information i found about the history of internet: Create a blog post about the history of the internet, write in English and publish in PDF format." @@ -125,7 +125,7 @@ async def write( if ev.is_good or too_many_attempts: # too many attempts or the blog post is good - stream final response if requested return PublishEvent( - input=f"Please publish this content: ```{ev.input}```. The user request was: ```{ctx.data['user_input']}```", + input=f"Please publish this content: ```{ctx.data['result'].response.message.content}```. The user request was: ```{ctx.data['user_input']}```", ) result: AgentRunResult = await self.run_agent(ctx, writer, ev.input) ctx.data["result"] = result From c09de15f4339374fc1472ce181e9b1f570562bcb Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 13:50:34 +0700 Subject: [PATCH 69/77] fix prompt grammar --- .../python/app/examples/choreography.py | 6 +- .../python/app/examples/orchestrator.py | 12 +-- .../python/app/examples/researcher.py | 26 +++--- .../python/app/examples/workflow.py | 46 +++++----- .../multiagent/typescript/workflow/agents.ts | 84 +++++++++---------- 5 files changed, 87 insertions(+), 87 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/choreography.py b/templates/components/multiagent/python/app/examples/choreography.py index 2ad180f47..13da60e53 100644 --- a/templates/components/multiagent/python/app/examples/choreography.py +++ b/templates/components/multiagent/python/app/examples/choreography.py @@ -23,9 +23,9 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None): description="expert in writing blog posts, needs researched information and images to write a blog post", system_prompt=dedent(""" You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself. - After creating a draft for the post, send it to the reviewer agent to receive some feedback and make sure to incorporate the feedback from the reviewer. - You can consult the reviewer and researcher maximal two times. Your output should just contain the blog post. - Finally, always request the publisher to create an document (pdf, html) and publish the blog post. + After creating a draft for the post, send it to the reviewer agent to receive feedback and make sure to incorporate the feedback from the reviewer. + You can consult the reviewer and researcher a maximum of two times. Your output should contain only the blog post. + Finally, always request the publisher to create a document (PDF, HTML) and publish the blog post. """), # TODO: add chat_history support to AgentCallingAgent # chat_history=chat_history, diff --git a/templates/components/multiagent/python/app/examples/orchestrator.py b/templates/components/multiagent/python/app/examples/orchestrator.py index fb6a336d9..8786dcd3f 100644 --- a/templates/components/multiagent/python/app/examples/orchestrator.py +++ b/templates/components/multiagent/python/app/examples/orchestrator.py @@ -14,10 +14,10 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): name="writer", description="expert in writing blog posts, need information and images to write a post", system_prompt=dedent(""" - You are an expert in writing blog posts. - You are given a task to write a blog post. Don't make up any information yourself. - If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". - If you need to use images, reply "I need images about the topic to write the blog post". Don't use any dummy images made up by you. + You are an expert in writing blog posts. + You are given a task to write a blog post. Do not make up any information yourself. + If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post". + If you need to use images, reply "I need images about the topic to write the blog post". Do not use any dummy images made up by you. If you have all the information needed, write the blog post. """), chat_history=chat_history, @@ -26,8 +26,8 @@ def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None): name="reviewer", description="expert in reviewing blog posts, needs a written blog post to review", system_prompt=dedent(""" - You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix the issues found yourself. You must output a final blog post. - A post must include at lease one valid image, if not, reply "I need images about the topic to write the blog post". An image URL start with example or your website is not valid. + You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix any issues found yourself. You must output a final blog post. + A post must include at least one valid image. If not, reply "I need images about the topic to write the blog post". An image URL starting with "example" or "your website" is not valid. Especially check for logical inconsistencies and proofread the post for grammar and spelling errors. """), chat_history=chat_history, diff --git a/templates/components/multiagent/python/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py index 043be6880..4ccfc3028 100644 --- a/templates/components/multiagent/python/app/examples/researcher.py +++ b/templates/components/multiagent/python/app/examples/researcher.py @@ -58,22 +58,22 @@ def create_researcher(chat_history: List[ChatMessage]): tools=tools, description="expert in retrieving any unknown content or searching for images from the internet", system_prompt=dedent(""" - You are a researcher agent. You are given a researching task. - If the conversation already included the information and there is no new request for a new information from the user, you should return the appropriate content to the writer. + You are a researcher agent. You are given a research task. + If the conversation already includes the information and there is no new request for additional information from the user, you should return the appropriate content to the writer. Otherwise, you must use tools to retrieve information needed for the task. - It's normal that the task include some ambiguity which you must always think carefully about the context of the user request to understand what is the real request that need to retrieve information - If you called the tools but don't found any related information, please return "I didn't find any new information for {the topic}.". Don't try to make up information yourself. - If the request don't need for any new information because it was in the conversation history, please return "The task don't need any new information. Please reuse the old content in the conversation history.". + It's normal for the task to include some ambiguity. You must always think carefully about the context of the user's request to understand what information needs to be retrieved. + If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. + If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history." Example: - Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." - -> - Your task: Looking for information in English about the history of the Internet - This is not your task: Create blog post, looking for how to create a PDF + Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." + -> + Your task: Look for information in English about the history of the Internet. + This is not your task: Create a blog post or look for how to create a PDF. - Next request: "Publish the blog post in HTML format." - -> - Your task: Return the previous content of the post to the writer. Don't need to do any research. - This is not your task: looking for how to create a HTML file. + Next request: "Publish the blog post in HTML format." + -> + Your task: Return the previous content of the post to the writer. No need to do any research. + This is not your task: Look for how to create an HTML file. """), chat_history=chat_history, ) diff --git a/templates/components/multiagent/python/app/examples/workflow.py b/templates/components/multiagent/python/app/examples/workflow.py index a0528cfe9..d916a3c02 100644 --- a/templates/components/multiagent/python/app/examples/workflow.py +++ b/templates/components/multiagent/python/app/examples/workflow.py @@ -26,20 +26,20 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): name="writer", description="expert in writing blog posts, need information and images to write a post.", system_prompt=dedent(""" - You are an expert in writing blog posts. - You are given a task to write a blog post from the research content provided by the researcher agent. Don't make up any information yourself. - It's important to read the whole conversation history to write the blog post correctly. - If you received a review from the reviewer, update the post with the review and return the new post content. - If user request for an update with an new thing but there is no research content provided, you must return "I don't have any research content to write about." - If the content is not valid (ex: broken link, broken image, etc.) don't use it. - It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. - If you updated the post for the reviewer, please firstly reply what did you change in the post and then return the new post content, don't include the review from reviewer. + You are an expert in writing blog posts. + You are given the task of writing a blog post based on research content provided by the researcher agent. Do not invent any information yourself. + It's important to read the entire conversation history to write the blog post accurately. + If you receive a review from the reviewer, update the post according to the feedback and return the new post content. + If the user requests an update with new information but no research content is provided, you must respond with: "I don't have any research content to write about." + If the content is not valid (e.g., broken link, broken image, etc.), do not use it. + It's normal for the task to include some ambiguity, so you must define the user's initial request to write the post correctly. + If you update the post based on the reviewer's feedback, first explain what changes you made to the post, then provide the new post content. Do not include the reviewer's comments. Example: - Task: "Here is the information i found about the history of internet: - Create a blog post about the history of the internet, write in English and publish in PDF format." - -> Your task: Use the research content {...} to write a blog post in English. - -> This is not your task: Create PDF - Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid. + Task: "Here is the information I found about the history of the internet: + Create a blog post about the history of the internet, write in English, and publish in PDF format." + -> Your task: Use the research content {...} to write a blog post in English. + -> This is not your task: Create a PDF + Please note that a localhost link is acceptable, but dummy links like "example.com" or "your-website.com" are not valid. """), chat_history=chat_history, ) @@ -47,17 +47,17 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): name="reviewer", description="expert in reviewing blog posts, needs a written blog post to review.", system_prompt=dedent(""" - You are an expert in reviewing blog posts. - You are given a task to review a blog post. As a reviewer, it's important that your review is matching with the user request. Please focus on the user request to review the post. - Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. - Furthermore, proofread the post for grammar and spelling errors. - Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. - It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. - Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid. + You are an expert in reviewing blog posts. + You are given a task to review a blog post. As a reviewer, it's important that your review aligns with the user's request. Please focus on the user's request when reviewing the post. + Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. + Furthermore, proofread the post for grammar and spelling errors. + Only if the post is good enough for publishing should you return 'The post is good.' In all other cases, return your review. + It's normal for the task to include some ambiguity, so you must define the user's initial request to review the post correctly. + Please note that a localhost link is acceptable, but dummy links like "example.com" or "your-website.com" are not valid. Example: - Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." - -> Your task: Review is the main content of the post is about the history of the internet, is it written in English. - -> This is not your task: Create blog post, create PDF, write in English. + Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." + -> Your task: Review whether the main content of the post is about the history of the internet and if it is written in English. + -> This is not your task: Create blog post, create PDF, write in English. """), chat_history=chat_history, ) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 838f39b8f..ec9e6b5de 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -13,22 +13,22 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "researcher", tools: tools, - systemPrompt: `You are a researcher agent. You are given a researching task. -If the conversation already included the information and there is no new request for a new information from the user, you should return the appropriate content to the writer. -Otherwise, you must use tools to retrieve information needed for the task. -It's normal that the task include some ambiguity which you must always think carefully about the context of the user request to understand what is the real request that need to retrieve information -If you called the tools but don't found any related information, please return "I didn't find any new information for {the topic}.". Don't try to make up information yourself. -If the request don't need for any new information because it was in the conversation history, please return "The task don't need any new information. Please reuse the old content in the conversation history.". + systemPrompt: `You are a researcher agent. You are given a research task. +If the conversation already includes the required information and there is no new request for additional information from the user, you should return the appropriate content to the writer. +Otherwise, you must use tools to retrieve the information needed for the task. +It's normal for the task to include some ambiguity. You must always think carefully about the context of the user's request to understand what information needs to be retrieved. +If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. +If the request doesn't require any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history." Example: -Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." --> -Your task: Looking for information in English about the history of the Internet -This is not your task: Create blog post, looking for how to create a PDF - -Next request: "Publish the blog post in HTML format." --> -Your task: Return the previous content of the post to the writer. Don't need to do any research. -This is not your task: looking for how to create a HTML file. + Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." + -> + Your task: Look for information in English about the history of the Internet. + This is not your task: Create a blog post or look for how to create a PDF. + + Next request: "Publish the blog post in HTML format." + -> + Your task: Return the previous content of the post to the writer. No need to do any research. + This is not your task: Look for how to create an HTML file. `, chatHistory, }); @@ -37,20 +37,20 @@ This is not your task: looking for how to create a HTML file. export const createWriter = (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "writer", - systemPrompt: `You are an expert in writing blog posts. -You are given a task to write a blog post from the research content provided by the researcher agent. Don't make up any information yourself. -It's important to read the whole conversation history to write the blog post correctly. -If you received a review from the reviewer, update the post with the review and return the new post content. -If user request for an update with an new thing but there is no research content provided, you must return "I don't have any research content to write about." -If the content is not valid (ex: broken link, broken image, etc.) don't use it. -It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to write the post correctly. -If you updated the post for the reviewer, please firstly reply what did you change in the post and then return the new post content. + systemPrompt: `You are an expert in writing blog posts. +You are given the task of writing a blog post based on research content provided by the researcher agent. Do not invent any information yourself. +It's important to read the entire conversation history to write the blog post accurately. +If you receive a review from the reviewer, update the post according to the feedback and return the new post content. +If the user requests an update with new information but no research content is provided, you must respond with: "I don't have any research content to write about." +If the content is not valid (e.g., broken link, broken image, etc.), do not use it. +It's normal for the task to include some ambiguity, so you must define the user's initial request to write the post correctly. +If you update the post based on the reviewer's feedback, first explain what changes you made to the post, then provide the new post content. Do not include the reviewer's comments. Example: -Task: "Here is the information i found about the history of internet: -Create a blog post about the history of the internet, write in English and publish in PDF format." --> Your task: Use the research content {...} to write a blog post in English. --> This is not your task: Create PDF -Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid.`, + Task: "Here is the information I found about the history of the internet: + Create a blog post about the history of the internet, write in English, and publish in PDF format." + -> Your task: Use the research content {...} to write a blog post in English. + -> This is not your task: Create a PDF + Please note that a localhost link is acceptable, but dummy links like "example.com" or "your-website.com" are not valid.`, chatHistory, }); }; @@ -58,29 +58,29 @@ Please note that a localhost link is fine, but a dummy one like "example.com" or export const createReviewer = (chatHistory: ChatMessage[]) => { return new FunctionCallingAgent({ name: "reviewer", - systemPrompt: `You are an expert in reviewing blog posts. -You are given a task to review a blog post. As a reviewer, it's important that your review is matching with the user request. Please focus on the user request to review the post. -Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. -Furthermore, proofread the post for grammar and spelling errors. -Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review. -It's normal that the task include some ambiguity, so you must be define what is the starter request of the user to review the post correctly. -Please note that a localhost link is fine, but a dummy one like "example.com" or "your-website.com" is not valid. + systemPrompt: `You are an expert in reviewing blog posts. +You are given a task to review a blog post. As a reviewer, it's important that your review aligns with the user's request. Please focus on the user's request when reviewing the post. +Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. +Furthermore, proofread the post for grammar and spelling errors. +Only if the post is good enough for publishing should you return 'The post is good.' In all other cases, return your review. +It's normal for the task to include some ambiguity, so you must define the user's initial request to review the post correctly. +Please note that a localhost link is acceptable, but dummy links like "example.com" or "your-website.com" are not valid. Example: -Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." --> Your task: Review is the main content of the post is about the history of the internet, is it written in English. --> This is not your task: Create blog post, create PDF, write in English.`, + Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." + -> Your task: Review whether the main content of the post is about the history of the internet and if it is written in English. + -> This is not your task: Create blog post, create PDF, write in English.`, chatHistory, }); }; export const createPublisher = async (chatHistory: ChatMessage[]) => { const tools = await lookupTools(["document_generator"]); - let systemPrompt = `You are an expert in publishing blog posts. You are given a task to publish a blog post. -If the writer say that there was an error you should reply with the error and not publish the post.`; + let systemPrompt = `ou are an expert in publishing blog posts. You are given a task to publish a blog post. +If the writer says that there was an error, you should reply with the error and not publish the post.`; if (tools.length > 0) { systemPrompt = `${systemPrompt}. -If user requests to generate a file, use the document_generator tool to generate the file and reply the link to the file. -Otherwise, just return the content of the post.`; +If the user requests to generate a file, use the document_generator tool to generate the file and reply with the link to the file. +Otherwise, simply return the content of the post.`; } return new FunctionCallingAgent({ name: "publisher", From be0deb5b87f7eafe10fa72ce236c8c8fd029b2c0 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 14:03:06 +0700 Subject: [PATCH 70/77] fix prompt --- helpers/tools.ts | 2 +- templates/components/multiagent/typescript/workflow/agents.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/tools.ts b/helpers/tools.ts index ac8072325..97bde8b61 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -72,7 +72,7 @@ export const supportedTools: Tool[] = [ name: TOOL_SYSTEM_PROMPT_ENV_VAR, description: "System prompt for DuckDuckGo search tool.", value: `You are a DuckDuckGo search agent. -You can use the duckduckgo search tool to get information or images from the web to answer user questions. +You can use the duckduckgo search tool to get information from the web to answer user questions. For better results, you can specify the region parameter to get results from a specific region but it's optional.`, }, ], diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index ec9e6b5de..ee79fc04e 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -75,7 +75,7 @@ Example: export const createPublisher = async (chatHistory: ChatMessage[]) => { const tools = await lookupTools(["document_generator"]); - let systemPrompt = `ou are an expert in publishing blog posts. You are given a task to publish a blog post. + let systemPrompt = `You are an expert in publishing blog posts. You are given a task to publish a blog post. If the writer says that there was an error, you should reply with the error and not publish the post.`; if (tools.length > 0) { systemPrompt = `${systemPrompt}. From af84764d40c6e418b58bd133420c415645c2f5c6 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 14:41:09 +0700 Subject: [PATCH 71/77] remove checking finished task event --- .../multiagent/typescript/workflow/factory.ts | 5 +---- .../streaming/fastapi/app/api/routers/models.py | 13 ++++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/templates/components/multiagent/typescript/workflow/factory.ts b/templates/components/multiagent/typescript/workflow/factory.ts index d265eb29e..01161303b 100644 --- a/templates/components/multiagent/typescript/workflow/factory.ts +++ b/templates/components/multiagent/typescript/workflow/factory.ts @@ -36,10 +36,7 @@ const prepareChatHistory = (chatHistory: ChatMessage[]) => { const agentAnnotations = chatHistory .filter((msg) => msg.role === "assistant") .flatMap((msg) => msg.annotations || []) - .filter( - (annotation) => - annotation.type === "agent" && annotation.data.text !== "Finished task", - ) + .filter((annotation) => annotation.type === "agent") .slice(-MAX_AGENT_MESSAGES); const agentMessages = agentAnnotations diff --git a/templates/types/streaming/fastapi/app/api/routers/models.py b/templates/types/streaming/fastapi/app/api/routers/models.py index 12568b4d0..123f97ba9 100644 --- a/templates/types/streaming/fastapi/app/api/routers/models.py +++ b/templates/types/streaming/fastapi/app/api/routers/models.py @@ -124,7 +124,7 @@ def get_last_message_content(self) -> str: break return message_content - def _get_agent_messages(self, max_messages: int = 5) -> List[str]: + def _get_agent_messages(self, max_messages: int = 10) -> List[str]: """ Construct agent messages from the annotations in the chat messages """ @@ -139,12 +139,11 @@ def _get_agent_messages(self, max_messages: int = 5) -> List[str]: annotation.data, AgentAnnotation ): text = annotation.data.text - if not text.startswith("Finished task"): - agent_messages.append( - f"\nAgent: {annotation.data.agent}\nsaid: {text}\n" - ) - if len(agent_messages) >= max_messages: - break + agent_messages.append( + f"\nAgent: {annotation.data.agent}\nsaid: {text}\n" + ) + if len(agent_messages) >= max_messages: + break return agent_messages def get_history_messages( From 7c687ac2043d97abc56044bccfc48ba0f643feac Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 14:57:33 +0700 Subject: [PATCH 72/77] fix missing rename readme in python template --- helpers/python.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/python.ts b/helpers/python.ts index c2f1a0a96..4af474342 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -438,6 +438,7 @@ export const installPythonTemplate = async ({ await copy("**", path.join(root), { parents: true, cwd: path.join(compPath, "multiagent", "python"), + rename: assetRelocator, }); } From c4ce77a6d31ab0f838f44fc9fef31ab42725e819 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 15:28:07 +0700 Subject: [PATCH 73/77] remove rendering system prompt env for multiagent template --- helpers/env-variables.ts | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/helpers/env-variables.ts b/helpers/env-variables.ts index 11beb0e85..49e0ab878 100644 --- a/helpers/env-variables.ts +++ b/helpers/env-variables.ts @@ -426,34 +426,35 @@ const getToolEnvs = (tools?: Tool[]): EnvVar[] => { const getSystemPromptEnv = ( tools?: Tool[], dataSources?: TemplateDataSource[], - framework?: TemplateFramework, + template?: TemplateType, ): EnvVar[] => { const defaultSystemPrompt = "You are a helpful assistant who helps users with their questions."; + const systemPromptEnv: EnvVar[] = []; // build tool system prompt by merging all tool system prompts - let toolSystemPrompt = ""; - tools?.forEach((tool) => { - const toolSystemPromptEnv = tool.envVars?.find( - (env) => env.name === TOOL_SYSTEM_PROMPT_ENV_VAR, - ); - if (toolSystemPromptEnv) { - toolSystemPrompt += toolSystemPromptEnv.value + "\n"; - } - }); + // multiagent template doesn't need system prompt + if (template !== "multiagent") { + let toolSystemPrompt = ""; + tools?.forEach((tool) => { + const toolSystemPromptEnv = tool.envVars?.find( + (env) => env.name === TOOL_SYSTEM_PROMPT_ENV_VAR, + ); + if (toolSystemPromptEnv) { + toolSystemPrompt += toolSystemPromptEnv.value + "\n"; + } + }); - const systemPrompt = toolSystemPrompt - ? `\"${toolSystemPrompt}\"` - : defaultSystemPrompt; + const systemPrompt = toolSystemPrompt + ? `\"${toolSystemPrompt}\"` + : defaultSystemPrompt; - const systemPromptEnv = [ - { + systemPromptEnv.push({ name: "SYSTEM_PROMPT", description: "The system prompt for the AI model.", value: systemPrompt, - }, - ]; - + }); + } if (tools?.length == 0 && (dataSources?.length ?? 0 > 0)) { const citationPrompt = `'You have provided information from a knowledge base that has been passed to you in nodes of information. Each node has useful metadata such as node ID, file name, page, etc. @@ -559,7 +560,7 @@ export const createBackendEnvFile = async ( ...getToolEnvs(opts.tools), ...getTemplateEnvs(opts.template), ...getObservabilityEnvs(opts.observability), - ...getSystemPromptEnv(opts.tools, opts.dataSources, opts.framework), + ...getSystemPromptEnv(opts.tools, opts.dataSources, opts.template), ]; // Render and write env file const content = renderEnvVar(envVars); From 18879501f88d2cdf362a3080d7b02709cc94cb92 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Wed, 2 Oct 2024 15:45:06 +0700 Subject: [PATCH 74/77] fix: overlapping messages in executor (start only one sub task) --- .../multiagent/python/app/agents/planner.py | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/templates/components/multiagent/python/app/agents/planner.py b/templates/components/multiagent/python/app/agents/planner.py index 71152c1dc..ce9ba01ee 100644 --- a/templates/components/multiagent/python/app/agents/planner.py +++ b/templates/components/multiagent/python/app/agents/planner.py @@ -128,11 +128,12 @@ async def execute_plan(self, ctx: Context, ev: ExecutePlanEvent) -> SubTaskEvent ctx.data["act_plan_id"] ) - ctx.data["num_sub_tasks"] = len(upcoming_sub_tasks) - # send an event per sub task - events = [SubTaskEvent(sub_task=sub_task) for sub_task in upcoming_sub_tasks] - for event in events: - ctx.send_event(event) + if upcoming_sub_tasks: + # Execute only the first sub-task + # otherwise the executor will get over-lapping messages + # alternatively, we could use one executor for all sub tasks + next_sub_task = upcoming_sub_tasks[0] + return SubTaskEvent(sub_task=next_sub_task) return None @@ -142,7 +143,7 @@ async def execute_sub_task( ) -> SubTaskResultEvent: if self._verbose: print(f"=== Executing sub task: {ev.sub_task.name} ===") - is_last_tasks = ctx.data["num_sub_tasks"] == self.get_remaining_subtasks(ctx) + is_last_tasks = self.get_remaining_subtasks(ctx) == 1 # TODO: streaming only works without plan refining streaming = is_last_tasks and ctx.data["streaming"] and not self.refine_plan handler = self.executor.run( @@ -164,22 +165,17 @@ async def execute_sub_task( async def gather_results( self, ctx: Context, ev: SubTaskResultEvent ) -> ExecutePlanEvent | StopEvent: - # wait for all sub tasks to finish - num_sub_tasks = ctx.data["num_sub_tasks"] - results = ctx.collect_events(ev, [SubTaskResultEvent] * num_sub_tasks) - if results is None: - return None + result = ev upcoming_sub_tasks = self.get_upcoming_sub_tasks(ctx) # if no more tasks to do, stop workflow and send result of last step if upcoming_sub_tasks == 0: - return StopEvent(result=results[-1].result) + return StopEvent(result=result.result) if self.refine_plan: - # store all results for refining the plan + # store the result for refining the plan ctx.data["results"] = ctx.data.get("results", {}) - for result in results: - ctx.data["results"][result.sub_task.name] = result.result + ctx.data["results"][result.sub_task.name] = result.result new_plan = await self.planner.refine_plan( ctx.data["task"], ctx.data["act_plan_id"], ctx.data["results"] From 1100274fe7d31f9467e4f4eb4e35f5d904d2bfa8 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Wed, 2 Oct 2024 16:38:09 +0700 Subject: [PATCH 75/77] added selector to workflow --- .../python/app/examples/workflow.py | 67 ++++++++++++++++--- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/workflow.py b/templates/components/multiagent/python/app/examples/workflow.py index d916a3c02..9fffb4500 100644 --- a/templates/components/multiagent/python/app/examples/workflow.py +++ b/templates/components/multiagent/python/app/examples/workflow.py @@ -1,5 +1,7 @@ from textwrap import dedent from typing import AsyncGenerator, List, Optional +from llama_index.core.settings import Settings +from llama_index.core.prompts import PromptTemplate from app.agents.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent from app.examples.publisher import create_publisher @@ -25,7 +27,8 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): writer = FunctionCallingAgent( name="writer", description="expert in writing blog posts, need information and images to write a post.", - system_prompt=dedent(""" + system_prompt=dedent( + """ You are an expert in writing blog posts. You are given the task of writing a blog post based on research content provided by the researcher agent. Do not invent any information yourself. It's important to read the entire conversation history to write the blog post accurately. @@ -40,13 +43,15 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): -> Your task: Use the research content {...} to write a blog post in English. -> This is not your task: Create a PDF Please note that a localhost link is acceptable, but dummy links like "example.com" or "your-website.com" are not valid. - """), + """ + ), chat_history=chat_history, ) reviewer = FunctionCallingAgent( name="reviewer", description="expert in reviewing blog posts, needs a written blog post to review.", - system_prompt=dedent(""" + system_prompt=dedent( + """ You are an expert in reviewing blog posts. You are given a task to review a blog post. As a reviewer, it's important that your review aligns with the user's request. Please focus on the user's request when reviewing the post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. @@ -58,10 +63,13 @@ def create_workflow(chat_history: Optional[List[ChatMessage]] = None): Task: "Create a blog post about the history of the internet, write in English and publish in PDF format." -> Your task: Review whether the main content of the post is about the history of the internet and if it is written in English. -> This is not your task: Create blog post, create PDF, write in English. - """), + """ + ), chat_history=chat_history, ) - workflow = BlogPostWorkflow(timeout=360) + workflow = BlogPostWorkflow( + timeout=360, chat_history=chat_history + ) # Pass chat_history here workflow.add_workflows( researcher=researcher, writer=writer, @@ -89,14 +97,52 @@ class PublishEvent(Event): class BlogPostWorkflow(Workflow): + def __init__( + self, timeout: int = 360, chat_history: Optional[List[ChatMessage]] = None + ): + super().__init__(timeout=timeout) + self.chat_history = chat_history or [] + @step() - async def start(self, ctx: Context, ev: StartEvent) -> ResearchEvent: + async def start(self, ctx: Context, ev: StartEvent) -> ResearchEvent | PublishEvent: # set streaming ctx.data["streaming"] = getattr(ev, "streaming", False) # start the workflow with researching about a topic ctx.data["task"] = ev.input ctx.data["user_input"] = ev.input - return ResearchEvent(input=f"Research for this task: {ev.input}") + + # Decision-making process + decision = await self._decide_workflow(ev.input, self.chat_history) + + if decision != "publish": + return ResearchEvent(input=f"Research for this task: {ev.input}") + else: + chat_history_str = "\n".join( + [f"{msg.role}: {msg.content}" for msg in self.chat_history] + ) + return PublishEvent( + input=f"Please publish content based on the chat history\n{chat_history_str}\n\n and task: {ev.input}" + ) + + async def _decide_workflow( + self, input: str, chat_history: List[ChatMessage] + ) -> str: + prompt_template = PromptTemplate( + "Given the following chat history and new task, decide whether to publish based on existing information.\n" + "Chat history:\n{chat_history}\n" + "New task: {input}\n" + "Decision (respond with either 'not_publish' or 'publish'):" + ) + + chat_history_str = "\n".join( + [f"{msg.role}: {msg.content}" for msg in chat_history] + ) + prompt = prompt_template.format(chat_history=chat_history_str, input=input) + + output = await Settings.llm.acomplete(prompt) + decision = output.text.strip().lower() + + return "publish" if decision == "publish" else "research" @step() async def research( @@ -111,7 +157,7 @@ async def research( @step() async def write( self, ctx: Context, ev: WriteEvent, writer: FunctionCallingAgent - ) -> ReviewEvent | PublishEvent: + ) -> ReviewEvent | StopEvent: MAX_ATTEMPTS = 2 ctx.data["attempts"] = ctx.data.get("attempts", 0) + 1 too_many_attempts = ctx.data["attempts"] > MAX_ATTEMPTS @@ -124,9 +170,10 @@ async def write( ) if ev.is_good or too_many_attempts: # too many attempts or the blog post is good - stream final response if requested - return PublishEvent( - input=f"Please publish this content: ```{ctx.data['result'].response.message.content}```. The user request was: ```{ctx.data['user_input']}```", + result = await self.run_agent( + ctx, writer, ev.input, streaming=ctx.data["streaming"] ) + return StopEvent(result=result) result: AgentRunResult = await self.run_agent(ctx, writer, ev.input) ctx.data["result"] = result return ReviewEvent(input=result.response.message.content) From 0263aefa1abcd09f942683529b8f4684b30ca6f9 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 16:44:14 +0700 Subject: [PATCH 76/77] revise researcher prompt --- .../python/app/examples/researcher.py | 15 +++++++++------ .../multiagent/typescript/workflow/agents.ts | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py index 4ccfc3028..29da10a1e 100644 --- a/templates/components/multiagent/python/app/examples/researcher.py +++ b/templates/components/multiagent/python/app/examples/researcher.py @@ -59,21 +59,24 @@ def create_researcher(chat_history: List[ChatMessage]): description="expert in retrieving any unknown content or searching for images from the internet", system_prompt=dedent(""" You are a researcher agent. You are given a research task. + If the conversation already includes the information and there is no new request for additional information from the user, you should return the appropriate content to the writer. - Otherwise, you must use tools to retrieve information needed for the task. - It's normal for the task to include some ambiguity. You must always think carefully about the context of the user's request to understand what information needs to be retrieved. - If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. - If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history." + Otherwise, you must use tools to retrieve information or images needed for the task. + + It's normal for the task to include some ambiguity. You must always think carefully about the context of the user's request to understand what are the main content needs to be retrieved. Example: Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." - -> + ->Though: The main content is "history of the internet", while "write in English and publish in PDF format" is a requirement for other agents. Your task: Look for information in English about the history of the Internet. This is not your task: Create a blog post or look for how to create a PDF. Next request: "Publish the blog post in HTML format." - -> + ->Though: User just asking for a format change, the previous content is still valid. Your task: Return the previous content of the post to the writer. No need to do any research. This is not your task: Look for how to create an HTML file. + + If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. + If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history." """), chat_history=chat_history, ) diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index ee79fc04e..29b126758 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -14,21 +14,24 @@ export const createResearcher = async (chatHistory: ChatMessage[]) => { name: "researcher", tools: tools, systemPrompt: `You are a researcher agent. You are given a research task. -If the conversation already includes the required information and there is no new request for additional information from the user, you should return the appropriate content to the writer. -Otherwise, you must use tools to retrieve the information needed for the task. -It's normal for the task to include some ambiguity. You must always think carefully about the context of the user's request to understand what information needs to be retrieved. -If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. -If the request doesn't require any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history." + +If the conversation already includes the information and there is no new request for additional information from the user, you should return the appropriate content to the writer. +Otherwise, you must use tools to retrieve information or images needed for the task. + +It's normal for the task to include some ambiguity. You must always think carefully about the context of the user's request to understand what are the main content needs to be retrieved. Example: Request: "Create a blog post about the history of the internet, write in English and publish in PDF format." - -> + ->Though: The main content is "history of the internet", while "write in English and publish in PDF format" is a requirement for other agents. Your task: Look for information in English about the history of the Internet. This is not your task: Create a blog post or look for how to create a PDF. - + Next request: "Publish the blog post in HTML format." - -> + ->Though: User just asking for a format change, the previous content is still valid. Your task: Return the previous content of the post to the writer. No need to do any research. This is not your task: Look for how to create an HTML file. + +If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. +If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history. `, chatHistory, }); From 4a1b9657296595a4578c35f137c2e92c7dc92441 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 2 Oct 2024 17:25:40 +0700 Subject: [PATCH 77/77] small tweak --- .../components/multiagent/python/app/examples/researcher.py | 2 +- templates/components/multiagent/typescript/workflow/agents.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/components/multiagent/python/app/examples/researcher.py b/templates/components/multiagent/python/app/examples/researcher.py index 29da10a1e..6efa70e9e 100644 --- a/templates/components/multiagent/python/app/examples/researcher.py +++ b/templates/components/multiagent/python/app/examples/researcher.py @@ -75,7 +75,7 @@ def create_researcher(chat_history: List[ChatMessage]): Your task: Return the previous content of the post to the writer. No need to do any research. This is not your task: Look for how to create an HTML file. - If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. + If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." along with the content you found. Don't try to make up information yourself. If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history." """), chat_history=chat_history, diff --git a/templates/components/multiagent/typescript/workflow/agents.ts b/templates/components/multiagent/typescript/workflow/agents.ts index 29b126758..b62bd360a 100644 --- a/templates/components/multiagent/typescript/workflow/agents.ts +++ b/templates/components/multiagent/typescript/workflow/agents.ts @@ -30,7 +30,7 @@ Example: Your task: Return the previous content of the post to the writer. No need to do any research. This is not your task: Look for how to create an HTML file. -If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." Don't try to make up information yourself. +If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." along with the content you found. Don't try to make up information yourself. If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history. `, chatHistory,