From b4f07672d588e777227d55a1de4d2ce7a31e1bb1 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 11 Feb 2025 17:14:01 +0700 Subject: [PATCH 01/30] stg --- .../components/engines/python/agent/engine.py | 18 +- .../python/app/api/routers/vercel_response.py | 99 --------- .../fastapi}/app/api/callbacks/__init__.py | 0 .../fastapi}/app/api/callbacks/base.py | 0 .../fastapi}/app/api/callbacks/llamacloud.py | 0 .../app/api/callbacks/next_question.py | 0 .../app/api/callbacks/stream_handler.py | 0 .../streaming/fastapi/app/api/routers/chat.py | 28 ++- .../fastapi/app/api/routers/events.py | 2 + .../app/api/routers/vercel_response.py | 204 ++++++------------ 10 files changed, 92 insertions(+), 259 deletions(-) delete mode 100644 templates/components/multiagent/python/app/api/routers/vercel_response.py rename templates/{components/multiagent/python => types/streaming/fastapi}/app/api/callbacks/__init__.py (100%) rename templates/{components/multiagent/python => types/streaming/fastapi}/app/api/callbacks/base.py (100%) rename templates/{components/multiagent/python => types/streaming/fastapi}/app/api/callbacks/llamacloud.py (100%) rename templates/{components/multiagent/python => types/streaming/fastapi}/app/api/callbacks/next_question.py (100%) rename templates/{components/multiagent/python => types/streaming/fastapi}/app/api/callbacks/stream_handler.py (100%) diff --git a/templates/components/engines/python/agent/engine.py b/templates/components/engines/python/agent/engine.py index 3cc52e1ed..b808f4a3d 100644 --- a/templates/components/engines/python/agent/engine.py +++ b/templates/components/engines/python/agent/engine.py @@ -1,8 +1,9 @@ import os from typing import List -from llama_index.core.agent import AgentRunner -from llama_index.core.callbacks import CallbackManager +from llama_index.core.agent.workflow import AgentWorkflow + +# from llama_index.core.agent import AgentRunner from llama_index.core.settings import Settings from llama_index.core.tools import BaseTool @@ -11,13 +12,14 @@ from app.engine.tools.query_engine import get_query_engine_tool -def get_chat_engine(params=None, event_handlers=None, **kwargs): +def get_engine(params=None, **kwargs): + if params is None: + params = {} system_prompt = os.getenv("SYSTEM_PROMPT") tools: List[BaseTool] = [] - callback_manager = CallbackManager(handlers=event_handlers or []) # Add query tool if index exists - index_config = IndexConfig(callback_manager=callback_manager, **(params or {})) + index_config = IndexConfig(**params) index = get_index(index_config) if index is not None: query_engine_tool = get_query_engine_tool(index, **kwargs) @@ -27,10 +29,8 @@ def get_chat_engine(params=None, event_handlers=None, **kwargs): configured_tools: List[BaseTool] = ToolFactory.from_env() tools.extend(configured_tools) - return AgentRunner.from_llm( + return AgentWorkflow.from_tools_or_functions( + tools_or_functions=tools, # type: ignore llm=Settings.llm, - tools=tools, system_prompt=system_prompt, - callback_manager=callback_manager, - verbose=True, ) diff --git a/templates/components/multiagent/python/app/api/routers/vercel_response.py b/templates/components/multiagent/python/app/api/routers/vercel_response.py deleted file mode 100644 index a5f1d7a01..000000000 --- a/templates/components/multiagent/python/app/api/routers/vercel_response.py +++ /dev/null @@ -1,99 +0,0 @@ -import asyncio -import json -import logging -from typing import AsyncGenerator - -from fastapi.responses import StreamingResponse -from llama_index.core.agent.workflow.workflow_events import AgentStream -from llama_index.core.workflow import StopEvent - -from app.api.callbacks.stream_handler import StreamHandler - -logger = logging.getLogger("uvicorn") - - -class VercelStreamResponse(StreamingResponse): - """ - Converts preprocessed events into Vercel-compatible streaming response format. - """ - - TEXT_PREFIX = "0:" - DATA_PREFIX = "8:" - ERROR_PREFIX = "3:" - - def __init__( - self, - stream_handler: StreamHandler, - *args, - **kwargs, - ): - self.handler = stream_handler - super().__init__(content=self.content_generator()) - - async def content_generator(self): - """Generate Vercel-formatted content from preprocessed events.""" - stream_started = False - try: - async for event in self.handler.stream_events(): - if not stream_started: - # Start the stream with an empty message - stream_started = True - yield self.convert_text("") - - # Handle different types of events - if isinstance(event, (AgentStream, StopEvent)): - async for chunk in self._stream_text(event): - await self.handler.accumulate_text(chunk) - yield self.convert_text(chunk) - elif isinstance(event, dict): - yield self.convert_data(event) - elif hasattr(event, "to_response"): - event_response = event.to_response() - yield self.convert_data(event_response) - else: - yield self.convert_data(event.model_dump()) - - except asyncio.CancelledError: - logger.warning("Client cancelled the request!") - await self.handler.cancel_run() - except Exception as e: - logger.error(f"Error in stream response: {e}") - yield self.convert_error(str(e)) - await self.handler.cancel_run() - - async def _stream_text( - self, event: AgentStream | StopEvent - ) -> AsyncGenerator[str, None]: - """ - Accept stream text from either AgentStream or StopEvent with string or AsyncGenerator result - """ - if isinstance(event, AgentStream): - yield self.convert_text(event.delta) - elif isinstance(event, StopEvent): - if isinstance(event.result, str): - yield event.result - elif isinstance(event.result, AsyncGenerator): - async for chunk in event.result: - if isinstance(chunk, str): - yield chunk - elif hasattr(chunk, "delta"): - yield chunk.delta - - @classmethod - def convert_text(cls, token: str) -> str: - """Convert text event to Vercel format.""" - # Escape newlines and double quotes to avoid breaking the stream - token = json.dumps(token) - return f"{cls.TEXT_PREFIX}{token}\n" - - @classmethod - def convert_data(cls, data: dict) -> str: - """Convert data event to Vercel format.""" - data_str = json.dumps(data) - return f"{cls.DATA_PREFIX}[{data_str}]\n" - - @classmethod - def convert_error(cls, error: str) -> str: - """Convert error event to Vercel format.""" - error_str = json.dumps(error) - return f"{cls.ERROR_PREFIX}{error_str}\n" diff --git a/templates/components/multiagent/python/app/api/callbacks/__init__.py b/templates/types/streaming/fastapi/app/api/callbacks/__init__.py similarity index 100% rename from templates/components/multiagent/python/app/api/callbacks/__init__.py rename to templates/types/streaming/fastapi/app/api/callbacks/__init__.py diff --git a/templates/components/multiagent/python/app/api/callbacks/base.py b/templates/types/streaming/fastapi/app/api/callbacks/base.py similarity index 100% rename from templates/components/multiagent/python/app/api/callbacks/base.py rename to templates/types/streaming/fastapi/app/api/callbacks/base.py diff --git a/templates/components/multiagent/python/app/api/callbacks/llamacloud.py b/templates/types/streaming/fastapi/app/api/callbacks/llamacloud.py similarity index 100% rename from templates/components/multiagent/python/app/api/callbacks/llamacloud.py rename to templates/types/streaming/fastapi/app/api/callbacks/llamacloud.py diff --git a/templates/components/multiagent/python/app/api/callbacks/next_question.py b/templates/types/streaming/fastapi/app/api/callbacks/next_question.py similarity index 100% rename from templates/components/multiagent/python/app/api/callbacks/next_question.py rename to templates/types/streaming/fastapi/app/api/callbacks/next_question.py diff --git a/templates/components/multiagent/python/app/api/callbacks/stream_handler.py b/templates/types/streaming/fastapi/app/api/callbacks/stream_handler.py similarity index 100% rename from templates/components/multiagent/python/app/api/callbacks/stream_handler.py rename to templates/types/streaming/fastapi/app/api/callbacks/stream_handler.py diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index c024dad02..0377bb6a6 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -3,15 +3,16 @@ from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status from llama_index.core.llms import MessageRole -from app.api.routers.events import EventCallbackHandler +from app.api.callbacks.llamacloud import LlamaCloudFileDownload +from app.api.callbacks.next_question import SuggestNextQuestions +from app.api.callbacks.stream_handler import StreamHandler from app.api.routers.models import ( ChatData, Message, Result, SourceNodes, ) -from app.api.routers.vercel_response import VercelStreamResponse -from app.engine.engine import get_chat_engine +from app.engine.engine import get_engine from app.engine.query_filter import generate_filters chat_router = r = APIRouter() @@ -36,15 +37,19 @@ async def chat( logger.info( f"Creating chat engine with filters: {str(filters)}", ) - event_handler = EventCallbackHandler() - chat_engine = get_chat_engine( - filters=filters, params=params, event_handlers=[event_handler] - ) - response = chat_engine.astream_chat(last_message_content, messages) - - return VercelStreamResponse( - request, event_handler, response, data, background_tasks + engine = get_engine(filters=filters, params=params) + handler = engine.run( + user_msg=last_message_content, + chat_history=messages, + stream=True, ) + return StreamHandler.from_default( + handler=handler, + callbacks=[ + LlamaCloudFileDownload.from_default(background_tasks), + SuggestNextQuestions.from_default(data), + ], + ).vercel_stream() except Exception as e: logger.exception("Error in chat engine", exc_info=True) raise HTTPException( @@ -53,6 +58,7 @@ async def chat( ) from e +# TODO: Update non-streaming endpoint # non-streaming endpoint - delete if not needed @r.post("/request") async def chat_request( diff --git a/templates/types/streaming/fastapi/app/api/routers/events.py b/templates/types/streaming/fastapi/app/api/routers/events.py index d19196a72..9d8c0eac5 100644 --- a/templates/types/streaming/fastapi/app/api/routers/events.py +++ b/templates/types/streaming/fastapi/app/api/routers/events.py @@ -99,6 +99,8 @@ def to_response(self): return None +# TODO: Add an adapter for workflow events +# and remove callback handler class EventCallbackHandler(BaseCallbackHandler): _aqueue: asyncio.Queue is_done: bool = False diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 0d41d893e..339ca9959 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -1,23 +1,20 @@ +import asyncio import json import logging -from typing import Awaitable, List +from typing import AsyncGenerator -from aiostream import stream -from fastapi import BackgroundTasks, Request from fastapi.responses import StreamingResponse -from llama_index.core.chat_engine.types import StreamingAgentChatResponse -from llama_index.core.schema import NodeWithScore +from llama_index.core.agent.workflow.workflow_events import AgentStream +from llama_index.core.workflow import StopEvent -from app.api.routers.events import EventCallbackHandler -from app.api.routers.models import ChatData, Message, SourceNodes -from app.api.services.suggestion import NextQuestionSuggestion +from app.api.callbacks.stream_handler import StreamHandler logger = logging.getLogger("uvicorn") class VercelStreamResponse(StreamingResponse): """ - Class to convert the response from the chat engine to the streaming format expected by Vercel + Converts preprocessed events into Vercel-compatible streaming response format. """ TEXT_PREFIX = "0:" @@ -26,152 +23,79 @@ class VercelStreamResponse(StreamingResponse): def __init__( self, - request: Request, - event_handler: EventCallbackHandler, - response: Awaitable[StreamingAgentChatResponse], - chat_data: ChatData, - background_tasks: BackgroundTasks, + stream_handler: StreamHandler, + *args, + **kwargs, ): - content = VercelStreamResponse.content_generator( - request, event_handler, response, chat_data, background_tasks - ) - super().__init__(content=content) + self.handler = stream_handler + super().__init__(content=self.content_generator()) - @classmethod - async def content_generator( - cls, - request: Request, - event_handler: EventCallbackHandler, - response: Awaitable[StreamingAgentChatResponse], - chat_data: ChatData, - background_tasks: BackgroundTasks, - ): - chat_response_generator = cls._chat_response_generator( - response, background_tasks, event_handler, chat_data - ) - event_generator = cls._event_generator(event_handler) - - # Merge the chat response generator and the event generator - combine = stream.merge(chat_response_generator, event_generator) - is_stream_started = False + async def content_generator(self): + """Generate Vercel-formatted content from preprocessed events.""" + stream_started = False try: - async with combine.stream() as streamer: - async for output in streamer: - if await request.is_disconnected(): - break - - if not is_stream_started: - is_stream_started = True - # Stream a blank message to start displaying the response in the UI - yield cls.convert_text("") - - yield output - except Exception: - logger.exception("Error in stream response") - yield cls.convert_error( - "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again." - ) - finally: - # Ensure event handler is marked as done even if connection breaks - event_handler.is_done = True - - @classmethod - async def _event_generator(cls, event_handler: EventCallbackHandler): - """ - Yield the events from the event handler - """ - async for event in event_handler.async_event_gen(): - event_response = event.to_response() - if event_response is not None: - yield cls.convert_data(event_response) - - @classmethod - async def _chat_response_generator( - cls, - response: Awaitable[StreamingAgentChatResponse], - background_tasks: BackgroundTasks, - event_handler: EventCallbackHandler, - chat_data: ChatData, - ): + async for event in self.handler.stream_events(): + if not stream_started: + # Start the stream with an empty message + stream_started = True + yield self.convert_text("") + + # Handle different types of events + if isinstance(event, (AgentStream, StopEvent)): + async for chunk in self._stream_text(event): + await self.handler.accumulate_text(chunk) + yield self.convert_text(chunk) + elif isinstance(event, dict): + yield self.convert_data(event) + elif hasattr(event, "to_response"): + event_response = event.to_response() + yield self.convert_data(event_response) + else: + yield self.convert_data( + {"type": "agent", "data": event.model_dump()} + ) + + except asyncio.CancelledError: + logger.warning("Client cancelled the request!") + await self.handler.cancel_run() + except Exception as e: + logger.error(f"Error in stream response: {e}") + yield self.convert_error(str(e)) + await self.handler.cancel_run() + + async def _stream_text( + self, event: AgentStream | StopEvent + ) -> AsyncGenerator[str, None]: """ - Yield the text response and source nodes from the chat engine + Accept stream text from either AgentStream or StopEvent with string or AsyncGenerator result """ - # Wait for the response from the chat engine - result = await response - - # Once we got a source node, start a background task to download the files (if needed) - cls._process_response_nodes(result.source_nodes, background_tasks) - - # Yield the source nodes - yield cls.convert_data( - { - "type": "sources", - "data": { - "nodes": [ - SourceNodes.from_source_node(node).model_dump() - for node in result.source_nodes - ] - }, - } - ) - - final_response = "" - async for token in result.async_response_gen(): - final_response += token - yield cls.convert_text(token) - - # Generate next questions if next question prompt is configured - question_data = await cls._generate_next_questions( - chat_data.messages, final_response - ) - if question_data: - yield cls.convert_data(question_data) - - # the text_generator is the leading stream, once it's finished, also finish the event stream - event_handler.is_done = True + if isinstance(event, AgentStream): + yield event.delta + elif isinstance(event, StopEvent): + if isinstance(event.result, str): + yield event.result + elif isinstance(event.result, AsyncGenerator): + async for chunk in event.result: + if isinstance(chunk, str): + yield chunk + elif hasattr(chunk, "delta"): + yield chunk.delta @classmethod - def convert_text(cls, token: str): + def convert_text(cls, token: str) -> str: + """Convert text event to Vercel format.""" # Escape newlines and double quotes to avoid breaking the stream token = json.dumps(token) return f"{cls.TEXT_PREFIX}{token}\n" @classmethod - def convert_data(cls, data: dict): + def convert_data(cls, data: dict) -> str: + """Convert data event to Vercel format.""" data_str = json.dumps(data) return f"{cls.DATA_PREFIX}[{data_str}]\n" @classmethod - def convert_error(cls, error: str): + def convert_error(cls, error: str) -> str: + """Convert error event to Vercel format.""" error_str = json.dumps(error) return f"{cls.ERROR_PREFIX}{error_str}\n" - - @staticmethod - def _process_response_nodes( - source_nodes: List[NodeWithScore], - background_tasks: BackgroundTasks, - ): - try: - # Start background tasks to download documents from LlamaCloud if needed - from app.engine.service import LLamaCloudFileService # type: ignore - - LLamaCloudFileService.download_files_from_nodes( - source_nodes, background_tasks - ) - except ImportError: - logger.debug( - "LlamaCloud is not configured. Skipping post processing of nodes" - ) - pass - - @staticmethod - async def _generate_next_questions(chat_history: List[Message], response: str): - questions = await NextQuestionSuggestion.suggest_next_questions( - chat_history, response - ) - if questions: - return { - "type": "suggested_questions", - "data": questions, - } - return None From bc2d503fd8140f1f774c7de8611ef792a907dc09 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 11 Feb 2025 17:18:58 +0700 Subject: [PATCH 02/30] raise error if there is no tools --- templates/components/engines/python/agent/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/components/engines/python/agent/engine.py b/templates/components/engines/python/agent/engine.py index b808f4a3d..c1fc25f2f 100644 --- a/templates/components/engines/python/agent/engine.py +++ b/templates/components/engines/python/agent/engine.py @@ -3,7 +3,6 @@ from llama_index.core.agent.workflow import AgentWorkflow -# from llama_index.core.agent import AgentRunner from llama_index.core.settings import Settings from llama_index.core.tools import BaseTool @@ -29,6 +28,9 @@ def get_engine(params=None, **kwargs): configured_tools: List[BaseTool] = ToolFactory.from_env() tools.extend(configured_tools) + if len(tools) == 0: + raise RuntimeError("Please provide at least one tool!") + return AgentWorkflow.from_tools_or_functions( tools_or_functions=tools, # type: ignore llm=Settings.llm, From cbebd031bc8d64eeba939dc7631ee46ac8009ceb Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 12 Feb 2025 09:00:20 +0700 Subject: [PATCH 03/30] stg --- helpers/python.ts | 6 --- questions/datasources.ts | 10 ++-- .../components/engines/python/chat/engine.py | 47 ------------------- .../python/chat/node_postprocessors.py | 21 --------- .../streaming/fastapi/app/api/routers/chat.py | 39 +++++++-------- .../streaming/fastapi/app/engine}/engine.py | 0 .../fastapi/app/engine}/tools/__init__.py | 0 .../fastapi/app/engine}/tools/artifact.py | 0 .../app/engine}/tools/document_generator.py | 0 .../fastapi/app/engine}/tools/duckduckgo.py | 0 .../fastapi/app/engine}/tools/form_filling.py | 0 .../fastapi/app/engine}/tools/img_gen.py | 0 .../fastapi/app/engine}/tools/interpreter.py | 0 .../app/engine}/tools/openapi_action.py | 0 .../fastapi/app/engine}/tools/query_engine.py | 0 .../fastapi/app/engine}/tools/weather.py | 0 16 files changed, 26 insertions(+), 97 deletions(-) delete mode 100644 templates/components/engines/python/chat/engine.py delete mode 100644 templates/components/engines/python/chat/node_postprocessors.py rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/engine.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/__init__.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/artifact.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/document_generator.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/duckduckgo.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/form_filling.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/img_gen.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/interpreter.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/openapi_action.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/query_engine.py (100%) rename templates/{components/engines/python/agent => types/streaming/fastapi/app/engine}/tools/weather.py (100%) diff --git a/helpers/python.ts b/helpers/python.ts index 3569a81df..72183b804 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -470,12 +470,6 @@ export const installPythonTemplate = async ({ } } - // Copy engine code - await copy("**", enginePath, { - parents: true, - cwd: path.join(compPath, "engines", "python", engine), - }); - // Copy router code await copyRouterCode(root, tools ?? []); } diff --git a/questions/datasources.ts b/questions/datasources.ts index 1961e4c88..b750184c2 100644 --- a/questions/datasources.ts +++ b/questions/datasources.ts @@ -19,10 +19,12 @@ export const getDataSourceChoices = ( }); } if (selectedDataSource === undefined || selectedDataSource.length === 0) { - choices.push({ - title: "No datasource", - value: "none", - }); + if (framework !== "fastapi") { + choices.push({ + title: "No datasource", + value: "none", + }); + } choices.push({ title: process.platform !== "linux" diff --git a/templates/components/engines/python/chat/engine.py b/templates/components/engines/python/chat/engine.py deleted file mode 100644 index f3795afd3..000000000 --- a/templates/components/engines/python/chat/engine.py +++ /dev/null @@ -1,47 +0,0 @@ -import os - -from app.engine.index import IndexConfig, get_index -from app.engine.node_postprocessors import NodeCitationProcessor -from fastapi import HTTPException -from llama_index.core.callbacks import CallbackManager -from llama_index.core.chat_engine import CondensePlusContextChatEngine -from llama_index.core.memory import ChatMemoryBuffer -from llama_index.core.settings import Settings - - -def get_chat_engine(params=None, event_handlers=None, **kwargs): - system_prompt = os.getenv("SYSTEM_PROMPT") - citation_prompt = os.getenv("SYSTEM_CITATION_PROMPT", None) - top_k = int(os.getenv("TOP_K", 0)) - llm = Settings.llm - memory = ChatMemoryBuffer.from_defaults( - token_limit=llm.metadata.context_window - 256 - ) - callback_manager = CallbackManager(handlers=event_handlers or []) - - node_postprocessors = [] - if citation_prompt: - node_postprocessors = [NodeCitationProcessor()] - system_prompt = f"{system_prompt}\n{citation_prompt}" - - index_config = IndexConfig(callback_manager=callback_manager, **(params or {})) - index = get_index(index_config) - if index is None: - raise HTTPException( - status_code=500, - detail=str( - "StorageContext is empty - call 'poetry run generate' to generate the storage first" - ), - ) - if top_k != 0 and kwargs.get("similarity_top_k") is None: - kwargs["similarity_top_k"] = top_k - retriever = index.as_retriever(**kwargs) - - return CondensePlusContextChatEngine( - llm=llm, - memory=memory, - system_prompt=system_prompt, - retriever=retriever, - node_postprocessors=node_postprocessors, # type: ignore - callback_manager=callback_manager, - ) diff --git a/templates/components/engines/python/chat/node_postprocessors.py b/templates/components/engines/python/chat/node_postprocessors.py deleted file mode 100644 index 336cd0edc..000000000 --- a/templates/components/engines/python/chat/node_postprocessors.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import List, Optional - -from llama_index.core import QueryBundle -from llama_index.core.postprocessor.types import BaseNodePostprocessor -from llama_index.core.schema import NodeWithScore - - -class NodeCitationProcessor(BaseNodePostprocessor): - """ - Append node_id into metadata for citation purpose. - Config SYSTEM_CITATION_PROMPT in your runtime environment variable to enable this feature. - """ - - def _postprocess_nodes( - self, - nodes: List[NodeWithScore], - query_bundle: Optional[QueryBundle] = None, - ) -> List[NodeWithScore]: - for node_score in nodes: - node_score.node.metadata["node_id"] = node_score.node.node_id - return nodes diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 0377bb6a6..8c2c7f389 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -59,25 +59,26 @@ async def chat( # TODO: Update non-streaming endpoint -# non-streaming endpoint - delete if not needed -@r.post("/request") -async def chat_request( - data: ChatData, -) -> Result: - last_message_content = data.get_last_message_content() - messages = data.get_history_messages() +# Would be better if we use same chat.py endpoint for both agent and multiagent templates +# # non-streaming endpoint - delete if not needed +# @r.post("/request") +# async def chat_request( +# data: ChatData, +# ) -> Result: +# last_message_content = data.get_last_message_content() +# messages = data.get_history_messages() - doc_ids = data.get_chat_document_ids() - filters = generate_filters(doc_ids) - params = data.data or {} - logger.info( - f"Creating chat engine with filters: {str(filters)}", - ) +# doc_ids = data.get_chat_document_ids() +# filters = generate_filters(doc_ids) +# params = data.data or {} +# logger.info( +# f"Creating chat engine with filters: {str(filters)}", +# ) - chat_engine = get_chat_engine(filters=filters, params=params) +# chat_engine = get_chat_engine(filters=filters, params=params) - response = await chat_engine.achat(last_message_content, messages) - return Result( - result=Message(role=MessageRole.ASSISTANT, content=response.response), - nodes=SourceNodes.from_source_nodes(response.source_nodes), - ) +# response = await chat_engine.achat(last_message_content, messages) +# return Result( +# result=Message(role=MessageRole.ASSISTANT, content=response.response), +# nodes=SourceNodes.from_source_nodes(response.source_nodes), +# ) diff --git a/templates/components/engines/python/agent/engine.py b/templates/types/streaming/fastapi/app/engine/engine.py similarity index 100% rename from templates/components/engines/python/agent/engine.py rename to templates/types/streaming/fastapi/app/engine/engine.py diff --git a/templates/components/engines/python/agent/tools/__init__.py b/templates/types/streaming/fastapi/app/engine/tools/__init__.py similarity index 100% rename from templates/components/engines/python/agent/tools/__init__.py rename to templates/types/streaming/fastapi/app/engine/tools/__init__.py diff --git a/templates/components/engines/python/agent/tools/artifact.py b/templates/types/streaming/fastapi/app/engine/tools/artifact.py similarity index 100% rename from templates/components/engines/python/agent/tools/artifact.py rename to templates/types/streaming/fastapi/app/engine/tools/artifact.py diff --git a/templates/components/engines/python/agent/tools/document_generator.py b/templates/types/streaming/fastapi/app/engine/tools/document_generator.py similarity index 100% rename from templates/components/engines/python/agent/tools/document_generator.py rename to templates/types/streaming/fastapi/app/engine/tools/document_generator.py diff --git a/templates/components/engines/python/agent/tools/duckduckgo.py b/templates/types/streaming/fastapi/app/engine/tools/duckduckgo.py similarity index 100% rename from templates/components/engines/python/agent/tools/duckduckgo.py rename to templates/types/streaming/fastapi/app/engine/tools/duckduckgo.py diff --git a/templates/components/engines/python/agent/tools/form_filling.py b/templates/types/streaming/fastapi/app/engine/tools/form_filling.py similarity index 100% rename from templates/components/engines/python/agent/tools/form_filling.py rename to templates/types/streaming/fastapi/app/engine/tools/form_filling.py diff --git a/templates/components/engines/python/agent/tools/img_gen.py b/templates/types/streaming/fastapi/app/engine/tools/img_gen.py similarity index 100% rename from templates/components/engines/python/agent/tools/img_gen.py rename to templates/types/streaming/fastapi/app/engine/tools/img_gen.py diff --git a/templates/components/engines/python/agent/tools/interpreter.py b/templates/types/streaming/fastapi/app/engine/tools/interpreter.py similarity index 100% rename from templates/components/engines/python/agent/tools/interpreter.py rename to templates/types/streaming/fastapi/app/engine/tools/interpreter.py diff --git a/templates/components/engines/python/agent/tools/openapi_action.py b/templates/types/streaming/fastapi/app/engine/tools/openapi_action.py similarity index 100% rename from templates/components/engines/python/agent/tools/openapi_action.py rename to templates/types/streaming/fastapi/app/engine/tools/openapi_action.py diff --git a/templates/components/engines/python/agent/tools/query_engine.py b/templates/types/streaming/fastapi/app/engine/tools/query_engine.py similarity index 100% rename from templates/components/engines/python/agent/tools/query_engine.py rename to templates/types/streaming/fastapi/app/engine/tools/query_engine.py diff --git a/templates/components/engines/python/agent/tools/weather.py b/templates/types/streaming/fastapi/app/engine/tools/weather.py similarity index 100% rename from templates/components/engines/python/agent/tools/weather.py rename to templates/types/streaming/fastapi/app/engine/tools/weather.py From 5ec1947d4a89f6f067b26a469785145e88bfd67b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 12 Feb 2025 12:51:05 +0700 Subject: [PATCH 04/30] support request api --- .../streaming/fastapi/app/api/routers/chat.py | 51 +++++++++++-------- .../fastapi/app/api/routers/models.py | 1 - 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 8c2c7f389..8103a4b54 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -1,6 +1,8 @@ +import json import logging from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status +from llama_index.core.agent.workflow import AgentOutput from llama_index.core.llms import MessageRole from app.api.callbacks.llamacloud import LlamaCloudFileDownload @@ -10,7 +12,6 @@ ChatData, Message, Result, - SourceNodes, ) from app.engine.engine import get_engine from app.engine.query_filter import generate_filters @@ -58,27 +59,33 @@ async def chat( ) from e -# TODO: Update non-streaming endpoint -# Would be better if we use same chat.py endpoint for both agent and multiagent templates -# # non-streaming endpoint - delete if not needed -# @r.post("/request") -# async def chat_request( -# data: ChatData, -# ) -> Result: -# last_message_content = data.get_last_message_content() -# messages = data.get_history_messages() +# non-streaming endpoint - delete if not needed +@r.post("/request") +async def chat_request( + data: ChatData, +) -> Result: + last_message_content = data.get_last_message_content() + messages = data.get_history_messages() -# doc_ids = data.get_chat_document_ids() -# filters = generate_filters(doc_ids) -# params = data.data or {} -# logger.info( -# f"Creating chat engine with filters: {str(filters)}", -# ) + doc_ids = data.get_chat_document_ids() + filters = generate_filters(doc_ids) + params = data.data or {} + logger.info( + f"Creating chat engine with filters: {str(filters)}", + ) + engine = get_engine(filters=filters, params=params) -# chat_engine = get_chat_engine(filters=filters, params=params) + response = await engine.run( + user_msg=last_message_content, + chat_history=messages, + stream=False, + ) + output = response + if isinstance(output, AgentOutput): + content = output.response.content + else: + content = json.dumps(output) -# response = await chat_engine.achat(last_message_content, messages) -# return Result( -# result=Message(role=MessageRole.ASSISTANT, content=response.response), -# nodes=SourceNodes.from_source_nodes(response.source_nodes), -# ) + return Result( + result=Message(role=MessageRole.ASSISTANT, content=content), + ) diff --git a/templates/types/streaming/fastapi/app/api/routers/models.py b/templates/types/streaming/fastapi/app/api/routers/models.py index 31f2fa46f..6c766a427 100644 --- a/templates/types/streaming/fastapi/app/api/routers/models.py +++ b/templates/types/streaming/fastapi/app/api/routers/models.py @@ -317,7 +317,6 @@ def from_source_nodes(cls, source_nodes: List[NodeWithScore]): class Result(BaseModel): result: Message - nodes: List[SourceNodes] class ChatConfig(BaseModel): From 6d5749d6ae9b05b154ab4e5e8a42446570de63d0 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 12 Feb 2025 13:01:29 +0700 Subject: [PATCH 05/30] remove --no-files e2e test for python --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2aa812c08..755e278ef 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -19,7 +19,7 @@ jobs: python-version: ["3.11"] os: [macos-latest, windows-latest, ubuntu-22.04] frameworks: ["fastapi"] - datasources: ["--no-files", "--example-file", "--llamacloud"] + datasources: ["--example-file", "--llamacloud"] defaults: run: shell: bash From 22e4be931f9081341c17e36772af0dc9ff0a4b10 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 12 Feb 2025 16:57:37 +0700 Subject: [PATCH 06/30] use agent workflow for financial report use case --- .../app/workflows/financial_report.py | 347 ++++-------------- .../app/workflows/function_calling_agent.py | 121 ------ .../multiagent/python/app/workflows/tools.py | 230 ------------ 3 files changed, 72 insertions(+), 626 deletions(-) delete mode 100644 templates/components/multiagent/python/app/workflows/function_calling_agent.py delete mode 100644 templates/components/multiagent/python/app/workflows/tools.py diff --git a/templates/components/agents/python/financial_report/app/workflows/financial_report.py b/templates/components/agents/python/financial_report/app/workflows/financial_report.py index 8f2abb9ee..af39435bf 100644 --- a/templates/components/agents/python/financial_report/app/workflows/financial_report.py +++ b/templates/components/agents/python/financial_report/app/workflows/financial_report.py @@ -1,33 +1,86 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional, Tuple, Type from llama_index.core import Settings -from llama_index.core.base.llms.types import ChatMessage, MessageRole -from llama_index.core.llms.function_calling import FunctionCallingLLM -from llama_index.core.memory import ChatMemoryBuffer -from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection -from llama_index.core.workflow import ( - Context, - Event, - StartEvent, - StopEvent, - Workflow, - step, +from llama_index.core.agent.workflow import ( + AgentWorkflow, + FunctionAgent, + ReActAgent, ) +from llama_index.core.llms import LLM +from llama_index.core.tools import FunctionTool, QueryEngineTool from app.engine.index import IndexConfig, get_index from app.engine.tools import ToolFactory from app.engine.tools.query_engine import get_query_engine_tool -from app.workflows.events import AgentRunEvent -from app.workflows.tools import ( - call_tools, - chat_with_tools, -) def create_workflow( params: Optional[Dict[str, Any]] = None, **kwargs, -) -> Workflow: +): + query_engine_tool, code_interpreter_tool, document_generator_tool = _prepare_tools( + params, **kwargs + ) + agent_cls = _get_agent_cls_from_llm(Settings.llm) + research_agent = agent_cls( + name="researcher", + description="A financial researcher who are given a tasks that need to look up information from the financial documents about the user's request.", + tools=[query_engine_tool], + system_prompt=""" + You are a financial researcher who are given a tasks that need to look up information from the financial documents about the user's request. + You should use the query engine tool to look up the information and return the result to the user. + Always handoff the task to the `analyst` agent after gathering information. + """, + llm=Settings.llm, + can_handoff_to=["analyst"], + ) + + analyst_agent = agent_cls( + name="analyst", + description="A financial analyst who takes responsibility to analyze the financial data and generate a report.", + tools=[code_interpreter_tool], + system_prompt=""" + Use the given information, don't make up anything yourself. + If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report. + You can use the code interpreter tool to generate a report. + Always handoff the task and pass the researched information to the `reporter` agent. + """, + llm=Settings.llm, + can_handoff_to=["reporter"], + ) + + reporter_agent = agent_cls( + name="reporter", + description="A reporter who takes responsibility to generate a document for a report content.", + tools=[document_generator_tool], + system_prompt=""" + Use the document generator tool to generate the document and return the result to the user. + Don't update the content of the document, just generate a new one. + After generating the document, tell the user for the content or the issue if there is any. + """, + llm=Settings.llm, + ) + + workflow = AgentWorkflow( + agents=[research_agent, analyst_agent, reporter_agent], + root_agent="researcher", + verbose=True, + ) + + return workflow + + +def _get_agent_cls_from_llm(llm: LLM) -> Type[FunctionAgent | ReActAgent]: + if llm.metadata.is_function_calling_model: + return FunctionAgent + else: + return ReActAgent + + +def _prepare_tools( + params: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Tuple[QueryEngineTool, FunctionTool, FunctionTool]: # Create query engine tool index_config = IndexConfig(**params) index = get_index(index_config) @@ -41,260 +94,4 @@ def create_workflow( code_interpreter_tool = configured_tools.get("interpret") document_generator_tool = configured_tools.get("generate_document") - return FinancialReportWorkflow( - query_engine_tool=query_engine_tool, - code_interpreter_tool=code_interpreter_tool, - document_generator_tool=document_generator_tool, - ) - - -class InputEvent(Event): - input: List[ChatMessage] - response: bool = False - - -class ResearchEvent(Event): - input: list[ToolSelection] - - -class AnalyzeEvent(Event): - input: list[ToolSelection] | ChatMessage - - -class ReportEvent(Event): - input: list[ToolSelection] - - -class FinancialReportWorkflow(Workflow): - """ - A workflow to generate a financial report using indexed documents. - - Requirements: - - Indexed documents containing financial data and a query engine tool to search them - - A code interpreter tool to analyze data and generate reports - - A document generator tool to create report files - - Steps: - 1. LLM Input: The LLM determines the next step based on function calling. - For example, if the model requests the query engine tool, it returns a ResearchEvent; - if it requests document generation, it returns a ReportEvent. - 2. Research: Uses the query engine to find relevant chunks from indexed documents. - After gathering information, it requests analysis (step 3). - 3. Analyze: Uses a custom prompt to analyze research results and can call the code - interpreter tool for visualization or calculation. Returns results to the LLM. - 4. Report: Uses the document generator tool to create a report. Returns results to the LLM. - """ - - _default_system_prompt = """ - You are a financial analyst who are given a set of tools to help you. - It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself. - For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries. - """ - stream: bool = True - - def __init__( - self, - query_engine_tool: QueryEngineTool, - code_interpreter_tool: FunctionTool, - document_generator_tool: FunctionTool, - llm: Optional[FunctionCallingLLM] = None, - timeout: int = 360, - system_prompt: Optional[str] = None, - ): - super().__init__(timeout=timeout) - self.system_prompt = system_prompt or self._default_system_prompt - self.query_engine_tool = query_engine_tool - self.code_interpreter_tool = code_interpreter_tool - self.document_generator_tool = document_generator_tool - assert query_engine_tool is not None, ( - "Query engine tool is not found. Try run generation script or upload a document file first." - ) - assert code_interpreter_tool is not None, "Code interpreter tool is required" - assert document_generator_tool is not None, ( - "Document generator tool is required" - ) - self.tools = [ - self.query_engine_tool, - self.code_interpreter_tool, - self.document_generator_tool, - ] - self.llm: FunctionCallingLLM = llm or Settings.llm - assert isinstance(self.llm, FunctionCallingLLM) - self.memory = ChatMemoryBuffer.from_defaults(llm=self.llm) - - @step() - async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent: - self.stream = ev.get("stream", True) - user_msg = ev.get("user_msg") - chat_history = ev.get("chat_history") - - if chat_history is not None: - self.memory.put_messages(chat_history) - - # Add user message to memory - self.memory.put(ChatMessage(role=MessageRole.USER, content=user_msg)) - - if self.system_prompt: - system_msg = ChatMessage( - role=MessageRole.SYSTEM, content=self.system_prompt - ) - self.memory.put(system_msg) - - return InputEvent(input=self.memory.get()) - - @step() - async def handle_llm_input( # type: ignore - self, - ctx: Context, - ev: InputEvent, - ) -> ResearchEvent | AnalyzeEvent | ReportEvent | StopEvent: - """ - Handle an LLM input and decide the next step. - """ - # Always use the latest chat history from the input - chat_history: list[ChatMessage] = ev.input - - # Get tool calls - response = await chat_with_tools( - self.llm, - self.tools, # type: ignore - chat_history, - ) - if not response.has_tool_calls(): - if self.stream: - return StopEvent(result=response.generator) - else: - return StopEvent(result=await response.full_response()) - # calling different tools at the same time is not supported at the moment - # add an error message to tell the AI to process step by step - if response.is_calling_different_tools(): - self.memory.put( - ChatMessage( - role=MessageRole.ASSISTANT, - content="Cannot call different tools at the same time. Try calling one tool at a time.", - ) - ) - return InputEvent(input=self.memory.get()) - self.memory.put(response.tool_call_message) - match response.tool_name(): - case self.code_interpreter_tool.metadata.name: - return AnalyzeEvent(input=response.tool_calls) - case self.document_generator_tool.metadata.name: - return ReportEvent(input=response.tool_calls) - case self.query_engine_tool.metadata.name: - return ResearchEvent(input=response.tool_calls) - case _: - raise ValueError(f"Unknown tool: {response.tool_name()}") - - @step() - async def research(self, ctx: Context, ev: ResearchEvent) -> AnalyzeEvent: - """ - Do a research to gather information for the user's request. - A researcher should have these tools: query engine, search engine, etc. - """ - ctx.write_event_to_stream( - AgentRunEvent( - name="Researcher", - msg="Starting research", - ) - ) - tool_calls = ev.input - - tool_messages = await call_tools( - ctx=ctx, - agent_name="Researcher", - tools=[self.query_engine_tool], - tool_calls=tool_calls, - ) - self.memory.put_messages(tool_messages) - return AnalyzeEvent( - input=ChatMessage( - role=MessageRole.ASSISTANT, - content="I've finished the research. Please analyze the result.", - ), - ) - - @step() - async def analyze(self, ctx: Context, ev: AnalyzeEvent) -> InputEvent: - """ - Analyze the research result. - """ - ctx.write_event_to_stream( - AgentRunEvent( - name="Analyst", - msg="Starting analysis", - ) - ) - event_requested_by_workflow_llm = isinstance(ev.input, list) - # Requested by the workflow LLM Input step, it's a tool call - if event_requested_by_workflow_llm: - # Set the tool calls - tool_calls = ev.input - else: - # Otherwise, it's triggered by the research step - # Use a custom prompt and independent memory for the analyst agent - analysis_prompt = """ - You are a financial analyst, you are given a research result and a set of tools to help you. - Always use the given information, don't make up anything yourself. If there is not enough information, you can asking for more information. - If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report. - """ - # This is handled by analyst agent - # Clone the shared memory to avoid conflicting with the workflow. - chat_history = self.memory.get() - chat_history.append( - ChatMessage(role=MessageRole.SYSTEM, content=analysis_prompt) - ) - chat_history.append(ev.input) # type: ignore - # Check if the analyst agent needs to call tools - response = await chat_with_tools( - self.llm, - [self.code_interpreter_tool], - chat_history, - ) - if not response.has_tool_calls(): - # If no tool call, fallback analyst message to the workflow - analyst_msg = ChatMessage( - role=MessageRole.ASSISTANT, - content=await response.full_response(), - ) - self.memory.put(analyst_msg) - return InputEvent(input=self.memory.get()) - else: - # Set the tool calls and the tool call message to the memory - tool_calls = response.tool_calls - self.memory.put(response.tool_call_message) - - # Call tools - tool_messages = await call_tools( - ctx=ctx, - agent_name="Analyst", - tools=[self.code_interpreter_tool], - tool_calls=tool_calls, # type: ignore - ) - self.memory.put_messages(tool_messages) - - # Fallback to the input with the latest chat history - return InputEvent(input=self.memory.get()) - - @step() - async def report(self, ctx: Context, ev: ReportEvent) -> InputEvent: - """ - Generate a report based on the analysis result. - """ - ctx.write_event_to_stream( - AgentRunEvent( - name="Reporter", - msg="Starting report generation", - ) - ) - tool_calls = ev.input - tool_messages = await call_tools( - ctx=ctx, - agent_name="Reporter", - tools=[self.document_generator_tool], - tool_calls=tool_calls, - ) - self.memory.put_messages(tool_messages) - - # After the tool calls, fallback to the input with the latest chat history - return InputEvent(input=self.memory.get()) + return query_engine_tool, code_interpreter_tool, document_generator_tool diff --git a/templates/components/multiagent/python/app/workflows/function_calling_agent.py b/templates/components/multiagent/python/app/workflows/function_calling_agent.py deleted file mode 100644 index 452fc5e7b..000000000 --- a/templates/components/multiagent/python/app/workflows/function_calling_agent.py +++ /dev/null @@ -1,121 +0,0 @@ -from typing import Any, List, Optional - -from app.workflows.events import AgentRunEvent -from app.workflows.tools import ToolCallResponse, call_tools, chat_with_tools -from llama_index.core.base.llms.types import ChatMessage -from llama_index.core.llms.function_calling import FunctionCallingLLM -from llama_index.core.memory import ChatMemoryBuffer -from llama_index.core.settings import Settings -from llama_index.core.tools.types import BaseTool -from llama_index.core.workflow import ( - Context, - Event, - StartEvent, - StopEvent, - Workflow, - step, -) - - -class InputEvent(Event): - input: list[ChatMessage] - - -class ToolCallEvent(Event): - input: ToolCallResponse - - -class FunctionCallingAgent(Workflow): - """ - A simple workflow to request LLM with tools independently. - You can share the previous chat history to provide the context for the LLM. - """ - - def __init__( - self, - *args: Any, - llm: FunctionCallingLLM | None = None, - chat_history: Optional[List[ChatMessage]] = None, - tools: List[BaseTool] | None = None, - system_prompt: str | None = None, - verbose: bool = False, - timeout: float = 360.0, - name: str, - write_events: bool = True, - **kwargs: Any, - ) -> None: - super().__init__(*args, verbose=verbose, timeout=timeout, **kwargs) # type: ignore - self.tools = tools or [] - self.name = name - self.write_events = write_events - - if llm is None: - llm = Settings.llm - self.llm = llm - if not self.llm.metadata.is_function_calling_model: - raise ValueError("The provided LLM must support function calling.") - - self.system_prompt = system_prompt - - self.memory = ChatMemoryBuffer.from_defaults( - llm=self.llm, chat_history=chat_history - ) - self.sources = [] # type: ignore - - @step() - async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent: - # clear sources - self.sources = [] - - # set streaming - ctx.data["streaming"] = getattr(ev, "streaming", False) - - # set system prompt - if self.system_prompt is not None: - system_msg = ChatMessage(role="system", content=self.system_prompt) - self.memory.put(system_msg) - - # get user input - user_input = ev.input - user_msg = ChatMessage(role="user", content=user_input) - self.memory.put(user_msg) - - if self.write_events: - ctx.write_event_to_stream( - AgentRunEvent(name=self.name, msg=f"Start to work on: {user_input}") - ) - - return InputEvent(input=self.memory.get()) - - @step() - async def handle_llm_input( - self, - ctx: Context, - ev: InputEvent, - ) -> ToolCallEvent | StopEvent: - chat_history = ev.input - - response = await chat_with_tools( - self.llm, - self.tools, - chat_history, - ) - is_tool_call = isinstance(response, ToolCallResponse) - if not is_tool_call: - if ctx.data["streaming"]: - return StopEvent(result=response) - else: - full_response = "" - async for chunk in response.generator: - full_response += chunk.message.content - return StopEvent(result=full_response) - return ToolCallEvent(input=response) - - @step() - async def handle_tool_calls(self, ctx: Context, ev: ToolCallEvent) -> InputEvent: - tool_calls = ev.input.tool_calls - tool_call_message = ev.input.tool_call_message - self.memory.put(tool_call_message) - tool_messages = await call_tools(self.name, self.tools, ctx, tool_calls) - self.memory.put_messages(tool_messages) - return InputEvent(input=self.memory.get()) diff --git a/templates/components/multiagent/python/app/workflows/tools.py b/templates/components/multiagent/python/app/workflows/tools.py deleted file mode 100644 index faab45955..000000000 --- a/templates/components/multiagent/python/app/workflows/tools.py +++ /dev/null @@ -1,230 +0,0 @@ -import logging -import uuid -from abc import ABC, abstractmethod -from typing import Any, AsyncGenerator, Callable, Optional - -from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole -from llama_index.core.llms.function_calling import FunctionCallingLLM -from llama_index.core.tools import ( - BaseTool, - FunctionTool, - ToolOutput, - ToolSelection, -) -from llama_index.core.workflow import Context -from pydantic import BaseModel, ConfigDict - -from app.workflows.events import AgentRunEvent, AgentRunEventType - -logger = logging.getLogger("uvicorn") - - -class ContextAwareTool(FunctionTool, ABC): - @abstractmethod - async def acall(self, ctx: Context, input: Any) -> ToolOutput: # type: ignore - pass - - -class ChatWithToolsResponse(BaseModel): - """ - A tool call response from chat_with_tools. - """ - - tool_calls: Optional[list[ToolSelection]] - tool_call_message: Optional[ChatMessage] - generator: Optional[AsyncGenerator[ChatResponse | None, None]] - - model_config = ConfigDict(arbitrary_types_allowed=True) - - def is_calling_different_tools(self) -> bool: - tool_names = {tool_call.tool_name for tool_call in self.tool_calls} - return len(tool_names) > 1 - - def has_tool_calls(self) -> bool: - return self.tool_calls is not None and len(self.tool_calls) > 0 - - def tool_name(self) -> str: - assert self.has_tool_calls() - assert not self.is_calling_different_tools() - return self.tool_calls[0].tool_name - - async def full_response(self) -> str: - assert self.generator is not None - full_response = "" - async for chunk in self.generator: - content = chunk.message.content - if content: - full_response += content - return full_response - - -async def chat_with_tools( # type: ignore - llm: FunctionCallingLLM, - tools: list[BaseTool], - chat_history: list[ChatMessage], -) -> ChatWithToolsResponse: - """ - Request LLM to call tools or not. - This function doesn't change the memory. - """ - generator = _tool_call_generator(llm, tools, chat_history) - is_tool_call = await generator.__anext__() - if is_tool_call: - # Last chunk is the full response - # Wait for the last chunk - full_response = None - async for chunk in generator: - full_response = chunk - assert isinstance(full_response, ChatResponse) - return ChatWithToolsResponse( - tool_calls=llm.get_tool_calls_from_response(full_response), - tool_call_message=full_response.message, - generator=None, - ) - else: - return ChatWithToolsResponse( - tool_calls=None, - tool_call_message=None, - generator=generator, - ) - - -async def call_tools( - ctx: Context, - agent_name: str, - tools: list[BaseTool], - tool_calls: list[ToolSelection], - emit_agent_events: bool = True, -) -> list[ChatMessage]: - if len(tool_calls) == 0: - return [] - - tools_by_name = {tool.metadata.get_name(): tool for tool in tools} - if len(tool_calls) == 1: - return [ - await call_tool( - ctx, - tools_by_name[tool_calls[0].tool_name], - tool_calls[0], - lambda msg: ctx.write_event_to_stream( - AgentRunEvent( - name=agent_name, - msg=msg, - ) - ), - ) - ] - # Multiple tool calls, show progress - tool_msgs: list[ChatMessage] = [] - - progress_id = str(uuid.uuid4()) - total_steps = len(tool_calls) - if emit_agent_events: - ctx.write_event_to_stream( - AgentRunEvent( - name=agent_name, - msg=f"Making {total_steps} tool calls", - ) - ) - for i, tool_call in enumerate(tool_calls): - tool = tools_by_name.get(tool_call.tool_name) - if not tool: - tool_msgs.append( - ChatMessage( - role=MessageRole.ASSISTANT, - content=f"Tool {tool_call.tool_name} does not exist", - ) - ) - continue - tool_msg = await call_tool( - ctx, - tool, - tool_call, - event_emitter=lambda msg: ctx.write_event_to_stream( - AgentRunEvent( - name=agent_name, - msg=msg, - event_type=AgentRunEventType.PROGRESS, - data={ - "id": progress_id, - "total": total_steps, - "current": i, - }, - ) - ), - ) - tool_msgs.append(tool_msg) - return tool_msgs - - -async def call_tool( - ctx: Context, - tool: BaseTool, - tool_call: ToolSelection, - event_emitter: Optional[Callable[[str], None]], -) -> ChatMessage: - if event_emitter: - event_emitter( - f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}" - ) - try: - if isinstance(tool, ContextAwareTool): - if ctx is None: - raise ValueError("Context is required for context aware tool") - # inject context for calling an context aware tool - response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs) - else: - response = await tool.acall(**tool_call.tool_kwargs) # type: ignore - return ChatMessage( - role=MessageRole.TOOL, - content=str(response.raw_output), - additional_kwargs={ - "tool_call_id": tool_call.tool_id, - "name": tool.metadata.get_name(), - }, - ) - except Exception as e: - logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}") - if event_emitter: - event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}") - return ChatMessage( - role=MessageRole.TOOL, - content=f"Error: {str(e)}", - additional_kwargs={ - "tool_call_id": tool_call.tool_id, - "name": tool.metadata.get_name(), - }, - ) - - -async def _tool_call_generator( - llm: FunctionCallingLLM, - tools: list[BaseTool], - chat_history: list[ChatMessage], -) -> AsyncGenerator[ChatResponse | bool, None]: - response_stream = await llm.astream_chat_with_tools( - tools, - chat_history=chat_history, - allow_parallel_tool_calls=False, - ) - - full_response = None - yielded_indicator = False - async for chunk in response_stream: - if "tool_calls" not in chunk.message.additional_kwargs: - # Yield a boolean to indicate whether the response is a tool call - if not yielded_indicator: - yield False - yielded_indicator = True - - # if not a tool call, yield the chunks! - yield chunk # type: ignore - elif not yielded_indicator: - # Yield the indicator for a tool call - yield True - yielded_indicator = True - - full_response = chunk - - if full_response: - yield full_response # type: ignore From 6ba502331c2f5061cedab3b3fa568d8f760ea14e Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Thu, 13 Feb 2025 09:10:11 +0700 Subject: [PATCH 07/30] migrate form_filling to AgentWorkflow --- .../app/workflows/form_filling.py | 262 ++++-------------- 1 file changed, 57 insertions(+), 205 deletions(-) diff --git a/templates/components/agents/python/form_filling/app/workflows/form_filling.py b/templates/components/agents/python/form_filling/app/workflows/form_filling.py index 6078a5072..1eba7cb49 100644 --- a/templates/components/agents/python/form_filling/app/workflows/form_filling.py +++ b/templates/components/agents/python/form_filling/app/workflows/form_filling.py @@ -1,33 +1,22 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional, Type from llama_index.core import Settings -from llama_index.core.base.llms.types import ChatMessage, MessageRole -from llama_index.core.llms.function_calling import FunctionCallingLLM -from llama_index.core.memory import ChatMemoryBuffer -from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection -from llama_index.core.workflow import ( - Context, - Event, - StartEvent, - StopEvent, - Workflow, - step, +from llama_index.core.agent.workflow import ( + AgentWorkflow, + FunctionAgent, + ReActAgent, ) +from llama_index.core.llms import LLM from app.engine.index import IndexConfig, get_index from app.engine.tools import ToolFactory from app.engine.tools.query_engine import get_query_engine_tool -from app.workflows.events import AgentRunEvent -from app.workflows.tools import ( - call_tools, - chat_with_tools, -) def create_workflow( params: Optional[Dict[str, Any]] = None, **kwargs, -) -> Workflow: +) -> AgentWorkflow: # Create query engine tool index_config = IndexConfig(**params) index = get_index(index_config) @@ -40,197 +29,60 @@ def create_workflow( extractor_tool = configured_tools.get("extract_questions") # type: ignore filling_tool = configured_tools.get("fill_form") # type: ignore - workflow = FormFillingWorkflow( - query_engine_tool=query_engine_tool, - extractor_tool=extractor_tool, # type: ignore - filling_tool=filling_tool, # type: ignore + if extractor_tool is None or filling_tool is None: + raise ValueError("Extractor and filling tools are required.") + + agent_cls = _get_agent_cls_from_llm(Settings.llm) + + extractor_agent = agent_cls( + name="extractor", + description="An agent that extracts missing cells from CSV files and generates questions to fill them.", + tools=[extractor_tool], + system_prompt=""" + You are a helpful assistant who extracts missing cells from CSV files. + Only extract missing cells from CSV files and generate questions to fill them. + Always handoff the task to the `researcher` agent after extracting the questions. + """, + llm=Settings.llm, + can_handoff_to=["researcher"], ) - return workflow - - -class InputEvent(Event): - input: List[ChatMessage] - response: bool = False - - -class ExtractMissingCellsEvent(Event): - tool_calls: list[ToolSelection] - - -class FindAnswersEvent(Event): - tool_calls: list[ToolSelection] - - -class FillEvent(Event): - tool_calls: list[ToolSelection] - - -class FormFillingWorkflow(Workflow): - """ - A predefined workflow for filling missing cells in a CSV file. - Required tools: - - query_engine: A query engine to query for the answers to the questions. - - extract_question: Extract missing cells in a CSV file and generate questions to fill them. - - answer_question: Query for the answers to the questions. - - Flow: - 1. Extract missing cells in a CSV file and generate questions to fill them. - 2. Query for the answers to the questions. - 3. Fill the missing cells with the answers. - """ - - _default_system_prompt = """ - You are a helpful assistant who helps fill missing cells in a CSV file. - Only extract missing cells from CSV files. - Only use provided data - never make up any information yourself. Fill N/A if an answer is not found. - If there is no query engine tool or the gathered information has many N/A values indicating the questions don't match the data, respond with a warning and ask the user to upload a different file or connect to a knowledge base. - """ - stream: bool = True - - def __init__( - self, - query_engine_tool: Optional[QueryEngineTool], - extractor_tool: FunctionTool, - filling_tool: FunctionTool, - llm: Optional[FunctionCallingLLM] = None, - timeout: int = 360, - system_prompt: Optional[str] = None, - ): - super().__init__(timeout=timeout) - self.system_prompt = system_prompt or self._default_system_prompt - self.query_engine_tool = query_engine_tool - self.extractor_tool = extractor_tool - self.filling_tool = filling_tool - if self.extractor_tool is None or self.filling_tool is None: - raise ValueError("Extractor and filling tools are required.") - self.tools = [self.extractor_tool, self.filling_tool] - if self.query_engine_tool is not None: - self.tools.append(self.query_engine_tool) # type: ignore - self.llm: FunctionCallingLLM = llm or Settings.llm - if not isinstance(self.llm, FunctionCallingLLM): - raise ValueError("FormFillingWorkflow only supports FunctionCallingLLM.") - self.memory = ChatMemoryBuffer.from_defaults(llm=self.llm) - - @step() - async def start(self, ctx: Context, ev: StartEvent) -> InputEvent: - self.stream = ev.get("stream", True) - user_msg = ev.get("user_msg", "") - chat_history = ev.get("chat_history", []) - - if chat_history: - self.memory.put_messages(chat_history) - - self.memory.put(ChatMessage(role=MessageRole.USER, content=user_msg)) - - if self.system_prompt: - system_msg = ChatMessage( - role=MessageRole.SYSTEM, content=self.system_prompt - ) - self.memory.put(system_msg) + researcher_agent = agent_cls( + name="researcher", + description="An agent that finds answers to questions about missing cells.", + tools=[query_engine_tool] if query_engine_tool else [], + system_prompt=""" + You are a researcher who finds answers to questions about missing cells. + Only use provided data - never make up any information yourself. Use N/A if an answer is not found. + Always handoff the task to the `processor` agent after finding the answers. + """, + llm=Settings.llm, + can_handoff_to=["processor"], + ) - return InputEvent(input=self.memory.get()) + processor_agent = agent_cls( + name="processor", + description="An agent that fills missing cells with found answers.", + tools=[filling_tool], + system_prompt=""" + You are a processor who fills missing cells with found answers. + Fill N/A for any missing answers. + After filling the cells, tell the user about the results or any issues encountered. + """, + llm=Settings.llm, + ) - @step() - async def handle_llm_input( # type: ignore - self, - ctx: Context, - ev: InputEvent, - ) -> ExtractMissingCellsEvent | FillEvent | StopEvent: - """ - Handle an LLM input and decide the next step. - """ - chat_history: list[ChatMessage] = ev.input - response = await chat_with_tools( - self.llm, - self.tools, - chat_history, - ) - if not response.has_tool_calls(): - if self.stream: - return StopEvent(result=response.generator) - else: - return StopEvent(result=await response.full_response()) - # calling different tools at the same time is not supported at the moment - # add an error message to tell the AI to process step by step - if response.is_calling_different_tools(): - self.memory.put( - ChatMessage( - role=MessageRole.ASSISTANT, - content="Cannot call different tools at the same time. Try calling one tool at a time.", - ) - ) - return InputEvent(input=self.memory.get()) - self.memory.put(response.tool_call_message) - match response.tool_name(): - case self.extractor_tool.metadata.name: - return ExtractMissingCellsEvent(tool_calls=response.tool_calls) - case self.query_engine_tool.metadata.name: - return FindAnswersEvent(tool_calls=response.tool_calls) - case self.filling_tool.metadata.name: - return FillEvent(tool_calls=response.tool_calls) - case _: - raise ValueError(f"Unknown tool: {response.tool_name()}") + workflow = AgentWorkflow( + agents=[extractor_agent, researcher_agent, processor_agent], + root_agent="extractor", + verbose=True, + ) - @step() - async def extract_missing_cells( - self, ctx: Context, ev: ExtractMissingCellsEvent - ) -> InputEvent | FindAnswersEvent: - """ - Extract missing cells in a CSV file and generate questions to fill them. - """ - ctx.write_event_to_stream( - AgentRunEvent( - name="Extractor", - msg="Extracting missing cells", - ) - ) - # Call the extract questions tool - tool_messages = await call_tools( - agent_name="Extractor", - tools=[self.extractor_tool], - ctx=ctx, - tool_calls=ev.tool_calls, - ) - self.memory.put_messages(tool_messages) - return InputEvent(input=self.memory.get()) + return workflow - @step() - async def find_answers(self, ctx: Context, ev: FindAnswersEvent) -> InputEvent: - """ - Call answer questions tool to query for the answers to the questions. - """ - ctx.write_event_to_stream( - AgentRunEvent( - name="Researcher", - msg="Finding answers for missing cells", - ) - ) - tool_messages = await call_tools( - ctx=ctx, - agent_name="Researcher", - tools=[self.query_engine_tool], - tool_calls=ev.tool_calls, - ) - self.memory.put_messages(tool_messages) - return InputEvent(input=self.memory.get()) - @step() - async def fill_cells(self, ctx: Context, ev: FillEvent) -> InputEvent: - """ - Call fill cells tool to fill the missing cells with the answers. - """ - ctx.write_event_to_stream( - AgentRunEvent( - name="Processor", - msg="Filling missing cells", - ) - ) - tool_messages = await call_tools( - agent_name="Processor", - tools=[self.filling_tool], - ctx=ctx, - tool_calls=ev.tool_calls, - ) - self.memory.put_messages(tool_messages) - return InputEvent(input=self.memory.get()) +def _get_agent_cls_from_llm(llm: LLM) -> Type[FunctionAgent | ReActAgent]: + if llm.metadata.is_function_calling_model: + return FunctionAgent + else: + return ReActAgent From 0e4ee4a5c3fef713d3d7e27175e483225615c26e Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 13 Feb 2025 10:26:39 +0700 Subject: [PATCH 08/30] refactor: chat message content --- .../ui/chat/chat-message-content.tsx | 49 +++++-------------- .../ui/chat/custom/deep-research-card.tsx | 31 +++++------- .../components/ui/chat/tools/chat-tools.tsx | 4 +- 3 files changed, 27 insertions(+), 57 deletions(-) diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx index fa32f6f13..9d065b1f2 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx @@ -1,44 +1,19 @@ -import { - ChatMessage, - ContentPosition, - getSourceAnnotationData, - useChatMessage, - useChatUI, -} from "@llamaindex/chat-ui"; +import { ChatMessage } from "@llamaindex/chat-ui"; import { DeepResearchCard } from "./custom/deep-research-card"; -import { Markdown } from "./custom/markdown"; import { ToolAnnotations } from "./tools/chat-tools"; export function ChatMessageContent() { - const { isLoading, append } = useChatUI(); - const { message } = useChatMessage(); - const customContent = [ - { - // override the default markdown component - position: ContentPosition.MARKDOWN, - component: ( - - ), - }, - // add the deep research card - { - position: ContentPosition.CHAT_EVENTS, - component: , - }, - { - // add the tool annotations after events - position: ContentPosition.AFTER_EVENTS, - component: , - }, - ]; return ( - + + + + + + + + + + + ); } diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx index 03d0bd8c1..fc77cc70b 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx @@ -1,6 +1,6 @@ "use client"; -import { Message } from "@llamaindex/chat-ui"; +import { getCustomAnnotationData, useChatMessage } from "@llamaindex/chat-ui"; import { AlertCircle, CheckCircle2, @@ -54,7 +54,6 @@ type DeepResearchCardState = { }; interface DeepResearchCardProps { - message: Message; className?: string; } @@ -143,25 +142,19 @@ const deepResearchEventsToState = ( ); }; -export function DeepResearchCard({ - message, - className, -}: DeepResearchCardProps) { - const deepResearchEvents = message.annotations as - | DeepResearchEvent[] - | undefined; - const hasDeepResearchEvents = deepResearchEvents?.some( - (event) => event.type === "deep_research_event", - ); +export function DeepResearchCard({ className }: DeepResearchCardProps) { + const { message } = useChatMessage(); - const state = useMemo( - () => deepResearchEventsToState(deepResearchEvents), - [deepResearchEvents], - ); + const state = useMemo(() => { + const deepResearchEvents = getCustomAnnotationData( + message.annotations, + (annotation) => annotation?.type === "deep_research_event", + ); + if (!deepResearchEvents.length) return null; + return deepResearchEventsToState(deepResearchEvents); + }, [message.annotations]); - if (!hasDeepResearchEvents) { - return null; - } + if (!state) return null; return ( diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx index 71acda6d6..30606e5fd 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx @@ -2,6 +2,7 @@ import { Message, MessageAnnotation, getAnnotationData, + useChatMessage, useChatUI, } from "@llamaindex/chat-ui"; import { JSONValue } from "ai"; @@ -9,10 +10,11 @@ import { useMemo } from "react"; import { Artifact, CodeArtifact } from "./artifact"; import { WeatherCard, WeatherData } from "./weather-card"; -export function ToolAnnotations({ message }: { message: Message }) { +export function ToolAnnotations() { // TODO: This is a bit of a hack to get the artifact version. better to generate the version in the tool call and // store it in CodeArtifact const { messages } = useChatUI(); + const { message } = useChatMessage(); const artifactVersion = useMemo( () => getArtifactVersion(messages, message), [messages, message], From 86610e6c435f778b4f45bf9e155f89e4300f4543 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 14 Feb 2025 16:43:42 +0700 Subject: [PATCH 09/30] rename function in chat-ui --- .../app/components/ui/chat/custom/deep-research-card.tsx | 4 ++-- .../nextjs/app/components/ui/chat/tools/chat-tools.tsx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx index fc77cc70b..bc6118e61 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx @@ -1,6 +1,6 @@ "use client"; -import { getCustomAnnotationData, useChatMessage } from "@llamaindex/chat-ui"; +import { getCustomAnnotation, useChatMessage } from "@llamaindex/chat-ui"; import { AlertCircle, CheckCircle2, @@ -146,7 +146,7 @@ export function DeepResearchCard({ className }: DeepResearchCardProps) { const { message } = useChatMessage(); const state = useMemo(() => { - const deepResearchEvents = getCustomAnnotationData( + const deepResearchEvents = getCustomAnnotation( message.annotations, (annotation) => annotation?.type === "deep_research_event", ); diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx index 30606e5fd..a7e4eb218 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx @@ -1,7 +1,7 @@ import { Message, MessageAnnotation, - getAnnotationData, + getChatUIAnnotation, useChatMessage, useChatUI, } from "@llamaindex/chat-ui"; @@ -22,7 +22,7 @@ export function ToolAnnotations() { // Get the tool data from the message annotations const annotations = message.annotations as MessageAnnotation[] | undefined; const toolData = annotations - ? (getAnnotationData(annotations, "tools") as unknown as ToolData[]) + ? (getChatUIAnnotation(annotations, "tools") as unknown as ToolData[]) : null; return toolData?.[0] ? ( @@ -89,7 +89,7 @@ function getArtifactVersion( let versionIndex = 1; for (const m of messages) { const toolData = m.annotations - ? (getAnnotationData(m.annotations, "tools") as unknown as ToolData[]) + ? (getChatUIAnnotation(m.annotations, "tools") as unknown as ToolData[]) : null; if (toolData?.some((t) => t.toolCall.name === "artifact")) { From 8d3db71cdb4ef5e4549b133d1d250dc060bc68a2 Mon Sep 17 00:00:00 2001 From: Marcus Schiesser Date: Mon, 17 Feb 2025 11:12:42 +0700 Subject: [PATCH 10/30] Create cool-cars-promise.md --- .changeset/cool-cars-promise.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cool-cars-promise.md diff --git a/.changeset/cool-cars-promise.md b/.changeset/cool-cars-promise.md new file mode 100644 index 000000000..e09127e17 --- /dev/null +++ b/.changeset/cool-cars-promise.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +Migrate AgentRunner to Agent Workflow (Python) From 5a230becd894ca2822dd926ea87adf096a4b862c Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 18 Feb 2025 16:00:02 +0700 Subject: [PATCH 11/30] bump chat-ui --- .../streaming/fastapi/app/api/routers/vercel_response.py | 4 +--- templates/types/streaming/nextjs/package.json | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 339ca9959..83952b21e 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -51,9 +51,7 @@ async def content_generator(self): event_response = event.to_response() yield self.convert_data(event_response) else: - yield self.convert_data( - {"type": "agent", "data": event.model_dump()} - ) + yield self.convert_data(event.model_dump()) except asyncio.CancelledError: logger.warning("Client cancelled the request!") diff --git a/templates/types/streaming/nextjs/package.json b/templates/types/streaming/nextjs/package.json index 48c43c122..355fd4dfa 100644 --- a/templates/types/streaming/nextjs/package.json +++ b/templates/types/streaming/nextjs/package.json @@ -17,7 +17,7 @@ "@radix-ui/react-select": "^2.1.1", "@radix-ui/react-slot": "^1.0.2", "@radix-ui/react-tabs": "^1.1.0", - "@llamaindex/chat-ui": "0.0.14", + "@llamaindex/chat-ui": "0.1.0", "ai": "^4.0.3", "ajv": "^8.12.0", "class-variance-authority": "^0.7.1", From 7e23d779cb577065821ad27edf469fa389db666a Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 18 Feb 2025 16:39:58 +0700 Subject: [PATCH 12/30] add new query index and weather card for agent workflows --- .../ui/chat/chat-message-content.tsx | 4 + .../components/ui/chat/tools/query-index.tsx | 81 ++++++++++++++++++ .../components/ui/chat/tools/weather-card.tsx | 83 +++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx index 9d065b1f2..1eb12427e 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx @@ -1,11 +1,15 @@ import { ChatMessage } from "@llamaindex/chat-ui"; import { DeepResearchCard } from "./custom/deep-research-card"; import { ToolAnnotations } from "./tools/chat-tools"; +import { RetrieverComponent } from "./tools/query-index"; +import { WeatherToolComponent } from "./tools/weather-card"; export function ChatMessageContent() { return ( + + diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx new file mode 100644 index 000000000..203fffc63 --- /dev/null +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx @@ -0,0 +1,81 @@ +"use client"; + +import { getCustomAnnotation, useChatMessage } from "@llamaindex/chat-ui"; +import { ChatEvents } from "@llamaindex/chat-ui/widgets"; +import { useMemo } from "react"; +import { z } from "zod"; + +const QueryIndexSchema = z.object({ + tool_name: z.literal("query_index"), + tool_kwargs: z.object({ + input: z.string(), + }), + tool_id: z.string(), + tool_output: z.optional( + z + .object({ + content: z.string(), + tool_name: z.string(), + raw_input: z.record(z.unknown()), + raw_output: z.record(z.unknown()), + is_error: z.boolean().optional(), + }) + .optional(), + ), + return_direct: z.boolean().optional(), +}); +type QueryIndex = z.infer; + +type GroupedIndexQuery = { + initial: QueryIndex; + output?: QueryIndex; +}; + +export function RetrieverComponent() { + const { message } = useChatMessage(); + + const queryIndexEvents = getCustomAnnotation( + message.annotations, + (annotation) => { + const result = QueryIndexSchema.safeParse(annotation); + return result.success; + }, + ); + + // Group events by tool_id and render them in a single ChatEvents component + const groupedIndexQueries = useMemo(() => { + const groups = new Map(); + + queryIndexEvents?.forEach((event) => { + groups.set(event.tool_id, { initial: event }); + }); + + return Array.from(groups.values()); + }, [queryIndexEvents]); + + return ( +
+ {groupedIndexQueries.map(({ initial }) => { + const eventData = [ + { + title: `Searching index with query: ${initial.tool_kwargs.input}`, + }, + ]; + + if (initial.tool_output) { + eventData.push({ + title: `Got ${JSON.stringify((initial.tool_output?.raw_output as any).source_nodes?.length ?? 0)} sources for query: ${initial.tool_kwargs.input}`, + }); + } + + return ( + + ); + })} +
+ ); +} diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx index 8720c042a..1d126e81a 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx @@ -1,3 +1,8 @@ +import { getCustomAnnotation, useChatMessage } from "@llamaindex/chat-ui"; +import { ChatEvents } from "@llamaindex/chat-ui/widgets"; +import { useMemo } from "react"; +import { z } from "zod"; + export interface WeatherData { latitude: number; longitude: number; @@ -211,3 +216,81 @@ export function WeatherCard({ data }: { data: WeatherData }) { ); } + +// A new component for the weather tool which uses the WeatherCard component with the new data schema from agent workflow events +const WeatherToolSchema = z.object({ + tool_name: z.literal("get_weather_information"), + tool_kwargs: z.object({ + location: z.string(), + }), + tool_id: z.string(), + tool_output: z.optional( + z + .object({ + content: z.string(), + tool_name: z.string(), + raw_input: z.record(z.unknown()), + raw_output: z.custom(), + is_error: z.boolean().optional(), + }) + .optional(), + ), + return_direct: z.boolean().optional(), +}); + +type WeatherTool = z.infer; + +type GroupedWeatherQuery = { + initial: WeatherTool; + output?: WeatherTool; +}; + +export function WeatherToolComponent() { + const { message } = useChatMessage(); + + const weatherEvents = getCustomAnnotation( + message.annotations, + (annotation: unknown) => { + const result = WeatherToolSchema.safeParse(annotation); + return result.success; + }, + ); + + // Group events by tool_id + const groupedWeatherQueries = useMemo(() => { + const groups = new Map(); + + weatherEvents?.forEach((event: WeatherTool) => { + groups.set(event.tool_id, { initial: event }); + }); + + return Array.from(groups.values()); + }, [weatherEvents]); + + return ( +
+ {groupedWeatherQueries.map(({ initial }) => { + if (!initial.tool_output?.raw_output) { + return ( + + ); + } + + return ( + + ); + })} +
+ ); +} From 0139a1149364686bffe3413c8c162fb9c3144711 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 18 Feb 2025 18:09:42 +0700 Subject: [PATCH 13/30] support source nodes --- .../multiagent/python/app/api/routers/chat.py | 2 + .../fastapi/app/api/callbacks/source_nodes.py | 70 +++++++++++++++++++ .../streaming/fastapi/app/api/routers/chat.py | 2 + .../ui/chat/chat-message-content.tsx | 5 +- .../components/ui/chat/tools/query-index.tsx | 43 +++++++++++- templates/types/streaming/nextjs/package.json | 3 +- 6 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py index d7a44e691..37c305303 100644 --- a/templates/components/multiagent/python/app/api/routers/chat.py +++ b/templates/components/multiagent/python/app/api/routers/chat.py @@ -5,6 +5,7 @@ from app.api.callbacks.llamacloud import LlamaCloudFileDownload from app.api.callbacks.next_question import SuggestNextQuestions from app.api.callbacks.stream_handler import StreamHandler +from app.api.callbacks.add_node_url import AddNodeUrl from app.api.routers.models import ( ChatData, ) @@ -45,6 +46,7 @@ async def chat( callbacks=[ LlamaCloudFileDownload.from_default(background_tasks), SuggestNextQuestions.from_default(data), + AddNodeUrl.from_default(), ], ).vercel_stream() except Exception as e: diff --git a/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py b/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py new file mode 100644 index 000000000..01a351089 --- /dev/null +++ b/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py @@ -0,0 +1,70 @@ +import logging +import os +from typing import Any, Dict, Optional + +from app.api.callbacks.base import EventCallback +from app.config import DATA_DIR +from llama_index.core.agent.workflow.workflow_events import ToolCallResult + +logger = logging.getLogger("uvicorn") + + +class AddNodeUrl(EventCallback): + """ + Add URL to source nodes + """ + + async def run(self, event: Any) -> Any: + if self._is_retrieval_result_event(event): + for node_score in event.tool_output.raw_output.source_nodes: + node_score.node.metadata["url"] = self._get_url_from_metadata( + node_score.node.metadata + ) + return event + + def _is_retrieval_result_event(self, event: Any) -> bool: + if isinstance(event, ToolCallResult): + if event.tool_name == "query_index": + return True + return False + + def _get_url_from_metadata(self, metadata: Dict[str, Any]) -> Optional[str]: + url_prefix = os.getenv("FILESERVER_URL_PREFIX") + if not url_prefix: + logger.warning( + "Warning: FILESERVER_URL_PREFIX not set in environment variables. Can't use file server" + ) + file_name = metadata.get("file_name") + + if file_name and url_prefix: + # file_name exists and file server is configured + pipeline_id = metadata.get("pipeline_id") + if pipeline_id: + # file is from LlamaCloud + file_name = f"{pipeline_id}${file_name}" + return f"{url_prefix}/output/llamacloud/{file_name}" + is_private = metadata.get("private", "false") == "true" + if is_private: + # file is a private upload + return f"{url_prefix}/output/uploaded/{file_name}" + # file is from calling the 'generate' script + # Get the relative path of file_path to data_dir + file_path = metadata.get("file_path") + data_dir = os.path.abspath(DATA_DIR) + if file_path and data_dir: + relative_path = os.path.relpath(file_path, data_dir) + return f"{url_prefix}/data/{relative_path}" + # fallback to URL in metadata (e.g. for websites) + return metadata.get("URL") + + def convert_to_source_nodes(self, event: Any) -> Any: + if self._is_retrieval_result_event(event): + for node_score in event.tool_output.raw_output.source_nodes: + node_score.node.metadata["url"] = self._get_url_from_metadata( + node_score.node.metadata + ) + return event + + @classmethod + def from_default(cls) -> "AddNodeUrl": + return cls() diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 8103a4b54..5094adec0 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -7,6 +7,7 @@ from app.api.callbacks.llamacloud import LlamaCloudFileDownload from app.api.callbacks.next_question import SuggestNextQuestions +from app.api.callbacks.source_nodes import AddNodeUrl from app.api.callbacks.stream_handler import StreamHandler from app.api.routers.models import ( ChatData, @@ -49,6 +50,7 @@ async def chat( callbacks=[ LlamaCloudFileDownload.from_default(background_tasks), SuggestNextQuestions.from_default(data), + AddNodeUrl.from_default(), ], ).vercel_stream() except Exception as e: diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx index 1eb12427e..58507723a 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx @@ -1,21 +1,22 @@ import { ChatMessage } from "@llamaindex/chat-ui"; import { DeepResearchCard } from "./custom/deep-research-card"; import { ToolAnnotations } from "./tools/chat-tools"; -import { RetrieverComponent } from "./tools/query-index"; +import { ChatSourcesComponent, RetrieverComponent } from "./tools/query-index"; import { WeatherToolComponent } from "./tools/weather-card"; export function ChatMessageContent() { return ( + - + diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx index 203fffc63..2897a019a 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx @@ -1,7 +1,11 @@ "use client"; -import { getCustomAnnotation, useChatMessage } from "@llamaindex/chat-ui"; -import { ChatEvents } from "@llamaindex/chat-ui/widgets"; +import { + getCustomAnnotation, + SourceNode, + useChatMessage, +} from "@llamaindex/chat-ui"; +import { ChatEvents, ChatSources } from "@llamaindex/chat-ui/widgets"; import { useMemo } from "react"; import { z } from "zod"; @@ -79,3 +83,38 @@ export function RetrieverComponent() { ); } + +/** + * Render the source nodes whenever we got query_index tool with output + */ +export function ChatSourcesComponent() { + const { message } = useChatMessage(); + + const queryIndexEvents = getCustomAnnotation( + message.annotations, + (annotation) => { + const result = QueryIndexSchema.safeParse(annotation); + return result.success && !!result.data.tool_output; + }, + ); + + const sources: SourceNode[] = useMemo(() => { + return ( + queryIndexEvents?.flatMap((event) => { + const sourceNodes = + (event.tool_output?.raw_output?.source_nodes as any[]) || []; + return sourceNodes.map((node) => { + return { + id: node.node.id_, + metadata: node.node.metadata, + score: node.score, + text: node.node.text, + url: node.node.metadata.url, + }; + }); + }) || [] + ); + }, [queryIndexEvents]); + + return ; +} diff --git a/templates/types/streaming/nextjs/package.json b/templates/types/streaming/nextjs/package.json index 355fd4dfa..5de6e3a4c 100644 --- a/templates/types/streaming/nextjs/package.json +++ b/templates/types/streaming/nextjs/package.json @@ -37,7 +37,8 @@ "tiktoken": "^1.0.15", "uuid": "^9.0.1", "marked": "^14.1.2", - "wikipedia": "^2.1.2" + "wikipedia": "^2.1.2", + "zod": "^3.24.2" }, "devDependencies": { "@types/node": "^20.10.3", From dae32495dfa4130771c4171b81f3671c3b2de960 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 18 Feb 2025 18:21:12 +0700 Subject: [PATCH 14/30] remove unused function --- .../streaming/fastapi/app/api/callbacks/source_nodes.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py b/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py index 01a351089..91bf3664f 100644 --- a/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py +++ b/templates/types/streaming/fastapi/app/api/callbacks/source_nodes.py @@ -57,14 +57,6 @@ def _get_url_from_metadata(self, metadata: Dict[str, Any]) -> Optional[str]: # fallback to URL in metadata (e.g. for websites) return metadata.get("URL") - def convert_to_source_nodes(self, event: Any) -> Any: - if self._is_retrieval_result_event(event): - for node_score in event.tool_output.raw_output.source_nodes: - node_score.node.metadata["url"] = self._get_url_from_metadata( - node_score.node.metadata - ) - return event - @classmethod def from_default(cls) -> "AddNodeUrl": return cls() From 798f3785667d61a93f786b2469623ea7b2dd8683 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 19 Feb 2025 15:35:07 +0700 Subject: [PATCH 15/30] fix empty chunk --- .../streaming/fastapi/app/api/routers/vercel_response.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py index 83952b21e..a2d5c730d 100644 --- a/templates/types/streaming/fastapi/app/api/routers/vercel_response.py +++ b/templates/types/streaming/fastapi/app/api/routers/vercel_response.py @@ -68,7 +68,8 @@ async def _stream_text( Accept stream text from either AgentStream or StopEvent with string or AsyncGenerator result """ if isinstance(event, AgentStream): - yield event.delta + if event.delta.strip(): # Only yield non-empty deltas + yield event.delta elif isinstance(event, StopEvent): if isinstance(event.result, str): yield event.result @@ -76,7 +77,9 @@ async def _stream_text( async for chunk in event.result: if isinstance(chunk, str): yield chunk - elif hasattr(chunk, "delta"): + elif ( + hasattr(chunk, "delta") and chunk.delta.strip() + ): # Only yield non-empty deltas yield chunk.delta @classmethod From d09ae65ddb1fe8ac0c709448a2549afd42d9682a Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 19 Feb 2025 17:09:05 +0700 Subject: [PATCH 16/30] keep the old code for financial report and form-filling --- .../app/workflows/financial_report.py | 347 ++++++++++++++---- .../app/workflows/form_filling.py | 262 ++++++++++--- .../multiagent/python/app/api/routers/chat.py | 2 +- .../multiagent/python/app/workflows/tools.py | 230 ++++++++++++ 4 files changed, 711 insertions(+), 130 deletions(-) create mode 100644 templates/components/multiagent/python/app/workflows/tools.py diff --git a/templates/components/agents/python/financial_report/app/workflows/financial_report.py b/templates/components/agents/python/financial_report/app/workflows/financial_report.py index af39435bf..8f2abb9ee 100644 --- a/templates/components/agents/python/financial_report/app/workflows/financial_report.py +++ b/templates/components/agents/python/financial_report/app/workflows/financial_report.py @@ -1,86 +1,33 @@ -from typing import Any, Dict, Optional, Tuple, Type +from typing import Any, Dict, List, Optional from llama_index.core import Settings -from llama_index.core.agent.workflow import ( - AgentWorkflow, - FunctionAgent, - ReActAgent, +from llama_index.core.base.llms.types import ChatMessage, MessageRole +from llama_index.core.llms.function_calling import FunctionCallingLLM +from llama_index.core.memory import ChatMemoryBuffer +from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection +from llama_index.core.workflow import ( + Context, + Event, + StartEvent, + StopEvent, + Workflow, + step, ) -from llama_index.core.llms import LLM -from llama_index.core.tools import FunctionTool, QueryEngineTool from app.engine.index import IndexConfig, get_index from app.engine.tools import ToolFactory from app.engine.tools.query_engine import get_query_engine_tool +from app.workflows.events import AgentRunEvent +from app.workflows.tools import ( + call_tools, + chat_with_tools, +) def create_workflow( params: Optional[Dict[str, Any]] = None, **kwargs, -): - query_engine_tool, code_interpreter_tool, document_generator_tool = _prepare_tools( - params, **kwargs - ) - agent_cls = _get_agent_cls_from_llm(Settings.llm) - research_agent = agent_cls( - name="researcher", - description="A financial researcher who are given a tasks that need to look up information from the financial documents about the user's request.", - tools=[query_engine_tool], - system_prompt=""" - You are a financial researcher who are given a tasks that need to look up information from the financial documents about the user's request. - You should use the query engine tool to look up the information and return the result to the user. - Always handoff the task to the `analyst` agent after gathering information. - """, - llm=Settings.llm, - can_handoff_to=["analyst"], - ) - - analyst_agent = agent_cls( - name="analyst", - description="A financial analyst who takes responsibility to analyze the financial data and generate a report.", - tools=[code_interpreter_tool], - system_prompt=""" - Use the given information, don't make up anything yourself. - If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report. - You can use the code interpreter tool to generate a report. - Always handoff the task and pass the researched information to the `reporter` agent. - """, - llm=Settings.llm, - can_handoff_to=["reporter"], - ) - - reporter_agent = agent_cls( - name="reporter", - description="A reporter who takes responsibility to generate a document for a report content.", - tools=[document_generator_tool], - system_prompt=""" - Use the document generator tool to generate the document and return the result to the user. - Don't update the content of the document, just generate a new one. - After generating the document, tell the user for the content or the issue if there is any. - """, - llm=Settings.llm, - ) - - workflow = AgentWorkflow( - agents=[research_agent, analyst_agent, reporter_agent], - root_agent="researcher", - verbose=True, - ) - - return workflow - - -def _get_agent_cls_from_llm(llm: LLM) -> Type[FunctionAgent | ReActAgent]: - if llm.metadata.is_function_calling_model: - return FunctionAgent - else: - return ReActAgent - - -def _prepare_tools( - params: Optional[Dict[str, Any]] = None, - **kwargs, -) -> Tuple[QueryEngineTool, FunctionTool, FunctionTool]: +) -> Workflow: # Create query engine tool index_config = IndexConfig(**params) index = get_index(index_config) @@ -94,4 +41,260 @@ def _prepare_tools( code_interpreter_tool = configured_tools.get("interpret") document_generator_tool = configured_tools.get("generate_document") - return query_engine_tool, code_interpreter_tool, document_generator_tool + return FinancialReportWorkflow( + query_engine_tool=query_engine_tool, + code_interpreter_tool=code_interpreter_tool, + document_generator_tool=document_generator_tool, + ) + + +class InputEvent(Event): + input: List[ChatMessage] + response: bool = False + + +class ResearchEvent(Event): + input: list[ToolSelection] + + +class AnalyzeEvent(Event): + input: list[ToolSelection] | ChatMessage + + +class ReportEvent(Event): + input: list[ToolSelection] + + +class FinancialReportWorkflow(Workflow): + """ + A workflow to generate a financial report using indexed documents. + + Requirements: + - Indexed documents containing financial data and a query engine tool to search them + - A code interpreter tool to analyze data and generate reports + - A document generator tool to create report files + + Steps: + 1. LLM Input: The LLM determines the next step based on function calling. + For example, if the model requests the query engine tool, it returns a ResearchEvent; + if it requests document generation, it returns a ReportEvent. + 2. Research: Uses the query engine to find relevant chunks from indexed documents. + After gathering information, it requests analysis (step 3). + 3. Analyze: Uses a custom prompt to analyze research results and can call the code + interpreter tool for visualization or calculation. Returns results to the LLM. + 4. Report: Uses the document generator tool to create a report. Returns results to the LLM. + """ + + _default_system_prompt = """ + You are a financial analyst who are given a set of tools to help you. + It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself. + For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries. + """ + stream: bool = True + + def __init__( + self, + query_engine_tool: QueryEngineTool, + code_interpreter_tool: FunctionTool, + document_generator_tool: FunctionTool, + llm: Optional[FunctionCallingLLM] = None, + timeout: int = 360, + system_prompt: Optional[str] = None, + ): + super().__init__(timeout=timeout) + self.system_prompt = system_prompt or self._default_system_prompt + self.query_engine_tool = query_engine_tool + self.code_interpreter_tool = code_interpreter_tool + self.document_generator_tool = document_generator_tool + assert query_engine_tool is not None, ( + "Query engine tool is not found. Try run generation script or upload a document file first." + ) + assert code_interpreter_tool is not None, "Code interpreter tool is required" + assert document_generator_tool is not None, ( + "Document generator tool is required" + ) + self.tools = [ + self.query_engine_tool, + self.code_interpreter_tool, + self.document_generator_tool, + ] + self.llm: FunctionCallingLLM = llm or Settings.llm + assert isinstance(self.llm, FunctionCallingLLM) + self.memory = ChatMemoryBuffer.from_defaults(llm=self.llm) + + @step() + async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent: + self.stream = ev.get("stream", True) + user_msg = ev.get("user_msg") + chat_history = ev.get("chat_history") + + if chat_history is not None: + self.memory.put_messages(chat_history) + + # Add user message to memory + self.memory.put(ChatMessage(role=MessageRole.USER, content=user_msg)) + + if self.system_prompt: + system_msg = ChatMessage( + role=MessageRole.SYSTEM, content=self.system_prompt + ) + self.memory.put(system_msg) + + return InputEvent(input=self.memory.get()) + + @step() + async def handle_llm_input( # type: ignore + self, + ctx: Context, + ev: InputEvent, + ) -> ResearchEvent | AnalyzeEvent | ReportEvent | StopEvent: + """ + Handle an LLM input and decide the next step. + """ + # Always use the latest chat history from the input + chat_history: list[ChatMessage] = ev.input + + # Get tool calls + response = await chat_with_tools( + self.llm, + self.tools, # type: ignore + chat_history, + ) + if not response.has_tool_calls(): + if self.stream: + return StopEvent(result=response.generator) + else: + return StopEvent(result=await response.full_response()) + # calling different tools at the same time is not supported at the moment + # add an error message to tell the AI to process step by step + if response.is_calling_different_tools(): + self.memory.put( + ChatMessage( + role=MessageRole.ASSISTANT, + content="Cannot call different tools at the same time. Try calling one tool at a time.", + ) + ) + return InputEvent(input=self.memory.get()) + self.memory.put(response.tool_call_message) + match response.tool_name(): + case self.code_interpreter_tool.metadata.name: + return AnalyzeEvent(input=response.tool_calls) + case self.document_generator_tool.metadata.name: + return ReportEvent(input=response.tool_calls) + case self.query_engine_tool.metadata.name: + return ResearchEvent(input=response.tool_calls) + case _: + raise ValueError(f"Unknown tool: {response.tool_name()}") + + @step() + async def research(self, ctx: Context, ev: ResearchEvent) -> AnalyzeEvent: + """ + Do a research to gather information for the user's request. + A researcher should have these tools: query engine, search engine, etc. + """ + ctx.write_event_to_stream( + AgentRunEvent( + name="Researcher", + msg="Starting research", + ) + ) + tool_calls = ev.input + + tool_messages = await call_tools( + ctx=ctx, + agent_name="Researcher", + tools=[self.query_engine_tool], + tool_calls=tool_calls, + ) + self.memory.put_messages(tool_messages) + return AnalyzeEvent( + input=ChatMessage( + role=MessageRole.ASSISTANT, + content="I've finished the research. Please analyze the result.", + ), + ) + + @step() + async def analyze(self, ctx: Context, ev: AnalyzeEvent) -> InputEvent: + """ + Analyze the research result. + """ + ctx.write_event_to_stream( + AgentRunEvent( + name="Analyst", + msg="Starting analysis", + ) + ) + event_requested_by_workflow_llm = isinstance(ev.input, list) + # Requested by the workflow LLM Input step, it's a tool call + if event_requested_by_workflow_llm: + # Set the tool calls + tool_calls = ev.input + else: + # Otherwise, it's triggered by the research step + # Use a custom prompt and independent memory for the analyst agent + analysis_prompt = """ + You are a financial analyst, you are given a research result and a set of tools to help you. + Always use the given information, don't make up anything yourself. If there is not enough information, you can asking for more information. + If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report. + """ + # This is handled by analyst agent + # Clone the shared memory to avoid conflicting with the workflow. + chat_history = self.memory.get() + chat_history.append( + ChatMessage(role=MessageRole.SYSTEM, content=analysis_prompt) + ) + chat_history.append(ev.input) # type: ignore + # Check if the analyst agent needs to call tools + response = await chat_with_tools( + self.llm, + [self.code_interpreter_tool], + chat_history, + ) + if not response.has_tool_calls(): + # If no tool call, fallback analyst message to the workflow + analyst_msg = ChatMessage( + role=MessageRole.ASSISTANT, + content=await response.full_response(), + ) + self.memory.put(analyst_msg) + return InputEvent(input=self.memory.get()) + else: + # Set the tool calls and the tool call message to the memory + tool_calls = response.tool_calls + self.memory.put(response.tool_call_message) + + # Call tools + tool_messages = await call_tools( + ctx=ctx, + agent_name="Analyst", + tools=[self.code_interpreter_tool], + tool_calls=tool_calls, # type: ignore + ) + self.memory.put_messages(tool_messages) + + # Fallback to the input with the latest chat history + return InputEvent(input=self.memory.get()) + + @step() + async def report(self, ctx: Context, ev: ReportEvent) -> InputEvent: + """ + Generate a report based on the analysis result. + """ + ctx.write_event_to_stream( + AgentRunEvent( + name="Reporter", + msg="Starting report generation", + ) + ) + tool_calls = ev.input + tool_messages = await call_tools( + ctx=ctx, + agent_name="Reporter", + tools=[self.document_generator_tool], + tool_calls=tool_calls, + ) + self.memory.put_messages(tool_messages) + + # After the tool calls, fallback to the input with the latest chat history + return InputEvent(input=self.memory.get()) diff --git a/templates/components/agents/python/form_filling/app/workflows/form_filling.py b/templates/components/agents/python/form_filling/app/workflows/form_filling.py index 1eba7cb49..6078a5072 100644 --- a/templates/components/agents/python/form_filling/app/workflows/form_filling.py +++ b/templates/components/agents/python/form_filling/app/workflows/form_filling.py @@ -1,22 +1,33 @@ -from typing import Any, Dict, Optional, Type +from typing import Any, Dict, List, Optional from llama_index.core import Settings -from llama_index.core.agent.workflow import ( - AgentWorkflow, - FunctionAgent, - ReActAgent, +from llama_index.core.base.llms.types import ChatMessage, MessageRole +from llama_index.core.llms.function_calling import FunctionCallingLLM +from llama_index.core.memory import ChatMemoryBuffer +from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection +from llama_index.core.workflow import ( + Context, + Event, + StartEvent, + StopEvent, + Workflow, + step, ) -from llama_index.core.llms import LLM from app.engine.index import IndexConfig, get_index from app.engine.tools import ToolFactory from app.engine.tools.query_engine import get_query_engine_tool +from app.workflows.events import AgentRunEvent +from app.workflows.tools import ( + call_tools, + chat_with_tools, +) def create_workflow( params: Optional[Dict[str, Any]] = None, **kwargs, -) -> AgentWorkflow: +) -> Workflow: # Create query engine tool index_config = IndexConfig(**params) index = get_index(index_config) @@ -29,60 +40,197 @@ def create_workflow( extractor_tool = configured_tools.get("extract_questions") # type: ignore filling_tool = configured_tools.get("fill_form") # type: ignore - if extractor_tool is None or filling_tool is None: - raise ValueError("Extractor and filling tools are required.") - - agent_cls = _get_agent_cls_from_llm(Settings.llm) - - extractor_agent = agent_cls( - name="extractor", - description="An agent that extracts missing cells from CSV files and generates questions to fill them.", - tools=[extractor_tool], - system_prompt=""" - You are a helpful assistant who extracts missing cells from CSV files. - Only extract missing cells from CSV files and generate questions to fill them. - Always handoff the task to the `researcher` agent after extracting the questions. - """, - llm=Settings.llm, - can_handoff_to=["researcher"], + workflow = FormFillingWorkflow( + query_engine_tool=query_engine_tool, + extractor_tool=extractor_tool, # type: ignore + filling_tool=filling_tool, # type: ignore ) - researcher_agent = agent_cls( - name="researcher", - description="An agent that finds answers to questions about missing cells.", - tools=[query_engine_tool] if query_engine_tool else [], - system_prompt=""" - You are a researcher who finds answers to questions about missing cells. - Only use provided data - never make up any information yourself. Use N/A if an answer is not found. - Always handoff the task to the `processor` agent after finding the answers. - """, - llm=Settings.llm, - can_handoff_to=["processor"], - ) + return workflow - processor_agent = agent_cls( - name="processor", - description="An agent that fills missing cells with found answers.", - tools=[filling_tool], - system_prompt=""" - You are a processor who fills missing cells with found answers. - Fill N/A for any missing answers. - After filling the cells, tell the user about the results or any issues encountered. - """, - llm=Settings.llm, - ) - workflow = AgentWorkflow( - agents=[extractor_agent, researcher_agent, processor_agent], - root_agent="extractor", - verbose=True, - ) +class InputEvent(Event): + input: List[ChatMessage] + response: bool = False - return workflow +class ExtractMissingCellsEvent(Event): + tool_calls: list[ToolSelection] -def _get_agent_cls_from_llm(llm: LLM) -> Type[FunctionAgent | ReActAgent]: - if llm.metadata.is_function_calling_model: - return FunctionAgent - else: - return ReActAgent + +class FindAnswersEvent(Event): + tool_calls: list[ToolSelection] + + +class FillEvent(Event): + tool_calls: list[ToolSelection] + + +class FormFillingWorkflow(Workflow): + """ + A predefined workflow for filling missing cells in a CSV file. + Required tools: + - query_engine: A query engine to query for the answers to the questions. + - extract_question: Extract missing cells in a CSV file and generate questions to fill them. + - answer_question: Query for the answers to the questions. + + Flow: + 1. Extract missing cells in a CSV file and generate questions to fill them. + 2. Query for the answers to the questions. + 3. Fill the missing cells with the answers. + """ + + _default_system_prompt = """ + You are a helpful assistant who helps fill missing cells in a CSV file. + Only extract missing cells from CSV files. + Only use provided data - never make up any information yourself. Fill N/A if an answer is not found. + If there is no query engine tool or the gathered information has many N/A values indicating the questions don't match the data, respond with a warning and ask the user to upload a different file or connect to a knowledge base. + """ + stream: bool = True + + def __init__( + self, + query_engine_tool: Optional[QueryEngineTool], + extractor_tool: FunctionTool, + filling_tool: FunctionTool, + llm: Optional[FunctionCallingLLM] = None, + timeout: int = 360, + system_prompt: Optional[str] = None, + ): + super().__init__(timeout=timeout) + self.system_prompt = system_prompt or self._default_system_prompt + self.query_engine_tool = query_engine_tool + self.extractor_tool = extractor_tool + self.filling_tool = filling_tool + if self.extractor_tool is None or self.filling_tool is None: + raise ValueError("Extractor and filling tools are required.") + self.tools = [self.extractor_tool, self.filling_tool] + if self.query_engine_tool is not None: + self.tools.append(self.query_engine_tool) # type: ignore + self.llm: FunctionCallingLLM = llm or Settings.llm + if not isinstance(self.llm, FunctionCallingLLM): + raise ValueError("FormFillingWorkflow only supports FunctionCallingLLM.") + self.memory = ChatMemoryBuffer.from_defaults(llm=self.llm) + + @step() + async def start(self, ctx: Context, ev: StartEvent) -> InputEvent: + self.stream = ev.get("stream", True) + user_msg = ev.get("user_msg", "") + chat_history = ev.get("chat_history", []) + + if chat_history: + self.memory.put_messages(chat_history) + + self.memory.put(ChatMessage(role=MessageRole.USER, content=user_msg)) + + if self.system_prompt: + system_msg = ChatMessage( + role=MessageRole.SYSTEM, content=self.system_prompt + ) + self.memory.put(system_msg) + + return InputEvent(input=self.memory.get()) + + @step() + async def handle_llm_input( # type: ignore + self, + ctx: Context, + ev: InputEvent, + ) -> ExtractMissingCellsEvent | FillEvent | StopEvent: + """ + Handle an LLM input and decide the next step. + """ + chat_history: list[ChatMessage] = ev.input + response = await chat_with_tools( + self.llm, + self.tools, + chat_history, + ) + if not response.has_tool_calls(): + if self.stream: + return StopEvent(result=response.generator) + else: + return StopEvent(result=await response.full_response()) + # calling different tools at the same time is not supported at the moment + # add an error message to tell the AI to process step by step + if response.is_calling_different_tools(): + self.memory.put( + ChatMessage( + role=MessageRole.ASSISTANT, + content="Cannot call different tools at the same time. Try calling one tool at a time.", + ) + ) + return InputEvent(input=self.memory.get()) + self.memory.put(response.tool_call_message) + match response.tool_name(): + case self.extractor_tool.metadata.name: + return ExtractMissingCellsEvent(tool_calls=response.tool_calls) + case self.query_engine_tool.metadata.name: + return FindAnswersEvent(tool_calls=response.tool_calls) + case self.filling_tool.metadata.name: + return FillEvent(tool_calls=response.tool_calls) + case _: + raise ValueError(f"Unknown tool: {response.tool_name()}") + + @step() + async def extract_missing_cells( + self, ctx: Context, ev: ExtractMissingCellsEvent + ) -> InputEvent | FindAnswersEvent: + """ + Extract missing cells in a CSV file and generate questions to fill them. + """ + ctx.write_event_to_stream( + AgentRunEvent( + name="Extractor", + msg="Extracting missing cells", + ) + ) + # Call the extract questions tool + tool_messages = await call_tools( + agent_name="Extractor", + tools=[self.extractor_tool], + ctx=ctx, + tool_calls=ev.tool_calls, + ) + self.memory.put_messages(tool_messages) + return InputEvent(input=self.memory.get()) + + @step() + async def find_answers(self, ctx: Context, ev: FindAnswersEvent) -> InputEvent: + """ + Call answer questions tool to query for the answers to the questions. + """ + ctx.write_event_to_stream( + AgentRunEvent( + name="Researcher", + msg="Finding answers for missing cells", + ) + ) + tool_messages = await call_tools( + ctx=ctx, + agent_name="Researcher", + tools=[self.query_engine_tool], + tool_calls=ev.tool_calls, + ) + self.memory.put_messages(tool_messages) + return InputEvent(input=self.memory.get()) + + @step() + async def fill_cells(self, ctx: Context, ev: FillEvent) -> InputEvent: + """ + Call fill cells tool to fill the missing cells with the answers. + """ + ctx.write_event_to_stream( + AgentRunEvent( + name="Processor", + msg="Filling missing cells", + ) + ) + tool_messages = await call_tools( + agent_name="Processor", + tools=[self.filling_tool], + ctx=ctx, + tool_calls=ev.tool_calls, + ) + self.memory.put_messages(tool_messages) + return InputEvent(input=self.memory.get()) diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py index 37c305303..f46f43e19 100644 --- a/templates/components/multiagent/python/app/api/routers/chat.py +++ b/templates/components/multiagent/python/app/api/routers/chat.py @@ -5,7 +5,7 @@ from app.api.callbacks.llamacloud import LlamaCloudFileDownload from app.api.callbacks.next_question import SuggestNextQuestions from app.api.callbacks.stream_handler import StreamHandler -from app.api.callbacks.add_node_url import AddNodeUrl +from app.api.callbacks.source_nodes import AddNodeUrl from app.api.routers.models import ( ChatData, ) diff --git a/templates/components/multiagent/python/app/workflows/tools.py b/templates/components/multiagent/python/app/workflows/tools.py new file mode 100644 index 000000000..faab45955 --- /dev/null +++ b/templates/components/multiagent/python/app/workflows/tools.py @@ -0,0 +1,230 @@ +import logging +import uuid +from abc import ABC, abstractmethod +from typing import Any, AsyncGenerator, Callable, Optional + +from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole +from llama_index.core.llms.function_calling import FunctionCallingLLM +from llama_index.core.tools import ( + BaseTool, + FunctionTool, + ToolOutput, + ToolSelection, +) +from llama_index.core.workflow import Context +from pydantic import BaseModel, ConfigDict + +from app.workflows.events import AgentRunEvent, AgentRunEventType + +logger = logging.getLogger("uvicorn") + + +class ContextAwareTool(FunctionTool, ABC): + @abstractmethod + async def acall(self, ctx: Context, input: Any) -> ToolOutput: # type: ignore + pass + + +class ChatWithToolsResponse(BaseModel): + """ + A tool call response from chat_with_tools. + """ + + tool_calls: Optional[list[ToolSelection]] + tool_call_message: Optional[ChatMessage] + generator: Optional[AsyncGenerator[ChatResponse | None, None]] + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def is_calling_different_tools(self) -> bool: + tool_names = {tool_call.tool_name for tool_call in self.tool_calls} + return len(tool_names) > 1 + + def has_tool_calls(self) -> bool: + return self.tool_calls is not None and len(self.tool_calls) > 0 + + def tool_name(self) -> str: + assert self.has_tool_calls() + assert not self.is_calling_different_tools() + return self.tool_calls[0].tool_name + + async def full_response(self) -> str: + assert self.generator is not None + full_response = "" + async for chunk in self.generator: + content = chunk.message.content + if content: + full_response += content + return full_response + + +async def chat_with_tools( # type: ignore + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> ChatWithToolsResponse: + """ + Request LLM to call tools or not. + This function doesn't change the memory. + """ + generator = _tool_call_generator(llm, tools, chat_history) + is_tool_call = await generator.__anext__() + if is_tool_call: + # Last chunk is the full response + # Wait for the last chunk + full_response = None + async for chunk in generator: + full_response = chunk + assert isinstance(full_response, ChatResponse) + return ChatWithToolsResponse( + tool_calls=llm.get_tool_calls_from_response(full_response), + tool_call_message=full_response.message, + generator=None, + ) + else: + return ChatWithToolsResponse( + tool_calls=None, + tool_call_message=None, + generator=generator, + ) + + +async def call_tools( + ctx: Context, + agent_name: str, + tools: list[BaseTool], + tool_calls: list[ToolSelection], + emit_agent_events: bool = True, +) -> list[ChatMessage]: + if len(tool_calls) == 0: + return [] + + tools_by_name = {tool.metadata.get_name(): tool for tool in tools} + if len(tool_calls) == 1: + return [ + await call_tool( + ctx, + tools_by_name[tool_calls[0].tool_name], + tool_calls[0], + lambda msg: ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=msg, + ) + ), + ) + ] + # Multiple tool calls, show progress + tool_msgs: list[ChatMessage] = [] + + progress_id = str(uuid.uuid4()) + total_steps = len(tool_calls) + if emit_agent_events: + ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=f"Making {total_steps} tool calls", + ) + ) + for i, tool_call in enumerate(tool_calls): + tool = tools_by_name.get(tool_call.tool_name) + if not tool: + tool_msgs.append( + ChatMessage( + role=MessageRole.ASSISTANT, + content=f"Tool {tool_call.tool_name} does not exist", + ) + ) + continue + tool_msg = await call_tool( + ctx, + tool, + tool_call, + event_emitter=lambda msg: ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=msg, + event_type=AgentRunEventType.PROGRESS, + data={ + "id": progress_id, + "total": total_steps, + "current": i, + }, + ) + ), + ) + tool_msgs.append(tool_msg) + return tool_msgs + + +async def call_tool( + ctx: Context, + tool: BaseTool, + tool_call: ToolSelection, + event_emitter: Optional[Callable[[str], None]], +) -> ChatMessage: + if event_emitter: + event_emitter( + f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}" + ) + try: + if isinstance(tool, ContextAwareTool): + if ctx is None: + raise ValueError("Context is required for context aware tool") + # inject context for calling an context aware tool + response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs) + else: + response = await tool.acall(**tool_call.tool_kwargs) # type: ignore + return ChatMessage( + role=MessageRole.TOOL, + content=str(response.raw_output), + additional_kwargs={ + "tool_call_id": tool_call.tool_id, + "name": tool.metadata.get_name(), + }, + ) + except Exception as e: + logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}") + if event_emitter: + event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}") + return ChatMessage( + role=MessageRole.TOOL, + content=f"Error: {str(e)}", + additional_kwargs={ + "tool_call_id": tool_call.tool_id, + "name": tool.metadata.get_name(), + }, + ) + + +async def _tool_call_generator( + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> AsyncGenerator[ChatResponse | bool, None]: + response_stream = await llm.astream_chat_with_tools( + tools, + chat_history=chat_history, + allow_parallel_tool_calls=False, + ) + + full_response = None + yielded_indicator = False + async for chunk in response_stream: + if "tool_calls" not in chunk.message.additional_kwargs: + # Yield a boolean to indicate whether the response is a tool call + if not yielded_indicator: + yield False + yielded_indicator = True + + # if not a tool call, yield the chunks! + yield chunk # type: ignore + elif not yielded_indicator: + # Yield the indicator for a tool call + yield True + yielded_indicator = True + + full_response = chunk + + if full_response: + yield full_response # type: ignore From c7e4696191216dd02696b88f0f615bb557d88bba Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 19 Feb 2025 17:37:43 +0700 Subject: [PATCH 17/30] fix annotation message --- .../types/streaming/fastapi/app/api/routers/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/models.py b/templates/types/streaming/fastapi/app/api/routers/models.py index 6c766a427..0f647c325 100644 --- a/templates/types/streaming/fastapi/app/api/routers/models.py +++ b/templates/types/streaming/fastapi/app/api/routers/models.py @@ -103,7 +103,13 @@ def to_content(self) -> Optional[str]: class Message(BaseModel): role: MessageRole content: str - annotations: List[Annotation] | None = None + annotations: Optional[List[Annotation]] = None + + @validator("annotations", pre=True) + def validate_annotations(cls, v): + if v is None: + return v + return [item for item in v if isinstance(item, Annotation)] class ChatData(BaseModel): From c83fa960ad20e8d626c8a45e589cf336721513aa Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 19 Feb 2025 17:39:10 +0700 Subject: [PATCH 18/30] fix mypy --- templates/types/streaming/fastapi/app/api/routers/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/query.py b/templates/types/streaming/fastapi/app/api/routers/query.py index 70b9fb136..8a56b7d63 100644 --- a/templates/types/streaming/fastapi/app/api/routers/query.py +++ b/templates/types/streaming/fastapi/app/api/routers/query.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from app.engine.index import IndexConfig, get_index from llama_index.core.base.base_query_engine import BaseQueryEngine +from llama_index.core.base.response.schema import Response query_router = r = APIRouter() @@ -25,5 +26,5 @@ async def query_request( query: str, ) -> str: query_engine = get_query_engine() - response = await query_engine.aquery(query) + response: Response = await query_engine.aquery(query) return response.response From 25144dc37870e74e833f43c54719f79111f441cd Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 24 Feb 2025 16:19:01 +0700 Subject: [PATCH 19/30] add artifact tool component --- .../ui/chat/chat-message-content.tsx | 3 +- .../app/components/ui/chat/tools/artifact.tsx | 64 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx index 58507723a..44ae1dbc7 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx @@ -1,9 +1,9 @@ import { ChatMessage } from "@llamaindex/chat-ui"; import { DeepResearchCard } from "./custom/deep-research-card"; +import { ArtifactToolComponent } from "./tools/artifact"; import { ToolAnnotations } from "./tools/chat-tools"; import { ChatSourcesComponent, RetrieverComponent } from "./tools/query-index"; import { WeatherToolComponent } from "./tools/weather-card"; - export function ChatMessageContent() { return ( @@ -13,6 +13,7 @@ export function ChatMessageContent() { + diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx index fe6e81998..f17c273be 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx @@ -1,7 +1,13 @@ "use client"; +import { + getCustomAnnotation, + useChatMessage, + useChatUI, +} from "@llamaindex/chat-ui"; import { Check, ChevronDown, Code, Copy, Loader2 } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { z } from "zod"; import { Button, buttonVariants } from "../../button"; import { Collapsible, @@ -386,3 +392,59 @@ function closePanel() { panel.classList.add("hidden"); }); } + +const ArtifactToolSchema = z.object({ + tool_name: z.literal("artifact"), + tool_kwargs: z.object({ + query: z.string(), + }), + tool_id: z.string(), + tool_output: z.object({ + content: z.string(), + tool_name: z.string(), + raw_input: z.object({ + args: z.array(z.unknown()), + kwargs: z.object({ + query: z.string(), + }), + }), + raw_output: z.custom(), + is_error: z.boolean(), + }), + return_direct: z.boolean().optional(), +}); + +type ArtifactTool = z.infer; + +export function ArtifactToolComponent() { + const { message } = useChatMessage(); + const { messages } = useChatUI(); + + const artifactOutputEvent = getCustomAnnotation( + message.annotations, + (annotation: unknown) => { + const result = ArtifactToolSchema.safeParse(annotation); + return result.success; + }, + ).at(0); + + const artifactVersion = useMemo(() => { + const artifactToolCalls = messages.filter((m) => + m.annotations?.some( + (a: unknown) => (a as ArtifactTool).tool_name === "artifact", + ), + ); + return artifactToolCalls.length; + }, [messages]); + + return ( +
+ {artifactOutputEvent && ( + + )} +
+ ); +} From fe5982e4d2512feabafe022ae2441cb515b8b0f9 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 24 Feb 2025 17:25:10 +0700 Subject: [PATCH 20/30] fix render empty div --- .../ui/chat/custom/deep-research-card.tsx | 98 +++++++++-------- .../app/components/ui/chat/tools/artifact.tsx | 18 ++-- .../components/ui/chat/tools/query-index.tsx | 44 ++++---- .../components/ui/chat/tools/weather-card.tsx | 101 +++++++++--------- 4 files changed, 137 insertions(+), 124 deletions(-) diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx index bc6118e61..41aab5341 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/custom/deep-research-card.tsx @@ -157,53 +157,57 @@ export function DeepResearchCard({ className }: DeepResearchCardProps) { if (!state) return null; return ( - - - {state.retrieve.state !== null && ( - - - {state.retrieve.state === "inprogress" - ? "Searching..." - : "Search completed"} - - )} - {state.analyze.state !== null && ( - - - {state.analyze.state === "inprogress" ? "Analyzing..." : "Analysis"} - - )} - - - - {state.analyze.questions.length > 0 && ( - - {state.analyze.questions.map((question: QuestionState) => ( - - -
-
- {stateIcon[question.state]} + state.analyze.questions.length > 0 && ( + + + {state.retrieve.state !== null && ( + + + {state.retrieve.state === "inprogress" + ? "Searching..." + : "Search completed"} + + )} + {state.analyze.state !== null && ( + + + {state.analyze.state === "inprogress" + ? "Analyzing..." + : "Analysis"} + + )} + + + + {state.analyze.questions.length > 0 && ( + + {state.analyze.questions.map((question: QuestionState) => ( + + +
+
+ {stateIcon[question.state]} +
+ + {question.question} +
- - {question.question} - -
- - {question.answer && ( - - - - )} - - ))} - - )} - - + + {question.answer && ( + + + + )} + + ))} + + )} + + + ) ); } diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx index f17c273be..f3aff154e 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx @@ -438,13 +438,15 @@ export function ArtifactToolComponent() { }, [messages]); return ( -
- {artifactOutputEvent && ( - - )} -
+ artifactOutputEvent && ( +
+ {artifactOutputEvent && ( + + )} +
+ ) ); } diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx index 2897a019a..72328f59d 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx @@ -58,29 +58,31 @@ export function RetrieverComponent() { }, [queryIndexEvents]); return ( -
- {groupedIndexQueries.map(({ initial }) => { - const eventData = [ - { - title: `Searching index with query: ${initial.tool_kwargs.input}`, - }, - ]; + groupedIndexQueries.length > 0 && ( +
+ {groupedIndexQueries.map(({ initial }) => { + const eventData = [ + { + title: `Searching index with query: ${initial.tool_kwargs.input}`, + }, + ]; - if (initial.tool_output) { - eventData.push({ - title: `Got ${JSON.stringify((initial.tool_output?.raw_output as any).source_nodes?.length ?? 0)} sources for query: ${initial.tool_kwargs.input}`, - }); - } + if (initial.tool_output) { + eventData.push({ + title: `Got ${JSON.stringify((initial.tool_output?.raw_output as any).source_nodes?.length ?? 0)} sources for query: ${initial.tool_kwargs.input}`, + }); + } - return ( - - ); - })} -
+ return ( + + ); + })} +
+ ) ); } diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx index 1d126e81a..74e40cf70 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/weather-card.tsx @@ -182,38 +182,41 @@ export function WeatherCard({ data }: { data: WeatherData }) { ); return ( -
-
-
-
{currentDayString}
-
- - {data.current.temperature_2m} {data.current_units.temperature_2m} - - {weatherCodeDisplayMap[data.current.weather_code].icon} + data && ( +
+
+
+
{currentDayString}
+
+ + {data.current.temperature_2m}{" "} + {data.current_units.temperature_2m} + + {weatherCodeDisplayMap[data.current.weather_code].icon} +
+ + {weatherCodeDisplayMap[data.current.weather_code].status} +
- - {weatherCodeDisplayMap[data.current.weather_code].status} - -
-
- {data.daily.time.map((time, index) => { - if (index === 0) return null; // skip the current day - return ( -
- {displayDay(time)} -
- {weatherCodeDisplayMap[data.daily.weather_code[index]].icon} +
+ {data.daily.time.map((time, index) => { + if (index === 0) return null; // skip the current day + return ( +
+ {displayDay(time)} +
+ {weatherCodeDisplayMap[data.daily.weather_code[index]].icon} +
+ + {weatherCodeDisplayMap[data.daily.weather_code[index]].status} +
- - {weatherCodeDisplayMap[data.daily.weather_code[index]].status} - -
- ); - })} + ); + })} +
-
+ ) ); } @@ -268,29 +271,31 @@ export function WeatherToolComponent() { }, [weatherEvents]); return ( -
- {groupedWeatherQueries.map(({ initial }) => { - if (!initial.tool_output?.raw_output) { + groupedWeatherQueries.length > 0 && ( +
+ {groupedWeatherQueries.map(({ initial }) => { + if (!initial.tool_output?.raw_output) { + return ( + + ); + } + return ( - ); - } - - return ( - - ); - })} -
+ })} +
+ ) ); } From 1e90a6a1c55f506f8c7875239b3b58060b732834 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Mon, 24 Feb 2025 19:58:41 +0700 Subject: [PATCH 21/30] improve typing --- .../components/ui/chat/tools/query-index.tsx | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx index 72328f59d..23698d5d6 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/tools/query-index.tsx @@ -16,15 +16,25 @@ const QueryIndexSchema = z.object({ }), tool_id: z.string(), tool_output: z.optional( - z - .object({ - content: z.string(), - tool_name: z.string(), - raw_input: z.record(z.unknown()), - raw_output: z.record(z.unknown()), - is_error: z.boolean().optional(), - }) - .optional(), + z.object({ + content: z.string(), + tool_name: z.string(), + raw_output: z.object({ + source_nodes: z.array( + z.object({ + node: z.object({ + id_: z.string(), + metadata: z.object({ + url: z.string(), + }), + text: z.string(), + }), + score: z.number(), + }), + ), + }), + is_error: z.boolean().optional(), + }), ), return_direct: z.boolean().optional(), }); @@ -69,7 +79,7 @@ export function RetrieverComponent() { if (initial.tool_output) { eventData.push({ - title: `Got ${JSON.stringify((initial.tool_output?.raw_output as any).source_nodes?.length ?? 0)} sources for query: ${initial.tool_kwargs.input}`, + title: `Got ${JSON.stringify(initial.tool_output?.raw_output.source_nodes?.length ?? 0)} sources for query: ${initial.tool_kwargs.input}`, }); } @@ -103,8 +113,7 @@ export function ChatSourcesComponent() { const sources: SourceNode[] = useMemo(() => { return ( queryIndexEvents?.flatMap((event) => { - const sourceNodes = - (event.tool_output?.raw_output?.source_nodes as any[]) || []; + const sourceNodes = event.tool_output?.raw_output?.source_nodes || []; return sourceNodes.map((node) => { return { id: node.node.id_, From d38eb3c40506c293e27cc4daacbcd7e00a4449ee Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 11:02:10 +0700 Subject: [PATCH 22/30] unify chat.py file --- .../multiagent/python/app/api/routers/chat.py | 57 ------------------- .../streaming/fastapi/app/api/routers/chat.py | 54 +++--------------- .../fastapi/app/workflows/__init__.py | 1 + .../{engine/engine.py => workflows/agent.py} | 3 +- 4 files changed, 11 insertions(+), 104 deletions(-) delete mode 100644 templates/components/multiagent/python/app/api/routers/chat.py create mode 100644 templates/types/streaming/fastapi/app/workflows/__init__.py rename templates/types/streaming/fastapi/app/{engine/engine.py => workflows/agent.py} (96%) diff --git a/templates/components/multiagent/python/app/api/routers/chat.py b/templates/components/multiagent/python/app/api/routers/chat.py deleted file mode 100644 index f46f43e19..000000000 --- a/templates/components/multiagent/python/app/api/routers/chat.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging - -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status - -from app.api.callbacks.llamacloud import LlamaCloudFileDownload -from app.api.callbacks.next_question import SuggestNextQuestions -from app.api.callbacks.stream_handler import StreamHandler -from app.api.callbacks.source_nodes import AddNodeUrl -from app.api.routers.models import ( - ChatData, -) -from app.engine.query_filter import generate_filters -from app.workflows import create_workflow - -chat_router = r = APIRouter() - -logger = logging.getLogger("uvicorn") - - -@r.post("") -async def chat( - request: Request, - data: ChatData, - background_tasks: BackgroundTasks, -): - try: - last_message_content = data.get_last_message_content() - messages = data.get_history_messages(include_agent_messages=True) - - doc_ids = data.get_chat_document_ids() - filters = generate_filters(doc_ids) - params = data.data or {} - - workflow = create_workflow( - params=params, - filters=filters, - ) - - handler = workflow.run( - user_msg=last_message_content, - chat_history=messages, - stream=True, - ) - return StreamHandler.from_default( - handler=handler, - callbacks=[ - LlamaCloudFileDownload.from_default(background_tasks), - SuggestNextQuestions.from_default(data), - AddNodeUrl.from_default(), - ], - ).vercel_stream() - except Exception as e: - logger.exception("Error in chat engine", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error in chat engine: {e}", - ) from e diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 5094adec0..f46f43e19 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -1,28 +1,22 @@ -import json import logging from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status -from llama_index.core.agent.workflow import AgentOutput -from llama_index.core.llms import MessageRole from app.api.callbacks.llamacloud import LlamaCloudFileDownload from app.api.callbacks.next_question import SuggestNextQuestions -from app.api.callbacks.source_nodes import AddNodeUrl from app.api.callbacks.stream_handler import StreamHandler +from app.api.callbacks.source_nodes import AddNodeUrl from app.api.routers.models import ( ChatData, - Message, - Result, ) -from app.engine.engine import get_engine from app.engine.query_filter import generate_filters +from app.workflows import create_workflow chat_router = r = APIRouter() logger = logging.getLogger("uvicorn") -# streaming endpoint - delete if not needed @r.post("") async def chat( request: Request, @@ -31,16 +25,18 @@ async def chat( ): try: last_message_content = data.get_last_message_content() - messages = data.get_history_messages() + messages = data.get_history_messages(include_agent_messages=True) doc_ids = data.get_chat_document_ids() filters = generate_filters(doc_ids) params = data.data or {} - logger.info( - f"Creating chat engine with filters: {str(filters)}", + + workflow = create_workflow( + params=params, + filters=filters, ) - engine = get_engine(filters=filters, params=params) - handler = engine.run( + + handler = workflow.run( user_msg=last_message_content, chat_history=messages, stream=True, @@ -59,35 +55,3 @@ async def chat( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error in chat engine: {e}", ) from e - - -# non-streaming endpoint - delete if not needed -@r.post("/request") -async def chat_request( - data: ChatData, -) -> Result: - last_message_content = data.get_last_message_content() - messages = data.get_history_messages() - - doc_ids = data.get_chat_document_ids() - filters = generate_filters(doc_ids) - params = data.data or {} - logger.info( - f"Creating chat engine with filters: {str(filters)}", - ) - engine = get_engine(filters=filters, params=params) - - response = await engine.run( - user_msg=last_message_content, - chat_history=messages, - stream=False, - ) - output = response - if isinstance(output, AgentOutput): - content = output.response.content - else: - content = json.dumps(output) - - return Result( - result=Message(role=MessageRole.ASSISTANT, content=content), - ) diff --git a/templates/types/streaming/fastapi/app/workflows/__init__.py b/templates/types/streaming/fastapi/app/workflows/__init__.py new file mode 100644 index 000000000..f0172c6da --- /dev/null +++ b/templates/types/streaming/fastapi/app/workflows/__init__.py @@ -0,0 +1 @@ +from .agent import create_workflow diff --git a/templates/types/streaming/fastapi/app/engine/engine.py b/templates/types/streaming/fastapi/app/workflows/agent.py similarity index 96% rename from templates/types/streaming/fastapi/app/engine/engine.py rename to templates/types/streaming/fastapi/app/workflows/agent.py index c1fc25f2f..6dcfd76e5 100644 --- a/templates/types/streaming/fastapi/app/engine/engine.py +++ b/templates/types/streaming/fastapi/app/workflows/agent.py @@ -2,7 +2,6 @@ from typing import List from llama_index.core.agent.workflow import AgentWorkflow - from llama_index.core.settings import Settings from llama_index.core.tools import BaseTool @@ -11,7 +10,7 @@ from app.engine.tools.query_engine import get_query_engine_tool -def get_engine(params=None, **kwargs): +def create_workflow(params=None, **kwargs): if params is None: params = {} system_prompt = os.getenv("SYSTEM_PROMPT") From 9fd6d0c91daee9f4789673bce2bfd37b906534c8 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 11:15:11 +0700 Subject: [PATCH 23/30] remove multiagent folder (python) --- helpers/python.ts | 7 - .../app/workflows/deep_research.py | 2 +- .../deep_research/app/workflows/models.py | 15 ++ .../financial_report/app/workflows/events.py | 45 ++++ .../financial_report/app/workflows/tools.py | 230 ++++++++++++++++++ .../form_filling/app/workflows/events.py | 45 ++++ .../form_filling/app/workflows/tools.py | 230 ++++++++++++++++++ 7 files changed, 566 insertions(+), 8 deletions(-) create mode 100644 templates/components/agents/python/financial_report/app/workflows/events.py create mode 100644 templates/components/agents/python/financial_report/app/workflows/tools.py create mode 100644 templates/components/agents/python/form_filling/app/workflows/events.py create mode 100644 templates/components/agents/python/form_filling/app/workflows/tools.py diff --git a/helpers/python.ts b/helpers/python.ts index 72183b804..3937b7012 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -474,13 +474,6 @@ export const installPythonTemplate = async ({ await copyRouterCode(root, tools ?? []); } - // Copy multiagents overrides - if (template === "multiagent") { - await copy("**", path.join(root), { - cwd: path.join(compPath, "multiagent", "python"), - }); - } - if (template === "multiagent" || template === "reflex") { if (useCase) { const sourcePath = diff --git a/templates/components/agents/python/deep_research/app/workflows/deep_research.py b/templates/components/agents/python/deep_research/app/workflows/deep_research.py index 6af650826..17df08f73 100644 --- a/templates/components/agents/python/deep_research/app/workflows/deep_research.py +++ b/templates/components/agents/python/deep_research/app/workflows/deep_research.py @@ -18,13 +18,13 @@ from app.engine.index import IndexConfig, get_index from app.workflows.agents import plan_research, research, write_report -from app.workflows.events import SourceNodesEvent from app.workflows.models import ( CollectAnswersEvent, DataEvent, PlanResearchEvent, ReportEvent, ResearchEvent, + SourceNodesEvent, ) logger = logging.getLogger("uvicorn") diff --git a/templates/components/agents/python/deep_research/app/workflows/models.py b/templates/components/agents/python/deep_research/app/workflows/models.py index 0fe25b47a..34c40c91b 100644 --- a/templates/components/agents/python/deep_research/app/workflows/models.py +++ b/templates/components/agents/python/deep_research/app/workflows/models.py @@ -41,3 +41,18 @@ class DataEvent(Event): def to_response(self): return self.model_dump() + + +class SourceNodesEvent(Event): + nodes: List[NodeWithScore] + + def to_response(self): + return { + "type": "sources", + "data": { + "nodes": [ + SourceNodes.from_source_node(node).model_dump() + for node in self.nodes + ] + }, + } diff --git a/templates/components/agents/python/financial_report/app/workflows/events.py b/templates/components/agents/python/financial_report/app/workflows/events.py new file mode 100644 index 000000000..f74f26b7c --- /dev/null +++ b/templates/components/agents/python/financial_report/app/workflows/events.py @@ -0,0 +1,45 @@ +from enum import Enum +from typing import List, Optional + +from llama_index.core.schema import NodeWithScore +from llama_index.core.workflow import Event + +from app.api.routers.models import SourceNodes + + +class AgentRunEventType(Enum): + TEXT = "text" + PROGRESS = "progress" + + +class AgentRunEvent(Event): + name: str + msg: str + event_type: AgentRunEventType = AgentRunEventType.TEXT + data: Optional[dict] = None + + def to_response(self) -> dict: + return { + "type": "agent", + "data": { + "agent": self.name, + "type": self.event_type.value, + "text": self.msg, + "data": self.data, + }, + } + + +class SourceNodesEvent(Event): + nodes: List[NodeWithScore] + + def to_response(self): + return { + "type": "sources", + "data": { + "nodes": [ + SourceNodes.from_source_node(node).model_dump() + for node in self.nodes + ] + }, + } diff --git a/templates/components/agents/python/financial_report/app/workflows/tools.py b/templates/components/agents/python/financial_report/app/workflows/tools.py new file mode 100644 index 000000000..faab45955 --- /dev/null +++ b/templates/components/agents/python/financial_report/app/workflows/tools.py @@ -0,0 +1,230 @@ +import logging +import uuid +from abc import ABC, abstractmethod +from typing import Any, AsyncGenerator, Callable, Optional + +from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole +from llama_index.core.llms.function_calling import FunctionCallingLLM +from llama_index.core.tools import ( + BaseTool, + FunctionTool, + ToolOutput, + ToolSelection, +) +from llama_index.core.workflow import Context +from pydantic import BaseModel, ConfigDict + +from app.workflows.events import AgentRunEvent, AgentRunEventType + +logger = logging.getLogger("uvicorn") + + +class ContextAwareTool(FunctionTool, ABC): + @abstractmethod + async def acall(self, ctx: Context, input: Any) -> ToolOutput: # type: ignore + pass + + +class ChatWithToolsResponse(BaseModel): + """ + A tool call response from chat_with_tools. + """ + + tool_calls: Optional[list[ToolSelection]] + tool_call_message: Optional[ChatMessage] + generator: Optional[AsyncGenerator[ChatResponse | None, None]] + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def is_calling_different_tools(self) -> bool: + tool_names = {tool_call.tool_name for tool_call in self.tool_calls} + return len(tool_names) > 1 + + def has_tool_calls(self) -> bool: + return self.tool_calls is not None and len(self.tool_calls) > 0 + + def tool_name(self) -> str: + assert self.has_tool_calls() + assert not self.is_calling_different_tools() + return self.tool_calls[0].tool_name + + async def full_response(self) -> str: + assert self.generator is not None + full_response = "" + async for chunk in self.generator: + content = chunk.message.content + if content: + full_response += content + return full_response + + +async def chat_with_tools( # type: ignore + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> ChatWithToolsResponse: + """ + Request LLM to call tools or not. + This function doesn't change the memory. + """ + generator = _tool_call_generator(llm, tools, chat_history) + is_tool_call = await generator.__anext__() + if is_tool_call: + # Last chunk is the full response + # Wait for the last chunk + full_response = None + async for chunk in generator: + full_response = chunk + assert isinstance(full_response, ChatResponse) + return ChatWithToolsResponse( + tool_calls=llm.get_tool_calls_from_response(full_response), + tool_call_message=full_response.message, + generator=None, + ) + else: + return ChatWithToolsResponse( + tool_calls=None, + tool_call_message=None, + generator=generator, + ) + + +async def call_tools( + ctx: Context, + agent_name: str, + tools: list[BaseTool], + tool_calls: list[ToolSelection], + emit_agent_events: bool = True, +) -> list[ChatMessage]: + if len(tool_calls) == 0: + return [] + + tools_by_name = {tool.metadata.get_name(): tool for tool in tools} + if len(tool_calls) == 1: + return [ + await call_tool( + ctx, + tools_by_name[tool_calls[0].tool_name], + tool_calls[0], + lambda msg: ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=msg, + ) + ), + ) + ] + # Multiple tool calls, show progress + tool_msgs: list[ChatMessage] = [] + + progress_id = str(uuid.uuid4()) + total_steps = len(tool_calls) + if emit_agent_events: + ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=f"Making {total_steps} tool calls", + ) + ) + for i, tool_call in enumerate(tool_calls): + tool = tools_by_name.get(tool_call.tool_name) + if not tool: + tool_msgs.append( + ChatMessage( + role=MessageRole.ASSISTANT, + content=f"Tool {tool_call.tool_name} does not exist", + ) + ) + continue + tool_msg = await call_tool( + ctx, + tool, + tool_call, + event_emitter=lambda msg: ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=msg, + event_type=AgentRunEventType.PROGRESS, + data={ + "id": progress_id, + "total": total_steps, + "current": i, + }, + ) + ), + ) + tool_msgs.append(tool_msg) + return tool_msgs + + +async def call_tool( + ctx: Context, + tool: BaseTool, + tool_call: ToolSelection, + event_emitter: Optional[Callable[[str], None]], +) -> ChatMessage: + if event_emitter: + event_emitter( + f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}" + ) + try: + if isinstance(tool, ContextAwareTool): + if ctx is None: + raise ValueError("Context is required for context aware tool") + # inject context for calling an context aware tool + response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs) + else: + response = await tool.acall(**tool_call.tool_kwargs) # type: ignore + return ChatMessage( + role=MessageRole.TOOL, + content=str(response.raw_output), + additional_kwargs={ + "tool_call_id": tool_call.tool_id, + "name": tool.metadata.get_name(), + }, + ) + except Exception as e: + logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}") + if event_emitter: + event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}") + return ChatMessage( + role=MessageRole.TOOL, + content=f"Error: {str(e)}", + additional_kwargs={ + "tool_call_id": tool_call.tool_id, + "name": tool.metadata.get_name(), + }, + ) + + +async def _tool_call_generator( + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> AsyncGenerator[ChatResponse | bool, None]: + response_stream = await llm.astream_chat_with_tools( + tools, + chat_history=chat_history, + allow_parallel_tool_calls=False, + ) + + full_response = None + yielded_indicator = False + async for chunk in response_stream: + if "tool_calls" not in chunk.message.additional_kwargs: + # Yield a boolean to indicate whether the response is a tool call + if not yielded_indicator: + yield False + yielded_indicator = True + + # if not a tool call, yield the chunks! + yield chunk # type: ignore + elif not yielded_indicator: + # Yield the indicator for a tool call + yield True + yielded_indicator = True + + full_response = chunk + + if full_response: + yield full_response # type: ignore diff --git a/templates/components/agents/python/form_filling/app/workflows/events.py b/templates/components/agents/python/form_filling/app/workflows/events.py new file mode 100644 index 000000000..f74f26b7c --- /dev/null +++ b/templates/components/agents/python/form_filling/app/workflows/events.py @@ -0,0 +1,45 @@ +from enum import Enum +from typing import List, Optional + +from llama_index.core.schema import NodeWithScore +from llama_index.core.workflow import Event + +from app.api.routers.models import SourceNodes + + +class AgentRunEventType(Enum): + TEXT = "text" + PROGRESS = "progress" + + +class AgentRunEvent(Event): + name: str + msg: str + event_type: AgentRunEventType = AgentRunEventType.TEXT + data: Optional[dict] = None + + def to_response(self) -> dict: + return { + "type": "agent", + "data": { + "agent": self.name, + "type": self.event_type.value, + "text": self.msg, + "data": self.data, + }, + } + + +class SourceNodesEvent(Event): + nodes: List[NodeWithScore] + + def to_response(self): + return { + "type": "sources", + "data": { + "nodes": [ + SourceNodes.from_source_node(node).model_dump() + for node in self.nodes + ] + }, + } diff --git a/templates/components/agents/python/form_filling/app/workflows/tools.py b/templates/components/agents/python/form_filling/app/workflows/tools.py new file mode 100644 index 000000000..faab45955 --- /dev/null +++ b/templates/components/agents/python/form_filling/app/workflows/tools.py @@ -0,0 +1,230 @@ +import logging +import uuid +from abc import ABC, abstractmethod +from typing import Any, AsyncGenerator, Callable, Optional + +from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole +from llama_index.core.llms.function_calling import FunctionCallingLLM +from llama_index.core.tools import ( + BaseTool, + FunctionTool, + ToolOutput, + ToolSelection, +) +from llama_index.core.workflow import Context +from pydantic import BaseModel, ConfigDict + +from app.workflows.events import AgentRunEvent, AgentRunEventType + +logger = logging.getLogger("uvicorn") + + +class ContextAwareTool(FunctionTool, ABC): + @abstractmethod + async def acall(self, ctx: Context, input: Any) -> ToolOutput: # type: ignore + pass + + +class ChatWithToolsResponse(BaseModel): + """ + A tool call response from chat_with_tools. + """ + + tool_calls: Optional[list[ToolSelection]] + tool_call_message: Optional[ChatMessage] + generator: Optional[AsyncGenerator[ChatResponse | None, None]] + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def is_calling_different_tools(self) -> bool: + tool_names = {tool_call.tool_name for tool_call in self.tool_calls} + return len(tool_names) > 1 + + def has_tool_calls(self) -> bool: + return self.tool_calls is not None and len(self.tool_calls) > 0 + + def tool_name(self) -> str: + assert self.has_tool_calls() + assert not self.is_calling_different_tools() + return self.tool_calls[0].tool_name + + async def full_response(self) -> str: + assert self.generator is not None + full_response = "" + async for chunk in self.generator: + content = chunk.message.content + if content: + full_response += content + return full_response + + +async def chat_with_tools( # type: ignore + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> ChatWithToolsResponse: + """ + Request LLM to call tools or not. + This function doesn't change the memory. + """ + generator = _tool_call_generator(llm, tools, chat_history) + is_tool_call = await generator.__anext__() + if is_tool_call: + # Last chunk is the full response + # Wait for the last chunk + full_response = None + async for chunk in generator: + full_response = chunk + assert isinstance(full_response, ChatResponse) + return ChatWithToolsResponse( + tool_calls=llm.get_tool_calls_from_response(full_response), + tool_call_message=full_response.message, + generator=None, + ) + else: + return ChatWithToolsResponse( + tool_calls=None, + tool_call_message=None, + generator=generator, + ) + + +async def call_tools( + ctx: Context, + agent_name: str, + tools: list[BaseTool], + tool_calls: list[ToolSelection], + emit_agent_events: bool = True, +) -> list[ChatMessage]: + if len(tool_calls) == 0: + return [] + + tools_by_name = {tool.metadata.get_name(): tool for tool in tools} + if len(tool_calls) == 1: + return [ + await call_tool( + ctx, + tools_by_name[tool_calls[0].tool_name], + tool_calls[0], + lambda msg: ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=msg, + ) + ), + ) + ] + # Multiple tool calls, show progress + tool_msgs: list[ChatMessage] = [] + + progress_id = str(uuid.uuid4()) + total_steps = len(tool_calls) + if emit_agent_events: + ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=f"Making {total_steps} tool calls", + ) + ) + for i, tool_call in enumerate(tool_calls): + tool = tools_by_name.get(tool_call.tool_name) + if not tool: + tool_msgs.append( + ChatMessage( + role=MessageRole.ASSISTANT, + content=f"Tool {tool_call.tool_name} does not exist", + ) + ) + continue + tool_msg = await call_tool( + ctx, + tool, + tool_call, + event_emitter=lambda msg: ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=msg, + event_type=AgentRunEventType.PROGRESS, + data={ + "id": progress_id, + "total": total_steps, + "current": i, + }, + ) + ), + ) + tool_msgs.append(tool_msg) + return tool_msgs + + +async def call_tool( + ctx: Context, + tool: BaseTool, + tool_call: ToolSelection, + event_emitter: Optional[Callable[[str], None]], +) -> ChatMessage: + if event_emitter: + event_emitter( + f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}" + ) + try: + if isinstance(tool, ContextAwareTool): + if ctx is None: + raise ValueError("Context is required for context aware tool") + # inject context for calling an context aware tool + response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs) + else: + response = await tool.acall(**tool_call.tool_kwargs) # type: ignore + return ChatMessage( + role=MessageRole.TOOL, + content=str(response.raw_output), + additional_kwargs={ + "tool_call_id": tool_call.tool_id, + "name": tool.metadata.get_name(), + }, + ) + except Exception as e: + logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}") + if event_emitter: + event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}") + return ChatMessage( + role=MessageRole.TOOL, + content=f"Error: {str(e)}", + additional_kwargs={ + "tool_call_id": tool_call.tool_id, + "name": tool.metadata.get_name(), + }, + ) + + +async def _tool_call_generator( + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> AsyncGenerator[ChatResponse | bool, None]: + response_stream = await llm.astream_chat_with_tools( + tools, + chat_history=chat_history, + allow_parallel_tool_calls=False, + ) + + full_response = None + yielded_indicator = False + async for chunk in response_stream: + if "tool_calls" not in chunk.message.additional_kwargs: + # Yield a boolean to indicate whether the response is a tool call + if not yielded_indicator: + yield False + yielded_indicator = True + + # if not a tool call, yield the chunks! + yield chunk # type: ignore + elif not yielded_indicator: + # Yield the indicator for a tool call + yield True + yielded_indicator = True + + full_response = chunk + + if full_response: + yield full_response # type: ignore From d0f606d7f0c8c91ba84348780c7e8c0c3a35d7dc Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 11:16:48 +0700 Subject: [PATCH 24/30] fix linting --- templates/types/streaming/fastapi/app/workflows/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/types/streaming/fastapi/app/workflows/__init__.py b/templates/types/streaming/fastapi/app/workflows/__init__.py index f0172c6da..29c530646 100644 --- a/templates/types/streaming/fastapi/app/workflows/__init__.py +++ b/templates/types/streaming/fastapi/app/workflows/__init__.py @@ -1 +1,4 @@ from .agent import create_workflow + + +__all__ = ["create_workflow"] From 21b7df11d730ccbe5280d83a1a8f1980da7cb95a Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 11:20:51 +0700 Subject: [PATCH 25/30] fix missing import --- .../agents/python/deep_research/app/workflows/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/components/agents/python/deep_research/app/workflows/models.py b/templates/components/agents/python/deep_research/app/workflows/models.py index 34c40c91b..fa4414cef 100644 --- a/templates/components/agents/python/deep_research/app/workflows/models.py +++ b/templates/components/agents/python/deep_research/app/workflows/models.py @@ -4,6 +4,8 @@ from llama_index.core.workflow import Event from pydantic import BaseModel +from app.api.routers.models import SourceNodes + # Workflow events class PlanResearchEvent(Event): From c996508e2e75362425fa7a0fca84e1d6cc159155 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 11:38:43 +0700 Subject: [PATCH 26/30] support non-streaming api --- .../streaming/fastapi/app/api/routers/chat.py | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index f46f43e19..499a68875 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -4,8 +4,8 @@ from app.api.callbacks.llamacloud import LlamaCloudFileDownload from app.api.callbacks.next_question import SuggestNextQuestions -from app.api.callbacks.stream_handler import StreamHandler from app.api.callbacks.source_nodes import AddNodeUrl +from app.api.callbacks.stream_handler import StreamHandler from app.api.routers.models import ( ChatData, ) @@ -50,8 +50,41 @@ async def chat( ], ).vercel_stream() except Exception as e: - logger.exception("Error in chat engine", exc_info=True) + logger.exception("Error in chat", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error in chat: {e}", + ) from e + + +# non-streaming endpoint - delete if not needed +@r.post("/request") +async def chat_request( + request: Request, + data: ChatData, +): + try: + last_message_content = data.get_last_message_content() + messages = data.get_history_messages(include_agent_messages=True) + + doc_ids = data.get_chat_document_ids() + filters = generate_filters(doc_ids) + params = data.data or {} + + workflow = create_workflow( + params=params, + filters=filters, + ) + + handler = workflow.run( + user_msg=last_message_content, + chat_history=messages, + stream=False, + ) + return await handler + except Exception as e: + logger.exception("Error in chat request", exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error in chat engine: {e}", + detail=f"Error in chat request: {e}", ) from e From be5870c1fe834a07994c465e35433eafd6c62296 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 12:04:42 +0700 Subject: [PATCH 27/30] update citation prompt --- helpers/env-variables.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/helpers/env-variables.ts b/helpers/env-variables.ts index aa6103339..cf557695d 100644 --- a/helpers/env-variables.ts +++ b/helpers/env-variables.ts @@ -483,11 +483,12 @@ const getSystemPromptEnv = ( }); } if (tools?.length == 0 && (dataSources?.length ?? 0 > 0)) { - const citationPrompt = `'You have provided information from a knowledge base that has been passed to you in nodes of information. -Each node has useful metadata such as node ID, file name, page, etc. -Please add the citation to the data node for each sentence or paragraph that you reference in the provided information. -The citation format is: . [citation:]() -Where the is the unique identifier of the data node. + const citationPrompt = `'You have provided information from a knowledge base that separates the information into multiple nodes. +Always add a citation to each sentence or paragraph that you reference in the provided information using the node_id field in the header of each node. + +The citation format is: [citation:] +Where the is the node_id field in the header of each node. +Always separate the citation by a space. Example: We have two nodes: @@ -497,11 +498,9 @@ We have two nodes: node_id: abc file_name: animal.pdf -User question: Tell me a fun fact about Llama. -Your answer: -A baby llama is called "Cria" [citation:xyz](). -It often live in desert [citation:abc](). -It\\'s cute animal. +Your answer with citations: +A baby llama is called "Cria" [citation:xyz] +It often lives in desert [citation:abc] [citation:xyz] '`; systemPromptEnv.push({ name: "SYSTEM_CITATION_PROMPT", From 8004c9fe7dc36e034efd28c59d12a960f94e1437 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 13:30:23 +0700 Subject: [PATCH 28/30] remove dead code --- .../multiagent/python/app/workflows/events.py | 45 ---- .../multiagent/python/app/workflows/tools.py | 230 ------------------ 2 files changed, 275 deletions(-) delete mode 100644 templates/components/multiagent/python/app/workflows/events.py delete mode 100644 templates/components/multiagent/python/app/workflows/tools.py diff --git a/templates/components/multiagent/python/app/workflows/events.py b/templates/components/multiagent/python/app/workflows/events.py deleted file mode 100644 index f74f26b7c..000000000 --- a/templates/components/multiagent/python/app/workflows/events.py +++ /dev/null @@ -1,45 +0,0 @@ -from enum import Enum -from typing import List, Optional - -from llama_index.core.schema import NodeWithScore -from llama_index.core.workflow import Event - -from app.api.routers.models import SourceNodes - - -class AgentRunEventType(Enum): - TEXT = "text" - PROGRESS = "progress" - - -class AgentRunEvent(Event): - name: str - msg: str - event_type: AgentRunEventType = AgentRunEventType.TEXT - data: Optional[dict] = None - - def to_response(self) -> dict: - return { - "type": "agent", - "data": { - "agent": self.name, - "type": self.event_type.value, - "text": self.msg, - "data": self.data, - }, - } - - -class SourceNodesEvent(Event): - nodes: List[NodeWithScore] - - def to_response(self): - return { - "type": "sources", - "data": { - "nodes": [ - SourceNodes.from_source_node(node).model_dump() - for node in self.nodes - ] - }, - } diff --git a/templates/components/multiagent/python/app/workflows/tools.py b/templates/components/multiagent/python/app/workflows/tools.py deleted file mode 100644 index faab45955..000000000 --- a/templates/components/multiagent/python/app/workflows/tools.py +++ /dev/null @@ -1,230 +0,0 @@ -import logging -import uuid -from abc import ABC, abstractmethod -from typing import Any, AsyncGenerator, Callable, Optional - -from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole -from llama_index.core.llms.function_calling import FunctionCallingLLM -from llama_index.core.tools import ( - BaseTool, - FunctionTool, - ToolOutput, - ToolSelection, -) -from llama_index.core.workflow import Context -from pydantic import BaseModel, ConfigDict - -from app.workflows.events import AgentRunEvent, AgentRunEventType - -logger = logging.getLogger("uvicorn") - - -class ContextAwareTool(FunctionTool, ABC): - @abstractmethod - async def acall(self, ctx: Context, input: Any) -> ToolOutput: # type: ignore - pass - - -class ChatWithToolsResponse(BaseModel): - """ - A tool call response from chat_with_tools. - """ - - tool_calls: Optional[list[ToolSelection]] - tool_call_message: Optional[ChatMessage] - generator: Optional[AsyncGenerator[ChatResponse | None, None]] - - model_config = ConfigDict(arbitrary_types_allowed=True) - - def is_calling_different_tools(self) -> bool: - tool_names = {tool_call.tool_name for tool_call in self.tool_calls} - return len(tool_names) > 1 - - def has_tool_calls(self) -> bool: - return self.tool_calls is not None and len(self.tool_calls) > 0 - - def tool_name(self) -> str: - assert self.has_tool_calls() - assert not self.is_calling_different_tools() - return self.tool_calls[0].tool_name - - async def full_response(self) -> str: - assert self.generator is not None - full_response = "" - async for chunk in self.generator: - content = chunk.message.content - if content: - full_response += content - return full_response - - -async def chat_with_tools( # type: ignore - llm: FunctionCallingLLM, - tools: list[BaseTool], - chat_history: list[ChatMessage], -) -> ChatWithToolsResponse: - """ - Request LLM to call tools or not. - This function doesn't change the memory. - """ - generator = _tool_call_generator(llm, tools, chat_history) - is_tool_call = await generator.__anext__() - if is_tool_call: - # Last chunk is the full response - # Wait for the last chunk - full_response = None - async for chunk in generator: - full_response = chunk - assert isinstance(full_response, ChatResponse) - return ChatWithToolsResponse( - tool_calls=llm.get_tool_calls_from_response(full_response), - tool_call_message=full_response.message, - generator=None, - ) - else: - return ChatWithToolsResponse( - tool_calls=None, - tool_call_message=None, - generator=generator, - ) - - -async def call_tools( - ctx: Context, - agent_name: str, - tools: list[BaseTool], - tool_calls: list[ToolSelection], - emit_agent_events: bool = True, -) -> list[ChatMessage]: - if len(tool_calls) == 0: - return [] - - tools_by_name = {tool.metadata.get_name(): tool for tool in tools} - if len(tool_calls) == 1: - return [ - await call_tool( - ctx, - tools_by_name[tool_calls[0].tool_name], - tool_calls[0], - lambda msg: ctx.write_event_to_stream( - AgentRunEvent( - name=agent_name, - msg=msg, - ) - ), - ) - ] - # Multiple tool calls, show progress - tool_msgs: list[ChatMessage] = [] - - progress_id = str(uuid.uuid4()) - total_steps = len(tool_calls) - if emit_agent_events: - ctx.write_event_to_stream( - AgentRunEvent( - name=agent_name, - msg=f"Making {total_steps} tool calls", - ) - ) - for i, tool_call in enumerate(tool_calls): - tool = tools_by_name.get(tool_call.tool_name) - if not tool: - tool_msgs.append( - ChatMessage( - role=MessageRole.ASSISTANT, - content=f"Tool {tool_call.tool_name} does not exist", - ) - ) - continue - tool_msg = await call_tool( - ctx, - tool, - tool_call, - event_emitter=lambda msg: ctx.write_event_to_stream( - AgentRunEvent( - name=agent_name, - msg=msg, - event_type=AgentRunEventType.PROGRESS, - data={ - "id": progress_id, - "total": total_steps, - "current": i, - }, - ) - ), - ) - tool_msgs.append(tool_msg) - return tool_msgs - - -async def call_tool( - ctx: Context, - tool: BaseTool, - tool_call: ToolSelection, - event_emitter: Optional[Callable[[str], None]], -) -> ChatMessage: - if event_emitter: - event_emitter( - f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}" - ) - try: - if isinstance(tool, ContextAwareTool): - if ctx is None: - raise ValueError("Context is required for context aware tool") - # inject context for calling an context aware tool - response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs) - else: - response = await tool.acall(**tool_call.tool_kwargs) # type: ignore - return ChatMessage( - role=MessageRole.TOOL, - content=str(response.raw_output), - additional_kwargs={ - "tool_call_id": tool_call.tool_id, - "name": tool.metadata.get_name(), - }, - ) - except Exception as e: - logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}") - if event_emitter: - event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}") - return ChatMessage( - role=MessageRole.TOOL, - content=f"Error: {str(e)}", - additional_kwargs={ - "tool_call_id": tool_call.tool_id, - "name": tool.metadata.get_name(), - }, - ) - - -async def _tool_call_generator( - llm: FunctionCallingLLM, - tools: list[BaseTool], - chat_history: list[ChatMessage], -) -> AsyncGenerator[ChatResponse | bool, None]: - response_stream = await llm.astream_chat_with_tools( - tools, - chat_history=chat_history, - allow_parallel_tool_calls=False, - ) - - full_response = None - yielded_indicator = False - async for chunk in response_stream: - if "tool_calls" not in chunk.message.additional_kwargs: - # Yield a boolean to indicate whether the response is a tool call - if not yielded_indicator: - yield False - yielded_indicator = True - - # if not a tool call, yield the chunks! - yield chunk # type: ignore - elif not yielded_indicator: - # Yield the indicator for a tool call - yield True - yielded_indicator = True - - full_response = chunk - - if full_response: - yield full_response # type: ignore From b60618a11c189c9b77f1e595a1b1f26133adfd06 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 25 Feb 2025 15:28:18 +0700 Subject: [PATCH 29/30] remove dead code --- helpers/python.ts | 23 +------------------ .../financial_report/app/workflows/events.py | 20 +--------------- .../form_filling/app/workflows/events.py | 20 +--------------- .../fastapi/app/api/routers/events.py | 2 -- 4 files changed, 3 insertions(+), 62 deletions(-) diff --git a/helpers/python.ts b/helpers/python.ts index 3937b7012..701bc0832 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -444,32 +444,11 @@ export const installPythonTemplate = async ({ cwd: path.join(compPath, "settings", "python"), }); - // Copy services if (template == "streaming" || template == "multiagent") { + // Copy services await copy("**", path.join(root, "app", "api", "services"), { cwd: path.join(compPath, "services", "python"), }); - } - // Copy engine code - if (template === "streaming" || template === "multiagent") { - // Select and copy engine code based on data sources and tools - let engine; - // Multiagent always uses agent engine - if (template === "multiagent") { - engine = "agent"; - } else { - // For streaming, use chat engine by default - // Unless tools are selected, in which case use agent engine - if (dataSources.length > 0 && (!tools || tools.length === 0)) { - console.log( - "\nNo tools selected - use optimized context chat engine\n", - ); - engine = "chat"; - } else { - engine = "agent"; - } - } - // Copy router code await copyRouterCode(root, tools ?? []); } diff --git a/templates/components/agents/python/financial_report/app/workflows/events.py b/templates/components/agents/python/financial_report/app/workflows/events.py index f74f26b7c..f40e9e1ab 100644 --- a/templates/components/agents/python/financial_report/app/workflows/events.py +++ b/templates/components/agents/python/financial_report/app/workflows/events.py @@ -1,11 +1,8 @@ from enum import Enum -from typing import List, Optional +from typing import Optional -from llama_index.core.schema import NodeWithScore from llama_index.core.workflow import Event -from app.api.routers.models import SourceNodes - class AgentRunEventType(Enum): TEXT = "text" @@ -28,18 +25,3 @@ def to_response(self) -> dict: "data": self.data, }, } - - -class SourceNodesEvent(Event): - nodes: List[NodeWithScore] - - def to_response(self): - return { - "type": "sources", - "data": { - "nodes": [ - SourceNodes.from_source_node(node).model_dump() - for node in self.nodes - ] - }, - } diff --git a/templates/components/agents/python/form_filling/app/workflows/events.py b/templates/components/agents/python/form_filling/app/workflows/events.py index f74f26b7c..f40e9e1ab 100644 --- a/templates/components/agents/python/form_filling/app/workflows/events.py +++ b/templates/components/agents/python/form_filling/app/workflows/events.py @@ -1,11 +1,8 @@ from enum import Enum -from typing import List, Optional +from typing import Optional -from llama_index.core.schema import NodeWithScore from llama_index.core.workflow import Event -from app.api.routers.models import SourceNodes - class AgentRunEventType(Enum): TEXT = "text" @@ -28,18 +25,3 @@ def to_response(self) -> dict: "data": self.data, }, } - - -class SourceNodesEvent(Event): - nodes: List[NodeWithScore] - - def to_response(self): - return { - "type": "sources", - "data": { - "nodes": [ - SourceNodes.from_source_node(node).model_dump() - for node in self.nodes - ] - }, - } diff --git a/templates/types/streaming/fastapi/app/api/routers/events.py b/templates/types/streaming/fastapi/app/api/routers/events.py index 9d8c0eac5..d19196a72 100644 --- a/templates/types/streaming/fastapi/app/api/routers/events.py +++ b/templates/types/streaming/fastapi/app/api/routers/events.py @@ -99,8 +99,6 @@ def to_response(self): return None -# TODO: Add an adapter for workflow events -# and remove callback handler class EventCallbackHandler(BaseCallbackHandler): _aqueue: asyncio.Queue is_done: bool = False From 7514736bbde63df3d88f5a3faa31c9bbeea84bd4 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 26 Feb 2025 16:21:03 +0700 Subject: [PATCH 30/30] add comment --- .../nextjs/app/components/ui/chat/chat-message-content.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx index 44ae1dbc7..aed2b0df9 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx @@ -12,12 +12,18 @@ export function ChatMessageContent() { + {/* For backward compatibility with the events from AgentRunner + * ToolAnnotations will be removed when we migrate to AgentWorkflow completely + */} + {/* For backward compatibility with the events from AgentRunner. + * The Source component will be removed when we migrate to AgentWorkflow completely + */}