Merge pull request #50 from EoghanOConnor/mcp_server_script

UPDATE: Simple Agent script

Author: MichaelClifford

.env.example

@@ -1,3 +1,6 @@
-REMOTE_BASE_URL=""
-REMOTE_MCP_URL=""
-REMOTE_VDB_PROVIDER=""
+ANSIBLE_MCP_SERVER_URL= ""
+GITHUB_MCP_SERVER_URL= "" 
+OCP_MCP_SERVER_URL= "" 
+CUSTOM_MCP_SERVER_URL= "" 
+REMOTE_BASE_URL= ""
+REMOTE_VDB_PROVIDER= ""

mcp-servers/github/queries_github.json

@@ -38,5 +38,4 @@
       "tool_call": "list_inventories"
     }
   ]
-}
-
+}

mcp-servers/openshift/queries_openshift.json

@@ -19,4 +19,4 @@
         "tool_call": "approve_score"
       }
     ]
-  }
+  } 

mcp-servers/ansible/ansible_tool_queries.json renamed to tests/queries/anisble_queries.json

@@ -0,0 +1,61 @@
+{
+  "meta": {
+      "name": "GitHub",
+      "description": "A collection of queries for GitHub MCP Server.",
+      "note": "Each entry includes a user query and its corresponding tool function name. Useful for LLM tool selection evaluation."
+  },
+  "queries": [
+    {
+      "query": "Search for top 5 Python repositories related to 'llama', sorted by stars.",
+      "tool_call": "search_repositories"
+    },
+    {
+      "query": "Fetch the contents of the readme file from the main branch of the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_file_contents"
+    },
+    {
+      "query": "List the most recent 3 commits title of the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "list_commits"
+    },
+    {
+      "query": "List the current open issues in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "list_issues"
+    },
+    {
+      "query": "Search for issue mentioning 'streamlit app', only list first 3.",
+      "tool_call": "search_issues" 
+    },
+    {
+      "query": "Search for the user 'RHETbot' on GitHub and describe the user.",
+      "tool_call": "search_user"
+    },
+    {
+      "query": "Get the details of issue number 5 in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_issue"
+    },
+    {
+      "query": "Get the details of pull request number 18 in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_pull_request"
+    },
+    {
+      "query": "List the top 3 most recent pull requests in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "list_pull_requests"
+    },
+    {
+      "query": "Get the files changed in pull request number 20 in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_pull_request_files"
+    },
+    {
+      "query": "Get pull request status of pr number 20 in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_pull_request_status"
+    },
+    {
+      "query": "Get pull request comments of pr number 18 in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_pull_request_commits"
+    },
+    {
+      "query": "Get the reviews of pull request number 20 in the 'llama-stack-on-ocp' repository by 'redhat-et'.",
+      "tool_call": "get_pull_request_reviews"
+    }
+  ]
+}

mcp-servers/custom/queries_custom.json renamed to tests/queries/custom_queries.json

@@ -0,0 +1,30 @@
+{
+  "meta": {
+      "name": "OpenShift",
+      "description": "A collection of queries for OpenShift MCP Server.",
+      "model": "",
+      "note": "Each entry includes a user query and its corresponding tool function name. Useful for LLM tool selection evaluation."
+  },
+  "queries": [
+    {
+      "query": "What pods are running in the llama-serve namespace?",
+      "tool_call": "pods_list_in_namespace"
+    },
+    {
+      "query": "Can you delete the pod example in llama-serve namespace?",
+      "tool_call": "pods_delete"
+    },
+    {
+      "query": "Can you fetch the logs of the pod example in the llama-serve namespace?",
+      "tool_call": "pods_log"
+    },
+    {
+      "query": "Create a pod with the docker.io/hello-world image and expose port 8080 in the llama-serve namespace?",
+      "tool_call": "pods_run"
+    },
+    {
+      "query": "What are the events in llama-serve namespace??",
+      "tool_call": "events_list" 
+    }
+  ]
+}

tests/queries/github_queries.json

@@ -0,0 +1,100 @@
+# Code bellow written following examples here: https://llama-stack.readthedocs.io/en/latest/building_applications
+from llama_stack_client.lib.agents.agent import Agent
+from llama_stack_client.lib.agents.event_logger import EventLogger
+from llama_stack_client import LlamaStackClient
+import logging
+import os
+from dotenv import load_dotenv
+import json
+
+
+
+
+def test_mcp_server(file_path, REMOTE_MCP_URL, model, mcp_server_name):
+
+    # Open the JSON file and load its content
+    with open(file_path, 'r') as file:
+        data = json.load(file)
+
+    queries = data['queries']
+    user_prompts = [query['query'] for query in queries]
+
+    logger = logging.getLogger(__name__)
+    if not logger.hasHandlers():  
+        logger.setLevel(logging.INFO)
+        stream_handler = logging.StreamHandler()
+        stream_handler.setLevel(logging.INFO)
+        formatter = logging.Formatter('%(message)s')
+        stream_handler.setFormatter(formatter)
+        logger.addHandler(stream_handler)
+
+    logger.info(f"Testing {mcp_server_name} toolgroup ")
+    logger.info(f'Model is {model} \n \n')
+
+
+    # Connect to a llama stack server
+    base_url = os.getenv('REMOTE_BASE_URL')
+    client = LlamaStackClient(base_url=base_url)
+    logger.info(f"Connected to Llama Stack server @ {base_url} \n")
+
+    # Get tool info and register tools
+    registered_tools = client.tools.list()
+    registered_tools_identifiers = [t.identifier for t in registered_tools]
+    registered_toolgroups = [t.toolgroup_id for t in registered_tools]
+
+    if mcp_server_name not in registered_toolgroups:
+        # Register MCP tools
+        client.toolgroups.register(
+            toolgroup_id=mcp_server_name,
+            provider_id="model-context-protocol",
+            mcp_endpoint={"uri":REMOTE_MCP_URL},
+            )
+
+    logger.info(f"""Your Server has access the the following toolgroups: 
+    {set(registered_toolgroups)}
+    """)
+
+    for prompt in user_prompts:
+        # Create simple agent with tools
+        agent = Agent(
+            client,
+            model=model,
+            instructions = """You are a helpful assistant. You have access to a number of tools.
+            Whenever a tool is called, be sure return the Response in a friendly and helpful tone.
+            When you are asked to search the web you must use a tool. keep answer concise
+            """ ,
+            tools=[mcp_server_name],
+            tool_config={"tool_choice":"auto"},
+            sampling_params={"max_tokens":4096}
+        )
+        session_id = agent.create_session(session_name="Conversation_demo")
+        turn_response = agent.create_turn(
+            messages=[
+                {
+                    "role":"user",
+                    "content": prompt
+                }
+            ],
+            session_id=session_id
+        )
+        for log in EventLogger().log(turn_response):
+            log.print()
+    
+
+if __name__ == "__main__":
+    load_dotenv()
+
+    ansible_mcp_server_url = os.getenv('ANSIBLE_MCP_SERVER_URL')
+    github_mcp_server_url = os.getenv('GITHUB_MCP_SERVER_URL')
+    ocp_mcp_server_url = os.getenv('OCP_MCP_SERVER_URL')
+    custom_mcp_server_url = os.getenv('CUSTOM_MCP_SERVER_URL')
+    # Call the function with the file path
+    ansilbe_file_path = './queries/anisble_queries.json'
+    custom_file_path = './queries/custom_queries.json'
+    github_file_path = './queries/github_queries.json'
+    openshift_file_path = './queries/ocp_queries.json'
+
+    test_mcp_server(ansilbe_file_path,ansible_mcp_server_url,"meta-llama/Llama-3.2-3B-Instruct","mcp::ansible")
+    test_mcp_server(github_file_path,github_mcp_server_url,"ibm-granite/granite-3.2-8b-instruct","mcp::github")
+    test_mcp_server(openshift_file_path,ocp_mcp_server_url,"ibm-granite/granite-3.2-8b-instruct","mcp::openshift")
+    test_mcp_server(custom_file_path,custom_mcp_server_url,"meta-llama/Llama-3.2-3B-Instruct","mcp::custom_tool")