From fcce1417f556a0d0225033a79ec0af4701091e69 Mon Sep 17 00:00:00 2001 From: koch3092 Date: Wed, 12 Mar 2025 02:41:38 +0800 Subject: [PATCH 01/14] feat: add mcp sample --- owl/run_mcp.py | 107 ++++++++++++ owl/utils/async_role_playing.py | 240 ++++++++++++++++++++++++++ owl/utils/mcp/__init__.py | 0 owl/utils/mcp/mcp_servers_config.json | 20 +++ owl/utils/mcp/mcp_toolkit_manager.py | 77 +++++++++ owl/utils/mcp/servers/__init__.py | 0 owl/utils/mcp/servers/mcp_server.py | 136 +++++++++++++++ requirements.txt | 2 +- 8 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 owl/run_mcp.py create mode 100644 owl/utils/async_role_playing.py create mode 100644 owl/utils/mcp/__init__.py create mode 100644 owl/utils/mcp/mcp_servers_config.json create mode 100644 owl/utils/mcp/mcp_toolkit_manager.py create mode 100644 owl/utils/mcp/servers/__init__.py create mode 100755 owl/utils/mcp/servers/mcp_server.py diff --git a/owl/run_mcp.py b/owl/run_mcp.py new file mode 100644 index 0000000..a8f8e76 --- /dev/null +++ b/owl/run_mcp.py @@ -0,0 +1,107 @@ +# run_mcp.py + +import asyncio +import sys +from pathlib import Path +from typing import List + +from dotenv import load_dotenv + +from camel.models import ModelFactory +from camel.toolkits import MCPToolkit, FunctionTool +from camel.types import ModelPlatformType, ModelType +from camel.logger import set_log_level + +from utils.async_role_playing import OwlRolePlaying, run_society + +from utils.mcp.mcp_toolkit_manager import MCPToolkitManager + + +load_dotenv() +set_log_level(level="DEBUG") + + +async def construct_society( + question: str, + tools: List[FunctionTool], +) -> OwlRolePlaying: + """ + 构建一个多Agent的OwlRolePlaying实例。 + 这里的tools已经是用户想交给assistant使用的全部Tool集合。 + """ + # 1. 创建模型 + models = { + "user": ModelFactory.create( + model_platform=ModelPlatformType.OPENAI, + model_type=ModelType.GPT_4O, + model_config_dict={"temperature": 0}, + ), + "assistant": ModelFactory.create( + model_platform=ModelPlatformType.OPENAI, + model_type=ModelType.GPT_4O, + model_config_dict={"temperature": 0}, + ), + } + + # 2. 配置User和Assistant + user_agent_kwargs = {"model": models["user"]} + assistant_agent_kwargs = { + "model": models["assistant"], + "tools": tools, # 直接使用外部提供的全部tools + } + + # 3. 设置任务参数 + task_kwargs = { + "task_prompt": question, + "with_task_specify": False, + } + + # 4. 构造并返回OwlRolePlaying + society = OwlRolePlaying( + **task_kwargs, + user_role_name="user", + user_agent_kwargs=user_agent_kwargs, + assistant_role_name="assistant", + assistant_agent_kwargs=assistant_agent_kwargs, + ) + return society + + +async def main(): + # 准备MCP Servers + config_path = str( + Path(__file__).parent / "utils/mcp/mcp_servers_config.json" + ) + + manager = MCPToolkitManager.from_config(config_path) + + # 示例问题 + question = ( + "I'd like a academic report about Guohao Li, including his research " + "direction, published papers (up to 20), institutions, etc." + "Then organize the report in Markdown format and save it to my desktop" + ) + + # 在main中统一用async with把所有MCP连接打开 + async with manager.connection(): + # 这里 manager.is_connected() = True + # 获取合并后的tools + tools = manager.get_all_tools() + + # 构造Society + society = await construct_society(question, tools) + + # 运行对话 + answer, chat_history, token_count = await run_society(society) + + # 出了 with 块，这些toolkit就全部关闭 + # manager.is_connected() = False + + # 打印结果 + print(f"\033[94mAnswer: {answer}\033[0m") + print("Chat History:", chat_history) + print("Token Count:", token_count) + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/owl/utils/async_role_playing.py b/owl/utils/async_role_playing.py new file mode 100644 index 0000000..7a8edd2 --- /dev/null +++ b/owl/utils/async_role_playing.py @@ -0,0 +1,240 @@ +from typing import Optional, Tuple, List + +from camel.agents import ChatAgent +from camel.responses import ChatAgentResponse +from camel.messages.base import BaseMessage +from camel.societies import RolePlaying +from camel.logger import get_logger + + +from copy import deepcopy + +logger = get_logger(__name__) + + +class OwlRolePlaying(RolePlaying): + def __init__(self, **kwargs): + self.user_role_name = kwargs.get("user_role_name", "user") + self.assistant_role_name = kwargs.get("assistant_role_name", "assistant") + + self.output_language = kwargs.get("output_language", None) + + self.user_agent_kwargs = kwargs.get("user_agent_kwargs", {}) + self.assistant_agent_kwargs = kwargs.get("assistant_agent_kwargs", {}) + + super().__init__(**kwargs) + + init_user_sys_msg, init_assistant_sys_msg = self._construct_gaia_sys_msgs() + + self.assistant_agent: ChatAgent + self.user_agent: ChatAgent + self.assistant_sys_msg: Optional[BaseMessage] + self.user_sys_msg: Optional[BaseMessage] + + self._init_agents( + init_assistant_sys_msg, + init_user_sys_msg, + assistant_agent_kwargs=self.assistant_agent_kwargs, + user_agent_kwargs=self.user_agent_kwargs, + output_language=self.output_language, + # is_reasoning_task=self.is_reasoning_task + ) + + def _construct_gaia_sys_msgs(self): + user_system_prompt = f""" +===== RULES OF USER ===== +Never forget you are a user and I am a assistant. Never flip roles! You will always instruct me. We share a common interest in collaborating to successfully complete a task. +I must help you to complete a difficult task. +You must instruct me based on my expertise and your needs to solve the task step by step. The format of your instruction is: `Instruction: [YOUR INSTRUCTION]`, where "Instruction" describes a sub-task or question. +You must give me one instruction at a time. +I must write a response that appropriately solves the requested instruction. +You should instruct me not ask me questions. + +Please note that the task may be very complicated. Do not attempt to solve the task by single step. You must instruct me to find the answer step by step. +Here are some tips that will help you to give more valuable instructions about our task to me: + +- I have various tools to use, such as search toolkit, web browser simulation toolkit, document relevant toolkit, code execution toolkit, etc. Thus, You must think how human will solve the task step-by-step, and give me instructions just like that. For example, one may first use google search to get some initial information and the target url, then retrieve the content of the url, or do some web browser interaction to find the answer. +- Although the task is complex, the answer does exist. If you can’t find the answer using the current scheme, try to re-plan and use other ways to find the answer, e.g. using other tools or methods that can achieve similar results. +- Always remind me to verify my final answer about the overall task. This work can be done by using multiple tools(e.g., screenshots, webpage analysis, etc.), or something else. +- If I have written code, please remind me to run the code and get the result. +- Search results typically do not provide precise answers. It is not likely to find the answer directly using search toolkit only, the search query should be concise and focuses on finding sources rather than direct answers, as it always need to use other tools to further process the url, e.g. interact with the webpage, extract webpage content, etc. +- If the question mentions youtube video, in most cases you have to process the content of the mentioned video. +- For downloading files, you can either use the web browser simulation toolkit or write codes (for example, the github content can be downloaded via https://raw.githubusercontent.com/...). +- Flexibly write codes to solve some problems, such as excel relevant tasks. + + +Now, here is the overall task: {self.task_prompt}. Never forget our task! + +Now you must start to instruct me to solve the task step-by-step. Do not add anything else other than your instruction! +Keep giving me instructions until you think the task is completed. +When the task is completed, you must only reply with a single word . +Never say unless my responses have solved your task. + """ + + assistant_system_prompt = f""" +===== RULES OF ASSISTANT ===== +Never forget you are a assistant and I am a user. Never flip roles! Never instruct me! You have to utilize your available tools to solve the task I assigned. +We share a common interest in collaborating to successfully complete a complex task. +You must help me to complete the task. + +Here is our overall task: {self.task_prompt}. Never forget our task! + +I must instruct you based on your expertise and my needs to complete the task. An instruction is typically a sub-task or question. + +You must leverage your available tools, try your best to solve the problem, and explain your solutions. +Unless I say the task is completed, you should always start with: +Solution: [YOUR_SOLUTION] +[YOUR_SOLUTION] should be specific, including detailed explanations and provide preferable detailed implementations and examples and lists for task-solving. + +Please note that our overall task may be very complicated. Here are some tips that may help you solve the task: + +- If one way fails to provide an answer, try other ways or methods. The answer does exists. +- If the search snippet is unhelpful but the URL comes from an authoritative source, try visit the website for more details. +- When looking for specific numerical values (e.g., dollar amounts), prioritize reliable sources and avoid relying only on search snippets. +- When solving tasks that require web searches, check Wikipedia first before exploring other websites. +- When trying to solve math problems, you can try to write python code and use sympy library to solve the problem. +- Always verify the accuracy of your final answers! Try cross-checking the answers by other ways. (e.g., screenshots, webpage analysis, etc.). +- Do not be overly confident in your own knowledge. Searching can provide a broader perspective and help validate existing knowledge. +- After writing codes, do not forget to run the code and get the result. If it encounters an error, try to debug it. +- When a tool fails to run, or the code does not run correctly, never assume that it returns the correct result and continue to reason based on the assumption, because the assumed result cannot lead you to the correct answer. The right way is to think about the reason for the error and try again. +- Search results typically do not provide precise answers. It is not likely to find the answer directly using search toolkit only, the search query should be concise and focuses on finding sources rather than direct answers, as it always need to use other tools to further process the url, e.g. interact with the webpage, extract webpage content, etc. +- For downloading files, you can either use the web browser simulation toolkit or write codes. + + + """ + + user_sys_msg = BaseMessage.make_user_message( + role_name=self.user_role_name, content=user_system_prompt + ) + + assistant_sys_msg = BaseMessage.make_assistant_message( + role_name=self.assistant_role_name, content=assistant_system_prompt + ) + + return user_sys_msg, assistant_sys_msg + + async def astep( + self, + assistant_msg: BaseMessage + ) -> Tuple[ChatAgentResponse, ChatAgentResponse]: + user_response = await self.user_agent.astep(assistant_msg) + if user_response.terminated or user_response.msgs is None: + return ( + ChatAgentResponse(msgs=[], terminated=False, info={}), + ChatAgentResponse( + msgs=[], + terminated=user_response.terminated, + info=user_response.info, + ), + ) + user_msg = self._reduce_message_options(user_response.msgs) + + modified_user_msg = deepcopy(user_msg) + + if "TASK_DONE" not in user_msg.content: + modified_user_msg.content += f"""\n + Here are auxiliary information about the overall task, which may help you understand the intent of the current task: + + {self.task_prompt} + + If there are available tools and you want to call them, never say 'I will ...', but first call the tool and reply based on tool call's result, and tell me which tool you have called. + """ + + else: + # The task is done, and the assistant agent need to give the final answer about the original task + modified_user_msg.content += f"""\n + Now please make a final answer of the original task based on our conversation : {self.task_prompt} + """ + + assistant_response = await self.assistant_agent.astep(user_msg) + if assistant_response.terminated or assistant_response.msgs is None: + return ( + ChatAgentResponse( + msgs=[], + terminated=assistant_response.terminated, + info=assistant_response.info, + ), + ChatAgentResponse( + msgs=[user_msg], terminated=False, info=user_response.info + ), + ) + assistant_msg = self._reduce_message_options(assistant_response.msgs) + + modified_assistant_msg = deepcopy(assistant_msg) + if "TASK_DONE" not in user_msg.content: + modified_assistant_msg.content += f"""\n + Provide me with the next instruction and input (if needed) based on my response and our current task: {self.task_prompt} + Before producing the final answer, please check whether I have rechecked the final answer using different toolkit as much as possible. If not, please remind me to do that. + If I have written codes, remind me to run the codes. + If you think our task is done, reply with `TASK_DONE` to end our conversation. + """ + + return ( + ChatAgentResponse( + msgs=[assistant_msg], + terminated=assistant_response.terminated, + info=assistant_response.info, + ), + ChatAgentResponse( + msgs=[user_msg], + terminated=user_response.terminated, + info=user_response.info, + ), + ) + + +async def run_society( + society: OwlRolePlaying, + round_limit: int = 15, +) -> Tuple[str, List[dict], dict]: + overall_completion_token_count = 0 + overall_prompt_token_count = 0 + + chat_history = [] + init_prompt = """ + Now please give me instructions to solve over overall task step by step. If the task requires some specific knowledge, please instruct me to use tools to complete the task. + """ + input_msg = society.init_chat(init_prompt) + for _round in range(round_limit): + assistant_response, user_response = await society.astep(input_msg) + overall_prompt_token_count += ( + assistant_response.info["usage"]["completion_tokens"] + ) + overall_prompt_token_count += ( + assistant_response.info["usage"]["prompt_tokens"] + + user_response.info["usage"]["prompt_tokens"] + ) + + # convert tool call to dict + tool_call_records: List[dict] = [] + for tool_call in assistant_response.info["tool_calls"]: + tool_call_records.append(tool_call.as_dict()) + + _data = { + "user": user_response.msg.content, + "assistant": assistant_response.msg.content, + "tool_calls": tool_call_records, + } + + chat_history.append(_data) + logger.info(f"Round #{_round} user_response:\n {user_response.msgs[0].content}") + logger.info( + f"Round #{_round} assistant_response:\n {assistant_response.msgs[0].content}" + ) + + if ( + assistant_response.terminated + or user_response.terminated + or "TASK_DONE" in user_response.msg.content + ): + break + + input_msg = assistant_response.msg + + answer = chat_history[-1]["assistant"] + token_info = { + "completion_token_count": overall_completion_token_count, + "prompt_token_count": overall_prompt_token_count, + } + + return answer, chat_history, token_info \ No newline at end of file diff --git a/owl/utils/mcp/__init__.py b/owl/utils/mcp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/owl/utils/mcp/mcp_servers_config.json b/owl/utils/mcp/mcp_servers_config.json new file mode 100644 index 0000000..418f364 --- /dev/null +++ b/owl/utils/mcp/mcp_servers_config.json @@ -0,0 +1,20 @@ +{ + "mcpServers": { + "filesystem": { + "command": "mcp-filesystem-server", + "args": [ + "/Users/coco/Desktop", + "/Users/coco/Downloads" + ] + }, + "simple-arxiv": { + "command": "python", + "args": ["-m", "mcp_simple_arxiv"] + } + }, + "mcpWebServers": { + "weather": { + "url": "https://c9a9-89-185-25-132.ngrok-free.app/sse" + } + } +} \ No newline at end of file diff --git a/owl/utils/mcp/mcp_toolkit_manager.py b/owl/utils/mcp/mcp_toolkit_manager.py new file mode 100644 index 0000000..2e9fe8f --- /dev/null +++ b/owl/utils/mcp/mcp_toolkit_manager.py @@ -0,0 +1,77 @@ +import json +import os +from typing import List, Optional, AsyncGenerator + +from camel.toolkits import MCPToolkit +from contextlib import AsyncExitStack, asynccontextmanager + + +class MCPToolkitManager: + """ + 负责管理多个 MCPToolkit 实例，并提供统一的连接管理。 + """ + + def __init__(self, toolkits: List[MCPToolkit]): + self.toolkits = toolkits + self._exit_stack: Optional[AsyncExitStack] = None + self._connected = False + + + @staticmethod + def from_config(config_path: str) -> "MCPToolkitManager": + """从 JSON 配置文件加载 MCPToolkit 实例，并返回 MCPToolkitManager 实例。 + + :param config_path: JSON 配置文件路径 + :return: MCPToolkitManager 实例 + """ + with open(config_path, "r", encoding="utf-8") as f: + data = json.load(f) + + all_toolkits = [] + + # 处理本地 MCP 服务器 + mcp_servers = data.get("mcpServers", {}) + for name, cfg in mcp_servers.items(): + toolkit = MCPToolkit( + command_or_url=cfg["command"], + args=cfg.get("args", []), + env={**os.environ, **cfg.get("env", {})}, + timeout=cfg.get("timeout", None), + ) + all_toolkits.append(toolkit) + + # 处理远程 MCP Web 服务器 + mcp_web_servers = data.get("mcpWebServers", {}) + for name, cfg in mcp_web_servers.items(): + toolkit = MCPToolkit( + command_or_url=cfg["url"], + timeout=cfg.get("timeout", None), + ) + all_toolkits.append(toolkit) + + return MCPToolkitManager(all_toolkits) + + @asynccontextmanager + async def connection(self) -> AsyncGenerator["MCPToolkitManager", None]: + """统一打开多个 MCPToolkit 的连接，并在离开上下文时关闭。""" + self._exit_stack = AsyncExitStack() + try: + # 顺序进入每个 toolkit 的 async context + for tk in self.toolkits: + await self._exit_stack.enter_async_context(tk.connection()) + self._connected = True + yield self + finally: + self._connected = False + await self._exit_stack.aclose() + self._exit_stack = None + + def is_connected(self) -> bool: + return self._connected + + def get_all_tools(self): + """合并所有 MCPToolkit 提供的工具""" + all_tools = [] + for tk in self.toolkits: + all_tools.extend(tk.get_tools()) + return all_tools \ No newline at end of file diff --git a/owl/utils/mcp/servers/__init__.py b/owl/utils/mcp/servers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/owl/utils/mcp/servers/mcp_server.py b/owl/utils/mcp/servers/mcp_server.py new file mode 100755 index 0000000..7f02d57 --- /dev/null +++ b/owl/utils/mcp/servers/mcp_server.py @@ -0,0 +1,136 @@ +# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= +from typing import Any + +import httpx +from mcp.server.fastmcp import FastMCP + +mcp = FastMCP("weather") + +NWS_API_BASE = "https://api.weather.gov" +USER_AGENT = "weather-app/1.0" + + +async def make_nws_request(url: str) -> dict[str, Any] | None: + r"""Make a request to the NWS API with proper error handling.""" + headers = {"User-Agent": USER_AGENT, "Accept": "application/geo+json"} + async with httpx.AsyncClient() as client: + try: + response = await client.get(url, headers=headers, timeout=30.0) + response.raise_for_status() + return response.json() + except Exception: + return None + + +def format_alert(feature: dict) -> str: + r"""Format an alert feature into a readable string.""" + props = feature["properties"] + return f""" +Event: {props.get('event', 'Unknown')} +Area: {props.get('areaDesc', 'Unknown')} +Severity: {props.get('severity', 'Unknown')} +Description: {props.get('description', 'No description available')} +Instructions: {props.get('instruction', 'No specific instructions provided')} +""" + + +@mcp.tool() +async def get_alerts(state: str) -> str: + r"""Get weather alerts for a US state. + + Args: + state: Two-letter US state code (e.g. CA, NY) + """ + url = f"{NWS_API_BASE}/alerts/active/area/{state}" + data = await make_nws_request(url) + + if not data or "features" not in data: + return "Unable to fetch alerts or no alerts found." + + if not data["features"]: + return "No active alerts for this state." + + alerts = [format_alert(feature) for feature in data["features"]] + return "\n---\n".join(alerts) + + +@mcp.tool() +async def get_forecast(latitude: float, longitude: float) -> str: + r"""Get weather forecast for a location. + + Args: + latitude: Latitude of the location + longitude: Longitude of the location + """ + # First get the forecast grid endpoint + points_url = f"{NWS_API_BASE}/points/{latitude},{longitude}" + points_data = await make_nws_request(points_url) + + if not points_data: + return "Unable to fetch forecast data for this location." + + # Get the forecast URL from the points response + forecast_url = points_data["properties"]["forecast"] + forecast_data = await make_nws_request(forecast_url) + + if not forecast_data: + return "Unable to fetch detailed forecast." + + # Format the periods into a readable forecast + periods = forecast_data["properties"]["periods"] + forecasts = [] + for period in periods[:5]: # Only show next 5 periods + forecast = f""" +{period['name']}: +Temperature: {period['temperature']}°{period['temperatureUnit']} +Wind: {period['windSpeed']} {period['windDirection']} +Forecast: {period['detailedForecast']} +""" + forecasts.append(forecast) + + return "\n---\n".join(forecasts) + + +def main(transport: str = "stdio"): + r"""Weather MCP Server + + This server provides weather-related functionalities implemented via the Model Context Protocol (MCP). + It demonstrates how to establish interactions between AI models and external tools using MCP. + + The server supports two modes of operation: + + 1. stdio mode (default): + + - Communicates with clients via standard input/output streams, ideal for local command-line usage. + + - Example usage: python mcp_server.py [--transport stdio] + + 2. SSE mode (Server-Sent Events): + + - Communicates with clients over HTTP using server-sent events, suitable for persistent network connections. + + - Runs by default at http://127.0.0.1:8000. + + - Example usage: python mcp_server.py --transport sse + """ # noqa: E501 + if transport == 'stdio': + mcp.run(transport='stdio') + elif transport == 'sse': + mcp.run(transport='sse') + + +if __name__ == "__main__": + # Hardcoded to use stdio transport mode + main("stdio") diff --git a/requirements.txt b/requirements.txt index c279871..25f0fd7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -camel-ai[all]==0.2.23 +camel-ai[all]==0.2.24 chunkr-ai>=0.0.41 docx2markdown>=0.1.1 gradio>=3.50.2 From 76400e4f943b7df49792aa0e78edfe3e76941d77 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 12 Mar 2025 16:37:23 +0800 Subject: [PATCH 02/14] integrate class OwlRolePlaying --- owl/utils/async_role_playing.py | 240 ----------------------------- owl/utils/enhanced_role_playing.py | 85 +++++++++- 2 files changed, 77 insertions(+), 248 deletions(-) delete mode 100644 owl/utils/async_role_playing.py diff --git a/owl/utils/async_role_playing.py b/owl/utils/async_role_playing.py deleted file mode 100644 index 7a8edd2..0000000 --- a/owl/utils/async_role_playing.py +++ /dev/null @@ -1,240 +0,0 @@ -from typing import Optional, Tuple, List - -from camel.agents import ChatAgent -from camel.responses import ChatAgentResponse -from camel.messages.base import BaseMessage -from camel.societies import RolePlaying -from camel.logger import get_logger - - -from copy import deepcopy - -logger = get_logger(__name__) - - -class OwlRolePlaying(RolePlaying): - def __init__(self, **kwargs): - self.user_role_name = kwargs.get("user_role_name", "user") - self.assistant_role_name = kwargs.get("assistant_role_name", "assistant") - - self.output_language = kwargs.get("output_language", None) - - self.user_agent_kwargs = kwargs.get("user_agent_kwargs", {}) - self.assistant_agent_kwargs = kwargs.get("assistant_agent_kwargs", {}) - - super().__init__(**kwargs) - - init_user_sys_msg, init_assistant_sys_msg = self._construct_gaia_sys_msgs() - - self.assistant_agent: ChatAgent - self.user_agent: ChatAgent - self.assistant_sys_msg: Optional[BaseMessage] - self.user_sys_msg: Optional[BaseMessage] - - self._init_agents( - init_assistant_sys_msg, - init_user_sys_msg, - assistant_agent_kwargs=self.assistant_agent_kwargs, - user_agent_kwargs=self.user_agent_kwargs, - output_language=self.output_language, - # is_reasoning_task=self.is_reasoning_task - ) - - def _construct_gaia_sys_msgs(self): - user_system_prompt = f""" -===== RULES OF USER ===== -Never forget you are a user and I am a assistant. Never flip roles! You will always instruct me. We share a common interest in collaborating to successfully complete a task. -I must help you to complete a difficult task. -You must instruct me based on my expertise and your needs to solve the task step by step. The format of your instruction is: `Instruction: [YOUR INSTRUCTION]`, where "Instruction" describes a sub-task or question. -You must give me one instruction at a time. -I must write a response that appropriately solves the requested instruction. -You should instruct me not ask me questions. - -Please note that the task may be very complicated. Do not attempt to solve the task by single step. You must instruct me to find the answer step by step. -Here are some tips that will help you to give more valuable instructions about our task to me: - -- I have various tools to use, such as search toolkit, web browser simulation toolkit, document relevant toolkit, code execution toolkit, etc. Thus, You must think how human will solve the task step-by-step, and give me instructions just like that. For example, one may first use google search to get some initial information and the target url, then retrieve the content of the url, or do some web browser interaction to find the answer. -- Although the task is complex, the answer does exist. If you can’t find the answer using the current scheme, try to re-plan and use other ways to find the answer, e.g. using other tools or methods that can achieve similar results. -- Always remind me to verify my final answer about the overall task. This work can be done by using multiple tools(e.g., screenshots, webpage analysis, etc.), or something else. -- If I have written code, please remind me to run the code and get the result. -- Search results typically do not provide precise answers. It is not likely to find the answer directly using search toolkit only, the search query should be concise and focuses on finding sources rather than direct answers, as it always need to use other tools to further process the url, e.g. interact with the webpage, extract webpage content, etc. -- If the question mentions youtube video, in most cases you have to process the content of the mentioned video. -- For downloading files, you can either use the web browser simulation toolkit or write codes (for example, the github content can be downloaded via https://raw.githubusercontent.com/...). -- Flexibly write codes to solve some problems, such as excel relevant tasks. - - -Now, here is the overall task: {self.task_prompt}. Never forget our task! - -Now you must start to instruct me to solve the task step-by-step. Do not add anything else other than your instruction! -Keep giving me instructions until you think the task is completed. -When the task is completed, you must only reply with a single word . -Never say unless my responses have solved your task. - """ - - assistant_system_prompt = f""" -===== RULES OF ASSISTANT ===== -Never forget you are a assistant and I am a user. Never flip roles! Never instruct me! You have to utilize your available tools to solve the task I assigned. -We share a common interest in collaborating to successfully complete a complex task. -You must help me to complete the task. - -Here is our overall task: {self.task_prompt}. Never forget our task! - -I must instruct you based on your expertise and my needs to complete the task. An instruction is typically a sub-task or question. - -You must leverage your available tools, try your best to solve the problem, and explain your solutions. -Unless I say the task is completed, you should always start with: -Solution: [YOUR_SOLUTION] -[YOUR_SOLUTION] should be specific, including detailed explanations and provide preferable detailed implementations and examples and lists for task-solving. - -Please note that our overall task may be very complicated. Here are some tips that may help you solve the task: - -- If one way fails to provide an answer, try other ways or methods. The answer does exists. -- If the search snippet is unhelpful but the URL comes from an authoritative source, try visit the website for more details. -- When looking for specific numerical values (e.g., dollar amounts), prioritize reliable sources and avoid relying only on search snippets. -- When solving tasks that require web searches, check Wikipedia first before exploring other websites. -- When trying to solve math problems, you can try to write python code and use sympy library to solve the problem. -- Always verify the accuracy of your final answers! Try cross-checking the answers by other ways. (e.g., screenshots, webpage analysis, etc.). -- Do not be overly confident in your own knowledge. Searching can provide a broader perspective and help validate existing knowledge. -- After writing codes, do not forget to run the code and get the result. If it encounters an error, try to debug it. -- When a tool fails to run, or the code does not run correctly, never assume that it returns the correct result and continue to reason based on the assumption, because the assumed result cannot lead you to the correct answer. The right way is to think about the reason for the error and try again. -- Search results typically do not provide precise answers. It is not likely to find the answer directly using search toolkit only, the search query should be concise and focuses on finding sources rather than direct answers, as it always need to use other tools to further process the url, e.g. interact with the webpage, extract webpage content, etc. -- For downloading files, you can either use the web browser simulation toolkit or write codes. - - - """ - - user_sys_msg = BaseMessage.make_user_message( - role_name=self.user_role_name, content=user_system_prompt - ) - - assistant_sys_msg = BaseMessage.make_assistant_message( - role_name=self.assistant_role_name, content=assistant_system_prompt - ) - - return user_sys_msg, assistant_sys_msg - - async def astep( - self, - assistant_msg: BaseMessage - ) -> Tuple[ChatAgentResponse, ChatAgentResponse]: - user_response = await self.user_agent.astep(assistant_msg) - if user_response.terminated or user_response.msgs is None: - return ( - ChatAgentResponse(msgs=[], terminated=False, info={}), - ChatAgentResponse( - msgs=[], - terminated=user_response.terminated, - info=user_response.info, - ), - ) - user_msg = self._reduce_message_options(user_response.msgs) - - modified_user_msg = deepcopy(user_msg) - - if "TASK_DONE" not in user_msg.content: - modified_user_msg.content += f"""\n - Here are auxiliary information about the overall task, which may help you understand the intent of the current task: - - {self.task_prompt} - - If there are available tools and you want to call them, never say 'I will ...', but first call the tool and reply based on tool call's result, and tell me which tool you have called. - """ - - else: - # The task is done, and the assistant agent need to give the final answer about the original task - modified_user_msg.content += f"""\n - Now please make a final answer of the original task based on our conversation : {self.task_prompt} - """ - - assistant_response = await self.assistant_agent.astep(user_msg) - if assistant_response.terminated or assistant_response.msgs is None: - return ( - ChatAgentResponse( - msgs=[], - terminated=assistant_response.terminated, - info=assistant_response.info, - ), - ChatAgentResponse( - msgs=[user_msg], terminated=False, info=user_response.info - ), - ) - assistant_msg = self._reduce_message_options(assistant_response.msgs) - - modified_assistant_msg = deepcopy(assistant_msg) - if "TASK_DONE" not in user_msg.content: - modified_assistant_msg.content += f"""\n - Provide me with the next instruction and input (if needed) based on my response and our current task: {self.task_prompt} - Before producing the final answer, please check whether I have rechecked the final answer using different toolkit as much as possible. If not, please remind me to do that. - If I have written codes, remind me to run the codes. - If you think our task is done, reply with `TASK_DONE` to end our conversation. - """ - - return ( - ChatAgentResponse( - msgs=[assistant_msg], - terminated=assistant_response.terminated, - info=assistant_response.info, - ), - ChatAgentResponse( - msgs=[user_msg], - terminated=user_response.terminated, - info=user_response.info, - ), - ) - - -async def run_society( - society: OwlRolePlaying, - round_limit: int = 15, -) -> Tuple[str, List[dict], dict]: - overall_completion_token_count = 0 - overall_prompt_token_count = 0 - - chat_history = [] - init_prompt = """ - Now please give me instructions to solve over overall task step by step. If the task requires some specific knowledge, please instruct me to use tools to complete the task. - """ - input_msg = society.init_chat(init_prompt) - for _round in range(round_limit): - assistant_response, user_response = await society.astep(input_msg) - overall_prompt_token_count += ( - assistant_response.info["usage"]["completion_tokens"] - ) - overall_prompt_token_count += ( - assistant_response.info["usage"]["prompt_tokens"] - + user_response.info["usage"]["prompt_tokens"] - ) - - # convert tool call to dict - tool_call_records: List[dict] = [] - for tool_call in assistant_response.info["tool_calls"]: - tool_call_records.append(tool_call.as_dict()) - - _data = { - "user": user_response.msg.content, - "assistant": assistant_response.msg.content, - "tool_calls": tool_call_records, - } - - chat_history.append(_data) - logger.info(f"Round #{_round} user_response:\n {user_response.msgs[0].content}") - logger.info( - f"Round #{_round} assistant_response:\n {assistant_response.msgs[0].content}" - ) - - if ( - assistant_response.terminated - or user_response.terminated - or "TASK_DONE" in user_response.msg.content - ): - break - - input_msg = assistant_response.msg - - answer = chat_history[-1]["assistant"] - token_info = { - "completion_token_count": overall_completion_token_count, - "prompt_token_count": overall_prompt_token_count, - } - - return answer, chat_history, token_info \ No newline at end of file diff --git a/owl/utils/enhanced_role_playing.py b/owl/utils/enhanced_role_playing.py index 53b8b75..0382773 100644 --- a/owl/utils/enhanced_role_playing.py +++ b/owl/utils/enhanced_role_playing.py @@ -152,7 +152,7 @@ Please note that the task may be very complicated. Do not attempt to solve the t Here are some tips that will help you to give more valuable instructions about our task to me: - I have various tools to use, such as search toolkit, web browser simulation toolkit, document relevant toolkit, code execution toolkit, etc. Thus, You must think how human will solve the task step-by-step, and give me instructions just like that. For example, one may first use google search to get some initial information and the target url, then retrieve the content of the url, or do some web browser interaction to find the answer. -- Although the task is complex, the answer does exist. If you can’t find the answer using the current scheme, try to re-plan and use other ways to find the answer, e.g. using other tools or methods that can achieve similar results. +- Although the task is complex, the answer does exist. If you can't find the answer using the current scheme, try to re-plan and use other ways to find the answer, e.g. using other tools or methods that can achieve similar results. - Always remind me to verify my final answer about the overall task. This work can be done by using multiple tools(e.g., screenshots, webpage analysis, etc.), or something else. - If I have written code, please remind me to run the code and get the result. - Search results typically do not provide precise answers. It is not likely to find the answer directly using search toolkit only, the search query should be concise and focuses on finding sources rather than direct answers, as it always need to use other tools to further process the url, e.g. interact with the webpage, extract webpage content, etc. @@ -281,6 +281,75 @@ Please note that our overall task may be very complicated. Here are some tips th ), ) + async def astep( + self, + assistant_msg: BaseMessage + ) -> Tuple[ChatAgentResponse, ChatAgentResponse]: + user_response = await self.user_agent.astep(assistant_msg) + if user_response.terminated or user_response.msgs is None: + return ( + ChatAgentResponse(msgs=[], terminated=False, info={}), + ChatAgentResponse( + msgs=[], + terminated=user_response.terminated, + info=user_response.info, + ), + ) + user_msg = self._reduce_message_options(user_response.msgs) + + modified_user_msg = deepcopy(user_msg) + + if "TASK_DONE" not in user_msg.content: + modified_user_msg.content += f"""\n + Here are auxiliary information about the overall task, which may help you understand the intent of the current task: + + {self.task_prompt} + + If there are available tools and you want to call them, never say 'I will ...', but first call the tool and reply based on tool call's result, and tell me which tool you have called. + """ + + else: + # The task is done, and the assistant agent need to give the final answer about the original task + modified_user_msg.content += f"""\n + Now please make a final answer of the original task based on our conversation : {self.task_prompt} + """ + + assistant_response = await self.assistant_agent.astep(user_msg) + if assistant_response.terminated or assistant_response.msgs is None: + return ( + ChatAgentResponse( + msgs=[], + terminated=assistant_response.terminated, + info=assistant_response.info, + ), + ChatAgentResponse( + msgs=[user_msg], terminated=False, info=user_response.info + ), + ) + assistant_msg = self._reduce_message_options(assistant_response.msgs) + + modified_assistant_msg = deepcopy(assistant_msg) + if "TASK_DONE" not in user_msg.content: + modified_assistant_msg.content += f"""\n + Provide me with the next instruction and input (if needed) based on my response and our current task: {self.task_prompt} + Before producing the final answer, please check whether I have rechecked the final answer using different toolkit as much as possible. If not, please remind me to do that. + If I have written codes, remind me to run the codes. + If you think our task is done, reply with `TASK_DONE` to end our conversation. + """ + + return ( + ChatAgentResponse( + msgs=[assistant_msg], + terminated=assistant_response.terminated, + info=assistant_response.info, + ), + ChatAgentResponse( + msgs=[user_msg], + terminated=user_response.terminated, + info=user_response.info, + ), + ) + class OwlGAIARolePlaying(OwlRolePlaying): def __init__(self, **kwargs): @@ -374,22 +443,22 @@ class OwlGAIARolePlaying(OwlRolePlaying): ) -def run_society( - society: RolePlaying, round_limit: int = 15 +async def run_society( + society: OwlRolePlaying, + round_limit: int = 15, ) -> Tuple[str, List[dict], dict]: overall_completion_token_count = 0 overall_prompt_token_count = 0 chat_history = [] init_prompt = """ -Now please give me instructions to solve over overall task step by step. If the task requires some specific knowledge, please instruct me to use tools to complete the task. - """ + Now please give me instructions to solve over overall task step by step. If the task requires some specific knowledge, please instruct me to use tools to complete the task. + """ input_msg = society.init_chat(init_prompt) for _round in range(round_limit): - assistant_response, user_response = society.step(input_msg) - overall_completion_token_count += ( + assistant_response, user_response = await society.astep(input_msg) + overall_prompt_token_count += ( assistant_response.info["usage"]["completion_tokens"] - + user_response.info["usage"]["completion_tokens"] ) overall_prompt_token_count += ( assistant_response.info["usage"]["prompt_tokens"] From e5efc9cf7cebbbc27bb7bdb03cf348d1b44c95e7 Mon Sep 17 00:00:00 2001 From: koch3092 Date: Wed, 12 Mar 2025 16:42:33 +0800 Subject: [PATCH 03/14] style: reformat code style --- owl/run_mcp.py | 33 +++++++-------------------- owl/utils/mcp/mcp_servers_config.json | 6 +---- owl/utils/mcp/mcp_toolkit_manager.py | 29 +++++++++++++++-------- 3 files changed, 28 insertions(+), 40 deletions(-) diff --git a/owl/run_mcp.py b/owl/run_mcp.py index a8f8e76..a2d5832 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -1,14 +1,11 @@ -# run_mcp.py - import asyncio -import sys from pathlib import Path from typing import List from dotenv import load_dotenv from camel.models import ModelFactory -from camel.toolkits import MCPToolkit, FunctionTool +from camel.toolkits import FunctionTool from camel.types import ModelPlatformType, ModelType from camel.logger import set_log_level @@ -25,11 +22,12 @@ async def construct_society( question: str, tools: List[FunctionTool], ) -> OwlRolePlaying: + r"""build a multi-agent OwlRolePlaying instance. + + Args: + question (str): The question to ask. + tools (List[FunctionTool]): The MCP tools to use. """ - 构建一个多Agent的OwlRolePlaying实例。 - 这里的tools已经是用户想交给assistant使用的全部Tool集合。 - """ - # 1. 创建模型 models = { "user": ModelFactory.create( model_platform=ModelPlatformType.OPENAI, @@ -43,20 +41,17 @@ async def construct_society( ), } - # 2. 配置User和Assistant user_agent_kwargs = {"model": models["user"]} assistant_agent_kwargs = { "model": models["assistant"], - "tools": tools, # 直接使用外部提供的全部tools + "tools": tools, } - # 3. 设置任务参数 task_kwargs = { "task_prompt": question, "with_task_specify": False, } - # 4. 构造并返回OwlRolePlaying society = OwlRolePlaying( **task_kwargs, user_role_name="user", @@ -68,39 +63,27 @@ async def construct_society( async def main(): - # 准备MCP Servers config_path = str( Path(__file__).parent / "utils/mcp/mcp_servers_config.json" ) manager = MCPToolkitManager.from_config(config_path) - # 示例问题 question = ( "I'd like a academic report about Guohao Li, including his research " "direction, published papers (up to 20), institutions, etc." "Then organize the report in Markdown format and save it to my desktop" ) - # 在main中统一用async with把所有MCP连接打开 + # Connect to all MCP toolkits async with manager.connection(): - # 这里 manager.is_connected() = True - # 获取合并后的tools tools = manager.get_all_tools() - # 构造Society society = await construct_society(question, tools) - # 运行对话 answer, chat_history, token_count = await run_society(society) - # 出了 with 块，这些toolkit就全部关闭 - # manager.is_connected() = False - - # 打印结果 print(f"\033[94mAnswer: {answer}\033[0m") - print("Chat History:", chat_history) - print("Token Count:", token_count) if __name__ == "__main__": diff --git a/owl/utils/mcp/mcp_servers_config.json b/owl/utils/mcp/mcp_servers_config.json index 418f364..bf22d59 100644 --- a/owl/utils/mcp/mcp_servers_config.json +++ b/owl/utils/mcp/mcp_servers_config.json @@ -12,9 +12,5 @@ "args": ["-m", "mcp_simple_arxiv"] } }, - "mcpWebServers": { - "weather": { - "url": "https://c9a9-89-185-25-132.ngrok-free.app/sse" - } - } + "mcpWebServers": {} } \ No newline at end of file diff --git a/owl/utils/mcp/mcp_toolkit_manager.py b/owl/utils/mcp/mcp_toolkit_manager.py index 2e9fe8f..b7ceff6 100644 --- a/owl/utils/mcp/mcp_toolkit_manager.py +++ b/owl/utils/mcp/mcp_toolkit_manager.py @@ -7,8 +7,12 @@ from contextlib import AsyncExitStack, asynccontextmanager class MCPToolkitManager: - """ - 负责管理多个 MCPToolkit 实例，并提供统一的连接管理。 + r"""MCPToolkitManager is a class for managing multiple MCPToolkit + instances and providing unified connection management. + + Attributes: + toolkits (List[MCPToolkit]): A list of MCPToolkit instances to be + managed. """ def __init__(self, toolkits: List[MCPToolkit]): @@ -19,17 +23,21 @@ class MCPToolkitManager: @staticmethod def from_config(config_path: str) -> "MCPToolkitManager": - """从 JSON 配置文件加载 MCPToolkit 实例，并返回 MCPToolkitManager 实例。 + r"""Loads an MCPToolkit instance from a JSON configuration file and + returns an MCPToolkitManager instance. - :param config_path: JSON 配置文件路径 - :return: MCPToolkitManager 实例 + Args: + config_path (str): The path to the JSON configuration file. + + Returns: + MCPToolkitManager: The MCPToolkitManager instance. """ with open(config_path, "r", encoding="utf-8") as f: data = json.load(f) all_toolkits = [] - # 处理本地 MCP 服务器 + # "mcpServers" is the MCP server configuration running as stdio mode mcp_servers = data.get("mcpServers", {}) for name, cfg in mcp_servers.items(): toolkit = MCPToolkit( @@ -40,7 +48,7 @@ class MCPToolkitManager: ) all_toolkits.append(toolkit) - # 处理远程 MCP Web 服务器 + # "mcpWebServers" is the MCP server configuration running as sse mode mcp_web_servers = data.get("mcpWebServers", {}) for name, cfg in mcp_web_servers.items(): toolkit = MCPToolkit( @@ -53,10 +61,10 @@ class MCPToolkitManager: @asynccontextmanager async def connection(self) -> AsyncGenerator["MCPToolkitManager", None]: - """统一打开多个 MCPToolkit 的连接，并在离开上下文时关闭。""" + r"""Connect multiple MCPToolkit instances and close them when + leaving""" self._exit_stack = AsyncExitStack() try: - # 顺序进入每个 toolkit 的 async context for tk in self.toolkits: await self._exit_stack.enter_async_context(tk.connection()) self._connected = True @@ -67,10 +75,11 @@ class MCPToolkitManager: self._exit_stack = None def is_connected(self) -> bool: + r"""Returns whether the MCPToolkitManager is connected.""" return self._connected def get_all_tools(self): - """合并所有 MCPToolkit 提供的工具""" + r"""Returns all tools from all MCPToolkit instances.""" all_tools = [] for tk in self.toolkits: all_tools.extend(tk.get_tools()) From 9880cd7b8649d8a15746d6b6358801775258a7ee Mon Sep 17 00:00:00 2001 From: koch3092 Date: Wed, 12 Mar 2025 16:46:22 +0800 Subject: [PATCH 04/14] chore: update requirements.txt --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 25f0fd7..d281407 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -camel-ai[all]==0.2.24 +camel-ai[all]==0.2.26 chunkr-ai>=0.0.41 docx2markdown>=0.1.1 gradio>=3.50.2 +mcp==1.3.0 +mcp-simple-arxiv==0.2.2 \ No newline at end of file From e40839733fd05990173f3b5db0922fa21b6f7f1a Mon Sep 17 00:00:00 2001 From: koch3092 Date: Wed, 12 Mar 2025 17:36:03 +0800 Subject: [PATCH 05/14] chore: update mcp-filesystem-server's arguments to the common value --- owl/utils/mcp/mcp_servers_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/owl/utils/mcp/mcp_servers_config.json b/owl/utils/mcp/mcp_servers_config.json index bf22d59..accd678 100644 --- a/owl/utils/mcp/mcp_servers_config.json +++ b/owl/utils/mcp/mcp_servers_config.json @@ -3,8 +3,8 @@ "filesystem": { "command": "mcp-filesystem-server", "args": [ - "/Users/coco/Desktop", - "/Users/coco/Downloads" + "/Users/username/Desktop", + "/Users/username/Downloads" ] }, "simple-arxiv": { From 40574fb0b9e32ecdbce548facae68123c95b813d Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 11:09:18 +0800 Subject: [PATCH 06/14] add run_mcp instructions and new requirements --- owl/run_mcp.py | 75 ++++++++++++++++++++++++++- owl/utils/mcp/mcp_servers_config.json | 7 ++- requirements.txt | 3 +- 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/owl/run_mcp.py b/owl/run_mcp.py index a2d5832..84c201d 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -1,3 +1,74 @@ +"""MCP Multi-Agent System Example + +This example demonstrates how to use MCP (Model Context Protocol) with CAMEL agents +for advanced information retrieval and processing tasks. + +Environment Setup: +1. Configure the required dependencies of owl library. + +2. Go Environment (v1.23.2+): + ```bash + # Verify Go installation + go version + + # Add Go binary path to PATH + export PATH=$PATH:~/go/bin + # Note: Add to ~/.bashrc or ~/.zshrc for persistence + ``` + +3. Playwright Setup: + ```bash + # Install Node.js and npm first + npm install -g @executeautomation/playwright-mcp-server + npx playwright install-deps + + # Configure in mcp_servers_config.json: + { + "mcpServers": { + "playwright": { + "command": "npx", + "args": ["-y", "@executeautomation/playwright-mcp-server"] + } + } + } + ``` + +4. MCP Filesystem Server Setup: + ```bash + # Install MCP filesystem server + go install github.com/mark3labs/mcp-filesystem-server@latest + npm install -g @modelcontextprotocol/server-filesystem + + # Configure mcp_servers_config.json in owl/utils/mcp/ + { + "mcpServers": { + "filesystem": { + "command": "mcp-filesystem-server", + "args": [ + "/home/your_path", + "/home/your_path" + ], + "type": "filesystem" + } + } + } + ``` + +Usage: +1. Ensure all MCP servers are properly configured in mcp_servers_config.json +2. Run this script to create a multi-agent system that can: + - Access and manipulate files through MCP filesystem server + - Perform web automation tasks using Playwright + - Process and generate information using GPT-4o +3. The system will execute the specified task while maintaining security through + relative paths and controlled access + +Note: +- All file operations are restricted to configured directories +- System uses GPT-4o for both user and assistant roles +- Supports asynchronous operations for efficient processing +""" + import asyncio from pathlib import Path from typing import List @@ -9,7 +80,7 @@ from camel.toolkits import FunctionTool from camel.types import ModelPlatformType, ModelType from camel.logger import set_log_level -from utils.async_role_playing import OwlRolePlaying, run_society +from utils.enhanced_role_playing import OwlRolePlaying, run_society from utils.mcp.mcp_toolkit_manager import MCPToolkitManager @@ -71,7 +142,7 @@ async def main(): question = ( "I'd like a academic report about Guohao Li, including his research " - "direction, published papers (up to 20), institutions, etc." + "direction, published papers (At least 3), institutions, etc." "Then organize the report in Markdown format and save it to my desktop" ) diff --git a/owl/utils/mcp/mcp_servers_config.json b/owl/utils/mcp/mcp_servers_config.json index accd678..f51fac0 100644 --- a/owl/utils/mcp/mcp_servers_config.json +++ b/owl/utils/mcp/mcp_servers_config.json @@ -7,10 +7,15 @@ "/Users/username/Downloads" ] }, + "playwright": { + "command": "npx", + "args": ["-y", "@executeautomation/playwright-mcp-server"] + }, "simple-arxiv": { "command": "python", "args": ["-m", "mcp_simple_arxiv"] } }, "mcpWebServers": {} -} \ No newline at end of file +} + diff --git a/requirements.txt b/requirements.txt index 644a765..3a3e5e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ chunkr-ai>=0.0.41 docx2markdown>=0.1.1 gradio>=3.50.2 mcp==1.3.0 -mcp-simple-arxiv==0.2.2 \ No newline at end of file +mcp-simple-arxiv==0.2.2 +mcp-server-fetch==2025.1.17 \ No newline at end of file From 9349cda00811173ef70cfd943de0d56ab3b49706 Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 16:31:11 +0800 Subject: [PATCH 07/14] Modify the running example to adapt to the camel style --- owl/run_mcp.py | 16 +++--- owl/utils/mcp/mcp_toolkit_manager.py | 86 ---------------------------- 2 files changed, 9 insertions(+), 93 deletions(-) delete mode 100644 owl/utils/mcp/mcp_toolkit_manager.py diff --git a/owl/run_mcp.py b/owl/run_mcp.py index 84c201d..4ae556b 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -79,10 +79,10 @@ from camel.models import ModelFactory from camel.toolkits import FunctionTool from camel.types import ModelPlatformType, ModelType from camel.logger import set_log_level +from camel.toolkits import MCPToolkit from utils.enhanced_role_playing import OwlRolePlaying, run_society -from utils.mcp.mcp_toolkit_manager import MCPToolkitManager load_dotenv() @@ -138,7 +138,7 @@ async def main(): Path(__file__).parent / "utils/mcp/mcp_servers_config.json" ) - manager = MCPToolkitManager.from_config(config_path) + mcp_toolkit = MCPToolkit(config_path=config_path) question = ( "I'd like a academic report about Guohao Li, including his research " @@ -146,16 +146,18 @@ async def main(): "Then organize the report in Markdown format and save it to my desktop" ) - # Connect to all MCP toolkits - async with manager.connection(): - tools = manager.get_all_tools() + await mcp_toolkit.connect() - society = await construct_society(question, tools) + # # Connect to all MCP toolkits + tools = [*mcp_toolkit.get_tools()] - answer, chat_history, token_count = await run_society(society) + society = await construct_society(question, tools) + + answer, chat_history, token_count = await run_society(society) print(f"\033[94mAnswer: {answer}\033[0m") + await mcp_toolkit.disconnect() if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/owl/utils/mcp/mcp_toolkit_manager.py b/owl/utils/mcp/mcp_toolkit_manager.py deleted file mode 100644 index b7ceff6..0000000 --- a/owl/utils/mcp/mcp_toolkit_manager.py +++ /dev/null @@ -1,86 +0,0 @@ -import json -import os -from typing import List, Optional, AsyncGenerator - -from camel.toolkits import MCPToolkit -from contextlib import AsyncExitStack, asynccontextmanager - - -class MCPToolkitManager: - r"""MCPToolkitManager is a class for managing multiple MCPToolkit - instances and providing unified connection management. - - Attributes: - toolkits (List[MCPToolkit]): A list of MCPToolkit instances to be - managed. - """ - - def __init__(self, toolkits: List[MCPToolkit]): - self.toolkits = toolkits - self._exit_stack: Optional[AsyncExitStack] = None - self._connected = False - - - @staticmethod - def from_config(config_path: str) -> "MCPToolkitManager": - r"""Loads an MCPToolkit instance from a JSON configuration file and - returns an MCPToolkitManager instance. - - Args: - config_path (str): The path to the JSON configuration file. - - Returns: - MCPToolkitManager: The MCPToolkitManager instance. - """ - with open(config_path, "r", encoding="utf-8") as f: - data = json.load(f) - - all_toolkits = [] - - # "mcpServers" is the MCP server configuration running as stdio mode - mcp_servers = data.get("mcpServers", {}) - for name, cfg in mcp_servers.items(): - toolkit = MCPToolkit( - command_or_url=cfg["command"], - args=cfg.get("args", []), - env={**os.environ, **cfg.get("env", {})}, - timeout=cfg.get("timeout", None), - ) - all_toolkits.append(toolkit) - - # "mcpWebServers" is the MCP server configuration running as sse mode - mcp_web_servers = data.get("mcpWebServers", {}) - for name, cfg in mcp_web_servers.items(): - toolkit = MCPToolkit( - command_or_url=cfg["url"], - timeout=cfg.get("timeout", None), - ) - all_toolkits.append(toolkit) - - return MCPToolkitManager(all_toolkits) - - @asynccontextmanager - async def connection(self) -> AsyncGenerator["MCPToolkitManager", None]: - r"""Connect multiple MCPToolkit instances and close them when - leaving""" - self._exit_stack = AsyncExitStack() - try: - for tk in self.toolkits: - await self._exit_stack.enter_async_context(tk.connection()) - self._connected = True - yield self - finally: - self._connected = False - await self._exit_stack.aclose() - self._exit_stack = None - - def is_connected(self) -> bool: - r"""Returns whether the MCPToolkitManager is connected.""" - return self._connected - - def get_all_tools(self): - r"""Returns all tools from all MCPToolkit instances.""" - all_tools = [] - for tk in self.toolkits: - all_tools.extend(tk.get_tools()) - return all_tools \ No newline at end of file From e3e74011934364976535965c23906246a328742e Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 17:05:23 +0800 Subject: [PATCH 08/14] delete Old version of mcp implementation --- owl/{utils/mcp => }/mcp_servers_config.json | 0 owl/run_mcp.py | 6 +- owl/utils/mcp/__init__.py | 0 owl/utils/mcp/servers/__init__.py | 0 owl/utils/mcp/servers/mcp_server.py | 136 -------------------- 5 files changed, 2 insertions(+), 140 deletions(-) rename owl/{utils/mcp => }/mcp_servers_config.json (100%) delete mode 100644 owl/utils/mcp/__init__.py delete mode 100644 owl/utils/mcp/servers/__init__.py delete mode 100755 owl/utils/mcp/servers/mcp_server.py diff --git a/owl/utils/mcp/mcp_servers_config.json b/owl/mcp_servers_config.json similarity index 100% rename from owl/utils/mcp/mcp_servers_config.json rename to owl/mcp_servers_config.json diff --git a/owl/run_mcp.py b/owl/run_mcp.py index 4ae556b..570a03a 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -134,11 +134,9 @@ async def construct_society( async def main(): - config_path = str( - Path(__file__).parent / "utils/mcp/mcp_servers_config.json" - ) + config_path = Path(__file__).parent / "mcp_servers_config.json" - mcp_toolkit = MCPToolkit(config_path=config_path) + mcp_toolkit = MCPToolkit(config_path=str(config_path)) question = ( "I'd like a academic report about Guohao Li, including his research " diff --git a/owl/utils/mcp/__init__.py b/owl/utils/mcp/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/owl/utils/mcp/servers/__init__.py b/owl/utils/mcp/servers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/owl/utils/mcp/servers/mcp_server.py b/owl/utils/mcp/servers/mcp_server.py deleted file mode 100755 index 7f02d57..0000000 --- a/owl/utils/mcp/servers/mcp_server.py +++ /dev/null @@ -1,136 +0,0 @@ -# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= -from typing import Any - -import httpx -from mcp.server.fastmcp import FastMCP - -mcp = FastMCP("weather") - -NWS_API_BASE = "https://api.weather.gov" -USER_AGENT = "weather-app/1.0" - - -async def make_nws_request(url: str) -> dict[str, Any] | None: - r"""Make a request to the NWS API with proper error handling.""" - headers = {"User-Agent": USER_AGENT, "Accept": "application/geo+json"} - async with httpx.AsyncClient() as client: - try: - response = await client.get(url, headers=headers, timeout=30.0) - response.raise_for_status() - return response.json() - except Exception: - return None - - -def format_alert(feature: dict) -> str: - r"""Format an alert feature into a readable string.""" - props = feature["properties"] - return f""" -Event: {props.get('event', 'Unknown')} -Area: {props.get('areaDesc', 'Unknown')} -Severity: {props.get('severity', 'Unknown')} -Description: {props.get('description', 'No description available')} -Instructions: {props.get('instruction', 'No specific instructions provided')} -""" - - -@mcp.tool() -async def get_alerts(state: str) -> str: - r"""Get weather alerts for a US state. - - Args: - state: Two-letter US state code (e.g. CA, NY) - """ - url = f"{NWS_API_BASE}/alerts/active/area/{state}" - data = await make_nws_request(url) - - if not data or "features" not in data: - return "Unable to fetch alerts or no alerts found." - - if not data["features"]: - return "No active alerts for this state." - - alerts = [format_alert(feature) for feature in data["features"]] - return "\n---\n".join(alerts) - - -@mcp.tool() -async def get_forecast(latitude: float, longitude: float) -> str: - r"""Get weather forecast for a location. - - Args: - latitude: Latitude of the location - longitude: Longitude of the location - """ - # First get the forecast grid endpoint - points_url = f"{NWS_API_BASE}/points/{latitude},{longitude}" - points_data = await make_nws_request(points_url) - - if not points_data: - return "Unable to fetch forecast data for this location." - - # Get the forecast URL from the points response - forecast_url = points_data["properties"]["forecast"] - forecast_data = await make_nws_request(forecast_url) - - if not forecast_data: - return "Unable to fetch detailed forecast." - - # Format the periods into a readable forecast - periods = forecast_data["properties"]["periods"] - forecasts = [] - for period in periods[:5]: # Only show next 5 periods - forecast = f""" -{period['name']}: -Temperature: {period['temperature']}°{period['temperatureUnit']} -Wind: {period['windSpeed']} {period['windDirection']} -Forecast: {period['detailedForecast']} -""" - forecasts.append(forecast) - - return "\n---\n".join(forecasts) - - -def main(transport: str = "stdio"): - r"""Weather MCP Server - - This server provides weather-related functionalities implemented via the Model Context Protocol (MCP). - It demonstrates how to establish interactions between AI models and external tools using MCP. - - The server supports two modes of operation: - - 1. stdio mode (default): - - - Communicates with clients via standard input/output streams, ideal for local command-line usage. - - - Example usage: python mcp_server.py [--transport stdio] - - 2. SSE mode (Server-Sent Events): - - - Communicates with clients over HTTP using server-sent events, suitable for persistent network connections. - - - Runs by default at http://127.0.0.1:8000. - - - Example usage: python mcp_server.py --transport sse - """ # noqa: E501 - if transport == 'stdio': - mcp.run(transport='stdio') - elif transport == 'sse': - mcp.run(transport='sse') - - -if __name__ == "__main__": - # Hardcoded to use stdio transport mode - main("stdio") From af6de60e14cbbc94111f02e6d2cc40d88e9e666e Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 17:07:36 +0800 Subject: [PATCH 09/14] fix path in run_mcp --- owl/run_mcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owl/run_mcp.py b/owl/run_mcp.py index 570a03a..c09ffb5 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -39,7 +39,7 @@ Environment Setup: go install github.com/mark3labs/mcp-filesystem-server@latest npm install -g @modelcontextprotocol/server-filesystem - # Configure mcp_servers_config.json in owl/utils/mcp/ + # Configure mcp_servers_config.json in owl/ { "mcpServers": { "filesystem": { From cf1c01b978999fcdf413b235927773c68070c2cd Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 18:08:41 +0800 Subject: [PATCH 10/14] update mcp_servers_config.json --- owl/mcp_servers_config.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/owl/mcp_servers_config.json b/owl/mcp_servers_config.json index f51fac0..30192b6 100644 --- a/owl/mcp_servers_config.json +++ b/owl/mcp_servers_config.json @@ -14,6 +14,10 @@ "simple-arxiv": { "command": "python", "args": ["-m", "mcp_simple_arxiv"] + }, + "fetch": { + "command": "python", + "args": ["-m", "mcp_server_fetch"] } }, "mcpWebServers": {} From 73df0191f9a4929c313a2ecf6781a7944cf6aa49 Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 18:10:35 +0800 Subject: [PATCH 11/14] Update mcp_servers_config.json --- owl/mcp_servers_config.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/owl/mcp_servers_config.json b/owl/mcp_servers_config.json index f51fac0..30192b6 100644 --- a/owl/mcp_servers_config.json +++ b/owl/mcp_servers_config.json @@ -14,6 +14,10 @@ "simple-arxiv": { "command": "python", "args": ["-m", "mcp_simple_arxiv"] + }, + "fetch": { + "command": "python", + "args": ["-m", "mcp_server_fetch"] } }, "mcpWebServers": {} From 333ce841b94ba4a21dead6dbc22d0935cade0d53 Mon Sep 17 00:00:00 2001 From: Wendong Date: Thu, 13 Mar 2025 18:53:18 +0800 Subject: [PATCH 12/14] update dependency --- pyproject.toml | 4 ++- requirements.txt | 3 +- uv.lock | 81 +++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2fa6908..c9d77bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,10 +21,12 @@ keywords = [ "learning-systems" ] dependencies = [ - "camel-ai[all]==0.2.27", + "camel-ai[all]==0.2.28", "chunkr-ai>=0.0.41", "docx2markdown>=0.1.1", "gradio>=3.50.2", + "mcp-simple-arxiv==0.2.2", + "mcp-server-fetch==2025.1.17", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index 3a3e5e3..e66fdfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -camel-ai[all]==0.2.27 +camel-ai[all]==0.2.28 chunkr-ai>=0.0.41 docx2markdown>=0.1.1 gradio>=3.50.2 -mcp==1.3.0 mcp-simple-arxiv==0.2.2 mcp-server-fetch==2025.1.17 \ No newline at end of file diff --git a/uv.lock b/uv.lock index 5bb2dee..81d0dac 100644 --- a/uv.lock +++ b/uv.lock @@ -482,7 +482,7 @@ wheels = [ [[package]] name = "camel-ai" -version = "0.2.27" +version = "0.2.28" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama" }, @@ -499,9 +499,9 @@ dependencies = [ { name = "pyyaml" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ff/27/2bce666ae7f7d0db276d037b3afe84a460e782438e5cacc08de20417233b/camel_ai-0.2.27.tar.gz", hash = "sha256:4689245ad48f51e5e602d2651cf463afe212bcf046633a19c2189574c1f3481a", size = 441363 } +sdist = { url = "https://files.pythonhosted.org/packages/6a/3b/7f350ae3c5bf42263688d3a69333e3908af4d45ce8f5f838af634a2720b3/camel_ai-0.2.28.tar.gz", hash = "sha256:f47e12bdf59df6e789db4587f0c5bd0adf43b2029d6be1bfcc31bfd41cab9d9f", size = 443082 } wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/fa/94f5b41cb6babc81aac00494b170ec2bea058b6c00f477ceb3e886c49177/camel_ai-0.2.27-py3-none-any.whl", hash = "sha256:c4a6597791faf2f2161c56c2579e60850557b126135b29af77ebd08fa0774e0b", size = 746387 }, + { url = "https://files.pythonhosted.org/packages/5d/27/8a6e97f660354ce03413872268c7f4a40ceefdf39b20f161cb7f672dc67c/camel_ai-0.2.28-py3-none-any.whl", hash = "sha256:079e7e905a36b64be47a6a27ad4b99d21ca0403b27027a4d777744968a22040a", size = 748237 }, ] [package.optional-dependencies] @@ -2685,6 +2685,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, ] +[[package]] +name = "markdownify" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/78/c48fed23c7aebc2c16049062e72de1da3220c274de59d28c942acdc9ffb2/markdownify-1.1.0.tar.gz", hash = "sha256:449c0bbbf1401c5112379619524f33b63490a8fa479456d41de9dc9e37560ebd", size = 17127 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/11/b751af7ad41b254a802cf52f7bc1fca7cabe2388132f2ce60a1a6b9b9622/markdownify-1.1.0-py3-none-any.whl", hash = "sha256:32a5a08e9af02c8a6528942224c91b933b4bd2c7d078f9012943776fc313eeef", size = 13901 }, +] + [[package]] name = "markupsafe" version = "2.1.5" @@ -2806,6 +2819,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/d2/a9e87b506b2094f5aa9becc1af5178842701b27217fa43877353da2577e3/mcp-1.3.0-py3-none-any.whl", hash = "sha256:2829d67ce339a249f803f22eba5e90385eafcac45c94b00cab6cef7e8f217211", size = 70672 }, ] +[[package]] +name = "mcp-server-fetch" +version = "2025.1.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdownify" }, + { name = "mcp" }, + { name = "protego" }, + { name = "pydantic" }, + { name = "readabilipy" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/76/204ac83afe2000b1513b4741229586128361f376fab03832695e0179104d/mcp_server_fetch-2025.1.17.tar.gz", hash = "sha256:aa3a5dee358651103477bc121b98ada18a5c35840c56e4016cc3b40e7df1aa7d", size = 43468 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/34/c0dce3415b627f763a9b7a0202a6a0672446b49f5ca04827340c28d75c63/mcp_server_fetch-2025.1.17-py3-none-any.whl", hash = "sha256:53c4967572464c6329824c9b05cdfa5fe214004d577ae8700fdb04203844be52", size = 7991 }, +] + +[[package]] +name = "mcp-simple-arxiv" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "feedparser" }, + { name = "httpx" }, + { name = "mcp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/d3/d47bfce067ea85bc73154d8299549f84455e601f699fcff513f9d44cef0d/mcp_simple_arxiv-0.2.2.tar.gz", hash = "sha256:e27cfd58a470dcec7d733bd09b4219daddbdc3475a6d256e246a114e5b94e817", size = 12100 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/4e/6646a0004fc85b0c1df6e662db42f76fe5a0412179b7f65c066d7804370a/mcp_simple_arxiv-0.2.2-py3-none-any.whl", hash = "sha256:fcf607303c074ae5e88337b5bf3ea52cd781081f49ddf8fa0898eb3b8420dccb", size = 13686 }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -3571,14 +3616,18 @@ dependencies = [ { name = "chunkr-ai" }, { name = "docx2markdown" }, { name = "gradio" }, + { name = "mcp-server-fetch" }, + { name = "mcp-simple-arxiv" }, ] [package.metadata] requires-dist = [ - { name = "camel-ai", extras = ["all"], specifier = "==0.2.27" }, + { name = "camel-ai", extras = ["all"], specifier = "==0.2.28" }, { name = "chunkr-ai", specifier = ">=0.0.41" }, { name = "docx2markdown", specifier = ">=0.1.1" }, { name = "gradio", specifier = ">=3.50.2" }, + { name = "mcp-server-fetch", specifier = "==2025.1.17" }, + { name = "mcp-simple-arxiv", specifier = "==0.2.2" }, ] [[package]] @@ -3962,6 +4011,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/35/6c4c6fc8774a9e3629cd750dc24a7a4fb090a25ccd5c3246d127b70f9e22/propcache-0.3.0-py3-none-any.whl", hash = "sha256:67dda3c7325691c2081510e92c561f465ba61b975f481735aefdfc845d2cd043", size = 12101 }, ] +[[package]] +name = "protego" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/6b/84e878d0567dfc11538bad6ce2595cee7ae0c47cf6bf7293683c9ec78ef8/protego-0.4.0.tar.gz", hash = "sha256:93a5e662b61399a0e1f208a324f2c6ea95b23ee39e6cbf2c96246da4a656c2f6", size = 3246425 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/fd/8d84d75832b0983cecf3aff7ae48362fe96fc8ab6ebca9dcf3cefd87e79c/Protego-0.4.0-py2.py3-none-any.whl", hash = "sha256:37640bc0ebe37572d624453a21381d05e9d86e44f89ff1e81794d185a0491666", size = 8553 }, +] + [[package]] name = "proto-plus" version = "1.26.0" @@ -4673,6 +4731,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/f6/fa777f336629aee8938f3d5c95c09df38459d4eadbdbe34642889857fb6a/rapidfuzz-3.12.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:54bb69ebe5ca0bd7527357e348f16a4c0c52fe0c2fcc8a041010467dcb8385f7", size = 1555000 }, ] +[[package]] +name = "readabilipy" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "html5lib" }, + { name = "lxml" }, + { name = "regex" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/e4/260a202516886c2e0cc6e6ae96d1f491792d829098886d9529a2439fbe8e/readabilipy-0.3.0.tar.gz", hash = "sha256:e13313771216953935ac031db4234bdb9725413534bfb3c19dbd6caab0887ae0", size = 35491 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/46/8a640c6de1a6c6af971f858b2fb178ca5e1db91f223d8ba5f40efe1491e5/readabilipy-0.3.0-py3-none-any.whl", hash = "sha256:d106da0fad11d5fdfcde21f5c5385556bfa8ff0258483037d39ea6b1d6db3943", size = 22158 }, +] + [[package]] name = "redis" version = "5.2.1" From 7f5d356947b5a0e1b6437c1eaf57e24c0088c1b2 Mon Sep 17 00:00:00 2001 From: jjyaoao Date: Thu, 13 Mar 2025 21:25:36 +0800 Subject: [PATCH 13/14] update run_mcp and requirements --- owl/mcp_servers_config.json | 19 +++----- owl/run_mcp.py | 88 +++++++++++++++++++++---------------- requirements.txt | 2 - 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/owl/mcp_servers_config.json b/owl/mcp_servers_config.json index 30192b6..9291acc 100644 --- a/owl/mcp_servers_config.json +++ b/owl/mcp_servers_config.json @@ -1,25 +1,16 @@ { "mcpServers": { - "filesystem": { - "command": "mcp-filesystem-server", + "desktop-commander": { + "command": "npx", "args": [ - "/Users/username/Desktop", - "/Users/username/Downloads" + "-y", + "@wonderwhy-er/desktop-commander" ] }, "playwright": { "command": "npx", "args": ["-y", "@executeautomation/playwright-mcp-server"] - }, - "simple-arxiv": { - "command": "python", - "args": ["-m", "mcp_simple_arxiv"] - }, - "fetch": { - "command": "python", - "args": ["-m", "mcp_server_fetch"] } - }, - "mcpWebServers": {} + } } diff --git a/owl/run_mcp.py b/owl/run_mcp.py index c09ffb5..a8b1d88 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -4,21 +4,33 @@ This example demonstrates how to use MCP (Model Context Protocol) with CAMEL age for advanced information retrieval and processing tasks. Environment Setup: -1. Configure the required dependencies of owl library. +1. Configure the required dependencies of owl library + Refer to: https://github.com/camel-ai/owl for installation guide -2. Go Environment (v1.23.2+): +2. MCP Server Setup: + + 2.1 MCP Desktop Commander (File System Service): + Prerequisites: Node.js and npm ```bash - # Verify Go installation - go version + # Install MCP service + npx -y @smithery/cli install @wonderwhy-er/desktop-commander --client claude + npx @wonderwhy-er/desktop-commander setup - # Add Go binary path to PATH - export PATH=$PATH:~/go/bin - # Note: Add to ~/.bashrc or ~/.zshrc for persistence + # Configure in owl/mcp_servers_config.json: + { + "desktop-commander": { + "command": "npx", + "args": [ + "-y", + "@wonderwhy-er/desktop-commander" + ] + } + } ``` -3. Playwright Setup: + 2.2 MCP Playwright Service: ```bash - # Install Node.js and npm first + # Install MCP service npm install -g @executeautomation/playwright-mcp-server npx playwright install-deps @@ -33,22 +45,17 @@ Environment Setup: } ``` -4. MCP Filesystem Server Setup: + 2.3 MCP Fetch Service (Optional - for better retrieval): ```bash - # Install MCP filesystem server - go install github.com/mark3labs/mcp-filesystem-server@latest - npm install -g @modelcontextprotocol/server-filesystem + # Install MCP service + pip install mcp-server-fetch - # Configure mcp_servers_config.json in owl/ + # Configure in mcp_servers_config.json: { "mcpServers": { - "filesystem": { - "command": "mcp-filesystem-server", - "args": [ - "/home/your_path", - "/home/your_path" - ], - "type": "filesystem" + "fetch": { + "command": "python", + "args": ["-m", "mcp_server_fetch"] } } } @@ -57,11 +64,12 @@ Environment Setup: Usage: 1. Ensure all MCP servers are properly configured in mcp_servers_config.json 2. Run this script to create a multi-agent system that can: - - Access and manipulate files through MCP filesystem server + - Access and manipulate files through MCP Desktop Commander - Perform web automation tasks using Playwright - Process and generate information using GPT-4o + - Fetch web content (if fetch service is configured) 3. The system will execute the specified task while maintaining security through - relative paths and controlled access + controlled access Note: - All file operations are restricted to configured directories @@ -135,27 +143,29 @@ async def construct_society( async def main(): config_path = Path(__file__).parent / "mcp_servers_config.json" - mcp_toolkit = MCPToolkit(config_path=str(config_path)) - question = ( - "I'd like a academic report about Guohao Li, including his research " - "direction, published papers (At least 3), institutions, etc." - "Then organize the report in Markdown format and save it to my desktop" - ) + try: + await mcp_toolkit.connect() - await mcp_toolkit.connect() + question = ( + "I'd like a academic report about Andrew Ng, including his research " + "direction, published papers (At least 3), institutions, etc." + "Then organize the report in Markdown format and save it to my desktop" + ) - # # Connect to all MCP toolkits - tools = [*mcp_toolkit.get_tools()] + # Connect to all MCP toolkits + tools = [*mcp_toolkit.get_tools()] + society = await construct_society(question, tools) + answer, chat_history, token_count = await run_society(society) + print(f"\033[94mAnswer: {answer}\033[0m") - society = await construct_society(question, tools) - - answer, chat_history, token_count = await run_society(society) - - print(f"\033[94mAnswer: {answer}\033[0m") - - await mcp_toolkit.disconnect() + finally: + # Make sure to disconnect safely after all operations are completed. + try: + await mcp_toolkit.disconnect() + except Exception as e: + print(f"Warning: Error during disconnect: {e}") if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e66fdfa..ff6fc0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,3 @@ camel-ai[all]==0.2.28 chunkr-ai>=0.0.41 docx2markdown>=0.1.1 gradio>=3.50.2 -mcp-simple-arxiv==0.2.2 -mcp-server-fetch==2025.1.17 \ No newline at end of file From b44d5b9604c31608794e39c8825835d17afd43ed Mon Sep 17 00:00:00 2001 From: Wendong Date: Thu, 13 Mar 2025 22:33:21 +0800 Subject: [PATCH 14/14] update readme and format fix --- README.md | 29 ++++++++++++++++++++++++++++- README_zh.md | 27 ++++++++++++++++++++++++++- owl/run_deepseek_zh.py | 6 ++---- owl/run_mcp.py | 29 +++++++++++++++++++++-------- owl/run_terminal.py | 7 +++++-- owl/run_terminal_zh.py | 12 ++++++++---- owl/utils/enhanced_role_playing.py | 9 ++++----- owl/utils/gaia.py | 4 +++- run_app.py | 3 ++- run_app_zh.py | 3 ++- 10 files changed, 101 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 13640ab..71de92e 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,9 @@ https://private-user-images.githubusercontent.com/55657767/420212194-e813fc05-13 - **Browser Automation**: Utilize the Playwright framework for simulating browser interactions, including scrolling, clicking, input handling, downloading, navigation, and more. - **Document Parsing**: Extract content from Word, Excel, PDF, and PowerPoint files, converting them into text or Markdown format. - **Code Execution**: Write and execute Python code using interpreter. -- **Built-in Toolkits**: Access to a comprehensive set of built-in toolkits including ArxivToolkit, AudioAnalysisToolkit, CodeExecutionToolkit, DalleToolkit, DataCommonsToolkit, ExcelToolkit, GitHubToolkit, GoogleMapsToolkit, GoogleScholarToolkit, ImageAnalysisToolkit, MathToolkit, NetworkXToolkit, NotionToolkit, OpenAPIToolkit, RedditToolkit, SearchToolkit, SemanticScholarToolkit, SymPyToolkit, VideoAnalysisToolkit, WeatherToolkit, BrowserToolkit, and many more for specialized tasks. +- **Built-in Toolkits**: Access to a comprehensive set of built-in toolkits including: + - **Model Context Protocol (MCP)**: A universal protocol layer that standardizes AI model interactions with various tools and data sources + - **Core Toolkits**: ArxivToolkit, AudioAnalysisToolkit, CodeExecutionToolkit, DalleToolkit, DataCommonsToolkit, ExcelToolkit, GitHubToolkit, GoogleMapsToolkit, GoogleScholarToolkit, ImageAnalysisToolkit, MathToolkit, NetworkXToolkit, NotionToolkit, OpenAPIToolkit, RedditToolkit, SearchToolkit, SemanticScholarToolkit, SymPyToolkit, VideoAnalysisToolkit, WeatherToolkit, BrowserToolkit, and many more for specialized tasks # 🛠️ Installation @@ -275,6 +277,23 @@ For more detailed Docker usage instructions, including cross-platform support, o # 🚀 Quick Start +## Try MCP (Model Context Protocol) Integration + +Experience the power of MCP by running our example that demonstrates multi-agent information retrieval and processing: + +```bash +# Set up MCP servers (one-time setup) +npx -y @smithery/cli install @wonderwhy-er/desktop-commander --client claude +npx @wonderwhy-er/desktop-commander setup + +# Run the MCP example +python owl/run_mcp.py +``` + +This example showcases how OWL agents can seamlessly interact with file systems, web automation, and information retrieval through the MCP protocol. Check out `owl/run_mcp.py` for the full implementation. + +## Basic Usage + After installation and setting up your environment variables, you can start using OWL right away: ```bash @@ -355,6 +374,14 @@ Here are some tasks you can try with OWL: # 🧰 Toolkits and Capabilities +## Model Context Protocol (MCP) + +OWL's MCP integration provides a standardized way for AI models to interact with various tools and data sources: + +Try our comprehensive MCP example in `owl/run_mcp.py` to see these capabilities in action! + +## Available Toolkits + > **Important**: Effective use of toolkits requires models with strong tool calling capabilities. For multimodal toolkits (Web, Image, Video), models must also have multimodal understanding abilities. OWL supports various toolkits that can be customized by modifying the `tools` list in your script: diff --git a/README_zh.md b/README_zh.md index b7fd579..c7e24dd 100644 --- a/README_zh.md +++ b/README_zh.md @@ -105,7 +105,7 @@ - **[2025.03.12]**: 在SearchToolkit中添加了Bocha搜索功能，集成了火山引擎模型平台，并更新了Azure和OpenAI Compatible模型的结构化输出和工具调用能力。 -- **[2025.03.11]**: 我们添加了 MCPToolkit、FileWriteToolkit 和 TerminalToolkit，增强 OWL Agent的工具调用、文件写入能力和终端命令执行功能。 +- **[2025.03.11]**: 我们添加了 MCPToolkit、FileWriteToolkit 和 TerminalToolkit，增强了 OWL Agent 的 MCP（模型上下文协议）集成、文件写入能力和终端命令执行功能。MCP 作为一个通用协议层，标准化了 AI 模型与各种数据源和工具的交互方式。 - **[2025.03.09]**: 我们添加了基于网页的用户界面，使系统交互变得更加简便。 - **[2025.03.07]**: 我们开源了 🦉 OWL 项目的代码库。 - **[2025.03.03]**: OWL 在 GAIA 基准测试中取得 58.18 平均分，在开源框架中排名第一！ @@ -272,6 +272,23 @@ chmod +x build_docker.sh 更多详细的Docker使用说明，包括跨平台支持、优化配置和故障排除，请参阅 [DOCKER_README.md](.container/DOCKER_README.md) # 🚀 快速开始 + +## 尝试 MCP（模型上下文协议）集成 + +体验 MCP 的强大功能，运行我们的示例来展示多智能体信息检索和处理： + +```bash +# 设置 MCP 服务器（仅需一次性设置） +npx -y @smithery/cli install @wonderwhy-er/desktop-commander --client claude +npx @wonderwhy-er/desktop-commander setup + +# 运行 MCP 示例 +python owl/run_mcp.py +``` + +这个示例展示了 OWL 智能体如何通过 MCP 协议无缝地与文件系统、网页自动化和信息检索进行交互。查看 `owl/run_mcp.py` 了解完整实现。 + +## 基本用法 运行以下示例： @@ -349,6 +366,14 @@ OWL 将自动调用与文档相关的工具来处理文件并提取答案。 # 🧰 工具包与功能 +## 模型上下文协议（MCP） + +OWL 的 MCP 集成为 AI 模型与各种工具和数据源的交互提供了标准化的方式。 + +查看我们的综合示例 `owl/run_mcp.py` 来体验这些功能！ + +## 可用工具包 + > **重要提示**：有效使用工具包需要具备强大工具调用能力的模型。对于多模态工具包（Web、图像、视频），模型还必须具备多模态理解能力。 OWL支持多种工具包，可通过修改脚本中的`tools`列表进行自定义： diff --git a/owl/run_deepseek_zh.py b/owl/run_deepseek_zh.py index 52b4c34..0f14c58 100644 --- a/owl/run_deepseek_zh.py +++ b/owl/run_deepseek_zh.py @@ -31,7 +31,7 @@ from camel.toolkits import ( from camel.types import ModelPlatformType, ModelType -from utils import OwlRolePlaying, run_society, DocumentProcessingToolkit +from utils import OwlRolePlaying, run_society from camel.logger import set_log_level @@ -99,9 +99,7 @@ def construct_society(question: str) -> OwlRolePlaying: def main(): r"""Main function to run the OWL system with an example question.""" # Example research question - question = ( - "搜索OWL项目最近的新闻并生成一篇报告，最后保存到本地。" - ) + question = "搜索OWL项目最近的新闻并生成一篇报告，最后保存到本地。" # Construct and run the society society = construct_society(question) diff --git a/owl/run_mcp.py b/owl/run_mcp.py index a8b1d88..67849ee 100644 --- a/owl/run_mcp.py +++ b/owl/run_mcp.py @@ -1,3 +1,16 @@ +# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= """MCP Multi-Agent System Example This example demonstrates how to use MCP (Model Context Protocol) with CAMEL agents @@ -15,7 +28,7 @@ Environment Setup: # Install MCP service npx -y @smithery/cli install @wonderwhy-er/desktop-commander --client claude npx @wonderwhy-er/desktop-commander setup - + # Configure in owl/mcp_servers_config.json: { "desktop-commander": { @@ -33,7 +46,7 @@ Environment Setup: # Install MCP service npm install -g @executeautomation/playwright-mcp-server npx playwright install-deps - + # Configure in mcp_servers_config.json: { "mcpServers": { @@ -49,7 +62,7 @@ Environment Setup: ```bash # Install MCP service pip install mcp-server-fetch - + # Configure in mcp_servers_config.json: { "mcpServers": { @@ -92,7 +105,6 @@ from camel.toolkits import MCPToolkit from utils.enhanced_role_playing import OwlRolePlaying, run_society - load_dotenv() set_log_level(level="DEBUG") @@ -150,7 +162,7 @@ async def main(): question = ( "I'd like a academic report about Andrew Ng, including his research " - "direction, published papers (At least 3), institutions, etc." + "direction, published papers (At least 3), institutions, etc." "Then organize the report in Markdown format and save it to my desktop" ) @@ -164,8 +176,9 @@ async def main(): # Make sure to disconnect safely after all operations are completed. try: await mcp_toolkit.disconnect() - except Exception as e: - print(f"Warning: Error during disconnect: {e}") + except Exception: + print("Disconnect failed") + if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/owl/run_terminal.py b/owl/run_terminal.py index 3741a7e..d77f33a 100644 --- a/owl/run_terminal.py +++ b/owl/run_terminal.py @@ -18,7 +18,7 @@ from camel.toolkits import ( SearchToolkit, BrowserToolkit, FileWriteToolkit, - TerminalToolkit + TerminalToolkit, ) from camel.types import ModelPlatformType, ModelType from camel.logger import set_log_level @@ -30,6 +30,7 @@ set_log_level(level="DEBUG") # Get current script directory base_dir = os.path.dirname(os.path.abspath(__file__)) + def construct_society(question: str) -> OwlRolePlaying: r"""Construct a society of agents based on the given question. @@ -113,7 +114,9 @@ def main(): answer, chat_history, token_count = run_society(society) # Output the result - print(f"\033[94mAnswer: {answer}\nChat History: {chat_history}\ntoken_count:{token_count}\033[0m") + print( + f"\033[94mAnswer: {answer}\nChat History: {chat_history}\ntoken_count:{token_count}\033[0m" + ) if __name__ == "__main__": diff --git a/owl/run_terminal_zh.py b/owl/run_terminal_zh.py index 2582c24..bbddc5b 100644 --- a/owl/run_terminal_zh.py +++ b/owl/run_terminal_zh.py @@ -12,13 +12,13 @@ # limitations under the License. # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= from dotenv import load_dotenv - +import os from camel.models import ModelFactory from camel.toolkits import ( SearchToolkit, BrowserToolkit, FileWriteToolkit, - TerminalToolkit + TerminalToolkit, ) from camel.types import ModelPlatformType, ModelType from camel.logger import set_log_level @@ -27,10 +27,12 @@ from utils import OwlRolePlaying, run_society load_dotenv() set_log_level(level="DEBUG") -import os + + # Get current script directory base_dir = os.path.dirname(os.path.abspath(__file__)) + def construct_society(question: str) -> OwlRolePlaying: r"""Construct a society of agents based on the given question. @@ -112,7 +114,9 @@ def main(): answer, chat_history, token_count = run_society(society) # Output the result - print(f"\033[94mAnswer: {answer}\nChat History: {chat_history}\ntoken_count:{token_count}\033[0m") + print( + f"\033[94mAnswer: {answer}\nChat History: {chat_history}\ntoken_count:{token_count}\033[0m" + ) if __name__ == "__main__": diff --git a/owl/utils/enhanced_role_playing.py b/owl/utils/enhanced_role_playing.py index fe14efe..f8cf045 100644 --- a/owl/utils/enhanced_role_playing.py +++ b/owl/utils/enhanced_role_playing.py @@ -282,8 +282,7 @@ Please note that our overall task may be very complicated. Here are some tips th ) async def astep( - self, - assistant_msg: BaseMessage + self, assistant_msg: BaseMessage ) -> Tuple[ChatAgentResponse, ChatAgentResponse]: user_response = await self.user_agent.astep(assistant_msg) if user_response.terminated or user_response.msgs is None: @@ -452,9 +451,9 @@ async def run_society( input_msg = society.init_chat(init_prompt) for _round in range(round_limit): assistant_response, user_response = await society.astep(input_msg) - overall_prompt_token_count += ( - assistant_response.info["usage"]["completion_tokens"] - ) + overall_prompt_token_count += assistant_response.info["usage"][ + "completion_tokens" + ] overall_prompt_token_count += ( assistant_response.info["usage"]["prompt_tokens"] + user_response.info["usage"]["prompt_tokens"] diff --git a/owl/utils/gaia.py b/owl/utils/gaia.py index a133a26..83e8744 100644 --- a/owl/utils/gaia.py +++ b/owl/utils/gaia.py @@ -191,7 +191,9 @@ class GAIABenchmark(BaseBenchmark): except Exception as e: logger.warning(e) # raise FileNotFoundError(f"{self.save_to} does not exist.") - datas = [data for data in datas if not self._check_task_completed(data["task_id"])] + datas = [ + data for data in datas if not self._check_task_completed(data["task_id"]) + ] logger.info(f"Number of tasks to be processed: {len(datas)}") # Process tasks for task in tqdm(datas, desc="Running"): diff --git a/run_app.py b/run_app.py index ccea485..69af092 100644 --- a/run_app.py +++ b/run_app.py @@ -22,7 +22,8 @@ import os import sys from pathlib import Path -os.environ['PYTHONIOENCODING'] = 'utf-8' +os.environ["PYTHONIOENCODING"] = "utf-8" + def main(): """Main function to launch the OWL Intelligent Assistant Platform""" diff --git a/run_app_zh.py b/run_app_zh.py index 0ec4e7b..4edc38b 100644 --- a/run_app_zh.py +++ b/run_app_zh.py @@ -22,7 +22,8 @@ import os import sys from pathlib import Path -os.environ['PYTHONIOENCODING'] = 'utf-8' +os.environ["PYTHONIOENCODING"] = "utf-8" + def main(): """主函数，启动OWL智能助手运行平台"""