feat: astrbot agent sandbox env(improved code interpreter)

astrbot/core/config/default.py

@@ -113,6 +113,14 @@
             "provider": "moonshotai",
             "moonshotai_api_key": "",
         },
+        "sandbox": {
+            "enable": False,
+            "booter": "shipyard",
+            "shipyard_endpoint": "",
+            "shipyard_access_token": "",
+            "shipyard_ttl": 3600,
+            "shipyard_max_sessions": 10,
+        },
     },
     "provider_stt_settings": {
         "enable": False,
@@ -2539,6 +2547,62 @@ class ChatProviderTemplate(TypedDict):
             #         "provider_settings.enable": True,
             #     },
             # },
+            "sandbox": {
+                "description": "Agent 沙箱环境",
+                "type": "object",
+                "items": {
+                    "provider_settings.sandbox.enable": {
+                        "description": "启用沙箱环境",
+                        "type": "bool",
+                        "hint": "启用后，Agent 可以使用沙箱环境中的工具和资源，如 Python 代码执行、Shell 等。",
+                    },
+                    "provider_settings.sandbox.booter": {
+                        "description": "沙箱环境驱动器",
+                        "type": "string",
+                        "options": ["shipyard"],
+                        "condition": {
+                            "provider_settings.sandbox.enable": True,
+                        },
+                    },
+                    "provider_settings.sandbox.shipyard_endpoint": {
+                        "description": "Shipyard API Endpoint",
+                        "type": "string",
+                        "hint": "Shipyard 服务的 API 访问地址。",
+                        "condition": {
+                            "provider_settings.sandbox.enable": True,
+                            "provider_settings.sandbox.booter": "shipyard",
+                        },
+                        "_special": "check_shipyard_connection",
+                    },
+                    "provider_settings.sandbox.shipyard_access_token": {
+                        "description": "Shipyard Access Token",
+                        "type": "string",
+                        "hint": "用于访问 Shipyard 服务的访问令牌。",
+                        "condition": {
+                            "provider_settings.sandbox.enable": True,
+                            "provider_settings.sandbox.booter": "shipyard",
+                        },
+                    },
+                    "provider_settings.sandbox.shipyard_ttl": {
+                        "description": "Shipyard Session TTL",
+                        "type": "int",
+                        "hint": "Shipyard 会话的生存时间（秒）。",
+                        "condition": {
+                            "provider_settings.sandbox.enable": True,
+                            "provider_settings.sandbox.booter": "shipyard",
+                        },
+                    },
+                    "provider_settings.sandbox.shipyard_max_sessions": {
+                        "description": "Shipyard Max Sessions",
+                        "type": "int",
+                        "hint": "Shipyard 最大会话数量。",
+                        "condition": {
+                            "provider_settings.sandbox.enable": True,
+                            "provider_settings.sandbox.booter": "shipyard",
+                        },
+                    },
+                },
+            },
             "truncate_and_compress": {
                 "description": "上下文管理策略",
                 "type": "object",

astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py

@@ -2,10 +2,11 @@
 
 import asyncio
 import json
+import os
 from collections.abc import AsyncGenerator
 
 from astrbot.core import logger
-from astrbot.core.agent.message import Message
+from astrbot.core.agent.message import Message, TextPart
 from astrbot.core.agent.response import AgentStats
 from astrbot.core.agent.tool import ToolSet
 from astrbot.core.astr_agent_context import AstrAgentContext
@@ -35,8 +36,13 @@
 from ....context import PipelineContext, call_event_hook
 from ...stage import Stage
 from ...utils import (
+    EXECUTE_SHELL_TOOL,
+    FILE_DOWNLOAD_TOOL,
+    FILE_UPLOAD_TOOL,
     KNOWLEDGE_BASE_QUERY_TOOL,
     LLM_SAFETY_MODE_SYSTEM_PROMPT,
+    PYTHON_TOOL,
+    SANDBOX_MODE_PROMPT,
     decoded_blocked,
     retrieve_knowledge_base,
 )
@@ -94,6 +100,8 @@ async def initialize(self, ctx: PipelineContext) -> None:
             "safety_mode_strategy", "system_prompt"
         )
 
+        self.sandbox_cfg = settings.get("sandbox", {})
+
         self.conv_manager = ctx.plugin_manager.context.conversation_manager
 
     def _select_provider(self, event: AstrMessageEvent):
@@ -458,6 +466,24 @@ def _apply_llm_safety_mode(self, req: ProviderRequest) -> None:
                 f"Unsupported llm_safety_mode strategy: {self.safety_mode_strategy}.",
             )
 
+    def _apply_sandbox_tools(self, req: ProviderRequest, session_id: str) -> None:
+        """Add sandbox tools to the provider request."""
+        if req.func_tool is None:
+            req.func_tool = ToolSet()
+        if self.sandbox_cfg.get("booter") == "shipyard":
+            ep = self.sandbox_cfg.get("shipyard_endpoint", "")
+            at = self.sandbox_cfg.get("shipyard_access_token", "")
+            if not ep or not at:
+                logger.error("Shipyard sandbox configuration is incomplete.")
+                return
+            os.environ["SHIPYARD_ENDPOINT"] = ep
+            os.environ["SHIPYARD_ACCESS_TOKEN"] = at
+        req.func_tool.add_tool(EXECUTE_SHELL_TOOL)
+        req.func_tool.add_tool(PYTHON_TOOL)
+        req.func_tool.add_tool(FILE_UPLOAD_TOOL)
+        req.func_tool.add_tool(FILE_DOWNLOAD_TOOL)
+        req.system_prompt += f"\n{SANDBOX_MODE_PROMPT}\n"
+
     async def process(
         self, event: AstrMessageEvent, provider_wake_prefix: str
     ) -> AsyncGenerator[None, None]:
@@ -536,6 +562,20 @@ async def process(
                             image_path = await comp.convert_to_file_path()
                             req.image_urls.append(image_path)
 
+                            req.extra_user_content_parts.append(
+                                TextPart(text=f"[Image Attachment: path {image_path}]")
+                            )
+                        elif isinstance(comp, File) and self.sandbox_cfg.get(
+                            "enable", False
+                        ):
+                            file_path = await comp.get_file()
+                            file_name = comp.name or os.path.basename(file_path)
+                            req.extra_user_content_parts.append(
+                                TextPart(
+                                    text=f"[File Attachment: name {file_name}, path {file_path}]"
+                                )
+                            )
+
                     conversation = await self._get_session_conv(event)
                     req.conversation = conversation
                     req.contexts = json.loads(conversation.history)
@@ -586,6 +626,10 @@ async def process(
                 if self.llm_safety_mode:
                     self._apply_llm_safety_mode(req)
 
+                # apply sandbox tools
+                if self.sandbox_cfg.get("enable", False):
+                    self._apply_sandbox_tools(req, req.session_id)
+
                 stream_to_general = (
                     self.unsupported_streaming_strategy == "turn_off"
                     and not event.platform_meta.support_streaming_message

astrbot/core/pipeline/process_stage/utils.py

@@ -7,6 +7,12 @@
 from astrbot.core.agent.run_context import ContextWrapper
 from astrbot.core.agent.tool import FunctionTool, ToolExecResult
 from astrbot.core.astr_agent_context import AstrAgentContext
+from astrbot.core.sandbox.tools import (
+    ExecuteShellTool,
+    FileDownloadTool,
+    FileUploadTool,
+    PythonTool,
+)
 from astrbot.core.star.context import Context
 
 LLM_SAFETY_MODE_SYSTEM_PROMPT = """You are running in Safe Mode.
@@ -21,6 +27,20 @@
 - Output same language as the user's input.
 """
 
+SANDBOX_MODE_PROMPT = (
+    "You have access to a sandboxed environment and can execute shell commands and Python code securely."
+    # "Your have extended skills library, such as PDF processing, image generation, data analysis, etc. "
+    # "Before handling complex tasks, please retrieve and review the documentation in the in /app/skills/ directory. "
+    # "If the current task matches the description of a specific skill, prioritize following the workflow defined by that skill."
+    # "Use `ls /app/skills/` to list all available skills. "
+    # "Use `cat /app/skills/{skill_name}/SKILL.md` to read the documentation of a specific skill."
+    # "SKILL.md might be large, you can read the description first, which is located in the YAML frontmatter of the file."
+    # "Use shell commands such as grep, sed, awk to extract relevant information from the documentation as needed.\n"
+    "Note:\n"
+    "1. If you use shell, your command will always runs in the /home/<username>/workspace directory.\n"
+    "2. If you use IPython, you would better use absolute paths when dealing with files to avoid confusion.\n"
+)
+
 
 @dataclass
 class KnowledgeBaseQueryTool(FunctionTool[AstrAgentContext]):
@@ -138,6 +158,11 @@ async def retrieve_knowledge_base(
 
 KNOWLEDGE_BASE_QUERY_TOOL = KnowledgeBaseQueryTool()
 
+EXECUTE_SHELL_TOOL = ExecuteShellTool()
+PYTHON_TOOL = PythonTool()
+FILE_UPLOAD_TOOL = FileUploadTool()
+FILE_DOWNLOAD_TOOL = FileDownloadTool()
+
 # we prevent astrbot from connecting to known malicious hosts
 # these hosts are base64 encoded
 BLOCKED = {"dGZid2h2d3IuY2xvdWQuc2VhbG9zLmlv", "a291cmljaGF0"}

astrbot/core/platform/sources/webchat/webchat_adapter.py

@@ -93,7 +93,8 @@ async def send_by_session(
         session: MessageSesion,
         message_chain: MessageChain,
     ):
-        await WebChatMessageEvent._send(message_chain, session.session_id)
+        message_id = f"active_{str(uuid.uuid4())}"
+        await WebChatMessageEvent._send(message_id, message_chain, session.session_id)
         await super().send_by_session(session, message_chain)
 
     async def _get_message_history(
@@ -196,7 +197,7 @@ async def convert_message(self, data: tuple) -> AstrBotMessage:
 
         abm.session_id = f"webchat!{username}!{cid}"
 
-        abm.message_id = str(uuid.uuid4())
+        abm.message_id = payload.get("message_id")
 
         # 处理消息段列表
         message_parts = payload.get("message", [])

astrbot/core/platform/sources/webchat/webchat_event.py

@@ -21,7 +21,10 @@ def __init__(self, message_str, message_obj, platform_meta, session_id):
 
     @staticmethod
     async def _send(
-        message: MessageChain | None, session_id: str, streaming: bool = False
+        message_id: str,
+        message: MessageChain | None,
+        session_id: str,
+        streaming: bool = False,
     ) -> str | None:
         cid = session_id.split("!")[-1]
         web_chat_back_queue = webchat_queue_mgr.get_or_create_back_queue(cid)
@@ -31,6 +34,7 @@ async def _send(
                     "type": "end",
                     "data": "",
                     "streaming": False,
+                    "message_id": message_id,
                 },  # end means this request is finished
             )
             return
@@ -45,6 +49,7 @@ async def _send(
                         "data": data,
                         "streaming": streaming,
                         "chain_type": message.type,
+                        "message_id": message_id,
                     },
                 )
             elif isinstance(comp, Json):
@@ -54,6 +59,7 @@ async def _send(
                         "data": json.dumps(comp.data, ensure_ascii=False),
                         "streaming": streaming,
                         "chain_type": message.type,
+                        "message_id": message_id,
                     },
                 )
             elif isinstance(comp, Image):
@@ -69,6 +75,7 @@ async def _send(
                         "type": "image",
                         "data": data,
                         "streaming": streaming,
+                        "message_id": message_id,
                     },
                 )
             elif isinstance(comp, Record):
@@ -84,6 +91,7 @@ async def _send(
                         "type": "record",
                         "data": data,
                         "streaming": streaming,
+                        "message_id": message_id,
                     },
                 )
             elif isinstance(comp, File):
@@ -94,12 +102,13 @@ async def _send(
                 filename = f"{uuid.uuid4()!s}{ext}"
                 dest_path = os.path.join(imgs_dir, filename)
                 shutil.copy2(file_path, dest_path)
-                data = f"[FILE]{filename}|{original_name}"
+                data = f"[FILE]{filename}"
                 await web_chat_back_queue.put(
                     {
                         "type": "file",
                         "data": data,
                         "streaming": streaming,
+                        "message_id": message_id,
                     },
                 )
             else:
@@ -108,14 +117,16 @@ async def _send(
         return data
 
     async def send(self, message: MessageChain | None):
-        await WebChatMessageEvent._send(message, session_id=self.session_id)
+        message_id = self.message_obj.message_id
+        await WebChatMessageEvent._send(message_id, message, session_id=self.session_id)
         await super().send(MessageChain([]))
 
     async def send_streaming(self, generator, use_fallback: bool = False):
         final_data = ""
         reasoning_content = ""
         cid = self.session_id.split("!")[-1]
         web_chat_back_queue = webchat_queue_mgr.get_or_create_back_queue(cid)
+        message_id = self.message_obj.message_id
         async for chain in generator:
             # if chain.type == "break" and final_data:
             #     # 分割符
@@ -130,7 +141,8 @@ async def send_streaming(self, generator, use_fallback: bool = False):
             #     continue
 
             r = await WebChatMessageEvent._send(
-                chain,
+                message_id=message_id,
+                message=chain,
                 session_id=self.session_id,
                 streaming=True,
             )
@@ -147,6 +159,7 @@ async def send_streaming(self, generator, use_fallback: bool = False):
                 "data": final_data,
                 "reasoning": reasoning_content,
                 "streaming": True,
+                "message_id": message_id,
             },
         )
         await super().send_streaming(generator, use_fallback)