diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0020037cb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,36 @@ +# Root build context is only used by claudecode/codex runtime images. +# Keep it small so Docker does not send frontend node_modules, backend venv, +# local Rust targets, logs, or other large development artifacts. +* +!sandbox-runner/ +!sandbox-runner/** +!proto/ +!proto/** +!backend/ +!backend/app/ +!backend/app/joysafeter_orchestrator_rs/ +!backend/app/joysafeter_orchestrator_rs/** +!deploy/ +!deploy/docker/ +!deploy/docker/** + +**/target/ +**/.git/ +**/.venv/ +**/node_modules/ +**/__pycache__/ +**/.pytest_cache/ +**/logs/ +**/*.log + +# Runtime images copy the prebuilt runner binary from repo-root target/. +!target/ +!target/x86_64-unknown-linux-gnu/ +!target/x86_64-unknown-linux-gnu/release/ +!target/x86_64-unknown-linux-gnu/release/joysafeter-runner +!target/aarch64-unknown-linux-gnu/ +!target/aarch64-unknown-linux-gnu/release/ +!target/aarch64-unknown-linux-gnu/release/joysafeter-runner +!target/aarch64-unknown-linux-musl/ +!target/aarch64-unknown-linux-musl/release/ +!target/aarch64-unknown-linux-musl/release/joysafeter-runner diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index d3ea552e3..cf2f012a7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -27,7 +27,7 @@ If applicable, add screenshots to help explain your problem. - Browser: [e.g., Chrome 120, Safari 17] - Python version: [e.g., 3.12.1] - Node.js version: [e.g., 20.10.0] -- Project version: [e.g., 0.1.0] +- Project version: [e.g., 0.3.2] ## Additional context Add any other context about the problem here. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f44b728a..034ae9775 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,11 @@ jobs: run: | uv run mypy app --ignore-missing-imports + - name: Error-code catalog guard + working-directory: backend + run: | + uv run pytest tests/test_error_code_catalog_guard.py -q + # - name: Run tests # working-directory: backend # env: diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 28d1d006c..26b8574a3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -13,7 +13,7 @@ on: jobs: # ============================================================================= - # Main application images (backend, frontend, openclaw) + # Main application images (backend, frontend) # Build only (no push) — release.yml handles publishing versioned images # ============================================================================= build: @@ -29,10 +29,6 @@ jobs: - name: joysafeter-frontend context: ./frontend dockerfile: ./deploy/docker/frontend.Dockerfile - - name: joysafeter-openclaw - context: ./deploy/openclaw - dockerfile: ./deploy/openclaw/Dockerfile - steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 67f726689..6439a01e7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,10 +80,6 @@ jobs: context: ./frontend dockerfile: ./deploy/docker/frontend.Dockerfile push_dockerhub: true - - name: joysafeter-openclaw - context: ./deploy/openclaw - dockerfile: ./deploy/openclaw/Dockerfile - push_dockerhub: false - name: joysafeter-sandbox context: ./deploy/docker dockerfile: ./deploy/docker/sandbox.Dockerfile diff --git a/.gitignore b/.gitignore index 0596fac50..2c6e45ec4 100644 --- a/.gitignore +++ b/.gitignore @@ -34,8 +34,6 @@ share/python-wheels/ MANIFEST # PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec @@ -62,23 +60,10 @@ cover/ *.mo *.pot -# Crew stuff: -demo_stands/ - -# Django stuff: +# Logs *.log -local_settings.py -db.sqlite3 -db.sqlite3-journal -db/*sqlite3 -*_stand/ backend/logs/* -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy +backend/app/logs # Sphinx documentation docs/_build/ @@ -87,6 +72,10 @@ docs/_build/ .pybuilder/ target/ +# Rust / Cargo build + vendored caches (sandbox-runner) +.cargo-home*/ +**/.cargo-home*/ + # Jupyter Notebook .ipynb_checkpoints @@ -95,60 +84,33 @@ profile_default/ ipython_config.py # pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: # .python-version -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - # pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide .pdm.toml -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +# PEP 582 __pypackages__/ -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - # Environments *.env .env +# .env backups / variants (.env.bak, .env.local, .env.prod, ...). The bare +# `.env`/`*.env` rules above do NOT match `.env.`, so list it +# explicitly — these files routinely hold real secrets. +.env.* +**/.env.* .venv # env.* 会误伤 backend/alembic/env.py，故只忽略根目录下的 env.* /env.* !frontend/lib/core/config/env.ts # 允许前端配置文件 -src/ENV/ venv/ ENV/ env/ env.bak/ venv.bak/ -.django_venv -.crew_venv *venv venvs/ -run_program/manager.env !src/.env !src/debug.env @@ -158,6 +120,24 @@ run_program/manager.env !env.example !**/env.example +# Deploy sensitive files (contain real secrets / site-specific config) +deploy/.env.remote.example +deploy/docker-compose.remote.yml + +# Docker image archives +/deploy/*.tar +/deploy/docker/claude-code-best-*.tgz + +# Runtime data (user uploads, logs) +backend/data/ + +# Scripts, logs, docs (not for git) +scripts/ +logs/ +backend/scripts/ +backend/logs/ +backend/docs/ + # Spyder project settings .spyderproject .spyproject @@ -183,10 +163,6 @@ dmypy.json cython_debug/ # PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ # VSCode @@ -195,8 +171,6 @@ cython_debug/ # Logging files log.txt .telemetry_log/ - -backend/langgraph_app.* backend/app/logs *.jsonl @@ -221,17 +195,11 @@ api_keys.json .secrets.baseline.scan.json .secrets.baseline.tmp -# Additional tool -additional_tools/ -make_scripts/backups - # Savefiles **/savefiles # FRONTEND GIT ignore - - # Compiled output frontend/dist frontend/tmp @@ -281,19 +249,19 @@ frontend/typings frontend/.DS_Store Thumbs.db -executions -openai-realtime-console -src/django_app/staticfiles/* - - -#knowledge data -knowledge/graph_data/ +/executions # IDE/Editor folders .claude/ .cursor/ .specify/ +# File storage +backend/data/files/ + +# Local Rust orchestrator experiment (do not publish) +#backend/app/joysafeter_orchestrator_rs/ + deploy/docker/mcp/mcp_servers/jebmcp/jeb deploy/docker/mcp/mcp_servers/jebmcp/core deploy/docker/mcp/mcp_servers/jebmcp/logs diff --git a/.jdos/backend-build.yam b/.jdos/backend-build.yam new file mode 100644 index 000000000..cc9c0f1f5 --- /dev/null +++ b/.jdos/backend-build.yam @@ -0,0 +1,18 @@ +version: v1alpha #配置的版本，目前都是这个版本 + +architecture: x86 #镜像架构 (x86/arm64) +artifact_type: app_image # 制品类型 基础镜像/应用镜像/制品包 (base_image/app_image/package) +dockerfile_path: ./.jdos/docker/ # dockerfile文件所在路径(. 即表示在代码库根路径) +dockerfile_name: backend.Dockerfile # dockerfile文件名称 +build_context: ./backend # 执行dockerfile构建的context （. 即表示在代码库根路径） +build_args: # docker build args + RUN_ENV: Prod +build_target: production # 多阶段构建，要触发的target +resources: + requests: + cpu: 4 + mem: 32768 + limits: + cpu: 4 + mem: 32768 + diff --git a/.jdos/docker/backend.Dockerfile b/.jdos/docker/backend.Dockerfile new file mode 100644 index 000000000..53464a433 --- /dev/null +++ b/.jdos/docker/backend.Dockerfile @@ -0,0 +1,138 @@ +# base image +FROM is.jd.com/jdos_base/ubuntu-24.04:latest AS packages + +WORKDIR /home/export/App/api + + +# if you located in China, you can use aliyun mirror to speed up +# RUN sed -i 's@deb.debian.org@mirrors.aliyun.com@g' /etc/apt/sources.list.d/debian.sources + +RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://mirrors.jd.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources && \ + sed -i 's|http://security.ubuntu.com/ubuntu|http://mirrors.jd.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources && \ + apt-get update \ + && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev + +# Install uv +ENV UV_VERSION=0.8.9 +ENV UV_INDEX_URL=https://mirrors.jd.com/pypi/web/simple +ENV PIP_INDEX_URL=${UV_INDEX_URL} +ENV UV_TRUSTED_HOST=mirrors.jd.com +ENV PIP_TRUSTED_HOST=${UV_TRUSTED_HOST} + +RUN apt-get install -y --no-install-recommends \ + python3-dev \ + python3-pip \ + python3-venv \ + && pip install --no-cache-dir uv==${UV_VERSION} --break-system-packages + +# Install Python dependencies +COPY pyproject.toml uv.lock ./ +RUN uv lock && uv sync --locked --no-dev + +# production stage +FROM is.jd.com/jdos_base/ubuntu-24.04:latest AS production + +WORKDIR /home/export/App/api + +ENV FLASK_APP=app.py +ENV EDITION=SELF_HOSTED +ENV DEPLOY_ENV=PRODUCTION +ENV CONSOLE_API_URL=http://127.0.0.1:5001 +ENV CONSOLE_WEB_URL=http://127.0.0.1:3000 +ENV SERVICE_API_URL=http://127.0.0.1:5001 +ENV APP_WEB_URL=http://127.0.0.1:3000 + +EXPOSE 5001 + +# Install uv +ENV UV_VERSION=0.8.9 +ENV UV_INDEX_URL=https://mirrors.jd.com/pypi/web/simple +ENV PIP_INDEX_URL=${UV_INDEX_URL} +ENV UV_TRUSTED_HOST=mirrors.jd.com +ENV PIP_TRUSTED_HOST=${UV_TRUSTED_HOST} + +# set timezone +ENV TZ=Asia/Shanghai + +# Set UTF-8 locale +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV PYTHONIOENCODING=utf-8 + +RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://mirrors.jd.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources && \ + sed -i 's|http://security.ubuntu.com/ubuntu|http://mirrors.jd.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources && \ + apt-get update \ + # Install dependencies + && apt-get install -y --no-install-recommends \ + # basic environment + curl nodejs libgmp-dev libmpfr-dev libmpc-dev \ + # For Security + expat libldap2-dev perl libsqlite3-0 zlib1g \ + # install fonts to support the use of tools like pypdfium2 + fonts-noto-cjk \ + # install a package to improve the accuracy of guessing mime type and file extension + media-types \ + # install libmagic to support the use of python-magic guess MIMETYPE + libmagic1 \ + python3-dev \ + python3-pip \ + python3-venv \ + wget \ + openssl openssh-client openssh-server \ + netcat-traditional net-tools bash-completion vim tcpdump logrotate cron sudo dmidecode tzdata && \ + ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime && \ + echo ${TZ} > /etc/timezone && \ + mkdir -p /run/sshd && chmod 755 /run/sshd && \ + ssh-keygen -A && \ + ln -sf /bin/bash /bin/sh && \ + pip install --no-cache-dir uv==${UV_VERSION} --break-system-packages && \ + rm -rf /var/lib/apt/lists/* + + +# Copy Python environment and packages +ENV VIRTUAL_ENV=/home/export/App/api/.venv +COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Download nltk data +#RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')" +COPY docker/nltk_data /root/nltk_data + +ENV TIKTOKEN_CACHE_DIR=/home/export/App/api/.tiktoken_cache + +#RUN python -c "import tiktoken; tiktoken.encoding_for_model('gpt2')" +COPY docker/tiktoken_cache $TIKTOKEN_CACHE_DIR + +# Copy source code +COPY . /home/export/App/api/ + +# Copy entrypoint +COPY docker/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ARG COMMIT_SHA +ENV COMMIT_SHA=${COMMIT_SHA} + + +# Don't run production as root +ARG UID=1001 +ARG GID=1001 +RUN addgroup --system --gid ${GID} admin && adduser --system --uid ${UID} admin && usermod -s /bin/bash admin +RUN echo "* soft nofile 1048576" >> /etc/security/limits.conf && \ + echo "* hard nofile 1048576" >> /etc/security/limits.conf + +#ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] + +ENTRYPOINT /bin/sh -c '\ +# echo $PATH && ls -l $(which gunicorn) && \ +# export $(grep -v "^#" ./.env | xargs) && \ + /usr/sbin/sshd && \ + /usr/sbin/cron && \ + apt update && \ + curl -s http://storage.jd.local/tigagent/hufu_new_install.sh | bash -s && \ + rm -rf /export/App/api && mkdir -p /export/App/ && \ + cp -r /home/export/App/api /export/App/ && chown admin:admin -R /export/App/ && \ + rm -rf /export/Logs && mkdir -p /export/Logs && chown admin:admin -R /export/Logs && \ + mkdir -p /export/home && chown admin:admin -R /export/home && \ + su -p - admin -c "export PATH=/export/App/api/.venv/bin:$PATH; export MODE=${MODE};export CELERY_MIN_WORKERS=${CELERY_MIN_WORKERS};export CELERY_AUTO_SCALE=${CELERY_AUTO_SCALE};export CELERY_MAX_WORKERS=${CELERY_MAX_WORKERS};export SERVER_WORKER_AMOUNT=${SERVER_WORKER_AMOUNT};export SERVER_WORKER_CONNECTIONS=${SERVER_WORKER_CONNECTIONS};export CELERY_QUEUES=${CELERY_QUEUES};export GUNICORN_MAX_REQUESTS=${GUNICORN_MAX_REQUESTS};export GUNICORN_MAX_REQUESTS_JITTER=${GUNICORN_MAX_REQUESTS_JITTER};export GUNICORN_GRACEFUL_TIMEOUT=${GUNICORN_GRACEFUL_TIMEOUT};cd /export/App/api; /entrypoint.sh" \ +' diff --git a/.jdos/docker/frontend.Dockerfile b/.jdos/docker/frontend.Dockerfile new file mode 100644 index 000000000..d6d91fe5d --- /dev/null +++ b/.jdos/docker/frontend.Dockerfile @@ -0,0 +1,119 @@ +# base image +#FROM node:22-alpine3.21 AS base +#LABEL maintainer="takatost@gmail.com" + +# if you located in China, you can use aliyun mirror to speed up +# RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories + +# if you located in China, you can use taobao registry to speed up +# RUN npm config set registry https://registry.npmmirror.com + + +# 兼容 jdos 迁移的 base 镜像中 +# RUN apk add --no-cache tzdata +# RUN corepack enable +# ENV PNPM_HOME="/pnpm" +# ENV PATH="$PNPM_HOME:$PATH" +# ENV NEXT_PUBLIC_BASE_PATH= + + +# install packages +FROM is.jd.local/llm-app-dev-sys/autosec-frontend-base:v20260127.215140-457440ec-lSbHwH AS packages + +WORKDIR /home/export/App/web + +COPY package.json . +COPY pnpm-lock.yaml . + +# Use packageManager from package.json +RUN corepack install + +# 显式指定 registry，避免构建环境全局配置指向不可达的内网源 +RUN npm config set registry http://mirrors.jd.com/npm && \ + pnpm install --frozen-lockfile + +# build resources +FROM is.jd.local/llm-app-dev-sys/autosec-frontend-base:v20260127.215140-457440ec-lSbHwH AS builder +WORKDIR /home/export/App/web +COPY --from=packages /home/export/App/web . +COPY . . + +ENV NODE_OPTIONS="--max-old-space-size=4096" +RUN pnpm build:docker + +# production stage +FROM is.jd.local/llm-app-dev-sys/autosec-frontend-base:v20260127.215140-457440ec-lSbHwH AS production + +ENV NODE_ENV=production +ENV EDITION=SELF_HOSTED +ENV DEPLOY_ENV=PRODUCTION +ENV CONSOLE_API_URL=http://autosec-backend-pre.jd.com +ENV APP_API_URL=http://autosec-backend-pre.jd.com +ENV MARKETPLACE_API_URL=https://marketplace.dify.ai +ENV MARKETPLACE_URL=https://marketplace.dify.ai +ENV LANGFUSE_URL=https://autosec-langfuse-pre.jd.com +ENV X_SIGHT_CHAT_URL=https://autosec-deepresearch-frontend.jd.com/chat +ENV JDME_FEEDBACK_GROUP_ID=10216353116 +ENV PORT=3000 +ENV NEXT_TELEMETRY_DISABLED=1 +ENV PM2_INSTANCES=2 + +# set timezone +ENV TZ=Asia/Shanghai +#RUN ln -s /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone + +WORKDIR /home/export/App/web +COPY --from=builder /home/export/App/web/public ./public +COPY --from=builder /home/export/App/web/.next/standalone ./ +COPY --from=builder /home/export/App/web/.next/static ./.next/static + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget openssl openssh-client openssh-server \ + netcat-traditional net-tools bash-completion vim \ + tcpdump logrotate cron sudo dmidecode tzdata && \ + mkdir -p /run/sshd && chmod 755 /run/sshd && \ + ssh-keygen -A && \ + ln -sf /bin/bash /bin/sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Don't run production as root +RUN groupadd -r -g 900 admin && \ + useradd -r -u 900 -g admin -d /home/admin -s /bin/bash admin && \ + mkdir -p /home/admin && \ + chown -R admin:admin /home/admin + +# global runtime packages +# mirrors.jd.com/npm 返回的 tarball URL 指向不可达的 backend8091，改用 npmmirror +RUN npm config delete proxy 2>/dev/null; \ + npm config delete https-proxy 2>/dev/null; \ + http_proxy="" https_proxy="" HTTP_PROXY="" HTTPS_PROXY="" \ + npm install -g pm2 --registry https://registry.npmmirror.com && \ + mkdir -p /.pm2 && \ + chown -R admin:admin /.pm2 /home/export/App/web && \ + chmod -R g=u /.pm2 /home/export/App/web + +ENV PM2_HOME=/.pm2 +ENV HOME=/home/admin + +COPY docker/entrypoint.sh ./entrypoint.sh +COPY docker/pm2.json ./pm2.json + +ARG COMMIT_SHA +ENV COMMIT_SHA=${COMMIT_SHA} + +#ENTRYPOINT ["/bin/sh", "./entrypoint.sh"] + +EXPOSE 3000 + +ENTRYPOINT /bin/sh -c '\ + ls -l /home/export/App/ && \ + /usr/sbin/sshd && \ + /usr/sbin/cron && \ + apt update && \ + curl -s http://storage.jd.local/tigagent/hufu_new_install.sh | bash -s && \ + rm -rf /export/App/web && mkdir -p /export/App/ && \ + cp -r /home/export/App/web /export/App/ && chown -R admin:admin /export/App/ && \ + rm -rf /export/Logs && mkdir -p /export/Logs && chown -R admin:admin /export/Logs && \ + su -p - admin -c "export PATH=$PATH; export CONSOLE_API_URL=${CONSOLE_API_URL};export APP_API_URL=${APP_API_URL};export LANGFUSE_URL=${LANGFUSE_URL};export X_SIGHT_CHAT_URL=${X_SIGHT_CHAT_URL};export JDME_FEEDBACK_GROUP_ID=${JDME_FEEDBACK_GROUP_ID};export TEXT_GENERATION_TIMEOUT_MS=${TEXT_GENERATION_TIMEOUT_MS};cd /export/App/web; ./entrypoint.sh" \ +' diff --git a/.jdos/frontend-build.yam b/.jdos/frontend-build.yam new file mode 100644 index 000000000..f6406d560 --- /dev/null +++ b/.jdos/frontend-build.yam @@ -0,0 +1,18 @@ +version: v1alpha #配置的版本，目前都是这个版本 + +architecture: x86 #镜像架构 (x86/arm64) +artifact_type: app_image # 制品类型 基础镜像/应用镜像/制品包 (base_image/app_image/package) +dockerfile_path: ./.jdos/docker/ # dockerfile文件所在路径(. 即表示在代码库根路径) +dockerfile_name: frontend.Dockerfile # dockerfile文件名称 +build_context: ./frontend # 执行dockerfile构建的context （. 即表示在代码库根路径） +build_args: # docker build args + RUN_ENV: Prod +build_target: production # 多阶段构建，要触发的target +resources: + requests: + cpu: 4 + mem: 32768 + limits: + cpu: 4 + mem: 32768 + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c531b17ce..fb5477fb1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,8 +29,7 @@ repos: - id: ruff-format files: ^backend/ - # Backend - Ruff check (强制检查，不允许错误) - # 注意：此 hook 会检查整个 backend 目录，确保没有 lint 错误 + # Backend - Ruff strict check - repo: local hooks: - id: backend-ruff-check @@ -39,22 +38,10 @@ repos: language: system files: ^backend/.*\.py$ pass_filenames: false - always_run: false # 只在有 Python 文件更改时运行 - stages: [commit] + always_run: false + stages: [pre-commit] - # Python - Type checking (mypy runs in Backend CI job only; skipped in pre-commit) - # - repo: https://github.com/pre-commit/mirrors-mypy - # rev: v1.13.0 - # hooks: - # - id: mypy - # files: ^backend/app/ - # args: [--ignore-missing-imports] - # additional_dependencies: - # - types-requests - # - types-PyYAML - - # Frontend - ESLint (强制检查，不允许错误) - # 注意：此 hook 会检查整个 frontend 目录，确保没有 lint 错误 + # Frontend - ESLint + TypeScript type-check + Prettier format check - repo: local hooks: - id: frontend-lint @@ -63,8 +50,26 @@ repos: language: system files: ^frontend/.*\.(ts|tsx|js|jsx)$ pass_filenames: false - always_run: false # 只在有前端文件更改时运行 - stages: [commit] + always_run: false + stages: [pre-commit] + + - id: frontend-type-check + name: Frontend TypeScript Check + entry: bash -c 'cd frontend && bun run type-check || exit 1' + language: system + files: ^frontend/.*\.(ts|tsx)$ + pass_filenames: false + always_run: false + stages: [pre-commit] + + - id: frontend-format-check + name: Frontend Prettier Check + entry: bash -c 'cd frontend && bun run format:check || exit 1' + language: system + files: ^frontend/.*\.(ts|tsx|js|jsx|json|css|md)$ + pass_filenames: false + always_run: false + stages: [pre-commit] # CI configuration ci: diff --git a/.pre-commit-setup.md b/.pre-commit-setup.md index 375ef443b..7be2bf27a 100644 --- a/.pre-commit-setup.md +++ b/.pre-commit-setup.md @@ -12,6 +12,8 @@ ### 前端检查 1. **ESLint** - JavaScript/TypeScript 代码检查 +2. **TypeScript Check** - `tsc --noEmit` +3. **Prettier Format Check** - 检查前端 TS/JS/JSON/CSS/Markdown 格式 ### 通用检查 - 行尾空白检查 @@ -108,7 +110,7 @@ backend/.venv/bin/python -m pre_commit run frontend-lint --all-files backend/.venv/bin/python -m pre_commit run --all-files ``` -### 问题：`uv run ruff check` 找不到命令 +### 问题：`backend/.venv/bin/ruff check` 找不到命令 **解决方案：** 1. 确保已安装 uv：`curl -LsSf https://astral.sh/uv/install.sh | sh` @@ -150,8 +152,10 @@ backend/.venv/bin/python -m pre_commit install --install-hooks 配置文件：`.pre-commit-config.yaml` 主要配置： -- **后端 Ruff Check**: 使用 `uv run ruff check .` 检查整个 backend 目录 +- **后端 Ruff Check**: 使用 `backend/.venv/bin/ruff check .` 检查整个 backend 目录 - **前端 ESLint**: 使用 `bun run lint` 检查整个 frontend 目录 +- **前端 TypeScript Check**: 使用 `bun run type-check` +- **前端 Prettier Check**: 使用 `bun run format:check` - 两个检查都设置为 `always_run: false`，只在相关文件更改时运行 ## 最佳实践 diff --git a/.tmp/api.pid b/.tmp/api.pid new file mode 100644 index 000000000..8d75bf72b --- /dev/null +++ b/.tmp/api.pid @@ -0,0 +1 @@ +74225 diff --git a/.tmp/orchestrator.pid b/.tmp/orchestrator.pid new file mode 100644 index 000000000..4b08b2c59 --- /dev/null +++ b/.tmp/orchestrator.pid @@ -0,0 +1 @@ +74226 diff --git a/.tmp/worker.pid b/.tmp/worker.pid new file mode 100644 index 000000000..a02c677b7 --- /dev/null +++ b/.tmp/worker.pid @@ -0,0 +1 @@ +74227 diff --git a/CHANGELOG.md b/CHANGELOG.md index e1c57392a..2b4edaf04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,7 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this - Docker 构建流程优化：限制为仅构建，仅在 release 时推送镜像 - 新增 Docker Hub 双推送支持（ghcr.io + docker.io） -- 移除 MCP 镜像并跳过 openclaw 的 DockerHub 推送 +- 移除 MCP 镜像并跳过 DockerHub 推送 ### 其他 @@ -243,7 +243,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this ### System Architecture - **Multi-Tenant Sandbox Engine** - - Strict per-user isolation for code execution environments (OpenClaw) + - Strict per-user isolation for code execution environments - Guarantees data sovereignty and prevents state leakage between concurrent sessions - **Glass-Box Observability** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a9812474f..a0e7135df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,9 +35,9 @@ This project and everyone participating in it is governed by our [Code of Conduc ### Prerequisites - Python 3.12+ -- Node.js 20+ +- Node.js 20+ with Bun 1.2+ - PostgreSQL 15+ -- Redis (optional) +- Redis - Git ### Backend Setup @@ -69,7 +69,7 @@ pytest cd frontend # Install dependencies -bun install # or npm install +bun install # Copy and configure environment cp env.example .env.local @@ -124,7 +124,7 @@ Unsure where to begin? Look for issues labeled: cd backend && pytest # Frontend - cd frontend && npm run test + cd frontend && bun run test ``` 5. **Run linters**: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 29640ebe7..ed5571d62 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -7,7 +7,7 @@ This document provides detailed instructions for setting up and running the JoyS - **Python 3.12+** with [uv](https://docs.astral.sh/uv/) package manager - **Node.js 20+** with [bun](https://bun.sh) - **PostgreSQL 15+** -- **Redis** (optional, for caching) +- **Redis** (required for task wakeups and event stream mode; `deploy/local-test.sh` starts it) - **Docker** (optional, for containerized development) ## Quick Start @@ -23,18 +23,21 @@ This document provides detailed instructions for setting up and running the JoyS 执行后，每次 `git commit` 将自动运行代码校验。手动全量检查：`./scripts/run-pre-commit.sh` 或 `backend/.venv/bin/python -m pre_commit run --all-files`。 -### 1. Start Database Services - -Using Docker (recommended): +### 1. One-command local test ```bash -cd backend/docker -./start.sh +cd deploy +./local-test.sh ``` -Or manually start PostgreSQL and Redis on your system. +This starts PostgreSQL, Redis, and the backend (`api`, `orchestrator`, `worker`) plus the frontend for local testing. + +### 2. Start Backend Manually -### 2. Start Backend +The backend is one codebase split into three services by the `JOYSAFETER_SERVICE_ROLE` +environment variable. For local development you can run everything in a single process with +`JOYSAFETER_SERVICE_ROLE=all` (the default in `env.example`) via the compatibility entrypoint +`app.main:app`: ```bash cd backend @@ -49,19 +52,29 @@ uv sync --dev # Configure environment cp env.example .env -# Edit .env with your settings +# Edit .env with your settings (JOYSAFETER_SERVICE_ROLE=all runs all three roles in one process) # Note: UV uses Tsinghua mirror by default (configured in uv.toml) # You can customize via UV_INDEX_URL environment variable # Run database migrations alembic upgrade head -# Start development server +# Start development server (single-process, all roles) uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 ``` Backend will be available at http://localhost:8000 +To run the three services separately (as in the compose deployment), start each with its own +role and entrypoint: + +```bash +JOYSAFETER_SERVICE_ROLE=api uv run uvicorn app.joysafeter_api.main:app --port 8000 +JOYSAFETER_SERVICE_ROLE=orchestrator JOYSAFETER_GRPC_HOST=0.0.0.0 JOYSAFETER_GRPC_PORT=9090 \ + uv run uvicorn app.joysafeter_orchestrator.main:app --host 127.0.0.1 --port 8001 --workers 1 +JOYSAFETER_SERVICE_ROLE=worker uv run uvicorn app.joysafeter_worker.main:app --port 8002 +``` + #### PyPI 镜像源配置 (PyPI Mirror Configuration) 项目默认使用清华大学镜像源 (`https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple`) 以加速依赖安装。配置方式： @@ -95,7 +108,7 @@ cp env.example .env.local # Edit .env.local with your settings # Start development server -bun run dev # or: npm run dev +bun run dev ``` Frontend will be available at http://localhost:3000 @@ -114,7 +127,7 @@ pytest tests/ --cov=app --cov-report=html # Frontend tests cd frontend -npm run test +bun run test ``` ### Code Formatting & Linting @@ -128,8 +141,8 @@ mypy app # Type check # Frontend cd frontend -npm run lint # ESLint -npm run type-check # TypeScript +bun run lint # ESLint +bun run type-check # TypeScript ``` ### Using Pre-commit Hooks @@ -256,27 +269,29 @@ alembic downgrade -1 ## Architecture Overview +The backend is a single codebase split into three FastAPI services (`api` / `orchestrator` / +`worker`) selected at boot by `JOYSAFETER_SERVICE_ROLE`. See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) +for the full design. + ``` -agent-platform/ -├── backend/ # FastAPI backend -│ ├── app/ -│ │ ├── api/ # REST API routes -│ │ ├── core/ # Core business logic (agents, graphs) -│ │ ├── models/ # SQLAlchemy database models -│ │ ├── schemas/ # Pydantic schemas -│ │ ├── services/ # Business services -│ │ └── utils/ # Utilities -│ ├── alembic/ # Database migrations -│ └── tests/ # Test suite +JoySafeter/ +├── backend/app/ +│ ├── joysafeter_api/ # API service: REST routers, SSE, WS notifications, auth +│ ├── joysafeter_orchestrator/ # Orchestrator: gRPC AgentBridge, scheduler, sandbox lifecycle, event bus +│ ├── joysafeter_worker/ # Worker: Redis Stream consumer → event persistence +│ ├── joysafeter_domain/ # SQLAlchemy models, schemas, services, state machines +│ └── joysafeter_shared/ # Cross-service foundation (config, llm, security, storage, cache) │ -├── frontend/ # Next.js frontend +├── frontend/ # Next.js App Router UI (product surface under /managed/**) │ ├── app/ # App Router pages │ ├── components/ # React components │ ├── lib/ # Utilities, API clients -│ ├── stores/ # Zustand state stores -│ └── services/ # API service layer +│ └── stores/ # Zustand state stores │ -└── deploy/ # Deployment configurations +├── proto/ # AgentBridge gRPC contract (joysafeter.proto) +├── sandbox-runner/ # Rust workspace: types / runtime / runner / ctl +├── skills/ # Pre-built skill packs +└── deploy/ # Docker Compose + deployment configs ``` ## Environment Variables @@ -287,12 +302,15 @@ See `backend/env.example` and `frontend/env.example` for all available configura | Variable | Description | |----------|-------------| +| `JOYSAFETER_SERVICE_ROLE` | Service role: `api` / `orchestrator` / `worker`, or `all` for single-process dev | | `POSTGRES_HOST` | PostgreSQL host address | | `POSTGRES_PORT` | PostgreSQL port | | `POSTGRES_USER` | PostgreSQL username | | `POSTGRES_PASSWORD` | PostgreSQL password | | `POSTGRES_DB` | PostgreSQL database name | +| `REDIS_URL` | Redis connection URL (event streams, pub/sub, task queue) | | `SECRET_KEY` | JWT signing key | +| `CREDENTIAL_ENCRYPTION_KEY` | AES-256-GCM key for encrypting Secrets and Vault credentials | | `DEBUG` | Enable debug mode | | `CORS_ORIGINS` | Allowed CORS origins | @@ -300,8 +318,14 @@ See `backend/env.example` and `frontend/env.example` for all available configura | Variable | Description | |----------|-------------| -| `NEXT_PUBLIC_API_URL` | Backend API URL | -| `BETTER_AUTH_SECRET` | Auth secret key | +| `NEXT_PUBLIC_API_URL` | Optional backend API URL override; defaults to `http://localhost:8000` in `lib/api-client.ts` | +| `NEXT_PUBLIC_APP_URL` | Public frontend URL used for generated links in Docker/production | +| `NEXT_PUBLIC_MAX_UPLOAD_FILE_BYTES` | Browser-side upload size limit; keep aligned with backend `JOYSAFETER_MAX_UPLOAD_FILE_BYTES` | +| `NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED` | Shows the email/password signup entry point in the auth UI; backend auth policy still applies | +| `NEXT_PUBLIC_CSP_CONNECT_SRC_EXTRA` / `NEXT_PUBLIC_CSP_FRAME_SRC_EXTRA` | Adds third-party connect/frame domains to the generated CSP | +| `DISABLE_REGISTRATION` / `EMAIL_VERIFICATION_ENABLED` | Frontend server auth UI flags; backend enforcement still comes from backend auth config | +| `RESEND_API_KEY` / `AZURE_ACS_CONNECTION_STRING` / `FROM_EMAIL_ADDRESS` / `EMAIL_DOMAIN` | Frontend server email-service configuration | +| `FRONTEND_PORT` | Internal port the Next.js server listens on | ## Troubleshooting @@ -314,7 +338,7 @@ See `backend/env.example` and `frontend/env.example` for all available configura ### Frontend Build Errors 1. Clear Next.js cache: `rm -rf .next` -2. Reinstall dependencies: `rm -rf node_modules && npm install` +2. Reinstall dependencies: `rm -rf node_modules && bun install` 3. Check Node.js version: `node --version` (should be 20+) ### Import Errors diff --git a/INSTALL.md b/INSTALL.md index cbc42448a..92e2fcfcd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -8,131 +8,59 @@ Below you will find comprehensive setup instructions depending on your deploymen - Python 3.12+ and Node.js 20+ (only for local development) - PostgreSQL/Redis are included in Docker deployment -## Recommended: One-Click Run (Docker) - -```bash -./deploy/quick-start.sh -``` - -> For production and advanced deployment scenarios (pre-built images, custom registry, middleware-only, etc.), use the scenario scripts and docs under: [deploy/README.md](deploy/README.md). - -## Manual Deployment +## Recommended: Docker Compose ```bash cd deploy +cp .env.example .env +cd ../backend && cp env.example .env +cd ../frontend && cp env.example .env +cd ../deploy -# 1. Build images -sh deploy.sh build --all - -# 2. Initialize environment variables -cp ../frontend/env.example ../frontend/.env -cp ../backend/env.example ../backend/.env - -# IMPORTANT!! Configure TAVILY_API_KEY for search (Register at https://www.tavily.com/) -# Replace tvly-* with your actual API Key -echo 'TAVILY_API_KEY=tvly-*' >> ../backend/.env - -# 3. Initialize database -docker compose --profile init up - -# 4. Start services -docker compose -f docker-compose.yml up - -# Stop services -docker compose -f docker-compose.yml down -``` - -## Using Pre-built Docker Images - -We provide pre-built Docker images on Docker Hub: - -- `docker.io/jdopensource/joysafeter-backend:latest` -- `docker.io/jdopensource/joysafeter-frontend:latest` -- `docker.io/jdopensource/joysafeter-mcp:latest` - -Use them via: - -```bash -cd deploy -export DOCKER_REGISTRY=docker.io/jdopensource -docker-compose -f docker-compose.yml up -d +# Fully local: PostgreSQL + Redis + Python orchestrator +docker compose --profile local-redis --profile python-orchestrator up -d --build ``` -All images support multi-architecture (amd64, arm64). +Access points: -## Other setup methods +- Frontend: `http://localhost:3000` +- Backend API: `http://localhost:8000` +- API Docs: `http://localhost:8000/docs` -> The deploy module is the single source of truth for all Docker scenarios: -> [deploy/README.md](deploy/README.md) +The backend is one codebase split into three services by `JOYSAFETER_SERVICE_ROLE` and +deployed as separate containers: `api`, `orchestrator`, and `worker`, alongside PostgreSQL, +Redis, Envoy (per-sandbox egress proxy), and skillspector (skill security scanner). Local Redis +is started only when the `local-redis` profile is enabled; for cloud Redis, leave that profile +off and set `REDIS_URL` in `deploy/.env`. You must +use the `python-orchestrator` profile for the supported quick-start path. The +`rust-orchestrator` profile is experimental in this checkout: its Dockerfile expects +`backend/app/joysafeter_orchestrator_rs`, which is not present. For cloud +PostgreSQL/Redis, image building, and troubleshooting, see [deploy/README.md](deploy/README.md). -### Option 1: Interactive Installation - -Use the installation wizard to configure your environment: +## Using Pre-built Docker Images ```bash cd deploy - -# Interactive installation -./install.sh - -# Or quick install for development -./install.sh --mode dev --non-interactive -``` - -After installation, start services with scenario-specific scripts: - -```bash -# Development scenario -./scripts/dev.sh - -# Production scenario -./scripts/prod.sh - -# Test scenario -./scripts/test.sh - -# Minimal scenario (middleware only) -./scripts/minimal.sh - -# Local development (backend/frontend run locally) -./scripts/dev-local.sh +cp .env.example .env +# Point the image variables at the published registry, then start with a profile. +docker compose --profile local-redis --profile python-orchestrator up -d ``` -### Option 2: Manual Docker Compose - -For advanced users who want full control: +## Local Test One-Command Startup ```bash cd deploy - -# 1. Create configuration files -cp .env.example .env -cd ../backend && cp env.example .env - -# 2. Start middleware (PostgreSQL + Redis) -cd ../deploy -./scripts/start-middleware.sh - -# 3. Start full services -docker-compose up -d +./local-test.sh ``` -### Option 3: Environment Check -Before starting, you can check your environment: +## Environment Check ```bash cd deploy -./scripts/check-env.sh +docker compose config ``` -This will verify: -- Docker installation and status -- Docker Compose version -- Port availability -- Configuration files -- Disk space - ## Manual Setup
@@ -156,10 +84,15 @@ cp env.example .env createdb joysafeter alembic upgrade head -# Start server +# Start server (single-process, all three roles via JOYSAFETER_SERVICE_ROLE=all) uv run uvicorn app.main:app --reload --port 8000 ``` +> For a split, multi-service run (matching the compose deployment) start each role with its +> own entrypoint — `app.joysafeter_api.main:app`, `app.joysafeter_orchestrator.main:app`, +> `app.joysafeter_worker.main:app` — and set `JOYSAFETER_SERVICE_ROLE` accordingly. +> See [DEVELOPMENT.md](DEVELOPMENT.md). +
@@ -169,7 +102,7 @@ uv run uvicorn app.main:app --reload --port 8000 cd frontend # Install dependencies -bun install # or: npm install +bun install # Configure environment cp env.example .env.local diff --git a/INSTALL_CN.md b/INSTALL_CN.md index 743cef363..ca845bf45 100644 --- a/INSTALL_CN.md +++ b/INSTALL_CN.md @@ -8,132 +8,64 @@ - Python 3.12+ 与 Node.js 20+（仅本地开发需要） - PostgreSQL/Redis 在 Docker 部署场景下会自动包含 -## 推荐：一键启动（Docker） - -```bash -./deploy/quick-start.sh -``` - -> 生产与更多 Docker 场景（预构建镜像、自定义 registry、仅中间件等）请以 [deploy/README.md](deploy/README.md) 为准。 - -## 手动部署 +## 推荐：Docker Compose 部署 ```bash cd deploy +cp .env.example .env +cd ../backend && cp env.example .env +cd ../frontend && cp env.example .env +cd ../deploy -# 1. 编译镜像 -sh deploy.sh build --all - -# 2. 初始化环境变量 -cp ../frontend/env.example ../frontend/.env -cp ../backend/env.example ../backend/.env - -# 重要！！配置 TAVILY_API_KEY 搜索所用 key（自行注册 https://www.tavily.com/） -# 请将 tvly-* 替换为您实际的 API Key -echo 'TAVILY_API_KEY=tvly-*' >> ../backend/.env - -# 3. 初始化数据库 -docker compose --profile init up - -# 4. 启动服务 -docker compose -f docker-compose.yml up - -# 关闭服务 -docker compose -f docker-compose.yml down - -docker compose logs +# 全本地：PostgreSQL + Redis + Python orchestrator +docker compose --profile local-redis --profile python-orchestrator up -d --build ``` -## 使用预构建的 Docker 镜像 +访问地址： -我们提供预构建镜像（Docker Hub）： +- 前端：`http://localhost:3000` +- 后端 API：`http://localhost:8000` +- API 文档：`http://localhost:8000/docs` -- `docker.io/jdopensource/joysafeter-backend:latest` -- `docker.io/jdopensource/joysafeter-frontend:latest` -- `docker.io/jdopensource/joysafeter-mcp:latest` +后端是同一份代码，通过 `JOYSAFETER_SERVICE_ROLE` 拆成三个服务并作为独立容器部署：`api`、 +`orchestrator`、`worker`，同时配套 PostgreSQL、Redis、Envoy（每沙箱出站代理）与 +skillspector（Skill 安全扫描服务）。本地 Redis 只有启用 `local-redis` profile 时才会启动；如使用云 Redis， +不要启用该 profile，改 `deploy/.env` 的 `REDIS_URL` 即可。快速开始支持路径是 `python-orchestrator` +profile。当前 checkout 中 `rust-orchestrator` 仍是实验入口：其 Dockerfile 依赖 +`backend/app/joysafeter_orchestrator_rs`，但该源码目录当前不存在。生产、云数据库/云 Redis、镜像 +构建等场景请以 [deploy/README.md](deploy/README.md) 为准。 -使用方式： +## 使用预构建的 Docker 镜像 ```bash cd deploy -export DOCKER_REGISTRY=docker.io/jdopensource -docker-compose -f docker-compose.yml up -d +cp .env.example .env +# 将镜像相关变量指向已发布的镜像仓库，再带 profile 启动。 +docker compose --profile local-redis --profile python-orchestrator up -d ``` 所有镜像均支持多架构（amd64, arm64）。 -## 其他配置方式 - -> Docker 部署场景的单一入口： [deploy/README.md](deploy/README.md) - -### 方式 1：交互式安装 - -使用安装向导来配置您的环境： +## 本地测试一键启动 ```bash cd deploy - -# 交互式安装 -./install.sh - -# 或快速安装用于开发 -./install.sh --mode dev --non-interactive -``` - -安装完成后，使用针对特定场景的脚本运行服务： - -```bash -# 开发环境 -./scripts/dev.sh - -# 生产环境 -./scripts/prod.sh - -# 测试环境 -./scripts/test.sh - -# 极简运行（仅中间件） -./scripts/minimal.sh - -# 本地开发（后端和前端直接本地运行） -./scripts/dev-local.sh +./local-test.sh ``` -### 方式 2：手动执行 Docker Compose - -为希望获得完全控制权的高级用户准备： +停止： ```bash -cd deploy - -# 1. 创建配置文件 -cp .env.example .env -cd ../backend && cp env.example .env - -# 2. 启动中间件（PostgreSQL + Redis） -cd ../deploy -./scripts/start-middleware.sh - -# 3. 启动全量服务 -docker-compose up -d +docker compose down ``` -### 方式 3：环境检查 - -在启动之前，您可以检查您的环境情况： +## 环境检查 ```bash cd deploy -./scripts/check-env.sh +docker compose config ``` -这将验证： -- Docker 安装状态 -- Docker Compose 版本 -- 端口占用情况 -- 配置文件 -- 磁盘空间 - ## 手动安装（本地开发）
@@ -157,10 +89,15 @@ cp env.example .env createdb joysafeter alembic upgrade head -# 启动服务 +# 启动服务（单进程，JOYSAFETER_SERVICE_ROLE=all 在一个进程内运行三种角色） uv run uvicorn app.main:app --reload --port 8000 ``` +> 若需按服务拆分运行（与 compose 部署一致），请分别用各自的入口启动 —— +> `app.joysafeter_api.main:app`、`app.joysafeter_orchestrator.main:app`、 +> `app.joysafeter_worker.main:app`，并相应设置 `JOYSAFETER_SERVICE_ROLE`。 +> 详见 [DEVELOPMENT.md](DEVELOPMENT.md)。 +
@@ -170,7 +107,7 @@ uv run uvicorn app.main:app --reload --port 8000 cd frontend # 安装依赖 -bun install # 或: npm install +bun install # 配置环境变量 cp env.example .env.local diff --git a/README.md b/README.md index 32cf6a2e1..563561ed7 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,17 @@

- The AI-native platform for building, orchestrating, and running security agents at scale.
- From idea to production-grade security automation — in minutes, not months. + The open, self-hostable managed-agent platform for security.
+ Define an agent's tools, skills, and guardrails — JoySafeter runs it on your own hardened, observable infrastructure. From idea to production-grade security automation in minutes, not months.

License: Apache 2.0 Python 3.12+ Node.js 20+ - LangGraph FastAPI + Next.js 16 MCP Protocol - DeepAgents v0.4

@@ -26,16 +25,32 @@ ## Why JoySafeter -Traditional security tooling hits a ceiling: scripts are brittle, single agents lack context, and complex scenarios require 2–3 engineers working in parallel. JoySafeter breaks that ceiling. - -| Challenge | Traditional Approach | JoySafeter | -|-----------|---------------------|------------| -| APK vulnerability analysis | Manual MobSF + engineer review | Autonomous agent: upload → analyze → report | -| Penetration testing | Fixed scripts, static playbooks | Dynamic DeepAgents that adapt to findings in real time | -| Tool integration | Custom glue code per tool | 200+ tools via MCP Protocol, zero glue | -| Scale | Linear headcount growth | Agent teams that multiply capacity | - -> JoySafeter defines a new paradigm: **AI-driven Security Operations (AISecOps)** — where multi-agent collaboration, cognitive memory, and scenario-matched skills replace manual coordination. +The managed-agent operating model is the right one for autonomous work. But for **security** — +where you operate on client systems under NDA and run aggressive tooling — *where and how* the +agent runs is the whole decision. JoySafeter gives you that model **on your own infrastructure**: + +- **Your data and targets never leave your infra.** Fully self-hosted: prompts, findings, captured + traffic, and target details stay inside your network. No third party ever sees the engagement. +- **Network-contained execution.** Every session runs in a `NetworkMode=none` sandbox behind an + Envoy proxy with a **deny-all-by-default egress allowlist** — offensive tools can't phone home or + pivot into your network unless you explicitly permit it. +- **Runtime-closed skill supply-chain control.** Skills are code that runs in your environment; + skills are scanned by skillspector, and runtime packing blocks anything not approved, not + successfully scanned, blocked, failed, still scanning, or drifted from its last scan. +- **Engine-agnostic.** Claude Code, Codex, or the self-developed `ccb` engine behind one gRPC + contract — not locked to a single vendor or model. + +| | Cloud managed agents | Build it yourself | **JoySafeter** | +|---|---|---|---| +| Data / target residency | Vendor cloud | Yours | **Yours — fully self-hosted** | +| Engine / model | Single vendor | Whatever you wire | **Claude Code / Codex / native, per agent** | +| Network isolation | Vendor-managed | You build it | **Per-sandbox Envoy deny-all egress** | +| Skill & tool safety | Vendor-managed | You build it | **SkillSpector scan + runtime-closed gate** | +| Time to production | Days | Months | **Days, on your own hardware** | + +> JoySafeter frames this as **AI-driven Security Operations (AISecOps)**: the managed-agent model +> — multi-step autonomy, sandboxed tools, sessions, full-chain observability — specialized for +> security and run entirely under your control. --- @@ -49,12 +64,15 @@ Traditional security tooling hits a ceiling: scripts are brittle, single agents APK Vulnerability Detection Demo

+> **This is a demo workflow, not a built-in integration.** The agent is configured with the +> `pentest-mobile-app` skill and a sandbox image carrying mobile-analysis tooling — you choose the tools. + **How it works:** 1. User uploads the APK file -2. Agent invokes MobSF for static analysis +2. The agent runs static analysis with the mobile tooling in its sandbox (e.g. MobSF) 3. Extracts critical risk signals — permission abuse, hardcoded secrets, insecure network config -4. Deep-validates high-severity findings via Frida dynamic instrumentation +4. Deep-validates high-severity findings via dynamic instrumentation (e.g. Frida) 5. Auto-generates a structured report aligned to OWASP Mobile Top 10 The entire flow — from upload to report — requires zero manual intervention, covering work that traditionally takes 2–3 security engineers. @@ -72,10 +90,10 @@ The entire flow — from upload to report — requires zero manual intervention, **How it works:** 1. Open the Workbench and create a new agent -2. Enable **DeepAgents mode** → select penetration testing skills +2. Choose an engine (Claude Code / Codex / native) and select penetration testing skills 3. Provide an authorized target URL and test requirements -4. Agent runs autonomously — if it discovers a login page, it automatically triggers auth bypass testing -5. Download the final report when the run completes +4. Agent runs autonomously inside an isolated sandbox — if it discovers a login page, it automatically triggers auth bypass testing +5. Download the final report when the session completes > **Note:** Requires sandbox image `swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/jd-opensource/joysafeter-sandbox:latest` configured in Sandbox Settings. @@ -83,159 +101,204 @@ This dynamic decision-making — where the agent adapts its next step based on w --- -## Core Capabilities +## Core Capabilities — the managed-agent building blocks + +You declare an agent — engine, model, system prompt, tools, skills, MCP servers, guardrails — and +JoySafeter runs it end-to-end on the same managed-agent building blocks Anthropic ships behind +Claude Managed Agents, only **self-hosted and security-specialized**: + + + + + + + +
-### Visual Agent Builder +### 🧠 Managed harness & orchestration + +- The **orchestrator** + gRPC `AgentBridge` + in-sandbox Rust `sandbox-runner` decide when to call tools, manage context, and recover from errors +- **DB-backed scheduling** — tasks claimed from Postgres with `FOR UPDATE SKIP LOCKED`, with retries and timeouts +- **Engine-agnostic** — Claude Code CLI, Codex app-server, or the self-developed `native` (`ccb`) harness, selected per agent + + + +### 📦 Sandboxed execution + +- Every session runs in its **own hardened container** — dropped capabilities, non-root, no-new-privileges +- **Pluggable providers** — Docker (default), E2B (Firecracker), Daytona, behind one SPI +- **Egress control** — per-sandbox Envoy proxy with a deny-all-by-default domain allowlist + +
+ +### 🔧 Tools, custom tools & MCP -- **No-code workflow editor** — drag-and-drop nodes with loops, conditionals, and parallel execution -- **Rapid Mode** — describe in natural language, get a running agent team in minutes -- **Deep Mode** — visual debugging and step-by-step observability for complex security research +- Attach **builtin tools**, **custom tools** (name + JSON Schema), and **MCP servers** per agent +- MCP configs + Vault credentials are resolved at run time and delivered to the sandbox over gRPC +- **Security skill packs** drive tools like **Nmap / Nuclei / Trivy** inside the sandbox image; connect any external tool via the **MCP protocol** -### 200+ Security Tools, Ready to Use +### 📚 Skills -- Pre-integrated **Nmap, Nuclei, Trivy**, and more -- **MCP Protocol** — extend with any tool via Model Context Protocol -- **30+ pre-built skills** — penetration testing, document analysis, cloud security, and more +- **30 versioned capability packs** — penetration testing, document analysis, planning/meta +- **SkillSpector security scanning** + a runtime `is_skill_usable` gate (approved + `passed` / `warning` scan + no content drift) +- **AI skill authoring** — draft, edit, version, and diff skills with an LLM-assisted editor
-### DeepAgents Orchestration +### 💾 Sessions, memory & resume -- **Manager-Worker multi-level** agent collaboration -- **Memory evolution** — long/short-term memory for continuous learning across sessions -- **Skill system** — versioned, reusable capability units with progressive disclosure -- **LangGraph engine** — graph-based workflows with full state management +- **Sessions** are persistent conversations with an **append-only, seq-ordered event log** +- **Memory stores** — versioned, agent-writable KV stores synced bi-directionally with the sandbox +- **Resumable** — reattach a session's harness + work dir on reconnect -### Enterprise Ready +### 🛡️ Scoped permissions & guardrails -- **Multi-tenancy** — isolated workspaces with role-based access control -- **Full audit trail** — execution tracing and compliance governance -- **SSO integration** — GitHub, Google, Microsoft, OIDC (Keycloak, Authentik, GitLab), JD SSO -- **Multi-tenant sandbox** — per-user isolated code execution, zero state leakage +- **Per-tool authorization** — `always_ask` / `always_allow`, with human-in-the-loop confirmation for high-risk tools +- **Encrypted credentials** — provider keys in Secrets, MCP creds in Vaults, AES-256-GCM, injected as sandbox env +- **SSRF guard** — blocks cloud-metadata endpoints; opt-in private-range hardening + +
+ +### 🔎 Full-chain observability + +- **Live SSE event stream** of every message, thinking step, tool call, tool result, and model request +- **OpenTelemetry** traces + `observations` with token/cost aggregation, `trace_id` propagated end-to-end +- Append-only session event log doubles as a full **audit trail** + + + +### 🏢 Multi-tenancy & access + +- **Orgs / projects / RBAC** — isolated workspaces with role-based access control +- **SSO** — GitHub, Google, Microsoft, OIDC (Keycloak, Authentik, GitLab), JD SSO +- **Quickstart** — describe your goal in natural language and get a running agent in minutes
+> How these blocks compare to the Claude Managed Agents feature set — and what's on the +> roadmap — is tracked in [Managed-Agent Parity & Roadmap](#managed-agent-parity--roadmap). + --- ## Quick Start -### One-Click Launch (Recommended) +### Docker Compose (recommended) ```bash -./deploy/quick-start.sh +cd deploy +cp .env.example .env +cd ../backend && cp env.example .env +cd ../frontend && cp env.example .env +cd ../deploy + +# Fully local: PostgreSQL + Redis + Rust orchestrator +docker compose --profile local-redis --profile rust-orchestrator up -d --build ``` -The script provides an interactive menu to choose your startup mode and customize ports (with conflict detection): +Access points: -| Mode | Description | Ports Configured | -|------|-------------|-----------------| -| **(1) Docker Compose Full Stack** | All services in containers, supports localhost or remote server IP/domain | Frontend, Backend, PostgreSQL, Redis | -| **(2) Local Frontend Only** | `bun run dev`, supports connecting to remote backend | Frontend (can specify remote backend address) | -| **(3) Local Backend Only** | `uvicorn --reload`, supports remote DB/Redis | Backend (can specify remote DB/Redis/frontend address) | -| **(4) Local Frontend + Backend** | Auto-starts middleware, supports exposing via non-localhost address | Frontend, Backend | +| Service | URL | +|---------|-----| +| Frontend | http://localhost:3000 | +| Backend API | http://localhost:8000 | +| API Docs | http://localhost:8000/docs | -All modes support remote deployment scenarios: -- **Docker Compose Full Stack** — choose deployment address (localhost or IP/domain) + http/https -- **Local Frontend Only** — optionally connect to a remote backend API (enter backend IP + port + protocol) -- **Local Backend Only** — optionally connect to remote PostgreSQL, Redis, and frontend (enter each address and port) -- **Local Frontend + Backend** — optionally expose services via a non-localhost address -- Non-localhost deployments automatically update `frontend/.env` CSP whitelist (`NEXT_PUBLIC_CSP_CONNECT_SRC_EXTRA`) +The backend runs as two Python services plus the Rust orchestrator, deployed as separate +containers: -```bash -./deploy/quick-start.sh --skip-env # Skip .env file initialization -./deploy/quick-start.sh --skip-db-init # Skip database initialization -``` +- `api` — REST `/api/v1/*`, SSE event stream, notification WebSocket, auth + (`JOYSAFETER_SERVICE_ROLE=api`). +- `orchestrator-rs` — task scheduler, gRPC `AgentBridge`, and sandbox lifecycle. +- `worker` — consumes the Redis event stream and persists events to Postgres + (`JOYSAFETER_SERVICE_ROLE=worker`). -### Launch by Scenario +Supporting infrastructure: PostgreSQL, Redis, Envoy (per-sandbox egress proxy), and +skillspector (skill security scanner). The bundled Redis service is behind the `local-redis` +profile; for cloud Redis, leave that profile off and set `REDIS_URL` in `deploy/.env`. +The Python orchestrator package has been removed. Use the Rust orchestrator +profile for local and containerized orchestration. + +### Local test one-command startup ```bash -# ─── Development ──────────────────────────────────────── -./deploy/scripts/dev.sh # Docker full-stack dev (containerized frontend + backend) -./deploy/scripts/dev-local.sh # Local dev prep (start middleware, run backend/frontend on host) -./deploy/scripts/dev-backend.sh # Local backend only (requires middleware running) -./deploy/scripts/dev-frontend.sh # Local frontend only (requires backend running) - -# ─── Production ───────────────────────────────────────── -./deploy/scripts/prod.sh # Production deploy (pre-built images + docker-compose.prod.yml) -./deploy/scripts/prod.sh --skip-mcp # Production without MCP service -./deploy/scripts/prod.sh --skip-pull # Skip image pull, use local images - -# ─── Middleware / Infrastructure ──────────────────────── -./deploy/scripts/start-middleware.sh # Start middleware (PostgreSQL + Redis + MCP) -./deploy/scripts/minimal.sh # Minimal startup (PostgreSQL + Redis only) -./deploy/scripts/minimal.sh --with-mcp # Minimal + MCP service -./deploy/scripts/stop-middleware.sh # Stop middleware - -# ─── Test / CI ────────────────────────────────────────── -./deploy/scripts/test.sh # Test environment (minimal deps, automation-friendly) - -# ─── Install / Check ─────────────────────────────────── -./deploy/install.sh # Interactive installation wizard (generates config files) -./deploy/install.sh --mode dev --non-interactive # Non-interactive install -./deploy/scripts/check-env.sh # Environment preflight (Docker, ports, config files) - -# ─── Image Management ────────────────────────────────── -./deploy/deploy.sh build # Build frontend + backend images -./deploy/deploy.sh build --all # Build all images (including OpenClaw) -./deploy/deploy.sh push # Build and push to registry -./deploy/deploy.sh pull # Pull latest pre-built images +cd deploy +./local-test.sh ``` -### Default Ports +### Common deployment commands -| Service | Port | URL | -|---------|------|-----| -| Frontend | `3000` | http://localhost:3000 | -| Backend API | `8000` | http://localhost:8000 | -| API Docs | `8000/docs` | Swagger UI | -| PostgreSQL | `5432` | Database | -| Redis | `6379` | Cache | +```bash +./deploy/local-test.sh # local test one-command startup +./deploy/deploy.sh build # build frontend + backend images +./deploy/deploy.sh build --all # build all images +``` -> **Prerequisites:** Docker + Docker Compose. See [INSTALL.md](INSTALL.md) for detailed installation guide, [deploy/PRODUCTION_IP_GUIDE.md](deploy/PRODUCTION_IP_GUIDE.md) for production deployment. +> **Prerequisites:** Docker + Docker Compose. See [deploy/README.md](deploy/README.md) for deployment details. --- ## Architecture -

- JoySafeter System Architecture -

+![JoySafeter managed-agent architecture](docs/architecture-diagram.png) + +Overview infographic — ① control plane (REST · CLI) → ② Agent Harness (in-sandbox) → ③ Session state-layer. Interactive version: [`docs/architecture-diagram.html`](docs/architecture-diagram.html). + +```mermaid +flowchart LR + FE["Browser"] -->|"REST · SSE"| API["API service"] + API -->|"rpush task"| RLIST[("Redis list
global_queue")] + RLIST -.->|"wakeup"| SCHED["Orchestrator
scheduler (DB-authoritative)"] + SCHED -->|"claim / provision"| SBX["Sandbox (NetworkMode=none)
Rust runner + harness"] + SBX <-->|"gRPC AgentBridge"| ENVOY["Envoy
sole network conduit"] + ENVOY <--> GRPC["Orchestrator gRPC :9090"] + ENVOY -->|"egress allowlist"| EXT["Model API · MCP · targets"] + GRPC -->|"harness events"| BUS["Two-phase event bus"] + BUS -->|"① persist XADD"| RSTREAM[("Redis stream")] + BUS -->|"② broadcast PUBLISH"| RPUB[("Redis pub/sub")] + RSTREAM -->|"XREADGROUP"| WK["Worker → persist"] + WK --> PG[("PostgreSQL")] + WK -.->|"republish"| RPUB + RPUB -->|"SessionBroadcaster"| API + API -->|"SSE stream"| FE +``` -> Full architecture details: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) +> Full architecture (deployment topology, gRPC contract, engines, sandbox, event model, +> domain FSMs): **[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)** · layered view: +> [docs/architecture-unified-event-model.mmd](docs/architecture-unified-event-model.mmd) **Key design principles:** -- **Graph-based execution** — every agent workflow is a stateful LangGraph, enabling pause, resume, and branch -- **Unified Run Center** — Chat, Copilot, and Skill Creator share a single event-sourced run lifecycle (Run → Event → Snapshot) -- **Unified WebSocket layer** — BaseWsClient abstract class; Chat / Run / Notification clients share lifecycle, auth (ws-token), and reconnect logic -- **Full-chain trace_id propagation** — contextvars-based request tracing from HTTP/WS entry through LangGraph to persistence -- **Glass-box observability** — real-time Langfuse tracing of every agent decision and state transition -- **RAII sandbox isolation** — per-user Docker containers with automatic handle release, zero state leakage -- **Canonical model identifiers** — full-stack (provider_name, model_name) resolution via ModelService → ModelFactory -- **Layered skill system** — skills are versioned units that compose into workflows without coupling - -### User Journey — Quick Start in 9 Steps +- **Explicit service boundaries** — Python exposes `api` and `worker` roles; orchestration runs in the Rust `orchestrator-rs` service +- **DB is the source of truth for scheduling** — the API enqueues a task onto a Redis list as a wakeup signal; the orchestrator claims pending rows from Postgres with `FOR UPDATE SKIP LOCKED` +- **Decoupled persistence and live delivery** — a two-phase event bus fans out to Redis Streams (durable, consumed by the Worker → `joysafeter_session_events`) and Redis Pub/Sub (ephemeral, driving the SSE fan-out to the browser) +- **Live events over SSE** — the browser subscribes to `GET /api/v1/sessions/{id}/events/stream` (DB replay via `?after_seq`, then live); WebSocket is reserved for `/ws/notifications` +- **Sandboxed execution over gRPC** — agents never run in the orchestrator process; a Rust `sandbox-runner` inside a per-session container speaks the gRPC `AgentBridge` protocol back to the orchestrator +- **Pluggable engines** — `claude` (Claude Code CLI), `codex` (Codex app-server), and `native` (self-developed `ccb` binary), selected per agent by `engine_kind` +- **Pluggable sandboxes** — Docker (default, hardened), E2B, and Daytona providers behind one `SandboxProvider` SPI +- **Centralized state machines** — guarded FSMs for Task, Session, Sandbox, and Skill lifecycle +- **Normalized error system** — `AppError` produces a canonical `ErrorDescriptor` (`{code, message, data, source, retryable, user_action}`) consumed identically across HTTP and streaming paths +- **OTel-backed observation** — full-chain `trace_id` propagation with spans persisted to the database +- **Encrypted credentials** — provider API keys live in Secrets and MCP credentials in Vaults, both AES-256-GCM encrypted and injected into the sandbox at run time +- **Layered skill system** — skills are versioned capability packs; runtime only packs approved skills with an allowed scan verdict and no content drift -

- JoySafeter Quick Start User Journey -

+### User Journey — Quick Start -> **Login** → **Configure Models** → **MCP Tools** → **Skill Management** → **Build Agent** → **Self-Test (Langfuse Trace)** → **Publish** → **Chat UI** → **Run Center** +> **Login** → **Add provider keys (Secrets)** → **Configure MCP credentials (Vaults)** → **Skill Management** → **Build Agent** → **Open a Session** → **Chat & watch live events** → **Download report** --- @@ -243,16 +306,16 @@ All modes support remote deployment scenarios: | Layer | Technology | Purpose | |-------|------------|---------| -| **Frontend** | Next.js 16, React 19, TypeScript | Server-side rendering, App Router | -| **UI** | Radix UI, Tailwind CSS, Framer Motion | Accessible, animated components | +| **Frontend** | Next.js 16 (App Router), React 19, TypeScript | Server-side rendering, product surface under `/managed/**` | +| **UI** | Radix UI, Tailwind CSS | Accessible component primitives | | **State** | Zustand, TanStack Query | Client & server state | -| **Workflow Editor** | React Flow | Interactive node-based builder | -| **Backend** | FastAPI, Python 3.12+ | Async API with OpenAPI docs | -| **AI Framework** | LangChain, LangGraph, DeepAgents | Agent orchestration & workflows | +| **Backend** | FastAPI 0.122+, Python 3.12+ | Async API with OpenAPI docs, three services split by `JOYSAFETER_SERVICE_ROLE` | +| **Agent runtime** | Rust `sandbox-runner` + Claude Code / Codex / `ccb` harness | Per-session sandboxed execution over gRPC `AgentBridge` | | **MCP** | mcp 1.20+, fastmcp 2.14+ | Tool protocol support | -| **Database** | PostgreSQL, SQLAlchemy 2.0 | Async ORM with migrations | -| **Cache** | Redis | Session cache & rate limiting | -| **Observability** | Langfuse, Loguru | Tracing & structured logging | +| **Database** | PostgreSQL, SQLAlchemy 2.0 | Async ORM with Alembic migrations | +| **Event bus / cache** | Redis (Streams · Pub/Sub · list) | Durable event stream, live SSE fan-out, task queue | +| **Egress control** | Envoy | Per-sandbox deny-all-by-default domain allowlist | +| **Observability** | OpenTelemetry, Loguru | Full-chain tracing & structured logging | --- @@ -262,17 +325,61 @@ All modes support remote deployment scenarios: | Tag | Feature | What it means | |-----|---------|---------------| -| **NEW** | **Run Center Architecture** | Chat & Copilot fully integrated into Run Center — run details, session recovery, and live event replay on page refresh | -| **NEW** | **Dark Mode & Preferences** | System / Light / Dark theme switching; redesigned profile page with language & theme preferences | -| **NEW** | **Unified WebSocket Layer** | BaseWsClient abstract class — Chat, Run, and Notification clients share lifecycle, auth (ws-token), and reconnect logic | -| **NEW** | **Full-Chain trace_id Propagation** | End-to-end request tracing via contextvars for complete observability | -| **NEW** | **Ollama One-Click Integration** | Local Ollama model provider added out of the box | -| **NEW** | **Version Display** | In-app version info tied to bump-version.sh release pipeline | -| **NEW** | **Unified Model Identifiers** | Full-stack (provider_name, model_name) canonical form with data migration — no more legacy field ambiguity | -| **UPGRADE** | **Design Token Overhaul** | Hardcoded colors, font sizes, and border radii replaced with CSS variables and Tailwind tokens; z-index and typography scales unified | -| **UPGRADE** | **Sandbox Overhaul** | RAII handle management, adapter API uploads, security hardening | -| **UPGRADE** | **Frontend Component Extraction** | ConfirmDialog, UnifiedDialog, InlineRenameInput, SidebarContextMenu, AgentListContext — less prop drilling, more reuse | -| **UPGRADE** | **i18n & Code Quality** | Backend error messages internationalized; email templates moved to Jinja2; LLM prompts externalized to Markdown; 129 unused SVG icons removed | +| **NEW** | **Split Runtime Architecture** | The monolith was split into Python `api` / `worker` services and the Rust `orchestrator-rs`, deployed as separate containers | +| **NEW** | **Redis-Backed Event Bus** | A two-phase bus fans out to Redis Streams (durable, Worker-consumed) and Redis Pub/Sub (live SSE), replacing the old in-process WebSocket bus | +| **NEW** | **SSE Live Event Stream** | The browser subscribes to `GET /api/v1/sessions/{id}/events/stream` with `?after_seq` replay; WebSocket is reserved for notifications | +| **NEW** | **Sandboxed gRPC Execution** | A Rust `sandbox-runner` runs the harness inside a per-session container and speaks the gRPC `AgentBridge` protocol back to the orchestrator | +| **NEW** | **Pluggable Engines** | `claude` (Claude Code CLI), `codex` (Codex app-server), and `native` (self-developed `ccb`), selected per agent | +| **NEW** | **Pluggable Sandboxes** | Docker (default, hardened), E2B, and Daytona providers behind one SPI | +| **NEW** | **AI Skill Authoring** | LLM-assisted skill drafting, a code editor, and version diffs in the workspace UI | +| **NEW** | **Secrets & Vaults** | AES-256-GCM encrypted provider API keys (Secrets) and MCP credentials (Vaults), injected into the sandbox at run time | +| **NEW** | **Skill Security Scanning** | SkillSpector scans skill content; runtime blocks unapproved, unscanned, failed, blocked, scanning, or drifted skills before use | +| **NEW** | **Per-Sandbox Egress Control** | Envoy proxy enforces a deny-all-by-default domain allowlist per sandbox | +| **NEW** | **Full-Chain trace_id Propagation** | End-to-end request tracing via OpenTelemetry for complete observability | + +--- + +## Managed-Agent Parity & Roadmap + +JoySafeter implements the same **managed-agent operating model** that Anthropic describes for +[Claude Managed Agents](https://claude.com/blog/claude-managed-agents) — you declare an agent's +tools, skills, and guardrails, and the platform runs it on a managed harness with sandboxed +execution, sessions, scoped permissions, and full observability. The difference: JoySafeter is +**open-source, self-hostable, engine-agnostic** (Claude Code / Codex / native `ccb`), and +**specialized for security work**. This table maps the model concept-for-concept against what +JoySafeter ships today. + +**Legend:** ✅ shipped · 🟡 partial · ⬜ planned (see roadmap) + +| Managed-agent capability | JoySafeter | How we do it | +|---|:---:|---| +| Managed agent harness / orchestration | ✅ | Orchestrator + gRPC `AgentBridge` + in-sandbox Rust `sandbox-runner` harness | +| Sandboxed execution | ✅ | Per-session hardened containers; Docker (default) / E2B / Daytona behind one SPI | +| Tools, custom tools & MCP | ✅ | Per-agent builtin tools, custom tools, and `mcp_configs`, delivered to the sandbox over gRPC | +| Scoped permissions / guardrails | ✅ | Per-tool policy (`always_ask` / `always_allow`) with human-in-the-loop confirmation | +| Credential management | ✅ | Secrets (provider keys) + Vaults (MCP creds), AES-256-GCM encrypted, injected as sandbox env | +| Sessions & resumable work | ✅ | `JoySafeterSession` + append-only event log; harness session/work-dir resume on reconnect | +| Memory stores | ✅ | Versioned, agent-writable memory stores with bi-directional sandbox sync | +| Observability / session tracing | ✅ | OTel traces + `observations`, plus a live SSE event stream of every tool call & decision | +| Deployment CLI + console | ✅ | `joysafeterctl` (declarative REST CLI) + the web workspace | +| Multi-agent orchestration (lead → specialists) | 🟡 | Harness-driven sub-agents today, surfaced via `TaskNotification` events; first-class lead/specialist orchestration is on the roadmap | +| Durable checkpointing | 🟡 | Session-level resume today; step-level durable checkpoints are planned | +| Outcomes (rubric + grader self-correct loop) | ⬜ | Planned | +| Dreaming (scheduled memory consolidation / self-improvement) | ⬜ | Planned | +| Webhooks (notify on task/outcome completion) | ⬜ | Planned | + +### Roadmap / TODO + +Combining our current capabilities with the managed-agent frontier, the next work items are: + +- [ ] **Outcomes** — let a user define a rubric; an independent grader evaluates each result in its own context and the agent self-corrects until the criteria are met (no per-attempt human review). +- [ ] **First-class multi-agent orchestration** — a lead agent that delegates to specialist sub-agents, each with its own model / prompt / tools, running in parallel on a shared session workspace, with full per-sub-agent tracing (today sub-agents are spawned by the harness and only observed via `agent.bg_task_*` events). +- [ ] **Dreaming** — a scheduled job that reviews past sessions + memory stores, extracts recurring patterns and mistakes, and curates memory (opt-in auto-update or review-first). +- [ ] **Webhooks** — notify external systems (or trigger follow-on agents) when a task or outcome completes. +- [ ] **Durable step-level checkpointing** — resume a long-running task mid-flight beyond the current session/work-dir reattach. +- [ ] **Session-hour metering & cost analytics** — per-session runtime + token/cost accounting surfaced in the console. + +> Have a use case that needs one of these sooner? Open an issue — the roadmap is community-driven. --- @@ -282,10 +389,11 @@ All modes support remote deployment scenarios: - [INSTALL.md](INSTALL.md) — Installation guide (Docker / manual / pre-built images) - [DEVELOPMENT.md](DEVELOPMENT.md) — Local development setup - [deploy/README.md](deploy/README.md) — Docker deployment -- [deploy/PRODUCTION_IP_GUIDE.md](deploy/PRODUCTION_IP_GUIDE.md) — Production deployment ### Deep Dive +- [docs/README.md](docs/README.md) — Documentation map - [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — Architecture overview +- [docs/DOCUMENTATION_STATUS.md](docs/DOCUMENTATION_STATUS.md) — Current documentation review status - [backend/README.md](backend/README.md) — Backend guide - [frontend/README.md](frontend/README.md) — Frontend guide @@ -336,11 +444,11 @@ Third-party component licenses: [THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.m - - + +

LangChain
LangGraph
FastAPI
Next.js
Radix UI
Envoy
OpenTelemetry
diff --git a/README_CN.md b/README_CN.md index e9e4219c6..8403ae8ea 100644 --- a/README_CN.md +++ b/README_CN.md @@ -4,18 +4,17 @@

- AI 原生安全智能体平台 —— 构建、编排、规模化运行安全 Agent。
- 从想法到生产级安全自动化，只需几分钟，而非数月。 + 开源、可自托管的安全托管智能体（Managed Agent）平台。
+ 你只需声明 Agent 的工具、技能与护栏，JoySafeter 便在你自己的加固、可观测的基础设施上托管运行它。从想法到生产级安全自动化，只需几分钟，而非数月。

License: Apache 2.0 Python 3.12+ Node.js 20+ - LangGraph FastAPI + Next.js 16 MCP Protocol - DeepAgents v0.4

@@ -26,16 +25,29 @@ ## 为什么选择 JoySafeter -传统安全工具有天花板：脚本脆弱易断、单 Agent 缺乏上下文、复杂场景需要 2–3 名工程师并行协作。JoySafeter 打破这个天花板。 - -| 挑战 | 传统方式 | JoySafeter | -|------|---------|------------| -| APK 漏洞分析 | 手动 MobSF + 工程师人工审查 | 自主 Agent：上传 → 分析 → 出报告 | -| 渗透测试 | 固定脚本、静态 Playbook | DeepAgents 根据发现实时动态决策 | -| 工具集成 | 每个工具单独写胶水代码 | 200+ 工具通过 MCP 协议零胶水接入 | -| 规模扩展 | 人力线性增长 | Agent 团队倍增安全产能 | - -> JoySafeter 定义了全新范式：**AI 驱动安全运营（AISecOps）** —— 用多智能体协作、认知记忆进化、场景化战力速配，取代人工协调，实现安全能力的规模化运营。 +托管智能体运行模型对自主工作是正确选择。但对**安全**而言 —— 你在客户系统上、 +在 NDA 之下运行具攻击性的工具 —— Agent *在哪里、以何种方式*运行才是决定性的。 +JoySafeter 把这套模型交到你**自己的基础设施**上： + +- **数据与目标绝不出你的基础设施。** 完全自托管：提示词、发现、抓取的流量、目标细节都留在你的 + 网络内。任何第三方都不会看到本次工作。 +- **网络受限执行。** 每个会话跑在 `NetworkMode=none` 沙箱里，前置一个 Envoy 代理， + **默认全拒的出口白名单** —— 攻击性工具无法回连外部或横向进入你的网络，除非你显式放行。 +- **运行时 fail-closed 的技能供应链管控。** 技能是会在你环境里执行的代码；skillspector 负责扫描， + 运行时打包会拦截未审批、未成功扫描、blocked、failed、scanning 或与上次扫描发生漂移的技能。 +- **引擎无关。** Claude Code、Codex、或自研 `ccb` 引擎，统一在一个 gRPC 契约之后 —— 不锁定单一 + 厂商或模型。 + +| | 云托管智能体 | 自建 | **JoySafeter** | +|---|---|---|---| +| 数据 / 目标驻留 | 厂商云 | 你的 | **你的 —— 完全自托管** | +| 引擎 / 模型 | 单一厂商 | 自行拼装 | **Claude Code / Codex / native，按 Agent 选** | +| 网络隔离 | 厂商托管 | 自行搭建 | **每沙箱 Envoy 全拒出口** | +| 技能与工具安全 | 厂商托管 | 自行搭建 | **SkillSpector 扫描 + 运行时 fail-closed 闸门** | +| 上生产时间 | 数天 | 数月 | **数天，在你自己的硬件上** | + +> JoySafeter 将其定义为 **AI 驱动安全运营（AISecOps）**：把托管智能体模型 —— 多步自主、 +> 沙箱化工具、会话、全链路可观测 —— 面向安全场景专门化，并完全运行在你的掌控之下。 --- @@ -49,12 +61,15 @@ APK 漏洞检测演示

+> **这是一个演示流程，而非内置集成。** 该 Agent 配置了 `pentest-mobile-app` 技能， +> 并使用装有移动端分析工具的沙箱镜像 —— 工具由你自行选配。 + **运行流程：** 1. 用户上传 APK 文件 -2. Agent 调用 MobSF 进行静态分析 +2. Agent 用沙箱内的移动端工具做静态分析（例如 MobSF） 3. 提取关键风险点 —— 权限滥用、硬编码密钥、不安全的网络配置等 -4. 对高危项通过 Frida 动态插桩进行深度验证 +4. 对高危项通过动态插桩做深度验证（例如 Frida） 5. 自动生成符合 OWASP Mobile Top 10 格式的结构化检测报告 整个流程从上传到出报告，零人工干预，覆盖了传统需要 2–3 名安全工程师协作完成的工作量。 @@ -72,10 +87,10 @@ **操作流程：** 1. 进入工作台，创建新 Agent -2. 开启 **DeepAgents 模式** → 选择渗透测试相关 Skills +2. 选择执行引擎（Claude Code / Codex / native）→ 选择渗透测试相关 Skills 3. 输入经过授权的目标地址和测试要求 -4. Agent 自主运行 —— 若发现登录页面，自动触发认证绕过测试 -5. 运行结束后下载完整报告 +4. Agent 在隔离沙箱中自主运行 —— 若发现登录页面，自动触发认证绕过测试 +5. 会话结束后下载完整报告 > **备注：** 需在沙箱设置中配置镜像 `swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/jd-opensource/joysafeter-sandbox:latest`。 @@ -83,159 +98,203 @@ --- -## 核心能力 +## 核心能力 —— 托管智能体的构建块 + +你只需声明一个 Agent —— 引擎、模型、系统提示词、工具、Skills、MCP 服务、护栏 —— JoySafeter +就用与 Claude Managed Agents 背后同一套**托管智能体构建块**端到端地运行它，只不过是 +**自托管、面向安全场景**的： + + + + + + + +
-### 可视化 Agent 构建器 +### 🧠 托管 harness 与编排 + +- **Orchestrator** + gRPC `AgentBridge` + 沙箱内 Rust `sandbox-runner` 负责决定何时调用工具、管理上下文、从错误中恢复 +- **以 DB 为准的调度** —— 用 `FOR UPDATE SKIP LOCKED` 从 Postgres 认领任务，带重试与超时 +- **引擎无关** —— Claude Code CLI、Codex app-server、自研 `native`（`ccb`）harness，按 Agent 选择 + + + +### 📦 沙箱执行 + +- 每个会话运行在**独立加固容器**中 —— 丢弃能力、非 root、no-new-privileges +- **可插拔 Provider** —— Docker（默认）、E2B（Firecracker）、Daytona，统一 SPI +- **出站管控** —— 每沙箱 Envoy 代理，默认拒绝的域名白名单 + +
+ +### 🔧 工具、自定义工具与 MCP + +- 每个 Agent 可挂载**内置工具**、**自定义工具**（名称 + JSON Schema）与 **MCP 服务** +- MCP 配置 + Vault 凭据在运行时解析，经 gRPC 下发到沙箱 +- **安全技能包**在沙箱镜像内驱动 **Nmap / Nuclei / Trivy** 等工具；任意外部工具经 **MCP 协议**接入 + + + +### 📚 Skills + +- **30 个版本化能力包** —— 渗透测试、文档分析、规划/元技能 +- **SkillSpector 安全扫描** + 运行时 `is_skill_usable` 闸门（approved + `passed` / `warning` 扫描 + 内容未漂移） +- **AI Skill 创作** —— LLM 辅助的起草、编辑、版本化与 diff + +
+ +### 💾 会话、记忆与恢复 -- **无代码工作流编辑器** —— 拖拽节点，支持循环、条件、并行执行 -- **快速模式** —— 用自然语言描述需求，分钟级生成可运行的 Agent 团队 -- **深度模式** —— 可视化调试 + 逐步可观测，适用于复杂安全研究的持续迭代 +- **Session** 是持久对话，带**追加式、按 seq 排序的事件日志** +- **记忆库** —— 版本化、Agent 可写的 KV 存储，与沙箱双向同步 +- **可恢复** —— 重连时重新挂接会话的 harness 与工作目录 -### 200+ 安全工具开箱即用 +### 🛡️ 作用域权限与护栏 -- 预集成 **Nmap、Nuclei、Trivy** 等主流工具 -- **MCP 协议** —— 通过模型上下文协议扩展任意工具 -- **30+ 预置技能** —— 渗透测试、文档分析、云安全等 +- **每工具授权** —— `always_ask` / `always_allow`，高危工具触发人工确认（HITL） +- **凭据加密** —— Provider Key 存 Secrets、MCP 凭据存 Vaults，AES-256-GCM，作为沙箱 env 注入 +- **SSRF 守卫** —— 拦截云元数据端点；可选私网段加固
-### DeepAgents 编排引擎 +### 🔎 全链路可观测 -- **Manager-Worker 多层级**智能体协作 -- **记忆进化** —— 长短期记忆机制，跨会话持续学习 -- **技能体系** —— 版本化、可复用的能力单元，渐进式披露 -- **LangGraph 引擎** —— 基于图的工作流与完整状态管理 +- **实时 SSE 事件流** —— 每条消息、思考步、工具调用、工具结果、模型请求 +- **OpenTelemetry** traces + `observations`，带 token/成本聚合，`trace_id` 端到端传播 +- 追加式会话事件日志天然充当完整**审计轨迹** -### 企业级就绪 +### 🏢 多租户与访问控制 -- **多租户** —— 基于角色的工作区隔离与访问控制 -- **全链路审计** —— 执行追踪与合规治理 -- **SSO 集成** —— GitHub、Google、Microsoft、OIDC（Keycloak、Authentik、GitLab）、JD SSO -- **多租户沙箱** —— 用户级代码执行隔离，会话间零状态泄露 +- **组织 / 项目 / RBAC** —— 工作区隔离 + 基于角色的访问控制 +- **SSO** —— GitHub、Google、Microsoft、OIDC（Keycloak、Authentik、GitLab）、JD SSO +- **Quickstart** —— 用自然语言描述目标，分钟级生成可运行的 Agent
+> 这些构建块与 Claude Managed Agents 特性集的逐项对照、以及路线图，见 +> [与 Claude Managed Agents 的能力对齐与路线图](#与-claude-managed-agents-的能力对齐与路线图)。 + --- ## 快速开始 -### 一键启动（推荐） +### Docker Compose 部署（推荐） ```bash -./deploy/quick-start.sh +cd deploy +cp .env.example .env +cd ../backend && cp env.example .env +cd ../frontend && cp env.example .env +cd ../deploy + +# 全本地：PostgreSQL + Redis + Python orchestrator +docker compose --profile local-redis --profile python-orchestrator up -d --build ``` -脚本提供交互式菜单，选择启动模式并自定义端口（自动检测冲突）： +访问地址： -| 模式 | 说明 | 配置的端口 | -|------|------|-----------| -| **(1) Docker Compose 全栈** | 所有服务容器化运行，支持本机或远程服务器 IP/域名 | 前端、后端、PostgreSQL、Redis | -| **(2) 仅本地前端** | `bun run dev`，支持连接远程后端 | 前端（可指定远程后端地址） | -| **(3) 仅本地后端** | `uvicorn --reload`，支持连接远程 DB/Redis | 后端（可指定远程 DB/Redis/前端地址） | -| **(4) 本地前端 + 后端** | 自动启动中间件，支持对外暴露非 localhost 地址 | 前端、后端 | - -每种模式都支持远程部署场景： -- **Docker Compose 全栈** — 询问部署地址（localhost 或 IP/域名）+ http/https 协议 -- **仅本地前端** — 可选连接远程后端 API（输入后端 IP + 端口 + 协议） -- **仅本地后端** — 可选连接远程 PostgreSQL、Redis、前端（分别输入地址和端口） -- **本地前端 + 后端** — 可选通过非 localhost 地址对外提供服务 -- 非 localhost 部署时自动更新 `frontend/.env` 的 CSP 白名单（`NEXT_PUBLIC_CSP_CONNECT_SRC_EXTRA`） +| 服务 | 地址 | +|------|------| +| 前端 | http://localhost:3000 | +| 后端 API | http://localhost:8000 | +| API 文档 | http://localhost:8000/docs | -```bash -./deploy/quick-start.sh --skip-env # 跳过 .env 文件初始化 -./deploy/quick-start.sh --skip-db-init # 跳过数据库初始化 -``` +后端是同一份代码，通过 `JOYSAFETER_SERVICE_ROLE` 环境变量拆成三个服务，作为独立容器部署： -### 按场景启动 +- `api`：REST `/api/v1/*`、SSE 事件流、通知 WebSocket、鉴权。 +- `orchestrator`：任务调度、gRPC `AgentBridge`、sandbox 生命周期。 +- `worker`：消费 Redis 事件流并将事件落库到 Postgres。 + +配套基础设施：PostgreSQL、Redis、Envoy（每沙箱出站代理）、skillspector（Skill 安全扫描服务）。 +内置 Redis 服务由 `local-redis` profile 控制；如果使用云 Redis，不启用该 profile，改 `deploy/.env` +里的 `REDIS_URL` 即可。 +当前 checkout 中 `rust-orchestrator` compose profile 仍是实验入口：其 Dockerfile 依赖 +`backend/app/joysafeter_orchestrator_rs`，但该源码目录当前不存在，因此快速开始支持路径是 Python orchestrator。 + +### 本地测试一键启动 ```bash -# ─── 开发场景 ─────────────────────────────────────────── -./deploy/scripts/dev.sh # Docker 全栈开发（前后端容器化，适合联调） -./deploy/scripts/dev-local.sh # 本地开发准备（启动中间件，后端/前端在本地跑） -./deploy/scripts/dev-backend.sh # 仅启动本地后端（需中间件已启动） -./deploy/scripts/dev-frontend.sh # 仅启动本地前端（需后端已启动） - -# ─── 生产场景 ─────────────────────────────────────────── -./deploy/scripts/prod.sh # 生产部署（预构建镜像 + docker-compose.prod.yml） -./deploy/scripts/prod.sh --skip-mcp # 生产部署，不启动 MCP 服务 -./deploy/scripts/prod.sh --skip-pull # 跳过镜像拉取，使用本地已有镜像 - -# ─── 中间件 / 基础设施 ───────────────────────────────── -./deploy/scripts/start-middleware.sh # 启动中间件（PostgreSQL + Redis + MCP） -./deploy/scripts/minimal.sh # 最小化启动（仅 PostgreSQL + Redis） -./deploy/scripts/minimal.sh --with-mcp # 最小化 + MCP 服务 -./deploy/scripts/stop-middleware.sh # 停止中间件 - -# ─── 测试 / CI ────────────────────────────────────────── -./deploy/scripts/test.sh # 测试环境（最小依赖，适合自动化） - -# ─── 安装 / 检查 ──────────────────────────────────────── -./deploy/install.sh # 交互式安装向导（生成配置文件） -./deploy/install.sh --mode dev --non-interactive # 非交互式安装 -./deploy/scripts/check-env.sh # 环境预检（Docker、端口、配置文件） - -# ─── 镜像管理 ─────────────────────────────────────────── -./deploy/deploy.sh build # 构建前后端镜像 -./deploy/deploy.sh build --all # 构建所有镜像（含 OpenClaw） -./deploy/deploy.sh push # 构建并推送到仓库 -./deploy/deploy.sh pull # 拉取最新预构建镜像 +cd deploy +./local-test.sh ``` -### 默认端口 +### 常用部署命令 -| 服务 | 端口 | 说明 | -|------|------|------| -| 前端 | `3000` | http://localhost:3000 | -| 后端 API | `8000` | http://localhost:8000 | -| API 文档 | `8000/docs` | Swagger UI | -| PostgreSQL | `5432` | 数据库 | -| Redis | `6379` | 缓存 | +```bash +./deploy/local-test.sh # 本地测试一键启动 +./deploy/deploy.sh build # 构建前后端镜像 +./deploy/deploy.sh build --all # 构建全部镜像 +``` -> **环境要求：** Docker + Docker Compose。详细安装指南请参考 [INSTALL_CN.md](INSTALL_CN.md)，生产部署请参考 [deploy/PRODUCTION_IP_GUIDE.md](deploy/PRODUCTION_IP_GUIDE.md)。 +> **环境要求：** Docker + Docker Compose。部署细节请参考 [deploy/README.md](deploy/README.md)。 --- ## 架构概览 -

- JoySafeter 系统架构图 -

+![JoySafeter 托管智能体架构](docs/architecture-diagram.png) + +总览信息图 —— ① 控制面（REST · CLI）→ ② Agent Harness（沙箱内）→ ③ Session 状态层。交互版：[`docs/architecture-diagram.html`](docs/architecture-diagram.html)。 + +```mermaid +flowchart LR + FE["浏览器"] -->|"REST · SSE"| API["API 服务"] + API -->|"rpush task"| RLIST[("Redis list
global_queue")] + RLIST -.->|"唤醒"| SCHED["Orchestrator
调度器（DB 权威）"] + SCHED -->|"认领 / provision"| SBX["沙箱（NetworkMode=none）
Rust runner + harness"] + SBX <-->|"gRPC AgentBridge"| ENVOY["Envoy
唯一网络出入口"] + ENVOY <--> GRPC["Orchestrator gRPC :9090"] + ENVOY -->|"出口白名单"| EXT["模型 API · MCP · 目标"] + GRPC -->|"harness 事件"| BUS["两相事件总线"] + BUS -->|"① 持久化 XADD"| RSTREAM[("Redis stream")] + BUS -->|"② 广播 PUBLISH"| RPUB[("Redis pub/sub")] + RSTREAM -->|"XREADGROUP"| WK["Worker → 落库"] + WK --> PG[("PostgreSQL")] + WK -.->|"再发布"| RPUB + RPUB -->|"SessionBroadcaster"| API + API -->|"SSE 事件流"| FE +``` -> 详细架构：[docs/ARCHITECTURE_CN.md](docs/ARCHITECTURE_CN.md) +> **托管智能体总览信息图：** [docs/architecture-diagram.html](docs/architecture-diagram.html) —— 在浏览器打开（① 控制面 REST·CLI → ② Agent Harness → ③ Session 状态层）。 +> +> 完整架构（部署拓扑、gRPC 契约、引擎、沙箱、事件模型、领域 FSM）： +> **[docs/ARCHITECTURE_CN.md](docs/ARCHITECTURE_CN.md)** · 分层视图： +> [docs/architecture-unified-event-model.mmd](docs/architecture-unified-event-model.mmd) **核心设计原则：** -- **图式执行** —— 每个 Agent 工作流都是有状态的 LangGraph，支持暂停、恢复与分支 -- **统一 Run Center** —— Chat、Copilot、Skill Creator 共享同一套事件溯源运行生命周期（Run → Event → Snapshot） -- **统一 WebSocket 层** —— BaseWsClient 抽象基类；Chat / Run / Notification 三端客户端共享生命周期、认证（ws-token）与重连逻辑 -- **trace_id 全链路追踪** —— 基于 contextvars 的请求追踪，从 HTTP/WS 入口贯穿 LangGraph 直至持久化 -- **白盒可观测性** —— 基于 Langfuse 实时追踪每一步 Agent 决策与状态流转 -- **RAII 沙箱隔离** —— 用户级 Docker 容器，句柄自动释放，会话间零状态泄露 -- **规范化模型标识** —— 全栈统一 (provider_name, model_name) 解析路径：ModelService → ModelFactory -- **分层技能体系** —— 技能是版本化单元，可自由组合成工作流，互不耦合 +- **三服务、一份代码** —— `api`、`orchestrator`、`worker` 共享同一份代码，在启动时由 `JOYSAFETER_SERVICE_ROLE` 决定行为（`all` 表示单进程本地开发） +- **调度以 DB 为准** —— API 将任务推入 Redis list 作为唤醒信号；orchestrator 用 `FOR UPDATE SKIP LOCKED` 从 Postgres 认领待处理行 +- **持久化与实时投递解耦** —— 两阶段事件总线分别扇出到 Redis Streams（持久，Worker 消费 → `joysafeter_session_events`）和 Redis Pub/Sub（临时，驱动 SSE 扇出到浏览器） +- **实时事件走 SSE** —— 浏览器订阅 `GET /api/v1/sessions/{id}/events/stream`（先按 `?after_seq` 从 DB 回放，再接实时）；WebSocket 仅用于 `/ws/notifications` +- **沙箱内 gRPC 执行** —— Agent 从不在 orchestrator 进程中运行；每会话容器内的 Rust `sandbox-runner` 通过 gRPC `AgentBridge` 协议回连 orchestrator +- **可插拔引擎** —— `claude`（Claude Code CLI）、`codex`（Codex app-server）、`native`（自研 `ccb` 二进制），按 Agent 的 `engine_kind` 选择 +- **可插拔沙箱** —— Docker（默认，加固）、E2B、Daytona，统一的 `SandboxProvider` SPI +- **集中化状态机** —— Task、Session、Sandbox、Skill 生命周期均由受保护的 FSM 管理 +- **规范化错误系统** —— `AppError` 输出规范的 `ErrorDescriptor`（`{code, message, data, source, retryable, user_action}`），HTTP 与流式路径一致消费 +- **OTel 观测追踪** —— 全链路 `trace_id` 传播，span 落库 +- **凭据加密** —— Provider API Key 存于 Secrets、MCP 凭据存于 Vaults，均 AES-256-GCM 加密，运行时注入沙箱 +- **分层技能体系** —— Skills 是版本化能力包；运行时只打包已审批、扫描状态允许且内容未漂移的技能 -### 用户操作路径 —— 9 步快速入门 +### 用户操作路径 —— 快速入门 -

- JoySafeter 快速入门用户路径 -

- -> **登录** → **配置模型** → **MCP 工具** → **Skill 管理** → **构建 Agent** → **自测 (Langfuse Trace)** → **发布** → **Chat 运行** → **Run Center** +> **登录** → **添加 Provider Key（Secrets）** → **配置 MCP 凭据（Vaults）** → **Skill 管理** → **构建 Agent** → **开启 Session** → **对话并实时观看事件** → **下载报告** --- @@ -243,16 +302,16 @@ | 层级 | 技术 | 用途 | |------|------|------| -| **前端** | Next.js 16, React 19, TypeScript | 服务端渲染，App Router | -| **UI** | Radix UI, Tailwind CSS, Framer Motion | 无障碍、动画组件 | +| **前端** | Next.js 16（App Router）, React 19, TypeScript | 服务端渲染，产品界面位于 `/managed/**` | +| **UI** | Radix UI, Tailwind CSS | 无障碍组件基元 | | **状态管理** | Zustand, TanStack Query | 客户端与服务端状态 | -| **工作流编辑器** | React Flow | 交互式节点编辑器 | -| **后端** | FastAPI, Python 3.12+ | 异步 API，OpenAPI 文档 | -| **AI 框架** | LangChain, LangGraph, DeepAgents | Agent 编排与工作流 | +| **后端** | FastAPI 0.122+, Python 3.12+ | 异步 API + OpenAPI 文档，按 `JOYSAFETER_SERVICE_ROLE` 拆分为三服务 | +| **Agent 运行时** | Rust `sandbox-runner` + Claude Code / Codex / `ccb` harness | 每会话沙箱执行，经 gRPC `AgentBridge` | | **MCP** | mcp 1.20+, fastmcp 2.14+ | 工具协议支持 | -| **数据库** | PostgreSQL, SQLAlchemy 2.0 | 异步 ORM，数据库迁移 | -| **缓存** | Redis | 会话缓存与限流 | -| **可观测性** | Langfuse, Loguru | 追踪与结构化日志 | +| **数据库** | PostgreSQL, SQLAlchemy 2.0 | 异步 ORM，Alembic 迁移 | +| **事件总线/缓存** | Redis（Streams · Pub/Sub · list） | 持久事件流、实时 SSE 扇出、任务队列 | +| **出站管控** | Envoy | 每沙箱默认拒绝的域名白名单 | +| **可观测性** | OpenTelemetry, Loguru | 全链路追踪与结构化日志 | --- @@ -262,17 +321,59 @@ | 标签 | 功能 | 一句话说明 | |------|------|-----------| -| **NEW** | **Run Center 架构** | Chat 与 Copilot 全面迁入 Run Center——支持运行详情查看、会话恢复、页面刷新后实时事件回放 | -| **NEW** | **深色模式与偏好设置** | 系统/浅色/深色三种主题切换；重新设计个人资料页面，新增语言与主题偏好 | -| **NEW** | **统一 WebSocket 层** | 引入 BaseWsClient 抽象基类——Chat、Run、Notification 三端客户端共享生命周期、认证（ws-token）与重连逻辑 | -| **NEW** | **trace_id 全链路追踪** | 基于 contextvars 的端到端请求追踪，实现完整可观测性 | -| **NEW** | **Ollama 一键集成** | 开箱即用的本地 Ollama 模型供应商 | -| **NEW** | **版本信息展示** | 应用内版本信息展示，接入 bump-version.sh 发布管线 | -| **NEW** | **统一模型标识符** | 全栈统一为 (provider_name, model_name) 规范形式，含数据迁移——彻底消除遗留字段歧义 | -| **UPGRADE** | **设计令牌全面重构** | 硬编码颜色、字号、圆角替换为 CSS 变量与 Tailwind token；z-index 与排版体系统一 | -| **UPGRADE** | **沙箱架构重构** | RAII 句柄管理、适配器 API 上传、安全加固 | -| **UPGRADE** | **前端组件提取** | ConfirmDialog、UnifiedDialog、InlineRenameInput、SidebarContextMenu、AgentListContext——减少属性穿透，提升复用 | -| **UPGRADE** | **i18n 与代码质量** | 后端错误消息国际化；邮件模板迁移至 Jinja2；LLM 提示词外置为 Markdown；移除 129 个未使用 SVG 图标 | +| **NEW** | **三服务架构** | 单进程单体拆分为 `api` / `orchestrator` / `worker`，同一份代码由 `JOYSAFETER_SERVICE_ROLE` 选择，作为独立容器部署 | +| **NEW** | **Redis 事件总线** | 两阶段总线扇出到 Redis Streams（持久，Worker 消费）与 Redis Pub/Sub（实时 SSE），取代旧的进程内 WebSocket 总线 | +| **NEW** | **SSE 实时事件流** | 浏览器订阅 `GET /api/v1/sessions/{id}/events/stream`，支持 `?after_seq` 回放；WebSocket 仅用于通知 | +| **NEW** | **沙箱 gRPC 执行** | Rust `sandbox-runner` 在每会话容器内运行 harness，经 gRPC `AgentBridge` 协议回连 orchestrator | +| **NEW** | **可插拔引擎** | `claude`（Claude Code CLI）、`codex`（Codex app-server）、`native`（自研 `ccb`），按 Agent 选择 | +| **NEW** | **可插拔沙箱** | Docker（默认，加固）、E2B、Daytona，统一 SPI | +| **NEW** | **AI Skill 创作** | 工作区内 LLM 辅助的 Skill 起草、代码编辑与版本 diff | +| **NEW** | **Secrets 与 Vaults** | AES-256-GCM 加密的 Provider API Key（Secrets）与 MCP 凭据（Vaults），运行时注入沙箱 | +| **NEW** | **Skill 安全扫描** | SkillSpector 扫描技能内容；运行时拦截未审批、未扫描、failed、blocked、scanning 或漂移的技能 | +| **NEW** | **每沙箱出站管控** | Envoy 代理对每个沙箱执行默认拒绝的域名白名单 | +| **NEW** | **trace_id 全链路追踪** | 基于 OpenTelemetry 的端到端请求追踪，实现完整可观测性 | + +--- + +## 与 Claude Managed Agents 的能力对齐与路线图 + +JoySafeter 实现了 Anthropic 为 +[Claude Managed Agents](https://claude.com/blog/claude-managed-agents) 描述的同一套 +**托管智能体运行模型** —— 你声明 Agent 的工具、技能与护栏，平台在托管 harness 上以沙箱执行、 +会话、作用域权限与全链路可观测运行它。不同之处在于：JoySafeter 是**开源、可自托管、引擎无关** +（Claude Code / Codex / 自研 `ccb`）、并**面向安全场景专门化**的。下表逐项对照该模型与 JoySafeter 当前的落地情况。 + +**图例：** ✅ 已交付 · 🟡 部分实现 · ⬜ 规划中（见路线图） + +| Managed-agent 能力 | JoySafeter | 我们如何实现 | +|---|:---:|---| +| 托管 Agent harness / 编排 | ✅ | Orchestrator + gRPC `AgentBridge` + 沙箱内 Rust `sandbox-runner` harness | +| 沙箱执行 | ✅ | 每会话独立加固容器；Docker（默认）/ E2B / Daytona，统一 SPI | +| 工具、自定义工具 & MCP | ✅ | 每 Agent 的内置工具、自定义工具与 `mcp_configs`，经 gRPC 下发到沙箱 | +| 作用域权限 / 护栏 | ✅ | 每工具策略（`always_ask` / `always_allow`）+ 人工确认（HITL） | +| 凭据管理 | ✅ | Secrets（Provider Key）+ Vaults（MCP 凭据），AES-256-GCM 加密，作为沙箱 env 注入 | +| 会话与可恢复执行 | ✅ | `JoySafeterSession` + 追加式事件日志；重连时按 harness 会话/工作目录恢复 | +| 记忆库（Memory Store） | ✅ | 版本化、Agent 可写的记忆库，与沙箱双向同步 | +| 可观测性 / 会话追踪 | ✅ | OTel traces + `observations`，外加每个工具调用与决策的实时 SSE 事件流 | +| 部署 CLI + 控制台 | ✅ | `joysafeterctl`（声明式 REST CLI）+ Web 工作台 | +| 多智能体编排（lead → specialists） | 🟡 | 目前为 harness 驱动的子 Agent，经 `TaskNotification` 事件呈现；一等公民式的 lead/specialist 编排在路线图中 | +| 持久化 checkpoint | 🟡 | 目前为会话级恢复；步级持久 checkpoint 规划中 | +| Outcomes（rubric + grader 自我纠错闭环） | ⬜ | 规划中 | +| Dreaming（定时记忆整理 / 自我进化） | ⬜ | 规划中 | +| Webhooks（任务/结果完成通知） | ⬜ | 规划中 | + +### 路线图 / TODO + +结合我们已有的能力与 managed-agent 前沿特性，下一步工作项： + +- [ ] **Outcomes** —— 用户定义 rubric（评分标准），由独立 grader 在自己的上下文中评估每次结果，Agent 自我纠错直至达标（无需逐次人工审查）。 +- [ ] **一等公民式多智能体编排** —— lead agent 委派给各 specialist 子 Agent（各自独立的模型 / 提示词 / 工具），在共享会话工作区上并行执行，并对每个子 Agent 提供完整追踪（目前子 Agent 由 harness 拉起，仅通过 `agent.bg_task_*` 事件观测）。 +- [ ] **Dreaming** —— 定时任务回顾历史会话与记忆库，提炼反复出现的模式与错误，整理记忆（可选自动更新或先审后改）。 +- [ ] **Webhooks** —— 任务或 outcome 完成时通知外部系统（或触发后续 Agent）。 +- [ ] **步级持久 checkpoint** —— 支持长任务中途续跑，超越当前的会话/工作目录重连恢复。 +- [ ] **会话时长计量与成本分析** —— 在控制台呈现每会话运行时长 + token/成本核算。 + +> 有用例急需其中某项？欢迎提 issue —— 路线图由社区共同驱动。 --- @@ -282,10 +383,11 @@ - [INSTALL_CN.md](INSTALL_CN.md) — 安装指南（Docker / 手动 / 预构建镜像） - [DEVELOPMENT.md](DEVELOPMENT.md) — 本地开发 - [deploy/README.md](deploy/README.md) — Docker 部署 -- [deploy/PRODUCTION_IP_GUIDE.md](deploy/PRODUCTION_IP_GUIDE.md) — 生产环境部署 ### 深入了解 +- [docs/README.md](docs/README.md) — 文档地图 - [docs/ARCHITECTURE_CN.md](docs/ARCHITECTURE_CN.md) — 架构总览 +- [docs/DOCUMENTATION_STATUS.md](docs/DOCUMENTATION_STATUS.md) — 当前文档复核状态 - [backend/README.md](backend/README.md) — 后端指南 - [frontend/README.md](frontend/README.md) — 前端指南 @@ -336,11 +438,11 @@ Apache License 2.0 —— 详见 [LICENSE](LICENSE) 文件。 - - + +

LangChain
LangGraph
FastAPI
Next.js
Radix UI
Envoy
OpenTelemetry
diff --git a/SECURITY.md b/SECURITY.md index 416a79f03..489c53ccf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,8 +6,9 @@ We release patches for security vulnerabilities. Which versions are eligible for | Version | Supported | | ------- | ------------------ | -| 0.2.x | :white_check_mark: | -| 0.1.x | :white_check_mark: | +| 0.3.x | :white_check_mark: | +| 0.2.x | Critical fixes only | +| 0.1.x | :x: | ## Reporting a Vulnerability diff --git a/backend/.dockerignore b/backend/.dockerignore index 7174fdbfc..213a3952a 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -69,6 +69,9 @@ docker-compose*.yml # Alembic alembic.ini.bak +# Rust orchestrator source/cache is not needed by the Python backend image +app/joysafeter_orchestrator_rs/ + # 本地脚本（保留 db 初始化脚本） scripts/* !scripts/db/ diff --git a/backend/README.md b/backend/README.md index 5e52191c6..35928bd6e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # JoySafeter Backend -JoySafeter 的后端服务（FastAPI + LangGraph），提供 API、鉴权、多租户、技能系统、执行引擎等能力。 +JoySafeter 的后端服务（FastAPI），提供 API、鉴权、多租户、技能系统、任务调度、沙箱编排与事件落库能力。 > 说明：本文件只保留 **后端本地开发** 的最短路径；Docker/生产部署请统一以 `deploy/` 文档为准，避免重复与不一致。 @@ -27,14 +27,14 @@ cp env.example .env ### 3) 准备数据库并迁移 -> 推荐：直接用 Docker 启动中间件（PostgreSQL + Redis），避免本地安装依赖。 +如果要一键启动本地测试环境，直接运行： ```bash cd ../deploy -./scripts/minimal.sh +./local-test.sh ``` -然后在另一个终端执行迁移： +如果只想手动启动后端，请先自行准备 PostgreSQL/Redis，然后执行迁移： ```bash cd backend @@ -45,9 +45,80 @@ alembic upgrade head ```bash cd backend -uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 +uv run uvicorn app.joysafeter_api.main:app --reload --host 0.0.0.0 --port 8000 ``` +## 三服务启动方式 + +Python 进程只承载 API 与 worker；orchestrator 已迁移到 Rust。生产或压测时使用显式服务角色： + +```bash +# API：HTTP、WebSocket、管理接口 +JOYSAFETER_SERVICE_ROLE=api \ +uv run uvicorn app.joysafeter_api.main:app --host 0.0.0.0 --port 8000 + +# Orchestrator：gRPC AgentBridge、scheduler、sandbox/task lifecycle +JOYSAFETER_INSTANCE_ID=orchestrator-001 \ +JOYSAFETER_GRPC_HOST=0.0.0.0 \ +JOYSAFETER_GRPC_PORT=9090 \ +cargo run --manifest-path app/joysafeter_orchestrator_rs/Cargo.toml --release + +# Worker：Redis Stream consumer、批量事件落库 +JOYSAFETER_SERVICE_ROLE=worker \ +WORKER_HTTP_HOST=127.0.0.1 \ +uv run uvicorn app.joysafeter_worker.main:app --host 127.0.0.1 --port 8002 --workers 1 +``` + +入口说明： + +- 显式 Python 入口：`app.joysafeter_api.main:app`、`app.joysafeter_worker.main:app`。 +- Rust orchestrator 入口：`app/joysafeter_orchestrator_rs`。 +- 单机云虚拟机部署建议只对公网暴露 API；Worker HTTP 监听 `127.0.0.1:8002`，Orchestrator gRPC 监听 `0.0.0.0:9090` 供沙箱容器访问，并通过云安全组/防火墙禁止公网访问 `9090`。 + +注意事项： + +- Rust orchestrator 需要扩容时启动多个实例，并为每个实例配置唯一 `JOYSAFETER_INSTANCE_ID`。 +- 三个服务共享同一套 PostgreSQL / Redis；服务之间通过 DB 状态和 Redis 唤醒/协调通信。 +- 三服务模式建议开启 `JOYSAFETER_EVENT_STREAM_ENABLED=true`，让 orchestrator 把高频 JoySafeter event 写入 Redis Stream，再由 `worker` 批量消费落库。 +- `JOYSAFETER_EVENT_STREAM_FALLBACK_TO_DB=true` 时，如果 orchestrator 写 Redis Stream 失败，会自动降级为本地 DB 落库，避免事件直接丢失。 +- worker 只有在批量落库成功后才 ACK Redis Stream；未 ACK 的 pending 消息会在 `JOYSAFETER_EVENT_STREAM_PENDING_IDLE_MS` 后被其他 worker 自动认领恢复。 + +Docker Compose 可启动完整本地三服务栈： + +```bash +cd deploy +docker compose --profile local-redis --profile rust-orchestrator up -d db redis api orchestrator-rs worker frontend +``` + +## 项目结构 + +``` +app/ +├── joysafeter_api/ # API 服务：HTTP routes、WebSocket、API startup hooks、API service facade +│ ├── api/ # /api/v1 REST 路由（canonical 路径） +│ ├── websocket/ # /ws/notifications +│ ├── app.py # API app 组装：挂载 API / WebSocket +│ ├── main.py # API ASGI 入口 app.joysafeter_api.main:app +│ ├── services.py # API-facing service facade +│ └── startup.py # API 专属初始化 +├── joysafeter_orchestrator_rs/ # Rust orchestrator：gRPC、scheduler、sandbox/task lifecycle +│ ├── src/grpc/ # orchestrator gRPC server + protobuf +│ ├── src/kernel/ # queue、scheduler、task/sandbox controller、task runner +│ ├── src/events/ # JoySafeter event bus、mapping、Redis Stream publisher +│ ├── src/runtime/ # Claude/Codex/mock runtime adapters +│ └── src/sandbox/ # Docker/Daytona/E2B sandbox providers +├── joysafeter_worker/ # Worker 服务：Redis Stream consumer、批量事件落库 +│ ├── events/ # batch writer + stream consumer +│ ├── lifecycle.py # worker loops start/stop +│ ├── main.py # Worker ASGI 入口 app.joysafeter_worker.main:app +│ ├── services.py # worker-facing service facade +│ └── startup.py # worker 专属初始化/关闭 +├── joysafeter_shared/ # 跨服务共享基础设施（runtime/common/utils/storage/templates） +└── joysafeter_domain/ # 领域层真实实现（models/repositories/schemas/services/contracts/ports/state_machines） +``` + +> 完整架构文档：[`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md) | [中文版](../docs/ARCHITECTURE_CN.md) + ## API 文档 - Swagger UI: http://localhost:8000/docs @@ -79,7 +150,6 @@ pytest --cov=app ## 部署入口（统一文档） - 一键启动 / 场景化脚本 / 生产部署：[`deploy/README.md`](../deploy/README.md) -- 生产 IP/URL 配置最佳实践：[`deploy/PRODUCTION_IP_GUIDE.md`](../deploy/PRODUCTION_IP_GUIDE.md) ## License diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 99654a207..74b47f676 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -5,14 +5,19 @@ import asyncio from logging.config import fileConfig +from dotenv import load_dotenv from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import create_async_engine from alembic import context -from app import models # noqa: F401 - Import all models to ensure they are registered with Base.metadata -from app.core.database import Base -from app.core.settings import settings +from app.joysafeter_shared.config.settings import ENV_FILE + +load_dotenv(ENV_FILE, override=False) + +from app.joysafeter_domain import models # noqa: F401,E402 - register SQLAlchemy models +from app.joysafeter_shared.config.settings import settings # noqa: E402 +from app.joysafeter_shared.database import Base # noqa: E402 config = context.config diff --git a/backend/alembic/versions/20251225_000000_000000000000_base_tables.py b/backend/alembic/versions/20251225_000000_000000000000_base_tables.py deleted file mode 100644 index 93d4b1eda..000000000 --- a/backend/alembic/versions/20251225_000000_000000000000_base_tables.py +++ /dev/null @@ -1,967 +0,0 @@ -"""base_tables - -Revision ID: 000000000000 -Revises: -Create Date: 2025-12-25 00:00:00.000000+00:00 - -Initial migration: create all core database tables -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000000" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # =========================================== - # 1. Create enum types - # =========================================== - op.execute("CREATE TYPE workspacestatus AS ENUM ('active', 'deprecated', 'archived')") - op.execute("CREATE TYPE workspacetype AS ENUM ('personal', 'team')") - op.execute("CREATE TYPE workspacememberrole AS ENUM ('owner', 'admin', 'member', 'viewer')") - op.execute("CREATE TYPE workspaceinvitationstatus AS ENUM ('pending', 'accepted', 'rejected', 'cancelled')") - op.execute("CREATE TYPE permissiontype AS ENUM ('admin', 'write', 'read')") - - # =========================================== - # 2. Create core user tables (no foreign key dependencies) - # =========================================== - - # user table - op.create_table( - "user", - sa.Column("id", sa.String(255), primary_key=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("email", sa.String(255), nullable=False, unique=True), - sa.Column("email_verified", sa.Boolean(), nullable=False, default=False), - sa.Column("hashed_password", sa.String(255), nullable=True), - sa.Column("is_active", sa.Boolean(), nullable=False, default=True), - sa.Column("password_reset_token", sa.String(255), nullable=True), - sa.Column("password_reset_expires", sa.DateTime(timezone=True), nullable=True), - sa.Column("email_verify_token", sa.String(255), nullable=True), - sa.Column("email_verify_expires", sa.DateTime(timezone=True), nullable=True), - sa.Column("image", sa.String(1024), nullable=True), - sa.Column("stripe_customer_id", sa.String(255), nullable=True), - sa.Column("is_super_user", sa.Boolean(), nullable=False, default=False), - sa.Column("failed_login_attempts", sa.Integer(), nullable=False, default=0), - sa.Column("locked_until", sa.DateTime(timezone=True), nullable=True), - sa.Column("lock_reason", sa.String(255), nullable=True), - sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("last_login_ip", sa.String(255), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("ix_user_email", "user", ["email"], unique=True) - - # organization table - op.create_table( - "organization", - sa.Column("id", sa.String(255), primary_key=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("slug", sa.String(255), nullable=False), - sa.Column("logo", sa.String(500), nullable=True), - sa.Column("metadata", postgresql.JSONB(), nullable=True), - sa.Column("org_usage_limit", sa.Numeric(), nullable=True), - sa.Column("storage_used_bytes", sa.BigInteger(), nullable=False, default=0), - sa.Column("departed_member_usage", sa.Numeric(), nullable=False, default=0), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - - # =========================================== - # 3. Create tables that depend on user tables - # =========================================== - - # session table - op.create_table( - "session", - sa.Column("id", sa.String(255), primary_key=True), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("token", sa.String(255), nullable=False, unique=True), - sa.Column("ip_address", sa.String(255), nullable=True), - sa.Column("user_agent", sa.String(1024), nullable=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "active_organization_id", - sa.String(255), - sa.ForeignKey("organization.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column("last_activity_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("device_fingerprint", sa.String(255), nullable=True), - sa.Column("device_name", sa.String(255), nullable=True), - sa.Column("is_trusted", sa.Boolean(), nullable=False, default=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("session_user_id_idx", "session", ["user_id"]) - op.create_index("session_token_idx", "session", ["token"], unique=True) - - # member table - op.create_table( - "member", - sa.Column("id", sa.String(255), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "organization_id", sa.String(255), sa.ForeignKey("organization.id", ondelete="CASCADE"), nullable=False - ), - sa.Column("role", sa.String(50), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("member_user_id_idx", "member", ["user_id"]) - op.create_index("member_organization_id_idx", "member", ["organization_id"]) - - # workspaces table - op.create_table( - "workspaces", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column( - "status", - postgresql.ENUM("active", "deprecated", "archived", name="workspacestatus", create_type=False), - nullable=False, - default="active", - ), - sa.Column( - "type", - postgresql.ENUM("personal", "team", name="workspacetype", create_type=False), - nullable=False, - default="personal", - ), - sa.Column("settings", postgresql.JSONB(), nullable=True), - sa.Column("owner_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("allow_personal_api_keys", sa.Boolean(), nullable=False, default=True), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - - # workspace_members table - op.create_table( - "workspace_members", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "role", - postgresql.ENUM("owner", "admin", "member", "viewer", name="workspacememberrole", create_type=False), - nullable=False, - default="member", - ), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - - # workspace_folder table - op.create_table( - "workspace_folder", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column( - "parent_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspace_folder.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column("color", sa.String(32), nullable=True, default="#6B7280"), - sa.Column("is_expanded", sa.Boolean(), nullable=False, default=True), - sa.Column("sort_order", sa.Integer(), nullable=False, default=0), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("workspace_folder_user_idx", "workspace_folder", ["user_id"]) - op.create_index("workspace_folder_workspace_parent_idx", "workspace_folder", ["workspace_id", "parent_id"]) - op.create_index("workspace_folder_parent_sort_idx", "workspace_folder", ["parent_id", "sort_order"]) - op.create_index("workspace_folder_deleted_at_idx", "workspace_folder", ["deleted_at"]) - - # =========================================== - # 4. Create graph-related tables - # =========================================== - - # graphs table - op.create_table( - "graphs", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("name", sa.String(200), nullable=False), - sa.Column("description", sa.String(2000), nullable=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "folder_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspace_folder.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "parent_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("graphs.id", ondelete="SET NULL"), nullable=True - ), - sa.Column("color", sa.String(2000), nullable=True), - sa.Column("is_deployed", sa.Boolean(), nullable=False, default=False), - sa.Column("variables", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("deployed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("graphs_user_id_idx", "graphs", ["user_id"]) - op.create_index("graphs_workspace_id_idx", "graphs", ["workspace_id"]) - op.create_index("graphs_folder_id_idx", "graphs", ["folder_id"]) - op.create_index("graphs_parent_id_idx", "graphs", ["parent_id"]) - - # graph_nodes table - op.create_table( - "graph_nodes", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "graph_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("graphs.id", ondelete="CASCADE"), nullable=False - ), - sa.Column("tools", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("memory", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("prompt", sa.Text(), nullable=False), - sa.Column("position_x", sa.Numeric(), nullable=False), - sa.Column("position_y", sa.Numeric(), nullable=False), - sa.Column("position_absolute_x", sa.Numeric(), nullable=True), - sa.Column("position_absolute_y", sa.Numeric(), nullable=True), - sa.Column("width", sa.Numeric(), nullable=False, default=0), - sa.Column("height", sa.Numeric(), nullable=False, default=0), - sa.Column("data", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("type", sa.String(50), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("graph_nodes_graph_id_idx", "graph_nodes", ["graph_id"]) - op.create_index("graph_nodes_type_idx", "graph_nodes", ["type"]) - - # graph_edges table - op.create_table( - "graph_edges", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "graph_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("graphs.id", ondelete="CASCADE"), nullable=False - ), - sa.Column( - "source_node_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("graph_nodes.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column( - "target_node_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("graph_nodes.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("data", postgresql.JSONB(), nullable=False, server_default="{}"), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("graph_edges_graph_id_idx", "graph_edges", ["graph_id"]) - op.create_index("graph_edges_source_node_id_idx", "graph_edges", ["source_node_id"]) - op.create_index("graph_edges_target_node_id_idx", "graph_edges", ["target_node_id"]) - op.create_index("graph_edges_graph_source_idx", "graph_edges", ["graph_id", "source_node_id"]) - op.create_index("graph_edges_graph_target_idx", "graph_edges", ["graph_id", "target_node_id"]) - - # graph_deployment_version table - op.create_table( - "graph_deployment_version", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "graph_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("graphs.id", ondelete="CASCADE"), nullable=False - ), - sa.Column("version", sa.Integer(), nullable=False), - sa.Column("name", sa.String(255), nullable=True), - sa.Column("state", postgresql.JSONB(), nullable=False), - sa.Column("is_active", sa.Boolean(), nullable=False, server_default="false"), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("created_by", sa.String(255), nullable=True), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint("graph_id", "version", name="graph_deployment_version_graph_version_unique"), - ) - op.create_index("graph_deployment_version_graph_active_idx", "graph_deployment_version", ["graph_id", "is_active"]) - op.create_index("graph_deployment_version_created_at_idx", "graph_deployment_version", ["created_at"]) - - # =========================================== - # 5. Create conversation-related tables - # =========================================== - - # conversations table - op.create_table( - "conversations", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("thread_id", sa.String(100), nullable=False, unique=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("title", sa.String(200), nullable=False), - sa.Column("meta_data", postgresql.JSON(), nullable=True, default=dict), - sa.Column("is_active", sa.Integer(), nullable=False, default=1), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("ix_conversations_thread_id", "conversations", ["thread_id"], unique=True) - op.create_index("ix_conversations_user_id", "conversations", ["user_id"]) - - # messages table - op.create_table( - "messages", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "thread_id", sa.String(100), sa.ForeignKey("conversations.thread_id", ondelete="CASCADE"), nullable=False - ), - sa.Column("role", sa.String(20), nullable=False), - sa.Column("content", sa.Text(), nullable=False), - sa.Column("meta_data", postgresql.JSON(), nullable=True, default=dict), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("ix_messages_thread_id", "messages", ["thread_id"]) - - # chat table - op.create_table( - "chat", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("agent_graph_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("identifier", sa.String(255), nullable=False), - sa.Column("title", sa.String(255), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column("is_active", sa.Boolean(), nullable=False, default=True), - sa.Column("customizations", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("auth_type", sa.String(50), nullable=False, default="public"), - sa.Column("password", sa.String(255), nullable=True), - sa.Column("allowed_emails", postgresql.JSONB(), nullable=False, default=list), - sa.Column("output_configs", postgresql.JSONB(), nullable=False, default=list), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint("identifier", name="identifier_idx"), - ) - - # copilot_chats table - op.create_table( - "copilot_chats", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("agent_graph_id", postgresql.UUID(as_uuid=True), nullable=False), - sa.Column("title", sa.String(255), nullable=True), - sa.Column("messages", postgresql.JSONB(), nullable=False, default=list), - sa.Column("model", sa.String(100), nullable=False, default="claude-3-7-sonnet-latest"), - sa.Column("conversation_id", sa.String(255), nullable=True), - sa.Column("preview_yaml", sa.Text(), nullable=True), - sa.Column("plan_artifact", sa.Text(), nullable=True), - sa.Column("config", postgresql.JSONB(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("copilot_chats_user_id_idx", "copilot_chats", ["user_id"]) - op.create_index("copilot_chats_created_at_idx", "copilot_chats", ["created_at"]) - op.create_index("copilot_chats_updated_at_idx", "copilot_chats", ["updated_at"]) - - # =========================================== - # 6. Create permission and invitation tables - # =========================================== - - # workspace_invitation table - op.create_table( - "workspace_invitation", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("email", sa.String(255), nullable=False), - sa.Column("inviter_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("role", sa.String(50), nullable=False, default="member"), - sa.Column( - "status", - postgresql.ENUM( - "pending", "accepted", "rejected", "cancelled", name="workspaceinvitationstatus", create_type=False - ), - nullable=False, - default="pending", - ), - sa.Column("token", sa.String(255), nullable=False, unique=True), - sa.Column( - "permissions", - postgresql.ENUM("admin", "write", "read", name="permissiontype", create_type=False), - nullable=False, - default="admin", - ), - sa.Column("org_invitation_id", sa.String(255), nullable=True), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("workspace_invitation_email_status_idx", "workspace_invitation", ["email", "status"]) - op.create_index("workspace_invitation_expires_at_idx", "workspace_invitation", ["expires_at"]) - op.create_index("workspace_invitation_workspace_id_idx", "workspace_invitation", ["workspace_id"]) - - # permissions table - op.create_table( - "permissions", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("entity_type", sa.String(50), nullable=False), - sa.Column("entity_id", sa.String(255), nullable=False), - sa.Column( - "permission_type", - postgresql.ENUM("admin", "write", "read", name="permissiontype", create_type=False), - nullable=False, - ), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint("user_id", "entity_type", "entity_id", name="permissions_unique_constraint"), - ) - op.create_index("permissions_user_id_idx", "permissions", ["user_id"]) - op.create_index("permissions_entity_idx", "permissions", ["entity_type", "entity_id"]) - op.create_index("permissions_user_entity_type_idx", "permissions", ["user_id", "entity_type"]) - op.create_index( - "permissions_user_entity_permission_idx", "permissions", ["user_id", "entity_type", "permission_type"] - ) - op.create_index("permissions_user_entity_idx", "permissions", ["user_id", "entity_type", "entity_id"]) - - # =========================================== - # 7. Create settings-related tables - # =========================================== - - # environment table - op.create_table( - "environment", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False, unique=True), - sa.Column("variables", postgresql.JSONB(), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - - # workspace_environment table - op.create_table( - "workspace_environment", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("variables", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint("workspace_id", name="workspace_environment_workspace_unique"), - ) - - # settings table - op.create_table( - "settings", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False, unique=True), - sa.Column("theme", sa.String(50), nullable=False, default="system"), - sa.Column("auto_connect", sa.Boolean(), nullable=False, default=True), - sa.Column("auto_pan", sa.Boolean(), nullable=False, default=True), - sa.Column("console_expanded_by_default", sa.Boolean(), nullable=False, default=True), - sa.Column("telemetry_enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("email_preferences", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("billing_usage_notifications_enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("show_floating_controls", sa.Boolean(), nullable=False, default=True), - sa.Column("show_training_controls", sa.Boolean(), nullable=False, default=False), - sa.Column("super_user_mode_enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("error_notifications_enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("copilot_enabled_models", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("settings_user_id_idx", "settings", ["user_id"]) - - # =========================================== - # 8. Create file storage tables - # =========================================== - - # workspace_file table - op.create_table( - "workspace_file", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("key", sa.String(512), nullable=False, unique=True), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("type", sa.String(100), nullable=False), - sa.Column("uploaded_by", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("uploaded_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("workspace_file_workspace_id_idx", "workspace_file", ["workspace_id"]) - op.create_index("workspace_file_key_idx", "workspace_file", ["key"]) - - # workspace_files table - op.create_table( - "workspace_files", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("key", sa.String(512), nullable=False, unique=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=True, - ), - sa.Column("context", sa.String(50), nullable=False), - sa.Column("original_name", sa.String(255), nullable=False), - sa.Column("content_type", sa.String(255), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("uploaded_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("workspace_files_key_idx", "workspace_files", ["key"]) - op.create_index("workspace_files_user_id_idx", "workspace_files", ["user_id"]) - op.create_index("workspace_files_workspace_id_idx", "workspace_files", ["workspace_id"]) - op.create_index("workspace_files_context_idx", "workspace_files", ["context"]) - - # =========================================== - # 9. Create API key and tool tables - # =========================================== - - # api_key table - op.create_table( - "api_key", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=True, - ), - sa.Column("created_by", sa.String(255), sa.ForeignKey("user.id", ondelete="SET NULL"), nullable=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("key", sa.String(255), nullable=False, unique=True), - sa.Column("type", sa.String(50), nullable=False, default="personal"), - sa.Column("last_used", sa.DateTime(timezone=True), nullable=True), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.CheckConstraint( - "(type = 'workspace' AND workspace_id IS NOT NULL) OR (type = 'personal' AND workspace_id IS NULL)", - name="workspace_type_check", - ), - ) - op.create_index("api_key_user_id_idx", "api_key", ["user_id"]) - op.create_index("api_key_workspace_id_idx", "api_key", ["workspace_id"]) - op.create_index("api_key_key_idx", "api_key", ["key"]) - - # custom_tools table - op.create_table( - "custom_tools", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("owner_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("code", sa.Text(), nullable=False), - sa.Column("schema", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("runtime", sa.String(50), nullable=False, default="python"), - sa.Column("enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint("owner_id", "name", name="custom_tools_owner_name_unique"), - ) - op.create_index("custom_tools_owner_idx", "custom_tools", ["owner_id"]) - - # mcp_servers table - op.create_table( - "mcp_servers", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("created_by", sa.String(255), sa.ForeignKey("user.id", ondelete="SET NULL"), nullable=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column("transport", sa.String(50), nullable=False), - sa.Column("url", sa.String(2048), nullable=True), - sa.Column("headers", postgresql.JSONB(), nullable=False, default=dict), - sa.Column("timeout", sa.Integer(), nullable=True, default=30000), - sa.Column("retries", sa.Integer(), nullable=True, default=3), - sa.Column("enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("last_connected", sa.DateTime(timezone=True), nullable=True), - sa.Column("connection_status", sa.String(50), nullable=True, default="disconnected"), - sa.Column("last_error", sa.Text(), nullable=True), - sa.Column("tool_count", sa.Integer(), nullable=True, default=0), - sa.Column("last_tools_refresh", sa.DateTime(timezone=True), nullable=True), - sa.Column("total_requests", sa.Integer(), nullable=True, default=0), - sa.Column("last_used", sa.DateTime(timezone=True), nullable=True), - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("mcp_servers_user_id_idx", "mcp_servers", ["user_id"]) - op.create_index("mcp_servers_user_enabled_idx", "mcp_servers", ["user_id", "enabled"]) - op.create_index("mcp_servers_user_name_unique_idx", "mcp_servers", ["user_id", "name"], unique=True) - - # =========================================== - # 10. Create model-related tables - # =========================================== - - # model_provider table - op.create_table( - "model_provider", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("name", sa.String(100), nullable=False, unique=True), - sa.Column("display_name", sa.String(255), nullable=False), - sa.Column("icon", sa.String(500), nullable=True), - sa.Column("description", sa.String(1000), nullable=True), - sa.Column("supported_model_types", postgresql.JSON(), nullable=False, default=list), - sa.Column("credential_schema", postgresql.JSON(), nullable=False, default=dict), - sa.Column("config_schema", postgresql.JSON(), nullable=True), - sa.Column("is_enabled", sa.Boolean(), nullable=False, default=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("model_provider_name_idx", "model_provider", ["name"]) - op.create_index("model_provider_enabled_idx", "model_provider", ["is_enabled"]) - - # model_credential table - op.create_table( - "model_credential", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=True), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=True, - ), - sa.Column( - "provider_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("model_provider.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("credentials", sa.String(4096), nullable=False), - sa.Column("is_valid", sa.Boolean(), nullable=False, default=False), - sa.Column("last_validated_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("validation_error", sa.String(1000), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.CheckConstraint("(workspace_id IS NULL) OR (workspace_id IS NOT NULL)", name="model_credential_scope_check"), - ) - op.create_index("model_credential_user_id_idx", "model_credential", ["user_id"]) - op.create_index("model_credential_workspace_id_idx", "model_credential", ["workspace_id"]) - op.create_index("model_credential_provider_id_idx", "model_credential", ["provider_id"]) - op.create_index("model_credential_user_provider_idx", "model_credential", ["user_id", "provider_id"]) - - # model_instance table - op.create_table( - "model_instance", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=True), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=True, - ), - sa.Column( - "provider_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("model_provider.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("model_name", sa.String(255), nullable=False), - sa.Column("model_parameters", postgresql.JSON(), nullable=False, default=dict), - sa.Column("is_default", sa.Boolean(), nullable=False, default=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint( - "user_id", "workspace_id", "provider_id", "model_name", name="uq_model_instance_user_provider_model" - ), - ) - op.create_index("model_instance_user_id_idx", "model_instance", ["user_id"]) - op.create_index("model_instance_workspace_id_idx", "model_instance", ["workspace_id"]) - op.create_index("model_instance_provider_id_idx", "model_instance", ["provider_id"]) - op.create_index( - "model_instance_user_provider_model_idx", "model_instance", ["user_id", "provider_id", "model_name"] - ) - - # =========================================== - # 11. Create skill-related tables - # =========================================== - - # skills table - op.create_table( - "skills", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("description", sa.Text(), nullable=False), - sa.Column("content", sa.Text(), nullable=False), - sa.Column("tags", postgresql.JSONB(), nullable=False, default=list), - sa.Column("source_type", sa.String(50), nullable=False, default="local"), - sa.Column("source_url", sa.String(1024), nullable=True), - sa.Column("root_path", sa.String(512), nullable=True), - sa.Column("owner_id", sa.String(255), sa.ForeignKey("user.id", ondelete="SET NULL"), nullable=True), - sa.Column("created_by_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("is_public", sa.Boolean(), nullable=False, default=False), - sa.Column("license", sa.String(100), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.UniqueConstraint("owner_id", "name", name="skills_owner_name_unique"), - ) - op.create_index("skills_owner_idx", "skills", ["owner_id"]) - op.create_index("skills_created_by_idx", "skills", ["created_by_id"]) - op.create_index("skills_public_idx", "skills", ["is_public"]) - op.create_index("skills_tags_idx", "skills", ["tags"], postgresql_using="gin") - - # skill_files table - op.create_table( - "skill_files", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "skill_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("skills.id", ondelete="CASCADE"), nullable=False - ), - sa.Column("path", sa.String(512), nullable=False), - sa.Column("file_name", sa.String(255), nullable=False), - sa.Column("file_type", sa.String(50), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("storage_type", sa.String(20), nullable=False, default="database"), - sa.Column("storage_key", sa.String(512), nullable=True), - sa.Column("size", sa.Integer(), nullable=False, default=0), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("skill_files_skill_idx", "skill_files", ["skill_id"]) - op.create_index("skill_files_path_idx", "skill_files", ["skill_id", "path"]) - - # =========================================== - # 12. Create security audit and memory tables - # =========================================== - - # memories table - op.create_table( - "memories", - sa.Column("memory_id", sa.String(), primary_key=True), - sa.Column("memory", postgresql.JSON(), nullable=False), - sa.Column("feedback", sa.Text(), nullable=True), - sa.Column("input", sa.Text(), nullable=True), - sa.Column("agent_id", sa.String(), nullable=True), - sa.Column("team_id", sa.String(), nullable=True), - sa.Column("user_id", sa.String(), nullable=True), - sa.Column("topics", postgresql.JSON(), nullable=True), - sa.Column("created_at", sa.BigInteger(), nullable=False), - sa.Column("updated_at", sa.BigInteger(), nullable=True), - ) - op.create_index("ix_memories_user_id", "memories", ["user_id"]) - op.create_index("ix_memories_updated_at", "memories", ["updated_at"]) - op.create_index("ix_memories_created_at", "memories", ["created_at"]) - - # security_audit_log table - op.create_table( - "security_audit_log", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), nullable=True), - sa.Column("user_email", sa.String(255), nullable=True), - sa.Column("event_type", sa.String(50), nullable=False), - sa.Column("event_status", sa.String(20), nullable=False), - sa.Column("ip_address", sa.String(255), nullable=False), - sa.Column("user_agent", sa.String(1024), nullable=True), - sa.Column("device_fingerprint", sa.String(255), nullable=True), - sa.Column("location", sa.String(255), nullable=True), - sa.Column("country", sa.String(10), nullable=True), - sa.Column("details", postgresql.JSONB(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("ix_security_audit_log_user_id", "security_audit_log", ["user_id"]) - op.create_index("ix_security_audit_log_event_type", "security_audit_log", ["event_type"]) - op.create_index("ix_security_audit_log_event_status", "security_audit_log", ["event_status"]) - op.create_index("audit_log_user_id_idx", "security_audit_log", ["user_id"]) - op.create_index("audit_log_user_event_idx", "security_audit_log", ["user_id", "event_type"]) - op.create_index("audit_log_event_type_idx", "security_audit_log", ["event_type"]) - op.create_index("audit_log_event_status_idx", "security_audit_log", ["event_status"]) - op.create_index("audit_log_created_at_idx", "security_audit_log", ["created_at"]) - - -def downgrade() -> None: - # Drop tables in reverse dependency order - - # 12. Security audit and memory tables - op.drop_index("audit_log_created_at_idx", table_name="security_audit_log") - op.drop_index("audit_log_event_status_idx", table_name="security_audit_log") - op.drop_index("audit_log_event_type_idx", table_name="security_audit_log") - op.drop_index("audit_log_user_event_idx", table_name="security_audit_log") - op.drop_index("audit_log_user_id_idx", table_name="security_audit_log") - op.drop_index("ix_security_audit_log_event_status", table_name="security_audit_log") - op.drop_index("ix_security_audit_log_event_type", table_name="security_audit_log") - op.drop_index("ix_security_audit_log_user_id", table_name="security_audit_log") - op.drop_table("security_audit_log") - - op.drop_index("ix_memories_created_at", table_name="memories") - op.drop_index("ix_memories_updated_at", table_name="memories") - op.drop_index("ix_memories_user_id", table_name="memories") - op.drop_table("memories") - - # 11. Skill-related tables - op.drop_index("skill_files_path_idx", table_name="skill_files") - op.drop_index("skill_files_skill_idx", table_name="skill_files") - op.drop_table("skill_files") - - op.drop_index("skills_tags_idx", table_name="skills") - op.drop_index("skills_public_idx", table_name="skills") - op.drop_index("skills_created_by_idx", table_name="skills") - op.drop_index("skills_owner_idx", table_name="skills") - op.drop_table("skills") - - # 10. Model-related tables - op.drop_index("model_instance_user_provider_model_idx", table_name="model_instance") - op.drop_index("model_instance_provider_id_idx", table_name="model_instance") - op.drop_index("model_instance_workspace_id_idx", table_name="model_instance") - op.drop_index("model_instance_user_id_idx", table_name="model_instance") - op.drop_table("model_instance") - - op.drop_index("model_credential_user_provider_idx", table_name="model_credential") - op.drop_index("model_credential_provider_id_idx", table_name="model_credential") - op.drop_index("model_credential_workspace_id_idx", table_name="model_credential") - op.drop_index("model_credential_user_id_idx", table_name="model_credential") - op.drop_table("model_credential") - - op.drop_index("model_provider_enabled_idx", table_name="model_provider") - op.drop_index("model_provider_name_idx", table_name="model_provider") - op.drop_table("model_provider") - - # 9. API key and tool tables - op.drop_index("mcp_servers_user_name_unique_idx", table_name="mcp_servers") - op.drop_index("mcp_servers_user_enabled_idx", table_name="mcp_servers") - op.drop_index("mcp_servers_user_id_idx", table_name="mcp_servers") - op.drop_table("mcp_servers") - - op.drop_index("custom_tools_owner_idx", table_name="custom_tools") - op.drop_table("custom_tools") - - op.drop_index("api_key_key_idx", table_name="api_key") - op.drop_index("api_key_workspace_id_idx", table_name="api_key") - op.drop_index("api_key_user_id_idx", table_name="api_key") - op.drop_table("api_key") - - # 8. File storage tables - op.drop_index("workspace_files_context_idx", table_name="workspace_files") - op.drop_index("workspace_files_workspace_id_idx", table_name="workspace_files") - op.drop_index("workspace_files_user_id_idx", table_name="workspace_files") - op.drop_index("workspace_files_key_idx", table_name="workspace_files") - op.drop_table("workspace_files") - - op.drop_index("workspace_file_key_idx", table_name="workspace_file") - op.drop_index("workspace_file_workspace_id_idx", table_name="workspace_file") - op.drop_table("workspace_file") - - # 7. Settings-related tables - op.drop_index("settings_user_id_idx", table_name="settings") - op.drop_table("settings") - op.drop_table("workspace_environment") - op.drop_table("environment") - - # 6. Permission and invitation tables - op.drop_index("permissions_user_entity_idx", table_name="permissions") - op.drop_index("permissions_user_entity_permission_idx", table_name="permissions") - op.drop_index("permissions_user_entity_type_idx", table_name="permissions") - op.drop_index("permissions_entity_idx", table_name="permissions") - op.drop_index("permissions_user_id_idx", table_name="permissions") - op.drop_table("permissions") - - op.drop_index("workspace_invitation_workspace_id_idx", table_name="workspace_invitation") - op.drop_index("workspace_invitation_expires_at_idx", table_name="workspace_invitation") - op.drop_index("workspace_invitation_email_status_idx", table_name="workspace_invitation") - op.drop_table("workspace_invitation") - - # 5. Conversation-related tables - op.drop_index("copilot_chats_updated_at_idx", table_name="copilot_chats") - op.drop_index("copilot_chats_created_at_idx", table_name="copilot_chats") - op.drop_index("copilot_chats_user_id_idx", table_name="copilot_chats") - op.drop_table("copilot_chats") - op.drop_table("chat") - - op.drop_index("ix_messages_thread_id", table_name="messages") - op.drop_table("messages") - - op.drop_index("ix_conversations_user_id", table_name="conversations") - op.drop_index("ix_conversations_thread_id", table_name="conversations") - op.drop_table("conversations") - - # 4. Graph-related tables - op.drop_index("graph_deployment_version_created_at_idx", table_name="graph_deployment_version") - op.drop_index("graph_deployment_version_graph_active_idx", table_name="graph_deployment_version") - op.drop_table("graph_deployment_version") - - op.drop_index("graph_edges_graph_target_idx", table_name="graph_edges") - op.drop_index("graph_edges_graph_source_idx", table_name="graph_edges") - op.drop_index("graph_edges_target_node_id_idx", table_name="graph_edges") - op.drop_index("graph_edges_source_node_id_idx", table_name="graph_edges") - op.drop_index("graph_edges_graph_id_idx", table_name="graph_edges") - op.drop_table("graph_edges") - - op.drop_index("graph_nodes_type_idx", table_name="graph_nodes") - op.drop_index("graph_nodes_graph_id_idx", table_name="graph_nodes") - op.drop_table("graph_nodes") - - op.drop_index("graphs_parent_id_idx", table_name="graphs") - op.drop_index("graphs_folder_id_idx", table_name="graphs") - op.drop_index("graphs_workspace_id_idx", table_name="graphs") - op.drop_index("graphs_user_id_idx", table_name="graphs") - op.drop_table("graphs") - - # 3. Tables that depend on user tables - op.drop_index("workspace_folder_deleted_at_idx", table_name="workspace_folder") - op.drop_index("workspace_folder_parent_sort_idx", table_name="workspace_folder") - op.drop_index("workspace_folder_workspace_parent_idx", table_name="workspace_folder") - op.drop_index("workspace_folder_user_idx", table_name="workspace_folder") - op.drop_table("workspace_folder") - op.drop_table("workspace_members") - op.drop_table("workspaces") - - op.drop_index("member_organization_id_idx", table_name="member") - op.drop_index("member_user_id_idx", table_name="member") - op.drop_table("member") - - op.drop_index("session_token_idx", table_name="session") - op.drop_index("session_user_id_idx", table_name="session") - op.drop_table("session") - - # 2. Core user tables - op.drop_table("organization") - op.drop_index("ix_user_email", table_name="user") - op.drop_table("user") - - # 1. Drop enum types - op.execute("DROP TYPE IF EXISTS permissiontype") - op.execute("DROP TYPE IF EXISTS workspaceinvitationstatus") - op.execute("DROP TYPE IF EXISTS workspacememberrole") - op.execute("DROP TYPE IF EXISTS workspacetype") - op.execute("DROP TYPE IF EXISTS workspacestatus") diff --git a/backend/alembic/versions/20260108_000001_000000000001_add_copilot_chats_agent_graph_id_idx.py b/backend/alembic/versions/20260108_000001_000000000001_add_copilot_chats_agent_graph_id_idx.py deleted file mode 100644 index 6748c96bf..000000000 --- a/backend/alembic/versions/20260108_000001_000000000001_add_copilot_chats_agent_graph_id_idx.py +++ /dev/null @@ -1,30 +0,0 @@ -"""add_copilot_chats_agent_graph_id_idx - -Revision ID: 000000000001 -Revises: 000000000000 -Create Date: 2026-01-08 00:00:01.000000+00:00 - -Add missing index for copilot_chats.agent_graph_id to align ORM model with DB schema. -""" - -from typing import Sequence, Union - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000001" -down_revision: Union[str, None] = "000000000000" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_index( - "copilot_chats_agent_graph_id_idx", - "copilot_chats", - ["agent_graph_id"], - ) - - -def downgrade() -> None: - op.drop_index("copilot_chats_agent_graph_id_idx", table_name="copilot_chats") diff --git a/backend/alembic/versions/20260108_000002_000000000002_update_skills_agent_skills_spec.py b/backend/alembic/versions/20260108_000002_000000000002_update_skills_agent_skills_spec.py deleted file mode 100644 index 5ea149bea..000000000 --- a/backend/alembic/versions/20260108_000002_000000000002_update_skills_agent_skills_spec.py +++ /dev/null @@ -1,167 +0,0 @@ -"""update_skills_agent_skills_spec - -Revision ID: 000000000002 -Revises: 000000000001 -Create Date: 2026-01-08 00:00:02.000000+00:00 - -Update skills table to comply with Agent Skills specification: -- Change name from String(255) to String(64) -- Change description from Text to String(1024) -- Add compatibility field (String(500), nullable) -- Add metadata field (JSONB, default={}) -- Add allowed_tools field (JSONB, default=[]) -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000002" -down_revision: Union[str, None] = "000000000001" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Step 1: Add new columns (nullable first, then we'll backfill) - # Use DO block to check and add columns safely - op.execute(""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' AND column_name = 'compatibility' - ) THEN - ALTER TABLE skills ADD COLUMN compatibility VARCHAR(500); - END IF; - - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' AND column_name = 'metadata' - ) THEN - ALTER TABLE skills ADD COLUMN metadata JSONB; - END IF; - - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' AND column_name = 'allowed_tools' - ) THEN - ALTER TABLE skills ADD COLUMN allowed_tools JSONB; - END IF; - END $$; - """) - - # Step 2: Set default values for new columns - op.execute("UPDATE skills SET metadata = '{}'::jsonb WHERE metadata IS NULL") - op.execute("UPDATE skills SET allowed_tools = '[]'::jsonb WHERE allowed_tools IS NULL") - - # Step 3: Make new columns NOT NULL with defaults (only if they're still nullable) - op.execute(""" - DO $$ - BEGIN - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' - AND column_name = 'metadata' - AND is_nullable = 'YES' - ) THEN - ALTER TABLE skills - ALTER COLUMN metadata SET NOT NULL, - ALTER COLUMN metadata SET DEFAULT '{}'::jsonb; - END IF; - - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' - AND column_name = 'allowed_tools' - AND is_nullable = 'YES' - ) THEN - ALTER TABLE skills - ALTER COLUMN allowed_tools SET NOT NULL, - ALTER COLUMN allowed_tools SET DEFAULT '[]'::jsonb; - END IF; - END $$; - """) - - # Step 4: Truncate name and description if they exceed new limits - # This is safe because we're only truncating, not removing data - op.execute(""" - UPDATE skills - SET name = LEFT(name, 64) - WHERE LENGTH(name) > 64 - """) - op.execute(""" - UPDATE skills - SET description = LEFT(description, 1024) - WHERE LENGTH(description) > 1024 - """) - - # Step 5: Change column types - # For name: String(255) -> String(64) - op.alter_column("skills", "name", existing_type=sa.String(255), type_=sa.String(64), existing_nullable=False) - - # For description: Text -> String(1024) - # First convert to VARCHAR(1024), then we can change the type - op.execute(""" - ALTER TABLE skills - ALTER COLUMN description TYPE VARCHAR(1024) - USING LEFT(description, 1024) - """) - - -def downgrade() -> None: - # Revert column types (only if they exist and have the new types) - op.execute(""" - DO $$ - BEGIN - -- Check if name column exists and is VARCHAR(64) - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' - AND column_name = 'name' - AND character_maximum_length = 64 - ) THEN - ALTER TABLE skills ALTER COLUMN name TYPE VARCHAR(255); - END IF; - - -- Check if description column exists and is VARCHAR(1024) - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' - AND column_name = 'description' - AND character_maximum_length = 1024 - ) THEN - ALTER TABLE skills ALTER COLUMN description TYPE TEXT; - END IF; - END $$; - """) - - # Remove new columns (only if they exist) - op.execute(""" - DO $$ - BEGIN - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' AND column_name = 'allowed_tools' - ) THEN - ALTER TABLE skills DROP COLUMN allowed_tools; - END IF; - - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' AND column_name = 'metadata' - ) THEN - ALTER TABLE skills DROP COLUMN metadata; - END IF; - - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'skills' AND column_name = 'compatibility' - ) THEN - ALTER TABLE skills DROP COLUMN compatibility; - END IF; - END $$; - """) diff --git a/backend/alembic/versions/20260122_000003_000000000003_fix_memories_table_columns.py b/backend/alembic/versions/20260122_000003_000000000003_fix_memories_table_columns.py deleted file mode 100644 index 6dc7c67ca..000000000 --- a/backend/alembic/versions/20260122_000003_000000000003_fix_memories_table_columns.py +++ /dev/null @@ -1,75 +0,0 @@ -"""fix_memories_table_columns - -Revision ID: 000000000003 -Revises: 000000000002 -Create Date: 2026-01-22 00:00:03.000000+00:00 - -Fix missing columns in the memories table: -- Add memory column (JSON, NOT NULL) -- Add topics column (JSON, nullable) -Skip if columns already exist -""" - -from typing import Sequence, Union - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000003" -down_revision: Union[str, None] = "000000000002" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - """Add missing memory and topics columns to the memories table""" - # Use a DO block to safely check and add columns - op.execute(""" - DO $$ - BEGIN - -- Check and add the memory column - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'memories' AND column_name = 'memory' - ) THEN - -- If the table has data, add as nullable first, then set defaults - ALTER TABLE memories ADD COLUMN memory JSON; - -- Set default value for existing rows (empty JSON object) - UPDATE memories SET memory = '{}'::json WHERE memory IS NULL; - -- Set to NOT NULL - ALTER TABLE memories ALTER COLUMN memory SET NOT NULL; - END IF; - - -- Check and add the topics column - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'memories' AND column_name = 'topics' - ) THEN - ALTER TABLE memories ADD COLUMN topics JSON; - END IF; - END $$; - """) - - -def downgrade() -> None: - """Remove memory and topics columns (only if they exist)""" - op.execute(""" - DO $$ - BEGIN - -- Remove topics column if it exists - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'memories' AND column_name = 'topics' - ) THEN - ALTER TABLE memories DROP COLUMN topics; - END IF; - - -- Remove memory column if it exists - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'memories' AND column_name = 'memory' - ) THEN - ALTER TABLE memories DROP COLUMN memory; - END IF; - END $$; - """) diff --git a/backend/alembic/versions/20260203_000004_000000000004_add_oauth_account_table.py b/backend/alembic/versions/20260203_000004_000000000004_add_oauth_account_table.py deleted file mode 100644 index 47bd50c6c..000000000 --- a/backend/alembic/versions/20260203_000004_000000000004_add_oauth_account_table.py +++ /dev/null @@ -1,60 +0,0 @@ -"""add_oauth_account_table - -Revision ID: 000000000004 -Revises: 000000000003 -Create Date: 2026-02-03 00:00:00.000000+00:00 - -Add OAuth account association table to support GitHub, Google, and custom OIDC provider logins. -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000004" -down_revision: Union[str, None] = "000000000003" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Create oauth_account table - op.create_table( - "oauth_account", - sa.Column("id", sa.String(255), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("provider", sa.String(50), nullable=False), - sa.Column("provider_account_id", sa.String(255), nullable=False), - sa.Column("email", sa.String(255), nullable=True), - sa.Column("access_token", sa.Text(), nullable=True), - sa.Column("refresh_token", sa.Text(), nullable=True), - sa.Column("token_expires_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("raw_userinfo", postgresql.JSONB(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - - # Create unique index: ensure each provider account can only be linked to one user - op.create_index( - "ix_oauth_account_provider_account", - "oauth_account", - ["provider", "provider_account_id"], - unique=True, - ) - - # Create user index: speed up queries by user - op.create_index( - "ix_oauth_account_user_id", - "oauth_account", - ["user_id"], - ) - - -def downgrade() -> None: - op.drop_index("ix_oauth_account_user_id", table_name="oauth_account") - op.drop_index("ix_oauth_account_provider_account", table_name="oauth_account") - op.drop_table("oauth_account") diff --git a/backend/alembic/versions/20260206_000005_000000000005_add_execution_trace_tables.py b/backend/alembic/versions/20260206_000005_000000000005_add_execution_trace_tables.py deleted file mode 100644 index edca9379e..000000000 --- a/backend/alembic/versions/20260206_000005_000000000005_add_execution_trace_tables.py +++ /dev/null @@ -1,127 +0,0 @@ -"""add_execution_trace_tables - -Revision ID: 000000000005 -Revises: 000000000004 -Create Date: 2026-02-06 00:00:00.000000+00:00 - -Add execution trace table (execution_traces) and execution observation table (execution_observations) -for persisting LangGraph execution data, with support for hierarchical observation nesting. -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000005" -down_revision: Union[str, None] = "000000000004" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ============ execution_traces ============ - op.create_table( - "execution_traces", - sa.Column("id", sa.UUID(), primary_key=True), - sa.Column("workspace_id", sa.UUID(), nullable=True), - sa.Column("graph_id", sa.UUID(), nullable=True), - sa.Column("thread_id", sa.String(100), nullable=True), - sa.Column("user_id", sa.String(255), nullable=True), - sa.Column("name", sa.String(255), nullable=True), - sa.Column( - "status", - sa.Enum("RUNNING", "COMPLETED", "FAILED", "INTERRUPTED", name="tracestatus"), - nullable=False, - server_default="RUNNING", - ), - sa.Column("input", postgresql.JSON(), nullable=True), - sa.Column("output", postgresql.JSON(), nullable=True), - sa.Column("start_time", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("end_time", sa.DateTime(timezone=True), nullable=True), - sa.Column("duration_ms", sa.BigInteger(), nullable=True), - sa.Column("total_tokens", sa.Integer(), nullable=True), - sa.Column("total_cost", sa.Float(), nullable=True), - sa.Column("metadata", postgresql.JSON(), nullable=True), - sa.Column("tags", postgresql.JSON(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - ) - op.create_index("ix_execution_traces_workspace_id", "execution_traces", ["workspace_id"]) - op.create_index("ix_execution_traces_graph_id", "execution_traces", ["graph_id"]) - op.create_index("ix_execution_traces_thread_id", "execution_traces", ["thread_id"]) - op.create_index("ix_execution_traces_user_id", "execution_traces", ["user_id"]) - op.create_index("ix_execution_traces_graph_thread", "execution_traces", ["graph_id", "thread_id"]) - op.create_index("ix_execution_traces_start_time", "execution_traces", ["start_time"]) - - # ============ execution_observations ============ - op.create_table( - "execution_observations", - sa.Column("id", sa.UUID(), primary_key=True), - sa.Column( - "trace_id", - sa.UUID(), - sa.ForeignKey("execution_traces.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column( - "parent_observation_id", - sa.UUID(), - sa.ForeignKey("execution_observations.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "type", - sa.Enum("SPAN", "GENERATION", "TOOL", "EVENT", name="observationtype"), - nullable=False, - ), - sa.Column("name", sa.String(255), nullable=True), - sa.Column( - "level", - sa.Enum("DEBUG", "DEFAULT", "WARNING", "ERROR", name="observationlevel"), - nullable=False, - server_default="DEFAULT", - ), - sa.Column( - "status", - sa.Enum("RUNNING", "COMPLETED", "FAILED", "INTERRUPTED", name="observationstatus"), - nullable=False, - server_default="RUNNING", - ), - sa.Column("status_message", sa.Text(), nullable=True), - sa.Column("start_time", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("end_time", sa.DateTime(timezone=True), nullable=True), - sa.Column("duration_ms", sa.BigInteger(), nullable=True), - sa.Column("completion_start_time", sa.DateTime(timezone=True), nullable=True), - sa.Column("input", postgresql.JSON(), nullable=True), - sa.Column("output", postgresql.JSON(), nullable=True), - sa.Column("model_name", sa.String(255), nullable=True), - sa.Column("model_provider", sa.String(100), nullable=True), - sa.Column("model_parameters", postgresql.JSON(), nullable=True), - sa.Column("prompt_tokens", sa.Integer(), nullable=True), - sa.Column("completion_tokens", sa.Integer(), nullable=True), - sa.Column("total_tokens", sa.Integer(), nullable=True), - sa.Column("input_cost", sa.Float(), nullable=True), - sa.Column("output_cost", sa.Float(), nullable=True), - sa.Column("total_cost", sa.Float(), nullable=True), - sa.Column("metadata", postgresql.JSON(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - ) - op.create_index("ix_execution_observations_trace_id", "execution_observations", ["trace_id"]) - op.create_index( - "ix_execution_observations_parent_observation_id", "execution_observations", ["parent_observation_id"] - ) - op.create_index("ix_execution_observations_trace_start", "execution_observations", ["trace_id", "start_time"]) - op.create_index("ix_execution_observations_type", "execution_observations", ["type"]) - - -def downgrade() -> None: - op.drop_table("execution_observations") - op.drop_table("execution_traces") - op.execute("DROP TYPE IF EXISTS observationstatus") - op.execute("DROP TYPE IF EXISTS observationlevel") - op.execute("DROP TYPE IF EXISTS observationtype") - op.execute("DROP TYPE IF EXISTS tracestatus") diff --git a/backend/alembic/versions/20260206_000006_000000000006_add_observation_version_and_status.py b/backend/alembic/versions/20260206_000006_000000000006_add_observation_version_and_status.py deleted file mode 100644 index c04ac684b..000000000 --- a/backend/alembic/versions/20260206_000006_000000000006_add_observation_version_and_status.py +++ /dev/null @@ -1,62 +0,0 @@ -"""add_observation_version_and_status - -Revision ID: 000000000006 -Revises: 000000000005 -Create Date: 2026-02-06 00:00:06.000000+00:00 - -- Add version column (code/model version) to execution_observations -- Backfill status column and observationstatus enum for deployed environments missing it (idempotent) -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000006" -down_revision: Union[str, None] = "000000000005" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # 1. Add version column (all environments) - op.add_column( - "execution_observations", - sa.Column("version", sa.String(50), nullable=True, comment="Code/model version"), - ) - - # 2. If the table exists but lacks the status column (environments that ran 000000000005 before status was merged), add the enum and column - conn = op.get_bind() - if conn.dialect.name == "postgresql": - # Create observationstatus enum if it does not exist - op.execute( - sa.text(""" - DO $$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'observationstatus') THEN - CREATE TYPE observationstatus AS ENUM ( - 'RUNNING', 'COMPLETED', 'FAILED', 'INTERRUPTED' - ); - END IF; - END$$; - """) - ) - # Add status column if it does not exist - op.execute( - sa.text(""" - ALTER TABLE execution_observations - ADD COLUMN IF NOT EXISTS status observationstatus - NOT NULL DEFAULT 'RUNNING'::observationstatus - """) - ) - - -def downgrade() -> None: - op.drop_column("execution_observations", "version") - # Do not automatically drop the status column to avoid breaking applications that depend on it. - # To roll back manually: - # ALTER TABLE execution_observations DROP COLUMN IF EXISTS status; - # DROP TYPE IF EXISTS observationstatus; diff --git a/backend/alembic/versions/20260209_000007_000000000007_add_user_sandbox_table.py b/backend/alembic/versions/20260209_000007_000000000007_add_user_sandbox_table.py deleted file mode 100644 index 73489db83..000000000 --- a/backend/alembic/versions/20260209_000007_000000000007_add_user_sandbox_table.py +++ /dev/null @@ -1,49 +0,0 @@ -"""add_user_sandbox_table - -Revision ID: 000000000007 -Revises: 000000000006 -Create Date: 2026-02-09 12:00:00.000000 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000007" -down_revision: Union[str, None] = "000000000006" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - adjusted manually ### - op.create_table( - "user_sandbox", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column("user_id", sa.String(length=255), nullable=False), - sa.Column("container_id", sa.String(length=255), nullable=True), - sa.Column("status", sa.String(length=50), nullable=False), - sa.Column("image", sa.String(length=255), nullable=False), - sa.Column("runtime", sa.String(length=100), nullable=True), - sa.Column("last_active_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("cpu_limit", sa.Float(), nullable=True), - sa.Column("memory_limit", sa.Integer(), nullable=True), - sa.Column("idle_timeout", sa.Integer(), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("user_id"), - sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - adjusted manually ### - op.drop_table("user_sandbox") - # ### end Alembic commands ### diff --git a/backend/alembic/versions/20260226_000008_000000000008_add_openclaw_tables.py b/backend/alembic/versions/20260226_000008_000000000008_add_openclaw_tables.py deleted file mode 100644 index 6ac3f7a52..000000000 --- a/backend/alembic/versions/20260226_000008_000000000008_add_openclaw_tables.py +++ /dev/null @@ -1,68 +0,0 @@ -"""add_openclaw_tables - -Revision ID: 000000000008 -Revises: 000000000007 -Create Date: 2026-02-26 00:00:00.000000 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "000000000008" -down_revision: Union[str, None] = "000000000007" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "openclaw_worker", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column("name", sa.String(length=255), nullable=False), - sa.Column("endpoint_url", sa.String(length=512), nullable=False), - sa.Column("status", sa.String(length=50), nullable=False), - sa.Column("container_id", sa.String(length=255), nullable=True), - sa.Column("current_tasks", sa.Integer(), nullable=False, server_default="0"), - sa.Column("max_tasks", sa.Integer(), nullable=False, server_default="3"), - sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("endpoint_url"), - ) - - op.create_table( - "openclaw_task", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column("user_id", sa.String(length=255), nullable=False), - sa.Column("worker_id", sa.String(length=255), nullable=True), - sa.Column("title", sa.String(length=512), nullable=False), - sa.Column("input_data", sa.JSON(), nullable=True), - sa.Column("status", sa.String(length=50), nullable=False), - sa.Column("output", sa.Text(), nullable=True), - sa.Column("redis_channel", sa.String(length=255), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["worker_id"], ["openclaw_worker.id"], ondelete="SET NULL"), - ) - op.create_index("ix_openclaw_task_user_id", "openclaw_task", ["user_id"]) - op.create_index("ix_openclaw_task_worker_id", "openclaw_task", ["worker_id"]) - op.create_index("ix_openclaw_task_status", "openclaw_task", ["status"]) - - -def downgrade() -> None: - op.drop_index("ix_openclaw_task_status", table_name="openclaw_task") - op.drop_index("ix_openclaw_task_worker_id", table_name="openclaw_task") - op.drop_index("ix_openclaw_task_user_id", table_name="openclaw_task") - op.drop_table("openclaw_task") - op.drop_table("openclaw_worker") diff --git a/backend/alembic/versions/20260226_000009_000000000009_refactor_openclaw_to_instance.py b/backend/alembic/versions/20260226_000009_000000000009_refactor_openclaw_to_instance.py deleted file mode 100644 index e6efe8a9b..000000000 --- a/backend/alembic/versions/20260226_000009_000000000009_refactor_openclaw_to_instance.py +++ /dev/null @@ -1,102 +0,0 @@ -"""refactor_openclaw_worker_to_instance - -Revision ID: 000000000009 -Revises: 000000000008 -Create Date: 2026-02-26 12:00:00.000000 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "000000000009" -down_revision: Union[str, None] = "000000000008" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Drop old FK and indexes on openclaw_task that reference openclaw_worker - op.drop_index("ix_openclaw_task_worker_id", table_name="openclaw_task") - op.drop_constraint("fk_openclaw_task_worker_id_openclaw_worker", "openclaw_task", type_="foreignkey") - op.drop_column("openclaw_task", "worker_id") - - # Drop old openclaw_worker table - op.drop_table("openclaw_worker") - - # Create new openclaw_instance table - op.create_table( - "openclaw_instance", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column("user_id", sa.String(length=255), nullable=False), - sa.Column("name", sa.String(length=255), nullable=False), - sa.Column("status", sa.String(length=50), nullable=False, server_default="pending"), - sa.Column("container_id", sa.String(length=255), nullable=True), - sa.Column("gateway_port", sa.Integer(), nullable=False), - sa.Column("gateway_token", sa.String(length=512), nullable=False), - sa.Column("config_json", sa.JSON(), nullable=True), - sa.Column("last_active_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), - sa.UniqueConstraint("user_id"), - ) - op.create_index("ix_openclaw_instance_user_id", "openclaw_instance", ["user_id"]) - - # Add instance_id column to openclaw_task - op.add_column("openclaw_task", sa.Column("instance_id", sa.String(length=255), nullable=True)) - op.create_foreign_key( - "fk_openclaw_task_instance_id_openclaw_instance", - "openclaw_task", - "openclaw_instance", - ["instance_id"], - ["id"], - ondelete="SET NULL", - ) - op.create_index("ix_openclaw_task_instance_id", "openclaw_task", ["instance_id"]) - - -def downgrade() -> None: - # Remove instance_id from openclaw_task - op.drop_index("ix_openclaw_task_instance_id", table_name="openclaw_task") - op.drop_constraint("fk_openclaw_task_instance_id_openclaw_instance", "openclaw_task", type_="foreignkey") - op.drop_column("openclaw_task", "instance_id") - - # Drop openclaw_instance table - op.drop_index("ix_openclaw_instance_user_id", table_name="openclaw_instance") - op.drop_table("openclaw_instance") - - # Recreate openclaw_worker table - op.create_table( - "openclaw_worker", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column("name", sa.String(length=255), nullable=False), - sa.Column("endpoint_url", sa.String(length=512), nullable=False), - sa.Column("status", sa.String(length=50), nullable=False), - sa.Column("container_id", sa.String(length=255), nullable=True), - sa.Column("current_tasks", sa.Integer(), nullable=False, server_default="0"), - sa.Column("max_tasks", sa.Integer(), nullable=False, server_default="3"), - sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("endpoint_url"), - ) - - # Restore worker_id column on openclaw_task - op.add_column("openclaw_task", sa.Column("worker_id", sa.String(length=255), nullable=True)) - op.create_foreign_key( - "fk_openclaw_task_worker_id_openclaw_worker", - "openclaw_task", - "openclaw_worker", - ["worker_id"], - ["id"], - ondelete="SET NULL", - ) - op.create_index("ix_openclaw_task_worker_id", "openclaw_task", ["worker_id"]) diff --git a/backend/alembic/versions/20260228_000010_drop_openclaw_task_table.py b/backend/alembic/versions/20260228_000010_drop_openclaw_task_table.py deleted file mode 100644 index b253fbd87..000000000 --- a/backend/alembic/versions/20260228_000010_drop_openclaw_task_table.py +++ /dev/null @@ -1,46 +0,0 @@ -"""drop_openclaw_task_table - -Revision ID: 000000000010 -Revises: 000000000009 -Create Date: 2026-02-28 00:00:00.000000 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "000000000010" -down_revision: Union[str, None] = "000000000009" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_table("openclaw_task") - - -def downgrade() -> None: - op.create_table( - "openclaw_task", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column("user_id", sa.String(length=255), nullable=False), - sa.Column("instance_id", sa.String(length=255), nullable=True), - sa.Column("title", sa.String(length=512), nullable=False), - sa.Column("input_data", sa.JSON(), nullable=True), - sa.Column("status", sa.String(length=50), nullable=False), - sa.Column("output", sa.Text(), nullable=True), - sa.Column("redis_channel", sa.String(length=255), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["instance_id"], ["openclaw_instance.id"], ondelete="SET NULL"), - ) - op.create_index("ix_openclaw_task_user_id", "openclaw_task", ["user_id"]) - op.create_index("ix_openclaw_task_status", "openclaw_task", ["status"]) - op.create_index("ix_openclaw_task_instance_id", "openclaw_task", ["instance_id"]) diff --git a/backend/alembic/versions/20260228_085915_afb42e79eefb_add_graph_test_cases.py b/backend/alembic/versions/20260228_085915_afb42e79eefb_add_graph_test_cases.py deleted file mode 100644 index 935dc8d18..000000000 --- a/backend/alembic/versions/20260228_085915_afb42e79eefb_add_graph_test_cases.py +++ /dev/null @@ -1,47 +0,0 @@ -"""add graph_test_cases - -Revision ID: afb42e79eefb -Revises: 000000000010 -Create Date: 2026-02-28 08:59:15.229305+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "afb42e79eefb" -down_revision: Union[str, None] = "000000000010" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "graph_test_cases", - sa.Column("graph_id", sa.UUID(), nullable=False), - sa.Column("name", sa.String(length=200), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("expected_outputs", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("assertions", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("id", sa.UUID(), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.ForeignKeyConstraint( - ["graph_id"], ["graphs.id"], name=op.f("fk_graph_test_cases_graph_id_graphs"), ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_graph_test_cases")), - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("graph_test_cases") - # ### end Alembic commands ### diff --git a/backend/alembic/versions/20260307_111000_0faa0dc41210_add_model_provider_template_columns.py b/backend/alembic/versions/20260307_111000_0faa0dc41210_add_model_provider_template_columns.py deleted file mode 100644 index 6c7cecbca..000000000 --- a/backend/alembic/versions/20260307_111000_0faa0dc41210_add_model_provider_template_columns.py +++ /dev/null @@ -1,56 +0,0 @@ -"""add model provider template columns - -Revision ID: 0faa0dc41210 -Revises: afb42e79eefb -Create Date: 2026-03-07 11:10:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "0faa0dc41210" -down_revision: Union[str, None] = "afb42e79eefb" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.add_column( - "model_provider", - sa.Column( - "is_template", - sa.Boolean(), - nullable=False, - server_default="false", - comment="Whether this is a template (used to create custom providers)", - ), - ) - op.add_column( - "model_provider", - sa.Column("template_name", sa.String(length=100), nullable=True, comment="Name of the referenced template"), - ) - op.add_column( - "model_provider", - sa.Column( - "provider_type", - sa.String(length=50), - nullable=False, - server_default="system", - comment="Provider type: system, custom", - ), - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("model_provider", "provider_type") - op.drop_column("model_provider", "template_name") - op.drop_column("model_provider", "is_template") - # ### end Alembic commands ### diff --git a/backend/alembic/versions/20260307_120000_add_provider_name_to_credential_instance.py b/backend/alembic/versions/20260307_120000_add_provider_name_to_credential_instance.py deleted file mode 100644 index 5dd1f92d3..000000000 --- a/backend/alembic/versions/20260307_120000_add_provider_name_to_credential_instance.py +++ /dev/null @@ -1,110 +0,0 @@ -"""add provider_name to model_credential and model_instance (Direction A: template not in DB) - -Revision ID: d7e8f9a0b1c2 -Revises: 0faa0dc41210 -Create Date: 2026-03-07 12:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "d7e8f9a0b1c2" -down_revision: Union[str, None] = "0faa0dc41210" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # model_credential: add provider_name, make provider_id nullable - op.add_column( - "model_credential", - sa.Column( - "provider_name", - sa.String(length=100), - nullable=True, - comment="Template provider name, used when provider_id is null", - ), - ) - op.alter_column( - "model_credential", - "provider_id", - existing_type=sa.dialects.postgresql.UUID(as_uuid=True), - nullable=True, - ) - op.create_index("model_credential_provider_name_idx", "model_credential", ["provider_name"], unique=False) - # Partial unique: one credential per (user_id, workspace_id, provider_name) when template - op.execute( - """ - CREATE UNIQUE INDEX uq_model_credential_user_workspace_provider_name - ON model_credential (user_id, workspace_id, provider_name) - WHERE provider_id IS NULL AND provider_name IS NOT NULL - """ - ) - # When provider_id is set, keep existing uniqueness via application logic / existing index on provider_id - - # model_instance: add provider_name, make provider_id nullable, replace unique constraint - op.add_column( - "model_instance", - sa.Column( - "provider_name", - sa.String(length=100), - nullable=True, - comment="Template provider name, used when provider_id is null", - ), - ) - op.drop_constraint("uq_model_instance_user_provider_model", "model_instance", type_="unique") - op.alter_column( - "model_instance", - "provider_id", - existing_type=sa.dialects.postgresql.UUID(as_uuid=True), - nullable=True, - ) - op.create_index("model_instance_provider_name_idx", "model_instance", ["provider_name"], unique=False) - # Partial unique: (user_id, workspace_id, provider_id, model_name) when provider_id is not null - op.execute( - """ - CREATE UNIQUE INDEX uq_model_instance_user_workspace_provider_id_model - ON model_instance (user_id, workspace_id, provider_id, model_name) - WHERE provider_id IS NOT NULL - """ - ) - # Partial unique: (user_id, workspace_id, provider_name, model_name) when template - op.execute( - """ - CREATE UNIQUE INDEX uq_model_instance_user_workspace_provider_name_model - ON model_instance (user_id, workspace_id, provider_name, model_name) - WHERE provider_id IS NULL AND provider_name IS NOT NULL - """ - ) - - -def downgrade() -> None: - op.drop_index("uq_model_instance_user_workspace_provider_name_model", table_name="model_instance") - op.drop_index("uq_model_instance_user_workspace_provider_id_model", table_name="model_instance") - op.drop_index("model_instance_provider_name_idx", table_name="model_instance") - op.alter_column( - "model_instance", - "provider_id", - existing_type=sa.dialects.postgresql.UUID(as_uuid=True), - nullable=False, - ) - op.create_unique_constraint( - "uq_model_instance_user_provider_model", - "model_instance", - ["user_id", "workspace_id", "provider_id", "model_name"], - ) - op.drop_column("model_instance", "provider_name") - - op.drop_index("uq_model_credential_user_workspace_provider_name", table_name="model_credential") - op.drop_index("model_credential_provider_name_idx", table_name="model_credential") - op.alter_column( - "model_credential", - "provider_id", - existing_type=sa.dialects.postgresql.UUID(as_uuid=True), - nullable=False, - ) - op.drop_column("model_credential", "provider_name") diff --git a/backend/alembic/versions/20260309_000000_add_graph_node_secrets.py b/backend/alembic/versions/20260309_000000_add_graph_node_secrets.py deleted file mode 100644 index 22acdb4f8..000000000 --- a/backend/alembic/versions/20260309_000000_add_graph_node_secrets.py +++ /dev/null @@ -1,45 +0,0 @@ -"""add graph_node_secrets table for encrypted a2a_auth_headers - -Revision ID: a1b2c3d4e5f6 -Revises: d7e8f9a0b1c2 -Create Date: 2026-03-09 00:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -revision: str = "a1b2c3d4e5f6" -down_revision: Union[str, None] = "d7e8f9a0b1c2" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "graph_node_secrets", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "graph_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("graphs.id", ondelete="CASCADE"), nullable=False - ), - sa.Column( - "node_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("graph_nodes.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("key_slug", sa.String(64), nullable=False, server_default="a2a_auth_headers"), - sa.Column("encrypted_value", sa.Text(), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("graph_node_secrets_graph_node_idx", "graph_node_secrets", ["graph_id", "node_id"], unique=False) - - -def downgrade() -> None: - op.drop_index("graph_node_secrets_graph_node_idx", table_name="graph_node_secrets") - op.drop_table("graph_node_secrets") diff --git a/backend/alembic/versions/20260309_000001_drop_graph_nodes_redundant_columns.py b/backend/alembic/versions/20260309_000001_drop_graph_nodes_redundant_columns.py deleted file mode 100644 index 6df3461d6..000000000 --- a/backend/alembic/versions/20260309_000001_drop_graph_nodes_redundant_columns.py +++ /dev/null @@ -1,35 +0,0 @@ -"""drop graph_nodes redundant columns; add graphs.deleted_at for soft delete - -Revision ID: b2c3d4e5f6a7 -Revises: a1b2c3d4e5f6 -Create Date: 2026-03-09 00:00:01.000000+00:00 - -- Drop graph_nodes.prompt, tools, memory (compiler uses data.config only). -- Add graphs.deleted_at for AgentGraph soft delete. -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -revision: str = "b2c3d4e5f6a7" -down_revision: Union[str, None] = "a1b2c3d4e5f6" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column("graphs", sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True)) - op.drop_column("graph_nodes", "prompt") - op.drop_column("graph_nodes", "tools") - op.drop_column("graph_nodes", "memory") - - -def downgrade() -> None: - op.drop_column("graphs", "deleted_at") - op.add_column("graph_nodes", sa.Column("prompt", sa.Text(), nullable=True, server_default="")) - op.add_column("graph_nodes", sa.Column("tools", postgresql.JSONB(), nullable=True, server_default="{}")) - op.add_column("graph_nodes", sa.Column("memory", postgresql.JSONB(), nullable=True, server_default="{}")) diff --git a/backend/alembic/versions/20260310_000000_add_graph_executions_table.py b/backend/alembic/versions/20260310_000000_add_graph_executions_table.py deleted file mode 100644 index a6f9b2698..000000000 --- a/backend/alembic/versions/20260310_000000_add_graph_executions_table.py +++ /dev/null @@ -1,68 +0,0 @@ -"""add graph_executions table for OpenAPI graph run/status/abort/result - -Revision ID: c3d4e5f6a7b8 -Revises: b2c3d4e5f6a7 -Create Date: 2026-03-10 20:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -revision: str = "c3d4e5f6a7b8" -down_revision: Union[str, None] = "b2c3d4e5f6a7" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "graph_executions", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "graph_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("graphs.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column( - "user_id", - sa.String(255), - sa.ForeignKey("user.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column( - "api_key_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("api_key.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "status", - sa.Enum("init", "executing", "finish", "failed", name="executionstatus"), - nullable=False, - server_default="init", - ), - sa.Column("input_variables", postgresql.JSONB(), nullable=True), - sa.Column("output", postgresql.JSONB(), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - ) - op.create_index("graph_executions_graph_id_idx", "graph_executions", ["graph_id"], unique=False) - op.create_index("graph_executions_user_id_idx", "graph_executions", ["user_id"], unique=False) - op.create_index("graph_executions_status_idx", "graph_executions", ["status"], unique=False) - - -def downgrade() -> None: - op.drop_index("graph_executions_status_idx", table_name="graph_executions") - op.drop_index("graph_executions_user_id_idx", table_name="graph_executions") - op.drop_index("graph_executions_graph_id_idx", table_name="graph_executions") - op.drop_table("graph_executions") - op.execute("DROP TYPE IF EXISTS executionstatus") diff --git a/backend/alembic/versions/20260321_000000_d4e5f6a7b8c9_add_versioning_permissions_tokens.py b/backend/alembic/versions/20260321_000000_d4e5f6a7b8c9_add_versioning_permissions_tokens.py deleted file mode 100644 index 7b652c774..000000000 --- a/backend/alembic/versions/20260321_000000_d4e5f6a7b8c9_add_versioning_permissions_tokens.py +++ /dev/null @@ -1,125 +0,0 @@ -"""add skill_collaborators, skill_versions, skill_version_files, platform_tokens tables - -Revision ID: d4e5f6a7b8c9 -Revises: c3d4e5f6a7b8 -Create Date: 2026-03-21 00:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -revision: str = "d4e5f6a7b8c9" -down_revision: Union[str, None] = "c3d4e5f6a7b8" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # -- 1. collaborator_role enum type -- - collaborator_role = postgresql.ENUM( - "viewer", - "editor", - "publisher", - "admin", - name="collaborator_role", - create_type=False, - ) - collaborator_role.create(op.get_bind(), checkfirst=True) - - # -- 2. skill_collaborators -- - op.create_table( - "skill_collaborators", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "skill_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("skills.id", ondelete="CASCADE"), nullable=False - ), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("role", collaborator_role, nullable=False), - sa.Column("invited_by", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.UniqueConstraint("skill_id", "user_id", name="skill_collaborators_skill_user_unique"), - ) - op.create_index("skill_collaborators_user_skill_idx", "skill_collaborators", ["user_id", "skill_id"]) - - # -- 3. skill_versions -- - op.create_table( - "skill_versions", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "skill_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("skills.id", ondelete="CASCADE"), nullable=False - ), - sa.Column("version", sa.String(20), nullable=False), - sa.Column("release_notes", sa.Text, nullable=True), - sa.Column("skill_name", sa.String(64), nullable=False), - sa.Column("skill_description", sa.String(1024), nullable=False), - sa.Column("content", sa.Text, nullable=False), - sa.Column("tags", postgresql.JSONB, nullable=False, server_default="[]"), - sa.Column("metadata", postgresql.JSONB, nullable=False, server_default="{}"), - sa.Column("allowed_tools", postgresql.JSONB, nullable=False, server_default="[]"), - sa.Column("compatibility", sa.String(500), nullable=True), - sa.Column("license", sa.String(100), nullable=True), - sa.Column("published_by_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("published_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.UniqueConstraint("skill_id", "version", name="skill_versions_skill_version_unique"), - ) - op.create_index("skill_versions_skill_idx", "skill_versions", ["skill_id"]) - op.create_index("skill_versions_published_at_idx", "skill_versions", ["published_at"]) - - # -- 4. skill_version_files -- - op.create_table( - "skill_version_files", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "version_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("skill_versions.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("path", sa.String(512), nullable=False), - sa.Column("file_name", sa.String(255), nullable=False), - sa.Column("file_type", sa.String(50), nullable=False), - sa.Column("content", sa.Text, nullable=True), - sa.Column("storage_type", sa.String(20), nullable=False, server_default="database"), - sa.Column("storage_key", sa.String(512), nullable=True), - sa.Column("size", sa.Integer, nullable=False, server_default="0"), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - ) - op.create_index("skill_version_files_version_idx", "skill_version_files", ["version_id"]) - - # -- 5. platform_tokens -- - op.create_table( - "platform_tokens", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column("name", sa.String(255), nullable=False), - sa.Column("token_hash", sa.String(64), nullable=False, unique=True), - sa.Column("token_prefix", sa.String(12), nullable=False), - sa.Column("scopes", postgresql.JSONB, nullable=False, server_default="[]"), - sa.Column("resource_type", sa.String(50), nullable=True), - sa.Column("resource_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("is_active", sa.Boolean, nullable=False, server_default="true"), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - ) - op.create_index("platform_tokens_user_idx", "platform_tokens", ["user_id"]) - op.create_index("platform_tokens_hash_idx", "platform_tokens", ["token_hash"]) - op.create_index("platform_tokens_active_idx", "platform_tokens", ["is_active"]) - - -def downgrade() -> None: - op.drop_table("platform_tokens") - op.drop_table("skill_version_files") - op.drop_table("skill_versions") - op.drop_table("skill_collaborators") - sa.Enum(name="collaborator_role").drop(op.get_bind(), checkfirst=True) diff --git a/backend/alembic/versions/20260323_000000_e5f6a7b8c9d0_migrate_apikey_to_platform_token.py b/backend/alembic/versions/20260323_000000_e5f6a7b8c9d0_migrate_apikey_to_platform_token.py deleted file mode 100644 index c5963ba91..000000000 --- a/backend/alembic/versions/20260323_000000_e5f6a7b8c9d0_migrate_apikey_to_platform_token.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Migrate ApiKey system to unified PlatformToken system. - -Steps: -1. Migrate existing api_key records to platform_tokens -2. Drop api_key_id FK from graph_executions -3. Drop api_key_id column from graph_executions -4. Rename api_key table to api_keys_deprecated (30-day retention) - -Revision ID: e5f6a7b8c9d0 -Revises: d4e5f6a7b8c9 -Create Date: 2026-03-23 00:00:00.000000+00:00 - -""" - -import hashlib -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -revision: str = "e5f6a7b8c9d0" -down_revision: Union[str, None] = "d4e5f6a7b8c9" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - conn = op.get_bind() - - # 1. Migrate existing api_key records to platform_tokens - result = conn.execute( - sa.text(""" - SELECT id, user_id, workspace_id, key, name, created_at, last_used - FROM api_key - """) - ) - - migrated_count = 0 - for row in result: - token_hash = hashlib.sha256(row.key.encode()).hexdigest() - token_prefix = row.key[:12] if len(row.key) >= 12 else row.key - - conn.execute( - sa.text(""" - INSERT INTO platform_tokens - (id, user_id, name, token_hash, token_prefix, scopes, resource_type, resource_id, - is_active, created_at, last_used_at) - VALUES - (:id, :user_id, :name, :token_hash, :token_prefix, '["graphs:execute"]'::jsonb, :resource_type, :resource_id, - true, :created_at, :last_used_at) - """), - { - "id": row.id, - "user_id": row.user_id, - "name": row.name or "Migrated API Key", - "token_hash": token_hash, - "token_prefix": token_prefix, - "resource_type": "graph", - "resource_id": row.workspace_id, - "created_at": row.created_at, - "last_used_at": row.last_used, - }, - ) - migrated_count += 1 - - print(f"Migrated {migrated_count} active API keys to platform_tokens") - - # 2. Drop FK constraint and column from graph_executions - op.drop_constraint( - "fk_graph_executions_api_key_id_api_key", - "graph_executions", - type_="foreignkey", - ) - op.drop_column("graph_executions", "api_key_id") - - # 3. Rename api_key table for 30-day verification period - op.rename_table("api_key", "api_keys_deprecated") - op.execute( - "COMMENT ON TABLE api_keys_deprecated IS " - "'Deprecated: migrated to platform_tokens. Safe to drop after 2026-04-23.'" - ) - - -def downgrade() -> None: - # 1. Restore api_key table name - op.rename_table("api_keys_deprecated", "api_key") - op.execute("COMMENT ON TABLE api_key IS NULL") - - # 2. Re-add api_key_id column and FK to graph_executions - op.add_column( - "graph_executions", - sa.Column("api_key_id", postgresql.UUID(as_uuid=True), nullable=True), - ) - op.create_foreign_key( - "graph_executions_api_key_id_fkey", - "graph_executions", - "api_key", - ["api_key_id"], - ["id"], - ondelete="SET NULL", - ) diff --git a/backend/alembic/versions/20260323_000001_f6a7b8c9d0e1_drop_allow_personal_api_keys.py b/backend/alembic/versions/20260323_000001_f6a7b8c9d0e1_drop_allow_personal_api_keys.py deleted file mode 100644 index 55032b2c1..000000000 --- a/backend/alembic/versions/20260323_000001_f6a7b8c9d0e1_drop_allow_personal_api_keys.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Drop allow_personal_api_keys column from workspace table. - -This column controlled the legacy ApiKey system which has been fully -replaced by PlatformToken. The column is no longer referenced anywhere -in the application code. - -Revision ID: f6a7b8c9d0e1 -Revises: e5f6a7b8c9d0 -Create Date: 2026-03-23 00:00:01.000000+00:00 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "f6a7b8c9d0e1" -down_revision: Union[str, None] = "e5f6a7b8c9d0" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_column("workspaces", "allow_personal_api_keys") - - -def downgrade() -> None: - op.add_column( - "workspaces", - sa.Column("allow_personal_api_keys", sa.Boolean(), nullable=False, server_default=sa.text("true")), - ) diff --git a/backend/alembic/versions/20260323_000002_g7a8b9c0d1e2_drop_api_keys_deprecated.py b/backend/alembic/versions/20260323_000002_g7a8b9c0d1e2_drop_api_keys_deprecated.py deleted file mode 100644 index c6fb0eb8f..000000000 --- a/backend/alembic/versions/20260323_000002_g7a8b9c0d1e2_drop_api_keys_deprecated.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Drop api_keys_deprecated table. - -The api_key table was migrated to platform_tokens and renamed to -api_keys_deprecated on 2026-03-23. This migration drops the deprecated table. - -Revision ID: g7a8b9c0d1e2 -Revises: f6a7b8c9d0e1 -Create Date: 2026-03-23 00:00:02.000000+00:00 -""" - -from typing import Sequence, Union - -from alembic import op - -revision: str = "g7a8b9c0d1e2" -down_revision: Union[str, None] = "f6a7b8c9d0e1" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_table("api_keys_deprecated") - - -def downgrade() -> None: - # Cannot restore - data was already migrated to platform_tokens - pass diff --git a/backend/alembic/versions/20260325_000000_h8i9j0k1l2m3_add_agent_run_tables.py b/backend/alembic/versions/20260325_000000_h8i9j0k1l2m3_add_agent_run_tables.py deleted file mode 100644 index d237b60ee..000000000 --- a/backend/alembic/versions/20260325_000000_h8i9j0k1l2m3_add_agent_run_tables.py +++ /dev/null @@ -1,121 +0,0 @@ -"""add agent run persistence tables - -Revision ID: h8i9j0k1l2m3 -Revises: g7a8b9c0d1e2 -Create Date: 2026-03-25 12:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -revision: str = "h8i9j0k1l2m3" -down_revision: Union[str, None] = "g7a8b9c0d1e2" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "agent_runs", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("user_id", sa.String(length=255), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False), - sa.Column( - "workspace_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("workspaces.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "graph_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("graphs.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column("thread_id", sa.String(length=100), nullable=True), - sa.Column("run_type", sa.String(length=100), nullable=False), - sa.Column("source", sa.String(length=100), nullable=False), - sa.Column( - "status", - sa.Enum( - "queued", - "running", - "interrupt_wait", - "completed", - "failed", - "cancelled", - name="agentrunstatus", - ), - nullable=False, - server_default="queued", - ), - sa.Column("title", sa.String(length=255), nullable=True), - sa.Column("request_payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column("result_summary", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column("error_code", sa.String(length=100), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("trace_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("last_seq", sa.BigInteger(), nullable=False, server_default="0"), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - ) - op.create_index("agent_runs_user_created_idx", "agent_runs", ["user_id", "created_at"], unique=False) - op.create_index("agent_runs_thread_created_idx", "agent_runs", ["thread_id", "created_at"], unique=False) - op.create_index("agent_runs_graph_created_idx", "agent_runs", ["graph_id", "created_at"], unique=False) - op.create_index("agent_runs_status_updated_idx", "agent_runs", ["status", "updated_at"], unique=False) - - op.create_table( - "agent_run_events", - sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column( - "run_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("agent_runs.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("seq", sa.BigInteger(), nullable=False), - sa.Column("event_type", sa.String(length=100), nullable=False), - sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("trace_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("observation_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("parent_observation_id", postgresql.UUID(as_uuid=True), nullable=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - sa.UniqueConstraint("run_id", "seq", name="uq_agent_run_events_run_seq"), - ) - op.create_index("agent_run_events_run_seq_idx", "agent_run_events", ["run_id", "seq"], unique=False) - op.create_index("agent_run_events_run_created_idx", "agent_run_events", ["run_id", "created_at"], unique=False) - - op.create_table( - "agent_run_snapshots", - sa.Column( - "run_id", - postgresql.UUID(as_uuid=True), - sa.ForeignKey("agent_runs.id", ondelete="CASCADE"), - primary_key=True, - ), - sa.Column("last_seq", sa.BigInteger(), nullable=False, server_default="0"), - sa.Column("status", sa.String(length=100), nullable=False), - sa.Column("projection", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), - ) - - -def downgrade() -> None: - op.drop_table("agent_run_snapshots") - op.drop_index("agent_run_events_run_created_idx", table_name="agent_run_events") - op.drop_index("agent_run_events_run_seq_idx", table_name="agent_run_events") - op.drop_table("agent_run_events") - op.drop_index("agent_runs_status_updated_idx", table_name="agent_runs") - op.drop_index("agent_runs_graph_created_idx", table_name="agent_runs") - op.drop_index("agent_runs_thread_created_idx", table_name="agent_runs") - op.drop_index("agent_runs_user_created_idx", table_name="agent_runs") - op.drop_table("agent_runs") - op.execute("DROP TYPE IF EXISTS agentrunstatus") diff --git a/backend/alembic/versions/20260325_000001_i9j0k1l2m3n4_add_agent_run_runtime_ownership.py b/backend/alembic/versions/20260325_000001_i9j0k1l2m3n4_add_agent_run_runtime_ownership.py deleted file mode 100644 index 271c6d9f0..000000000 --- a/backend/alembic/versions/20260325_000001_i9j0k1l2m3n4_add_agent_run_runtime_ownership.py +++ /dev/null @@ -1,30 +0,0 @@ -"""add runtime ownership columns to agent runs - -Revision ID: i9j0k1l2m3n4 -Revises: h8i9j0k1l2m3 -Create Date: 2026-03-25 18:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "i9j0k1l2m3n4" -down_revision: Union[str, None] = "h8i9j0k1l2m3" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column("agent_runs", sa.Column("runtime_owner_id", sa.String(length=255), nullable=True)) - op.add_column("agent_runs", sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=True)) - op.create_index("agent_runs_owner_status_idx", "agent_runs", ["runtime_owner_id", "status"], unique=False) - - -def downgrade() -> None: - op.drop_index("agent_runs_owner_status_idx", table_name="agent_runs") - op.drop_column("agent_runs", "last_heartbeat_at") - op.drop_column("agent_runs", "runtime_owner_id") diff --git a/backend/alembic/versions/20260326_000000_add_agent_name_column.py b/backend/alembic/versions/20260326_000000_add_agent_name_column.py deleted file mode 100644 index d45581dc6..000000000 --- a/backend/alembic/versions/20260326_000000_add_agent_name_column.py +++ /dev/null @@ -1,30 +0,0 @@ -"""add agent name column to agent runs - -Revision ID: j0k1l2m3n4o5 -Revises: i9j0k1l2m3n4 -Create Date: 2026-03-26 00:00:00.000000+00:00 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "j0k1l2m3n4o5" -down_revision: Union[str, None] = "i9j0k1l2m3n4" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column("agent_runs", sa.Column("agent_name", sa.String(length=100), nullable=True)) - op.execute("UPDATE agent_runs SET agent_name = 'skill_creator' WHERE agent_name IS NULL") - op.alter_column("agent_runs", "agent_name", nullable=False) - op.create_index("agent_runs_agent_updated_idx", "agent_runs", ["agent_name", "updated_at"], unique=False) - - -def downgrade() -> None: - op.drop_index("agent_runs_agent_updated_idx", table_name="agent_runs") - op.drop_column("agent_runs", "agent_name") diff --git a/backend/alembic/versions/20260330_000000_add_provider_default_parameters.py b/backend/alembic/versions/20260330_000000_add_provider_default_parameters.py deleted file mode 100644 index 574f2a972..000000000 --- a/backend/alembic/versions/20260330_000000_add_provider_default_parameters.py +++ /dev/null @@ -1,28 +0,0 @@ -"""add default_parameters to model_provider - -Revision ID: p1q2r3s4t5u6 -Revises: j0k1l2m3n4o5 -Create Date: 2026-03-30 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "p1q2r3s4t5u6" -down_revision: Union[str, None] = "j0k1l2m3n4o5" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "model_provider", - sa.Column("default_parameters", sa.JSON(), nullable=False, server_default="{}"), - ) - - -def downgrade() -> None: - op.drop_column("model_provider", "default_parameters") diff --git a/backend/alembic/versions/20260330_000001_add_model_usage_log_table.py b/backend/alembic/versions/20260330_000001_add_model_usage_log_table.py deleted file mode 100644 index aa2e3377b..000000000 --- a/backend/alembic/versions/20260330_000001_add_model_usage_log_table.py +++ /dev/null @@ -1,51 +0,0 @@ -"""add model_usage_log table - -Revision ID: q2r3s4t5u6v7 -Revises: p1q2r3s4t5u6 -Create Date: 2026-03-30 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "q2r3s4t5u6v7" -down_revision: Union[str, None] = "p1q2r3s4t5u6" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "model_usage_log", - sa.Column("id", sa.dialects.postgresql.UUID(as_uuid=True), primary_key=True), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("provider_name", sa.String(100), nullable=False), - sa.Column("model_name", sa.String(255), nullable=False), - sa.Column("model_type", sa.String(50), nullable=False, server_default="chat"), - sa.Column("user_id", sa.String(255), nullable=True), - sa.Column("input_tokens", sa.Integer(), nullable=False, server_default="0"), - sa.Column("output_tokens", sa.Integer(), nullable=False, server_default="0"), - sa.Column("total_time_ms", sa.Float(), nullable=False, server_default="0.0"), - sa.Column("ttft_ms", sa.Float(), nullable=True), - sa.Column("status", sa.String(20), nullable=False, server_default="success"), - sa.Column("error_message", sa.String(2000), nullable=True), - sa.Column("source", sa.String(50), nullable=False, server_default="chat"), - ) - op.create_index("model_usage_log_created_at_idx", "model_usage_log", ["created_at"]) - op.create_index("model_usage_log_provider_model_idx", "model_usage_log", ["provider_name", "model_name"]) - op.create_index( - "model_usage_log_created_provider_model_idx", - "model_usage_log", - ["created_at", "provider_name", "model_name"], - ) - - -def downgrade() -> None: - op.drop_index("model_usage_log_created_provider_model_idx", table_name="model_usage_log") - op.drop_index("model_usage_log_provider_model_idx", table_name="model_usage_log") - op.drop_index("model_usage_log_created_at_idx", table_name="model_usage_log") - op.drop_table("model_usage_log") diff --git a/backend/alembic/versions/20260330_000002_backfill_provider_id.py b/backend/alembic/versions/20260330_000002_backfill_provider_id.py deleted file mode 100644 index c9422ba8a..000000000 --- a/backend/alembic/versions/20260330_000002_backfill_provider_id.py +++ /dev/null @@ -1,42 +0,0 @@ -"""backfill provider_id for model_instance and model_credential - -Revision ID: r3s4t5u6v7w8 -Revises: q2r3s4t5u6v7 -Create Date: 2026-03-30 -""" - -from typing import Sequence, Union - -from alembic import op - -revision: str = "r3s4t5u6v7w8" -down_revision: Union[str, None] = "q2r3s4t5u6v7" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Backfill provider_id for model_instance rows that only have provider_name - op.execute(""" - UPDATE model_instance mi - SET provider_id = mp.id - FROM model_provider mp - WHERE mi.provider_id IS NULL - AND mi.provider_name IS NOT NULL - AND mi.provider_name = mp.name - """) - - # Backfill provider_id for model_credential rows that only have provider_name - op.execute(""" - UPDATE model_credential mc - SET provider_id = mp.id - FROM model_provider mp - WHERE mc.provider_id IS NULL - AND mc.provider_name IS NOT NULL - AND mc.provider_name = mp.name - """) - - -def downgrade() -> None: - # No-op: we don't remove provider_id values on downgrade - pass diff --git a/backend/alembic/versions/20260330_000003_drop_deprecated_provider_name.py b/backend/alembic/versions/20260330_000003_drop_deprecated_provider_name.py deleted file mode 100644 index 51ba7ac77..000000000 --- a/backend/alembic/versions/20260330_000003_drop_deprecated_provider_name.py +++ /dev/null @@ -1,61 +0,0 @@ -"""drop deprecated provider_name columns from model_instance and model_credential - -Revision ID: s4t5u6v7w8x9 -Revises: r3s4t5u6v7w8 -Create Date: 2026-03-30 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -revision: str = "s4t5u6v7w8x9" -down_revision: Union[str, None] = "r3s4t5u6v7w8" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Drop ALL indexes that reference provider_name before dropping the column - # (includes partial unique indexes, named indexes, etc.) - for table in ("model_instance", "model_credential"): - op.execute(f""" - DO $$ - DECLARE idx RECORD; - BEGIN - FOR idx IN - SELECT i.relname AS index_name - FROM pg_index ix - JOIN pg_class i ON i.oid = ix.indexrelid - JOIN pg_class t ON t.oid = ix.indrelid - JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) - WHERE t.relname = '{table}' AND a.attname = 'provider_name' - LOOP - EXECUTE 'DROP INDEX IF EXISTS ' || idx.index_name; - END LOOP; - END $$; - """) - - # Make provider_id NOT NULL (all rows backfilled in previous migration) - op.alter_column("model_instance", "provider_id", nullable=False) - op.alter_column("model_credential", "provider_id", nullable=False) - - # Drop provider_name columns - op.drop_column("model_instance", "provider_name") - op.drop_column("model_credential", "provider_name") - - -def downgrade() -> None: - # Re-add provider_name to model_credential as nullable - op.add_column( - "model_credential", - sa.Column("provider_name", sa.String(100), nullable=True), - ) - - # Re-add provider_name to model_instance as nullable - op.add_column( - "model_instance", - sa.Column("provider_name", sa.String(100), nullable=True), - ) diff --git a/backend/alembic/versions/20260331_000000_remove_is_default_column.py b/backend/alembic/versions/20260331_000000_remove_is_default_column.py deleted file mode 100644 index 3df9f1fe6..000000000 --- a/backend/alembic/versions/20260331_000000_remove_is_default_column.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Remove is_default column from model_instance table. - -The default model mechanism is replaced by explicit model selection. - -Revision ID: t5u6v7w8x9y0 -Revises: s4t5u6v7w8x9 -Create Date: 2026-03-31 00:00:00.000000 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "t5u6v7w8x9y0" -down_revision: Union[str, None] = "s4t5u6v7w8x9" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_column("model_instance", "is_default") - - -def downgrade() -> None: - op.add_column( - "model_instance", - sa.Column("is_default", sa.Boolean(), nullable=False, server_default=sa.text("false")), - ) diff --git a/backend/alembic/versions/20260405_000000_drop_graph_test_cases.py b/backend/alembic/versions/20260405_000000_drop_graph_test_cases.py deleted file mode 100644 index ced5c3441..000000000 --- a/backend/alembic/versions/20260405_000000_drop_graph_test_cases.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Drop graph_test_cases table. - -Feature removed — will be redesigned later. - -Revision ID: 87be2dfd4240 -Revises: t5u6v7w8x9y0 -Create Date: 2026-04-05 00:00:00.000000 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "87be2dfd4240" -down_revision: Union[str, None] = "t5u6v7w8x9y0" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_table("graph_test_cases") - - -def downgrade() -> None: - op.create_table( - "graph_test_cases", - sa.Column("graph_id", sa.UUID(), nullable=False), - sa.Column("name", sa.String(length=200), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("expected_outputs", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("assertions", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column("id", sa.UUID(), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), - sa.ForeignKeyConstraint( - ["graph_id"], ["graphs.id"], name=op.f("fk_graph_test_cases_graph_id_graphs"), ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_graph_test_cases")), - ) diff --git a/backend/alembic/versions/20260406_000000_drop_copilot_chats_table.py b/backend/alembic/versions/20260406_000000_drop_copilot_chats_table.py deleted file mode 100644 index 152f508f5..000000000 --- a/backend/alembic/versions/20260406_000000_drop_copilot_chats_table.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Drop copilot_chats table. - -Copilot conversation history is now stored in agent_runs / agent_run_events / -agent_run_snapshots. The copilot_chats table, its model, and its repository are -no longer used. - -Revision ID: 4a6b5e9517ae -Revises: 87be2dfd4240 -Create Date: 2026-04-06 00:00:00.000000 -""" - -from typing import Sequence, Union - -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "4a6b5e9517ae" -down_revision: Union[str, None] = "87be2dfd4240" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_index("copilot_chats_agent_graph_id_idx", table_name="copilot_chats") - op.drop_index("copilot_chats_user_id_idx", table_name="copilot_chats") - op.drop_index("copilot_chats_created_at_idx", table_name="copilot_chats") - op.drop_index("copilot_chats_updated_at_idx", table_name="copilot_chats") - op.drop_table("copilot_chats") - - -def downgrade() -> None: - op.create_table( - "copilot_chats", - sa.Column("user_id", sa.String(length=255), nullable=False), - sa.Column("agent_graph_id", sa.UUID(), nullable=False), - sa.Column("title", sa.String(length=255), nullable=True), - sa.Column( - "messages", - postgresql.JSONB(astext_type=sa.Text()), - nullable=False, - server_default=sa.text("'[]'::jsonb"), - ), - sa.Column( - "model", - sa.String(length=100), - nullable=False, - server_default=sa.text("'claude-3-7-sonnet-latest'"), - ), - sa.Column("conversation_id", sa.String(length=255), nullable=True), - sa.Column("preview_yaml", sa.Text(), nullable=True), - sa.Column("plan_artifact", sa.Text(), nullable=True), - sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column("id", sa.UUID(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["user_id"], - ["user.id"], - name=op.f("fk_copilot_chats_user_id_user"), - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_copilot_chats")), - ) - op.create_index("copilot_chats_user_id_idx", "copilot_chats", ["user_id"]) - op.create_index("copilot_chats_agent_graph_id_idx", "copilot_chats", ["agent_graph_id"]) - op.create_index("copilot_chats_created_at_idx", "copilot_chats", ["created_at"]) - op.create_index("copilot_chats_updated_at_idx", "copilot_chats", ["updated_at"]) diff --git a/backend/alembic/versions/20260414_000000_unify_model_id_fields_in_node_config.py b/backend/alembic/versions/20260414_000000_unify_model_id_fields_in_node_config.py deleted file mode 100644 index 36be1e6eb..000000000 --- a/backend/alembic/versions/20260414_000000_unify_model_id_fields_in_node_config.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Unify model identifier fields in graph_nodes.data.config. - -Migrate legacy model fields to the canonical (provider_name, model_name) pair: - - "model" (combined "provider:model") → split into provider_name + model_name - - "provider" → provider_name - - "memoryModel" (combined) → split into memory_provider_name + memory_model_name - - "memoryProvider" → memory_provider_name - -After this migration, frontend/backend compat fallbacks for the old field names -can be removed. - -Revision ID: 0f7082711f20 -Revises: 4a6b5e9517ae -Create Date: 2026-04-14 00:00:00.000000 -""" - -from typing import Sequence, Union - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "0f7082711f20" -down_revision: Union[str, None] = "4a6b5e9517ae" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # 1. Migrate main model fields: split combined "model" into provider_name + model_name, - # and copy standalone "provider" to provider_name. - # Single UPDATE with CASE to avoid multiple table scans. - op.execute(""" - UPDATE graph_nodes - SET data = jsonb_set( - jsonb_set( - data, - '{config,provider_name}', - to_jsonb(COALESCE( - NULLIF(data->'config'->>'provider_name', ''), - NULLIF(data->'config'->>'provider', ''), - CASE - WHEN data->'config'->>'model' LIKE '%:%' - THEN split_part(data->'config'->>'model', ':', 1) - ELSE NULL - END, - '' - )) - ), - '{config,model_name}', - to_jsonb(COALESCE( - NULLIF(data->'config'->>'model_name', ''), - CASE - WHEN data->'config'->>'model' LIKE '%:%' - THEN substring(data->'config'->>'model' from position(':' in data->'config'->>'model') + 1) - WHEN data->'config'->>'model' IS NOT NULL AND data->'config'->>'model' != '' - THEN data->'config'->>'model' - ELSE NULL - END, - '' - )) - ) - WHERE data ? 'config' - AND ( - data->'config' ? 'model' - OR data->'config' ? 'provider' - ) - """) - - # 2. Migrate memory model fields: split combined "memoryModel" into - # memory_provider_name + memory_model_name, copy "memoryProvider". - op.execute(""" - UPDATE graph_nodes - SET data = jsonb_set( - jsonb_set( - data, - '{config,memory_provider_name}', - to_jsonb(COALESCE( - NULLIF(data->'config'->>'memory_provider_name', ''), - NULLIF(data->'config'->>'memoryProvider', ''), - CASE - WHEN data->'config'->>'memoryModel' LIKE '%:%' - THEN split_part(data->'config'->>'memoryModel', ':', 1) - ELSE NULL - END, - '' - )) - ), - '{config,memory_model_name}', - to_jsonb(COALESCE( - NULLIF(data->'config'->>'memory_model_name', ''), - CASE - WHEN data->'config'->>'memoryModel' LIKE '%:%' - THEN substring(data->'config'->>'memoryModel' from position(':' in data->'config'->>'memoryModel') + 1) - WHEN data->'config'->>'memoryModel' IS NOT NULL AND data->'config'->>'memoryModel' != '' - THEN data->'config'->>'memoryModel' - ELSE NULL - END, - '' - )) - ) - WHERE data ? 'config' - AND ( - data->'config' ? 'memoryModel' - OR data->'config' ? 'memoryProvider' - ) - """) - - # 3. Remove legacy fields from config - op.execute(""" - UPDATE graph_nodes - SET data = data #- '{config,model}' #- '{config,provider}' #- '{config,memoryModel}' #- '{config,memoryProvider}' - WHERE data ? 'config' - AND ( - data->'config' ? 'model' - OR data->'config' ? 'provider' - OR data->'config' ? 'memoryModel' - OR data->'config' ? 'memoryProvider' - ) - """) - - -def downgrade() -> None: - # Reconstruct combined "model" and "provider" from provider_name + model_name - op.execute(""" - UPDATE graph_nodes - SET data = jsonb_set( - jsonb_set( - data, - '{config,model}', - to_jsonb( - CASE - WHEN data->'config'->>'provider_name' IS NOT NULL - AND data->'config'->>'provider_name' != '' - THEN data->'config'->>'provider_name' || ':' || COALESCE(data->'config'->>'model_name', '') - ELSE COALESCE(data->'config'->>'model_name', '') - END - ) - ), - '{config,provider}', - to_jsonb(COALESCE(data->'config'->>'provider_name', '')) - ) - WHERE data ? 'config' - AND data->'config'->>'model_name' IS NOT NULL - AND data->'config'->>'model_name' != '' - """) - - # Reconstruct "memoryModel" and "memoryProvider" from memory_provider_name + memory_model_name - op.execute(""" - UPDATE graph_nodes - SET data = jsonb_set( - jsonb_set( - data, - '{config,memoryModel}', - to_jsonb( - CASE - WHEN data->'config'->>'memory_provider_name' IS NOT NULL - AND data->'config'->>'memory_provider_name' != '' - THEN data->'config'->>'memory_provider_name' || ':' || COALESCE(data->'config'->>'memory_model_name', '') - ELSE COALESCE(data->'config'->>'memory_model_name', '') - END - ) - ), - '{config,memoryProvider}', - to_jsonb(COALESCE(data->'config'->>'memory_provider_name', '')) - ) - WHERE data ? 'config' - AND data->'config'->>'memory_model_name' IS NOT NULL - AND data->'config'->>'memory_model_name' != '' - """) - - # Remove new fields added by upgrade - op.execute(""" - UPDATE graph_nodes - SET data = data #- '{config,provider_name}' #- '{config,model_name}' #- '{config,memory_provider_name}' #- '{config,memory_model_name}' - WHERE data ? 'config' - AND ( - data->'config' ? 'provider_name' - OR data->'config' ? 'model_name' - OR data->'config' ? 'memory_provider_name' - OR data->'config' ? 'memory_model_name' - ) - """) diff --git a/backend/alembic/versions/20260627_000001_init_joysafeter_schema.py b/backend/alembic/versions/20260627_000001_init_joysafeter_schema.py new file mode 100644 index 000000000..066f1c989 --- /dev/null +++ b/backend/alembic/versions/20260627_000001_init_joysafeter_schema.py @@ -0,0 +1,1219 @@ +"""init JoySafeter schema (squashed) + +Single squashed initial migration for fresh deployments. Replaces the prior +14-migration chain (initial v1 schema + skill feature additions + four +``drop_v1_dead_tables`` passes + data backfills) with one explicit-DDL step +that builds the final 33-table managed-agent schema. + +Generated via ``alembic revision --autogenerate`` against an empty database so +every table, column, server_default, conditional index and constraint matches +the current ORM models. Historical data-backfill steps are intentionally +dropped (a fresh database has no rows to migrate). + +Revision ID: 20260627_000001 +Revises: +Create Date: 2026-06-27 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260627_000001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "joysafeter_organizations", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=255), nullable=False), + sa.Column("logo", sa.String(length=500), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("org_usage_limit", sa.Numeric(), nullable=True), + sa.Column("storage_used_bytes", sa.BigInteger(), nullable=False), + sa.Column("departed_member_usage", sa.Numeric(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_organizations")), + ) + op.create_table( + "joysafeter_security_audit_logs", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.String(length=255), nullable=True), + sa.Column("user_email", sa.String(length=255), nullable=True), + sa.Column( + "event_type", + sa.String(length=50), + nullable=False, + comment="event type: login, logout, password_change, password_reset, 2fa_enable, account_lock, etc.", + ), + sa.Column( + "event_status", sa.String(length=20), nullable=False, comment="event status: success, failure, blocked" + ), + sa.Column("ip_address", sa.String(length=255), nullable=False), + sa.Column("user_agent", sa.String(length=1024), nullable=True), + sa.Column("device_fingerprint", sa.String(length=255), nullable=True), + sa.Column("location", sa.String(length=255), nullable=True), + sa.Column("country", sa.String(length=10), nullable=True), + sa.Column( + "details", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="extra info such as error reason, target entity, etc.", + ), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_security_audit_logs")), + ) + op.create_index( + "ix_joysafeter_security_audit_logs_created_at", "joysafeter_security_audit_logs", ["created_at"], unique=False + ) + op.create_index( + "ix_joysafeter_security_audit_logs_event_status", + "joysafeter_security_audit_logs", + ["event_status"], + unique=False, + ) + op.create_index( + "ix_joysafeter_security_audit_logs_event_type", "joysafeter_security_audit_logs", ["event_type"], unique=False + ) + op.create_index( + "ix_joysafeter_security_audit_logs_user_event", + "joysafeter_security_audit_logs", + ["user_id", "event_type"], + unique=False, + ) + op.create_index( + "ix_joysafeter_security_audit_logs_user_id", "joysafeter_security_audit_logs", ["user_id"], unique=False + ) + op.create_table( + "joysafeter_users", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("email_verified", sa.Boolean(), nullable=False), + sa.Column("hashed_password", sa.String(length=255), nullable=True), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column("password_reset_token", sa.String(length=255), nullable=True), + sa.Column("password_reset_expires", sa.DateTime(timezone=True), nullable=True), + sa.Column("email_verify_token", sa.String(length=255), nullable=True), + sa.Column("email_verify_expires", sa.DateTime(timezone=True), nullable=True), + sa.Column("image", sa.String(length=1024), nullable=True), + sa.Column("stripe_customer_id", sa.String(length=255), nullable=True), + sa.Column("is_super_user", sa.Boolean(), nullable=False), + sa.Column("failed_login_attempts", sa.Integer(), nullable=False), + sa.Column("locked_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("lock_reason", sa.String(length=255), nullable=True), + sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_login_ip", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_users")), + ) + op.create_index(op.f("ix_joysafeter_users_email"), "joysafeter_users", ["email"], unique=True) + op.create_table( + "joysafeter_auth_sessions", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("token", sa.String(length=255), nullable=False), + sa.Column("ip_address", sa.String(length=255), nullable=True), + sa.Column("user_agent", sa.String(length=1024), nullable=True), + sa.Column("user_id", sa.String(length=255), nullable=False), + sa.Column("active_organization_id", sa.String(length=255), nullable=True), + sa.Column("last_activity_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("device_fingerprint", sa.String(length=255), nullable=True), + sa.Column("device_name", sa.String(length=255), nullable=True), + sa.Column("is_trusted", sa.Boolean(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["active_organization_id"], + ["joysafeter_organizations.id"], + name=op.f("fk_joysafeter_auth_sessions_active_organization_id_joysafeter_organizations"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_auth_sessions_user_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_auth_sessions")), + sa.UniqueConstraint("token", name=op.f("uq_joysafeter_auth_sessions_token")), + ) + op.create_index("ix_joysafeter_auth_sessions_token", "joysafeter_auth_sessions", ["token"], unique=True) + op.create_index("ix_joysafeter_auth_sessions_user_id", "joysafeter_auth_sessions", ["user_id"], unique=False) + op.create_table( + "joysafeter_oauth_account", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("user_id", sa.String(length=255), nullable=False), + sa.Column("provider", sa.String(length=50), nullable=False), + sa.Column("provider_account_id", sa.String(length=255), nullable=False), + sa.Column("email", sa.String(length=255), nullable=True), + sa.Column("access_token", sa.Text(), nullable=True), + sa.Column("refresh_token", sa.Text(), nullable=True), + sa.Column("token_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("raw_userinfo", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_oauth_account_user_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_oauth_account")), + ) + op.create_index( + "ix_joysafeter_oauth_account_provider_account", + "joysafeter_oauth_account", + ["provider", "provider_account_id"], + unique=True, + ) + op.create_index("ix_joysafeter_oauth_account_user_id", "joysafeter_oauth_account", ["user_id"], unique=False) + op.create_table( + "joysafeter_organization_members", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("user_id", sa.String(length=255), nullable=False), + sa.Column("organization_id", sa.String(length=255), nullable=False), + sa.Column("role", sa.String(length=50), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["organization_id"], + ["joysafeter_organizations.id"], + name=op.f("fk_joysafeter_organization_members_organization_id_joysafeter_organizations"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_organization_members_user_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_organization_members")), + ) + op.create_index( + "ix_joysafeter_organization_members_organization_id", + "joysafeter_organization_members", + ["organization_id"], + unique=False, + ) + op.create_index( + "ix_joysafeter_organization_members_user_id", "joysafeter_organization_members", ["user_id"], unique=False + ) + op.create_table( + "joysafeter_organization_projects", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("org_id", sa.String(length=255), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=255), nullable=False), + sa.Column("is_default", sa.Boolean(), nullable=False), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["org_id"], + ["joysafeter_organizations.id"], + name=op.f("fk_joysafeter_organization_projects_org_id_joysafeter_organizations"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_organization_projects")), + sa.UniqueConstraint("org_id", "slug", name="uq_joysafeter_organization_projects_org_slug"), + ) + op.create_index( + "ix_joysafeter_organization_projects_org_id", "joysafeter_organization_projects", ["org_id"], unique=False + ) + op.create_table( + "joysafeter_agents", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("engine_kind", sa.Text(), nullable=False), + sa.Column("model", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("system_prompt", sa.Text(), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("env", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("mcp_configs", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("skills", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("tools", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("agents", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("commands", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("permission_mode", sa.Text(), nullable=False), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("multiagent", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("version", sa.Integer(), nullable=False), + sa.Column("environment_ref", sa.Text(), nullable=True), + sa.Column("secret_ref", sa.Text(), nullable=True), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_agents_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_agents")), + sa.UniqueConstraint("name", name="idx_ca_name_unique"), + ) + op.create_index("idx_ca_created_at", "joysafeter_agents", ["created_at"], unique=False) + op.create_index("idx_ca_project", "joysafeter_agents", ["project_id"], unique=False) + op.create_index(op.f("ix_joysafeter_agents_project_id"), "joysafeter_agents", ["project_id"], unique=False) + op.create_table( + "joysafeter_api_keys", + sa.Column("project_id", sa.String(length=255), nullable=False), + sa.Column("org_id", sa.String(length=255), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("key_hash", sa.Text(), nullable=False), + sa.Column("key_prefix", sa.Text(), nullable=False), + sa.Column("created_by", sa.String(length=255), nullable=False), + sa.Column("role", sa.Text(), nullable=False), + sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["created_by"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_api_keys_created_by_joysafeter_users"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["org_id"], + ["joysafeter_organizations.id"], + name=op.f("fk_joysafeter_api_keys_org_id_joysafeter_organizations"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_api_keys_project_id_joysafeter_organization_projects"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_api_keys")), + ) + op.create_index("idx_cak_key_hash", "joysafeter_api_keys", ["key_hash"], unique=False) + op.create_index("idx_cak_org", "joysafeter_api_keys", ["org_id"], unique=False) + op.create_index("idx_cak_project", "joysafeter_api_keys", ["project_id"], unique=False) + op.create_table( + "joysafeter_environments", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("image_tag", sa.Text(), nullable=True), + sa.Column("image_version", sa.Integer(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_environments_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_environments")), + sa.UniqueConstraint("name", name=op.f("uq_joysafeter_environments_name")), + ) + op.create_index( + op.f("ix_joysafeter_environments_project_id"), "joysafeter_environments", ["project_id"], unique=False + ) + op.create_table( + "joysafeter_memory_stores", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_memory_stores_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_memory_stores")), + ) + op.create_index( + op.f("ix_joysafeter_memory_stores_project_id"), "joysafeter_memory_stores", ["project_id"], unique=False + ) + op.create_table( + "joysafeter_project_members", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("project_id", sa.String(length=255), nullable=False), + sa.Column("user_id", sa.String(length=255), nullable=False), + sa.Column("role", sa.String(length=50), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_project_members_project_id_joysafeter_organization_projects"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_project_members_user_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_project_members")), + sa.UniqueConstraint("project_id", "user_id", name="uq_joysafeter_project_members_project_user"), + ) + op.create_index( + "ix_joysafeter_project_members_project_id", "joysafeter_project_members", ["project_id"], unique=False + ) + op.create_index("ix_joysafeter_project_members_user_id", "joysafeter_project_members", ["user_id"], unique=False) + op.create_table( + "joysafeter_sandboxes", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("external_id", sa.Text(), nullable=False), + sa.Column("provider", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("chat_session_id", sa.UUID(), nullable=True), + sa.Column("image", sa.Text(), nullable=False), + sa.Column("last_task_id", sa.UUID(), nullable=True), + sa.Column("last_used_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("idle_since", sa.DateTime(timezone=True), nullable=True), + sa.Column("disconnected_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("destroyed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("workspace_path", sa.Text(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_sandboxes_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_sandboxes")), + ) + op.create_index( + "idx_csb_active_session_unique", + "joysafeter_sandboxes", + ["chat_session_id"], + unique=True, + postgresql_where="chat_session_id IS NOT NULL AND destroyed_at IS NULL AND status IN ('creating', 'provisioning', 'idle', 'running', 'stopped', 'error')", + ) + op.create_index( + "idx_csb_pool", "joysafeter_sandboxes", ["created_at"], unique=False, postgresql_where="status = 'pooled'" + ) + op.create_index("idx_csb_project", "joysafeter_sandboxes", ["project_id"], unique=False) + op.create_index( + "idx_csb_session", + "joysafeter_sandboxes", + ["chat_session_id"], + unique=False, + postgresql_where="chat_session_id IS NOT NULL", + ) + op.create_index("idx_csb_status", "joysafeter_sandboxes", ["status"], unique=False) + op.create_index(op.f("ix_joysafeter_sandboxes_project_id"), "joysafeter_sandboxes", ["project_id"], unique=False) + op.create_table( + "joysafeter_secrets", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("provider", sa.String(length=64), server_default="custom", nullable=False), + sa.Column("protocol", sa.String(length=64), server_default="custom", nullable=False), + sa.Column("data", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("is_default", sa.Boolean(), server_default="false", nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_secrets_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_secrets")), + sa.UniqueConstraint("name", name="idx_cs_name_unique"), + ) + op.create_index(op.f("ix_joysafeter_secrets_project_id"), "joysafeter_secrets", ["project_id"], unique=False) + op.create_table( + "joysafeter_skills", + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("description", sa.String(length=1024), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("tags", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("source_type", sa.String(length=50), nullable=False), + sa.Column("source_url", sa.String(length=1024), nullable=True), + sa.Column("root_path", sa.String(length=512), nullable=True), + sa.Column("owner_id", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=False), + sa.Column("is_public", sa.Boolean(), nullable=False), + sa.Column("visibility", sa.String(length=16), nullable=False), + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("license", sa.String(length=100), nullable=True), + sa.Column("compatibility", sa.String(length=500), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("allowed_tools", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("security_status", sa.String(length=32), nullable=False), + sa.Column("security_score", sa.Integer(), nullable=True), + sa.Column("security_severity", sa.String(length=32), nullable=True), + sa.Column("security_recommendation", sa.String(length=32), nullable=True), + sa.Column("security_scanned_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("security_scan_id", sa.UUID(), nullable=True), + sa.Column("security_scan_hash", sa.String(length=64), nullable=True), + sa.Column("security_issues_count", sa.Integer(), nullable=False), + sa.Column("security_critical_count", sa.Integer(), nullable=False), + sa.Column("security_high_count", sa.Integer(), nullable=False), + sa.Column("security_medium_count", sa.Integer(), nullable=False), + sa.Column("security_low_count", sa.Integer(), nullable=False), + sa.Column("lifecycle_status", sa.String(length=16), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["created_by_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skills_created_by_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["owner_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skills_owner_id_joysafeter_users"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_skills_project_id_joysafeter_organization_projects"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skills")), + sa.UniqueConstraint("owner_id", "name", name="skills_owner_name_unique"), + ) + op.create_index("skills_created_by_idx", "joysafeter_skills", ["created_by_id"], unique=False) + op.create_index("skills_lifecycle_status_idx", "joysafeter_skills", ["lifecycle_status"], unique=False) + op.create_index("skills_owner_idx", "joysafeter_skills", ["owner_id"], unique=False) + op.create_index("skills_project_idx", "joysafeter_skills", ["project_id"], unique=False) + op.create_index("skills_public_idx", "joysafeter_skills", ["is_public"], unique=False) + op.create_index( + "skills_security_recommendation_idx", "joysafeter_skills", ["security_recommendation"], unique=False + ) + op.create_index("skills_security_severity_idx", "joysafeter_skills", ["security_severity"], unique=False) + op.create_index("skills_security_status_idx", "joysafeter_skills", ["security_status"], unique=False) + op.create_index("skills_tags_idx", "joysafeter_skills", ["tags"], unique=False, postgresql_using="gin") + op.create_index("skills_visibility_idx", "joysafeter_skills", ["visibility"], unique=False) + op.create_table( + "joysafeter_vaults", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=False), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_vaults_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_vaults")), + sa.UniqueConstraint("name", name="idx_cv_name"), + ) + op.create_index("idx_cv_project", "joysafeter_vaults", ["project_id"], unique=False) + op.create_index(op.f("ix_joysafeter_vaults_project_id"), "joysafeter_vaults", ["project_id"], unique=False) + op.create_table( + "joysafeter_agent_versions", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("agent_id", sa.UUID(), nullable=False), + sa.Column("version", sa.Integer(), nullable=False), + sa.Column("snapshot", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["agent_id"], ["joysafeter_agents.id"], name=op.f("fk_joysafeter_agent_versions_agent_id_joysafeter_agents") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_agent_versions")), + sa.UniqueConstraint("agent_id", "version", name=op.f("uq_joysafeter_agent_versions_agent_id")), + ) + op.create_table( + "joysafeter_memories", + sa.Column("store_id", sa.UUID(), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("content_sha256", sa.Text(), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("version", sa.Integer(), nullable=False), + sa.Column("current_version_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["store_id"], + ["joysafeter_memory_stores.id"], + name=op.f("fk_joysafeter_memories_store_id_joysafeter_memory_stores"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_memories")), + sa.UniqueConstraint("store_id", "path", name=op.f("uq_joysafeter_memories_store_id")), + ) + op.create_table( + "joysafeter_memory_versions", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("store_id", sa.UUID(), nullable=False), + sa.Column("memory_id", sa.UUID(), nullable=False), + sa.Column("operation", sa.Text(), nullable=False), + sa.Column("path", sa.Text(), nullable=True), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("content_sha256", sa.Text(), nullable=True), + sa.Column("content_size_bytes", sa.Integer(), nullable=True), + sa.Column("session_id", sa.UUID(), nullable=True), + sa.Column("api_key_id", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("redacted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("redacted_by", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint( + ["store_id"], + ["joysafeter_memory_stores.id"], + name=op.f("fk_joysafeter_memory_versions_store_id_joysafeter_memory_stores"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_memory_versions")), + ) + op.create_table( + "joysafeter_sessions", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("agent_id", sa.UUID(), nullable=False), + sa.Column("title", sa.Text(), nullable=True), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("stop_reason", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column( + "usage", + postgresql.JSONB(astext_type=sa.Text()), + server_default='{"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}', + nullable=False, + ), + sa.Column("active_seconds", sa.Float(), nullable=True), + sa.Column("duration_seconds", sa.Float(), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("vault_ids", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("agent_version", sa.Integer(), nullable=True), + sa.Column("agent_snapshot", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("environment_ref", sa.Text(), nullable=True), + sa.Column("last_harness_session_id", sa.Text(), nullable=True), + sa.Column("last_work_dir", sa.Text(), nullable=True), + sa.Column("last_sandbox_id", sa.UUID(), nullable=True), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["agent_id"], ["joysafeter_agents.id"], name=op.f("fk_joysafeter_sessions_agent_id_joysafeter_agents") + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_sessions_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_sessions")), + ) + op.create_index("idx_csess_agent", "joysafeter_sessions", ["agent_id"], unique=False) + op.create_index("idx_csess_created", "joysafeter_sessions", ["created_at"], unique=False) + op.create_index("idx_csess_project", "joysafeter_sessions", ["project_id"], unique=False) + op.create_index(op.f("ix_joysafeter_sessions_project_id"), "joysafeter_sessions", ["project_id"], unique=False) + op.create_table( + "joysafeter_skill_collaborators", + sa.Column("skill_id", sa.UUID(), nullable=False), + sa.Column("user_id", sa.String(length=255), nullable=False), + sa.Column( + "role", + sa.Enum("viewer", "editor", "publisher", "admin", name="collaborator_role", create_constraint=True), + nullable=False, + ), + sa.Column("invited_by", sa.String(length=255), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["invited_by"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skill_collaborators_invited_by_joysafeter_users"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["skill_id"], + ["joysafeter_skills.id"], + name=op.f("fk_joysafeter_skill_collaborators_skill_id_joysafeter_skills"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skill_collaborators_user_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skill_collaborators")), + sa.UniqueConstraint("skill_id", "user_id", name="skill_collaborators_skill_user_unique"), + ) + op.create_index( + "skill_collaborators_user_skill_idx", "joysafeter_skill_collaborators", ["user_id", "skill_id"], unique=False + ) + op.create_table( + "joysafeter_skill_files", + sa.Column("skill_id", sa.UUID(), nullable=False), + sa.Column("path", sa.String(length=512), nullable=False), + sa.Column("file_name", sa.String(length=255), nullable=False), + sa.Column("file_type", sa.String(length=50), nullable=False), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("storage_type", sa.String(length=20), nullable=False), + sa.Column("storage_key", sa.String(length=512), nullable=True), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["skill_id"], + ["joysafeter_skills.id"], + name=op.f("fk_joysafeter_skill_files_skill_id_joysafeter_skills"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skill_files")), + ) + op.create_index("skill_files_path_idx", "joysafeter_skill_files", ["skill_id", "path"], unique=False) + op.create_index("skill_files_skill_idx", "joysafeter_skill_files", ["skill_id"], unique=False) + op.create_table( + "joysafeter_skill_security_scans", + sa.Column("skill_id", sa.UUID(), nullable=True), + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("owner_id", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=False), + sa.Column("trigger", sa.String(length=32), nullable=False), + sa.Column("target_name", sa.String(length=128), nullable=True), + sa.Column("target_hash", sa.String(length=64), nullable=False), + sa.Column("scanner", sa.String(length=64), nullable=False), + sa.Column("scanner_version", sa.String(length=64), nullable=True), + sa.Column("ruleset_version", sa.String(length=64), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("score", sa.Integer(), nullable=True), + sa.Column("severity", sa.String(length=32), nullable=True), + sa.Column("recommendation", sa.String(length=32), nullable=True), + sa.Column("issues_count", sa.Integer(), nullable=False), + sa.Column("critical_count", sa.Integer(), nullable=False), + sa.Column("high_count", sa.Integer(), nullable=False), + sa.Column("medium_count", sa.Integer(), nullable=False), + sa.Column("low_count", sa.Integer(), nullable=False), + sa.Column("report", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["created_by_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skill_security_scans_created_by_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["owner_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skill_security_scans_owner_id_joysafeter_users"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_skill_security_scans_project_id_joysafeter_organization_projects"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["skill_id"], + ["joysafeter_skills.id"], + name=op.f("fk_joysafeter_skill_security_scans_skill_id_joysafeter_skills"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skill_security_scans")), + ) + op.create_index( + "skill_security_scans_owner_created_idx", + "joysafeter_skill_security_scans", + ["owner_id", "created_at"], + unique=False, + ) + op.create_index( + "skill_security_scans_project_created_idx", + "joysafeter_skill_security_scans", + ["project_id", "created_at"], + unique=False, + ) + op.create_index( + "skill_security_scans_recommendation_created_idx", + "joysafeter_skill_security_scans", + ["recommendation", "created_at"], + unique=False, + ) + op.create_index( + "skill_security_scans_severity_created_idx", + "joysafeter_skill_security_scans", + ["severity", "created_at"], + unique=False, + ) + op.create_index( + "skill_security_scans_skill_created_idx", + "joysafeter_skill_security_scans", + ["skill_id", "created_at"], + unique=False, + ) + op.create_index( + "skill_security_scans_status_created_idx", + "joysafeter_skill_security_scans", + ["status", "created_at"], + unique=False, + ) + op.create_index( + "skill_security_scans_target_hash_idx", "joysafeter_skill_security_scans", ["target_hash"], unique=False + ) + op.create_table( + "joysafeter_skill_usage_log", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("skill_id", sa.UUID(), nullable=True), + sa.Column("skill_version", sa.String(length=64), nullable=True), + sa.Column("session_id", sa.String(length=255), nullable=True), + sa.Column("agent_id", sa.String(length=255), nullable=True), + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("user_id", sa.String(length=255), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["skill_id"], + ["joysafeter_skills.id"], + name=op.f("fk_joysafeter_skill_usage_log_skill_id_joysafeter_skills"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skill_usage_log")), + ) + op.create_index( + "skill_usage_log_project_created_idx", "joysafeter_skill_usage_log", ["project_id", "created_at"], unique=False + ) + op.create_index( + "skill_usage_log_session_created_idx", "joysafeter_skill_usage_log", ["session_id", "created_at"], unique=False + ) + op.create_index( + "skill_usage_log_skill_created_idx", "joysafeter_skill_usage_log", ["skill_id", "created_at"], unique=False + ) + op.create_table( + "joysafeter_vault_credentials", + sa.Column("vault_id", sa.UUID(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("credential_type", sa.Text(), nullable=False), + sa.Column("mcp_server_url", sa.Text(), nullable=False), + sa.Column("token_value", sa.Text(), nullable=False), + sa.Column("oauth_config", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["vault_id"], + ["joysafeter_vaults.id"], + name=op.f("fk_joysafeter_vault_credentials_vault_id_joysafeter_vaults"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_vault_credentials")), + sa.UniqueConstraint("vault_id", "mcp_server_url", name="idx_cvc_url"), + ) + op.create_table( + "joysafeter_files", + sa.Column("project_id", sa.String(length=255), nullable=False), + sa.Column("filename", sa.String(length=512), nullable=False), + sa.Column("purpose", sa.String(length=50), nullable=False), + sa.Column("content_type", sa.String(length=255), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=False), + sa.Column("storage_key", sa.Text(), nullable=False), + sa.Column("downloadable", sa.Boolean(), nullable=False), + sa.Column("session_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_files_project_id_joysafeter_organization_projects"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["session_id"], + ["joysafeter_sessions.id"], + name=op.f("fk_joysafeter_files_session_id_joysafeter_sessions"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_files")), + ) + op.create_index( + "idx_joysafeter_files_project_created", "joysafeter_files", ["project_id", "created_at"], unique=False + ) + op.create_index("idx_joysafeter_files_session", "joysafeter_files", ["session_id"], unique=False) + op.create_index(op.f("ix_joysafeter_files_project_id"), "joysafeter_files", ["project_id"], unique=False) + op.create_table( + "joysafeter_session_events", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("session_id", sa.UUID(), nullable=False), + sa.Column("event_type", sa.Text(), nullable=False), + sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("seq", sa.BigInteger(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("processed_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["session_id"], + ["joysafeter_sessions.id"], + name=op.f("fk_joysafeter_session_events_session_id_joysafeter_sessions"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_session_events")), + sa.UniqueConstraint("session_id", "seq", name=op.f("uq_joysafeter_session_events_session_id")), + ) + op.create_index("idx_cse_created_at", "joysafeter_session_events", ["created_at"], unique=False) + op.create_index("idx_cse_event_created", "joysafeter_session_events", ["event_type", "created_at"], unique=False) + op.create_index( + "idx_cse_session_event_seq", "joysafeter_session_events", ["session_id", "event_type", "seq"], unique=False + ) + op.create_index( + "idx_cse_session_processed_event", + "joysafeter_session_events", + ["session_id", "processed_at", "event_type"], + unique=False, + ) + op.create_index("idx_cse_session_seq", "joysafeter_session_events", ["session_id", "seq"], unique=False) + op.create_table( + "joysafeter_session_memory_stores", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("session_id", sa.UUID(), nullable=False), + sa.Column("store_id", sa.UUID(), nullable=False), + sa.Column("access", sa.Text(), nullable=False), + sa.Column("instructions", sa.Text(), nullable=True), + sa.Column("mount_name", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["session_id"], + ["joysafeter_sessions.id"], + name=op.f("fk_joysafeter_session_memory_stores_session_id_joysafeter_sessions"), + ), + sa.ForeignKeyConstraint( + ["store_id"], + ["joysafeter_memory_stores.id"], + name=op.f("fk_joysafeter_session_memory_stores_store_id_joysafeter_memory_stores"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_session_memory_stores")), + sa.UniqueConstraint("session_id", "store_id", name=op.f("uq_joysafeter_session_memory_stores_session_id")), + ) + op.create_table( + "joysafeter_session_repos", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("session_id", sa.UUID(), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("branch", sa.String(length=255), nullable=False), + sa.Column("mount_path", sa.Text(), nullable=False), + sa.Column("mount_name", sa.String(length=255), nullable=False), + sa.Column("encrypted_token", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["session_id"], + ["joysafeter_sessions.id"], + name=op.f("fk_joysafeter_session_repos_session_id_joysafeter_sessions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_session_repos")), + ) + op.create_index("idx_session_repos_session", "joysafeter_session_repos", ["session_id"], unique=False) + op.create_table( + "joysafeter_skill_versions", + sa.Column("skill_id", sa.UUID(), nullable=False), + sa.Column("version", sa.String(length=20), nullable=False), + sa.Column("release_notes", sa.Text(), nullable=True), + sa.Column("skill_name", sa.String(length=64), nullable=False), + sa.Column("skill_description", sa.String(length=1024), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("tags", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("allowed_tools", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("compatibility", sa.String(length=500), nullable=True), + sa.Column("license", sa.String(length=100), nullable=True), + sa.Column("published_by_id", sa.String(length=255), nullable=False), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("security_scan_id", sa.UUID(), nullable=True), + sa.Column("target_hash", sa.String(length=64), nullable=True), + sa.Column("lifecycle_status", sa.String(length=16), nullable=False), + sa.Column("approved_by_id", sa.String(length=255), nullable=True), + sa.Column("approved_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["approved_by_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skill_versions_approved_by_id_joysafeter_users"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["published_by_id"], + ["joysafeter_users.id"], + name=op.f("fk_joysafeter_skill_versions_published_by_id_joysafeter_users"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["security_scan_id"], + ["joysafeter_skill_security_scans.id"], + name=op.f("fk_joysafeter_skill_versions_security_scan_id_joysafeter_skill_security_scans"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["skill_id"], + ["joysafeter_skills.id"], + name=op.f("fk_joysafeter_skill_versions_skill_id_joysafeter_skills"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skill_versions")), + sa.UniqueConstraint("skill_id", "version", name="skill_versions_skill_version_unique"), + ) + op.create_index( + "skill_versions_lifecycle_status_idx", "joysafeter_skill_versions", ["lifecycle_status"], unique=False + ) + op.create_index("skill_versions_published_at_idx", "joysafeter_skill_versions", ["published_at"], unique=False) + op.create_index("skill_versions_security_scan_idx", "joysafeter_skill_versions", ["security_scan_id"], unique=False) + op.create_index("skill_versions_skill_idx", "joysafeter_skill_versions", ["skill_id"], unique=False) + op.create_table( + "joysafeter_tasks", + sa.Column("project_id", sa.String(length=255), nullable=True), + sa.Column("agent_id", sa.UUID(), nullable=False), + sa.Column("chat_session_id", sa.UUID(), nullable=True), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("prompt", sa.Text(), nullable=False), + sa.Column("system_prompt", sa.Text(), nullable=True), + sa.Column("sandbox_id", sa.UUID(), nullable=True), + sa.Column("output", sa.Text(), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("usage", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("timeout_sec", sa.Integer(), nullable=False), + sa.Column("retry_count", sa.Integer(), nullable=False), + sa.Column("max_retries", sa.Integer(), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("duration_ms", sa.BigInteger(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["agent_id"], ["joysafeter_agents.id"], name=op.f("fk_joysafeter_tasks_agent_id_joysafeter_agents") + ), + sa.ForeignKeyConstraint( + ["chat_session_id"], + ["joysafeter_sessions.id"], + name=op.f("fk_joysafeter_tasks_chat_session_id_joysafeter_sessions"), + ), + sa.ForeignKeyConstraint( + ["project_id"], + ["joysafeter_organization_projects.id"], + name=op.f("fk_joysafeter_tasks_project_id_joysafeter_organization_projects"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_tasks")), + ) + op.create_index("idx_ct_agent", "joysafeter_tasks", ["agent_id"], unique=False) + op.create_index("idx_ct_created", "joysafeter_tasks", ["created_at"], unique=False) + op.create_index("idx_ct_project", "joysafeter_tasks", ["project_id"], unique=False) + op.create_index( + "idx_ct_project_status_created", "joysafeter_tasks", ["project_id", "status", "created_at"], unique=False + ) + op.create_index("idx_ct_sandbox_status", "joysafeter_tasks", ["sandbox_id", "status"], unique=False) + op.create_index( + "idx_ct_sandbox_status_created", "joysafeter_tasks", ["sandbox_id", "status", "created_at"], unique=False + ) + op.create_index("idx_ct_status", "joysafeter_tasks", ["status"], unique=False) + op.create_index("idx_ct_status_created", "joysafeter_tasks", ["status", "created_at"], unique=False) + op.create_index("idx_ct_status_updated", "joysafeter_tasks", ["status", "updated_at"], unique=False) + op.create_index(op.f("ix_joysafeter_tasks_project_id"), "joysafeter_tasks", ["project_id"], unique=False) + op.create_table( + "joysafeter_session_files", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("session_id", sa.UUID(), nullable=False), + sa.Column("file_id", sa.UUID(), nullable=False), + sa.Column("mount_path", sa.Text(), nullable=False), + sa.Column("access", sa.String(length=20), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["file_id"], + ["joysafeter_files.id"], + name=op.f("fk_joysafeter_session_files_file_id_joysafeter_files"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["session_id"], + ["joysafeter_sessions.id"], + name=op.f("fk_joysafeter_session_files_session_id_joysafeter_sessions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_session_files")), + ) + op.create_index("idx_session_files_file", "joysafeter_session_files", ["file_id"], unique=False) + op.create_index("idx_session_files_session", "joysafeter_session_files", ["session_id"], unique=False) + op.create_table( + "joysafeter_skill_version_files", + sa.Column("version_id", sa.UUID(), nullable=False), + sa.Column("path", sa.String(length=512), nullable=False), + sa.Column("file_name", sa.String(length=255), nullable=False), + sa.Column("file_type", sa.String(length=50), nullable=False), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("storage_type", sa.String(length=20), nullable=False), + sa.Column("storage_key", sa.String(length=512), nullable=True), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["version_id"], + ["joysafeter_skill_versions.id"], + name=op.f("fk_joysafeter_skill_version_files_version_id_joysafeter_skill_versions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_joysafeter_skill_version_files")), + ) + op.create_index("skill_version_files_version_idx", "joysafeter_skill_version_files", ["version_id"], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("skill_version_files_version_idx", table_name="joysafeter_skill_version_files") + op.drop_table("joysafeter_skill_version_files") + op.drop_index("idx_session_files_session", table_name="joysafeter_session_files") + op.drop_index("idx_session_files_file", table_name="joysafeter_session_files") + op.drop_table("joysafeter_session_files") + op.drop_index(op.f("ix_joysafeter_tasks_project_id"), table_name="joysafeter_tasks") + op.drop_index("idx_ct_status_updated", table_name="joysafeter_tasks") + op.drop_index("idx_ct_status_created", table_name="joysafeter_tasks") + op.drop_index("idx_ct_status", table_name="joysafeter_tasks") + op.drop_index("idx_ct_sandbox_status_created", table_name="joysafeter_tasks") + op.drop_index("idx_ct_sandbox_status", table_name="joysafeter_tasks") + op.drop_index("idx_ct_project_status_created", table_name="joysafeter_tasks") + op.drop_index("idx_ct_project", table_name="joysafeter_tasks") + op.drop_index("idx_ct_created", table_name="joysafeter_tasks") + op.drop_index("idx_ct_agent", table_name="joysafeter_tasks") + op.drop_table("joysafeter_tasks") + op.drop_index("skill_versions_skill_idx", table_name="joysafeter_skill_versions") + op.drop_index("skill_versions_security_scan_idx", table_name="joysafeter_skill_versions") + op.drop_index("skill_versions_published_at_idx", table_name="joysafeter_skill_versions") + op.drop_index("skill_versions_lifecycle_status_idx", table_name="joysafeter_skill_versions") + op.drop_table("joysafeter_skill_versions") + op.drop_index("idx_session_repos_session", table_name="joysafeter_session_repos") + op.drop_table("joysafeter_session_repos") + op.drop_table("joysafeter_session_memory_stores") + op.drop_index("idx_cse_session_seq", table_name="joysafeter_session_events") + op.drop_index("idx_cse_session_processed_event", table_name="joysafeter_session_events") + op.drop_index("idx_cse_session_event_seq", table_name="joysafeter_session_events") + op.drop_index("idx_cse_event_created", table_name="joysafeter_session_events") + op.drop_index("idx_cse_created_at", table_name="joysafeter_session_events") + op.drop_table("joysafeter_session_events") + op.drop_index(op.f("ix_joysafeter_files_project_id"), table_name="joysafeter_files") + op.drop_index("idx_joysafeter_files_session", table_name="joysafeter_files") + op.drop_index("idx_joysafeter_files_project_created", table_name="joysafeter_files") + op.drop_table("joysafeter_files") + op.drop_table("joysafeter_vault_credentials") + op.drop_index("skill_usage_log_skill_created_idx", table_name="joysafeter_skill_usage_log") + op.drop_index("skill_usage_log_session_created_idx", table_name="joysafeter_skill_usage_log") + op.drop_index("skill_usage_log_project_created_idx", table_name="joysafeter_skill_usage_log") + op.drop_table("joysafeter_skill_usage_log") + op.drop_index("skill_security_scans_target_hash_idx", table_name="joysafeter_skill_security_scans") + op.drop_index("skill_security_scans_status_created_idx", table_name="joysafeter_skill_security_scans") + op.drop_index("skill_security_scans_skill_created_idx", table_name="joysafeter_skill_security_scans") + op.drop_index("skill_security_scans_severity_created_idx", table_name="joysafeter_skill_security_scans") + op.drop_index("skill_security_scans_recommendation_created_idx", table_name="joysafeter_skill_security_scans") + op.drop_index("skill_security_scans_project_created_idx", table_name="joysafeter_skill_security_scans") + op.drop_index("skill_security_scans_owner_created_idx", table_name="joysafeter_skill_security_scans") + op.drop_table("joysafeter_skill_security_scans") + op.drop_index("skill_files_skill_idx", table_name="joysafeter_skill_files") + op.drop_index("skill_files_path_idx", table_name="joysafeter_skill_files") + op.drop_table("joysafeter_skill_files") + op.drop_index("skill_collaborators_user_skill_idx", table_name="joysafeter_skill_collaborators") + op.drop_table("joysafeter_skill_collaborators") + op.drop_index(op.f("ix_joysafeter_sessions_project_id"), table_name="joysafeter_sessions") + op.drop_index("idx_csess_project", table_name="joysafeter_sessions") + op.drop_index("idx_csess_created", table_name="joysafeter_sessions") + op.drop_index("idx_csess_agent", table_name="joysafeter_sessions") + op.drop_table("joysafeter_sessions") + op.drop_table("joysafeter_memory_versions") + op.drop_table("joysafeter_memories") + op.drop_table("joysafeter_agent_versions") + op.drop_index(op.f("ix_joysafeter_vaults_project_id"), table_name="joysafeter_vaults") + op.drop_index("idx_cv_project", table_name="joysafeter_vaults") + op.drop_table("joysafeter_vaults") + op.drop_index("skills_visibility_idx", table_name="joysafeter_skills") + op.drop_index("skills_tags_idx", table_name="joysafeter_skills", postgresql_using="gin") + op.drop_index("skills_security_status_idx", table_name="joysafeter_skills") + op.drop_index("skills_security_severity_idx", table_name="joysafeter_skills") + op.drop_index("skills_security_recommendation_idx", table_name="joysafeter_skills") + op.drop_index("skills_public_idx", table_name="joysafeter_skills") + op.drop_index("skills_project_idx", table_name="joysafeter_skills") + op.drop_index("skills_owner_idx", table_name="joysafeter_skills") + op.drop_index("skills_lifecycle_status_idx", table_name="joysafeter_skills") + op.drop_index("skills_created_by_idx", table_name="joysafeter_skills") + op.drop_table("joysafeter_skills") + op.drop_index(op.f("ix_joysafeter_secrets_project_id"), table_name="joysafeter_secrets") + op.drop_table("joysafeter_secrets") + op.drop_index(op.f("ix_joysafeter_sandboxes_project_id"), table_name="joysafeter_sandboxes") + op.drop_index("idx_csb_status", table_name="joysafeter_sandboxes") + op.drop_index("idx_csb_session", table_name="joysafeter_sandboxes", postgresql_where="chat_session_id IS NOT NULL") + op.drop_index("idx_csb_project", table_name="joysafeter_sandboxes") + op.drop_index("idx_csb_pool", table_name="joysafeter_sandboxes", postgresql_where="status = 'pooled'") + op.drop_index( + "idx_csb_active_session_unique", + table_name="joysafeter_sandboxes", + postgresql_where="chat_session_id IS NOT NULL AND destroyed_at IS NULL AND status IN ('creating', 'provisioning', 'idle', 'running', 'stopped', 'error')", + ) + op.drop_table("joysafeter_sandboxes") + op.drop_index("ix_joysafeter_project_members_user_id", table_name="joysafeter_project_members") + op.drop_index("ix_joysafeter_project_members_project_id", table_name="joysafeter_project_members") + op.drop_table("joysafeter_project_members") + op.drop_index(op.f("ix_joysafeter_memory_stores_project_id"), table_name="joysafeter_memory_stores") + op.drop_table("joysafeter_memory_stores") + op.drop_index(op.f("ix_joysafeter_environments_project_id"), table_name="joysafeter_environments") + op.drop_table("joysafeter_environments") + op.drop_index("idx_cak_project", table_name="joysafeter_api_keys") + op.drop_index("idx_cak_org", table_name="joysafeter_api_keys") + op.drop_index("idx_cak_key_hash", table_name="joysafeter_api_keys") + op.drop_table("joysafeter_api_keys") + op.drop_index(op.f("ix_joysafeter_agents_project_id"), table_name="joysafeter_agents") + op.drop_index("idx_ca_project", table_name="joysafeter_agents") + op.drop_index("idx_ca_created_at", table_name="joysafeter_agents") + op.drop_table("joysafeter_agents") + op.drop_index("ix_joysafeter_organization_projects_org_id", table_name="joysafeter_organization_projects") + op.drop_table("joysafeter_organization_projects") + op.drop_index("ix_joysafeter_organization_members_user_id", table_name="joysafeter_organization_members") + op.drop_index("ix_joysafeter_organization_members_organization_id", table_name="joysafeter_organization_members") + op.drop_table("joysafeter_organization_members") + op.drop_index("ix_joysafeter_oauth_account_user_id", table_name="joysafeter_oauth_account") + op.drop_index("ix_joysafeter_oauth_account_provider_account", table_name="joysafeter_oauth_account") + op.drop_table("joysafeter_oauth_account") + op.drop_index("ix_joysafeter_auth_sessions_user_id", table_name="joysafeter_auth_sessions") + op.drop_index("ix_joysafeter_auth_sessions_token", table_name="joysafeter_auth_sessions") + op.drop_table("joysafeter_auth_sessions") + op.drop_index(op.f("ix_joysafeter_users_email"), table_name="joysafeter_users") + op.drop_table("joysafeter_users") + op.drop_index("ix_joysafeter_security_audit_logs_user_id", table_name="joysafeter_security_audit_logs") + op.drop_index("ix_joysafeter_security_audit_logs_user_event", table_name="joysafeter_security_audit_logs") + op.drop_index("ix_joysafeter_security_audit_logs_event_type", table_name="joysafeter_security_audit_logs") + op.drop_index("ix_joysafeter_security_audit_logs_event_status", table_name="joysafeter_security_audit_logs") + op.drop_index("ix_joysafeter_security_audit_logs_created_at", table_name="joysafeter_security_audit_logs") + op.drop_table("joysafeter_security_audit_logs") + op.drop_table("joysafeter_organizations") + # ### end Alembic commands ### diff --git a/backend/alembic/versions/20260702_000002_add_task_idempotency_key.py b/backend/alembic/versions/20260702_000002_add_task_idempotency_key.py new file mode 100644 index 000000000..63ca5a04e --- /dev/null +++ b/backend/alembic/versions/20260702_000002_add_task_idempotency_key.py @@ -0,0 +1,32 @@ +"""add task idempotency_key (Foundation 2 — effectively-once) + +Adds a nullable ``idempotency_key`` to ``joysafeter_tasks`` with a unique +constraint so a retried submission (client or HA API replica) maps to exactly +one task instead of double-executing pentest tooling. + +Revision ID: 20260702_000002 +Revises: 20260627_000001 +Create Date: 2026-07-02 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260702_000002" +down_revision: Union[str, None] = "20260627_000001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("joysafeter_tasks", sa.Column("idempotency_key", sa.Text(), nullable=True)) + op.create_unique_constraint("uq_joysafeter_tasks_idempotency_key", "joysafeter_tasks", ["idempotency_key"]) + + +def downgrade() -> None: + op.drop_constraint("uq_joysafeter_tasks_idempotency_key", "joysafeter_tasks", type_="unique") + op.drop_column("joysafeter_tasks", "idempotency_key") diff --git a/backend/alembic/versions/20260702_000003_add_task_lease.py b/backend/alembic/versions/20260702_000003_add_task_lease.py new file mode 100644 index 000000000..ee6e4c41f --- /dev/null +++ b/backend/alembic/versions/20260702_000003_add_task_lease.py @@ -0,0 +1,49 @@ +"""add running-task lease (Foundation 1 — fast reclaim) + +Adds ``owner_instance_id`` + ``lease_expires_at`` to ``joysafeter_tasks``. The +orchestrator instance that transitions a task to ``running`` stamps its +instance id and a short lease; it renews the lease while it holds the task. +When an instance crashes, its running tasks' leases lapse and any instance's +watchdog reclaims them in seconds — instead of waiting for the ~2h +``timeout_sec`` upper bound. + +A partial index on ``(lease_expires_at)`` filtered to ``status = 'running'`` +keeps the reclaim scan (the only query on these columns) cheap without +bloating the index with terminal/pending rows. + +Revision ID: 20260702_000003 +Revises: 20260702_000002 +Create Date: 2026-07-02 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260702_000003" +down_revision: Union[str, None] = "20260702_000002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("joysafeter_tasks", sa.Column("owner_instance_id", sa.Text(), nullable=True)) + op.add_column( + "joysafeter_tasks", + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "idx_ct_running_lease", + "joysafeter_tasks", + ["lease_expires_at"], + postgresql_where=sa.text("status = 'running'"), + ) + + +def downgrade() -> None: + op.drop_index("idx_ct_running_lease", table_name="joysafeter_tasks") + op.drop_column("joysafeter_tasks", "lease_expires_at") + op.drop_column("joysafeter_tasks", "owner_instance_id") diff --git a/backend/alembic/versions/20260702_000004_add_task_owner_epoch.py b/backend/alembic/versions/20260702_000004_add_task_owner_epoch.py new file mode 100644 index 000000000..3a4e8e1b5 --- /dev/null +++ b/backend/alembic/versions/20260702_000004_add_task_owner_epoch.py @@ -0,0 +1,42 @@ +"""add task owner_epoch fencing token (Foundation 1 — fencing) + +Adds ``owner_epoch`` to ``joysafeter_tasks`` plus a dedicated Postgres +``SEQUENCE``. Each →RUNNING claim stamps a fresh ``nextval`` — a globally +monotonic fencing token for that ownership grant. Every mutating write for a +running task is conditioned on the epoch it was granted; a stalled zombie whose +task was reclaimed and re-run (bumping the epoch) writes with a stale token, +matches zero rows, and is dropped instead of corrupting the row a new owner now +holds. + +Postgres SEQUENCE is the only durable monotonic source available (Redis has no +durable monotonic primitive, and a task-local counter cannot survive a requeue +across instances). + +Revision ID: 20260702_000004 +Revises: 20260702_000003 +Create Date: 2026-07-02 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260702_000004" +down_revision: Union[str, None] = "20260702_000003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +SEQUENCE_NAME = "joysafeter_task_owner_epoch_seq" + + +def upgrade() -> None: + op.execute(sa.schema.CreateSequence(sa.Sequence(SEQUENCE_NAME))) + op.add_column("joysafeter_tasks", sa.Column("owner_epoch", sa.BigInteger(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("joysafeter_tasks", "owner_epoch") + op.execute(sa.schema.DropSequence(sa.Sequence(SEQUENCE_NAME))) diff --git a/backend/alembic/versions/20260703_000005_add_project_task_limit.py b/backend/alembic/versions/20260703_000005_add_project_task_limit.py new file mode 100644 index 000000000..110aea09f --- /dev/null +++ b/backend/alembic/versions/20260703_000005_add_project_task_limit.py @@ -0,0 +1,35 @@ +"""add per-project concurrent-task limit override (Foundation 3 — tenancy) + +Adds a nullable ``max_concurrent_tasks`` to ``joysafeter_organization_projects``. +The project is the tenant boundary; admission control caps a project's live +(non-terminal) task count so one tenant cannot starve the shared HA fleet. NULL +means "use the global default" (settings.max_concurrent_per_project); a value +overrides it for that project (e.g. a paid/trusted tenant). + +Revision ID: 20260703_000005 +Revises: 20260702_000004 +Create Date: 2026-07-03 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260703_000005" +down_revision: Union[str, None] = "20260702_000004" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "joysafeter_organization_projects", + sa.Column("max_concurrent_tasks", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("joysafeter_organization_projects", "max_concurrent_tasks") diff --git a/backend/alembic/versions/20260703_000006_add_project_resource_limits.py b/backend/alembic/versions/20260703_000006_add_project_resource_limits.py new file mode 100644 index 000000000..39588c857 --- /dev/null +++ b/backend/alembic/versions/20260703_000006_add_project_resource_limits.py @@ -0,0 +1,34 @@ +"""add per-project sandbox resource-limit overrides (Foundation 3 — tenancy) + +Adds nullable ``max_cpu`` (cores) and ``max_memory_mb`` (MiB) to +``joysafeter_organization_projects``. NULL for a field means "use the global +default" (settings.sandbox_cpu / sandbox_memory_mb); a value overrides that +field for the project. Enforced per-sandbox via the Docker provider so one +tenant cannot exhaust host CPU/RAM on the shared fleet. + +Revision ID: 20260703_000006 +Revises: 20260703_000005 +Create Date: 2026-07-03 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260703_000006" +down_revision: Union[str, None] = "20260703_000005" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("joysafeter_organization_projects", sa.Column("max_cpu", sa.Float(), nullable=True)) + op.add_column("joysafeter_organization_projects", sa.Column("max_memory_mb", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("joysafeter_organization_projects", "max_memory_mb") + op.drop_column("joysafeter_organization_projects", "max_cpu") diff --git a/backend/alembic/versions/20260703_000007_add_task_submitter_identity.py b/backend/alembic/versions/20260703_000007_add_task_submitter_identity.py new file mode 100644 index 000000000..0cbbc0170 --- /dev/null +++ b/backend/alembic/versions/20260703_000007_add_task_submitter_identity.py @@ -0,0 +1,35 @@ +"""add task submitter identity user_id/org_id (Foundation 3 — tenancy) + +Denormalizes the submitting user's tenant identity (user_id, org_id) onto each +task for attribution/audit and per-user admission control. Both nullable and +NOT FK-constrained so a task's audit record survives user/org deletion. A +composite index on (user_id, status) backs the per-user active-task count. + +Revision ID: 20260703_000007 +Revises: 20260703_000006 +Create Date: 2026-07-03 00:00:00.000000+00:00 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "20260703_000007" +down_revision: Union[str, None] = "20260703_000006" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("joysafeter_tasks", sa.Column("user_id", sa.String(length=255), nullable=True)) + op.add_column("joysafeter_tasks", sa.Column("org_id", sa.String(length=255), nullable=True)) + op.create_index("idx_ct_user_status", "joysafeter_tasks", ["user_id", "status"]) + + +def downgrade() -> None: + op.drop_index("idx_ct_user_status", table_name="joysafeter_tasks") + op.drop_column("joysafeter_tasks", "org_id") + op.drop_column("joysafeter_tasks", "user_id") diff --git a/backend/alembic/versions/20260703_000008_add_session_message_idempotency_index.py b/backend/alembic/versions/20260703_000008_add_session_message_idempotency_index.py new file mode 100644 index 000000000..de5366ff7 --- /dev/null +++ b/backend/alembic/versions/20260703_000008_add_session_message_idempotency_index.py @@ -0,0 +1,27 @@ +"""add session user.message idempotency index + +Revision ID: 20260703_000008 +Revises: 20260703_000007 +Create Date: 2026-07-03 +""" + +from alembic import op + +revision = "20260703_000008" +down_revision = "20260703_000007" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS uq_cse_session_user_message_idempotency_key + ON joysafeter_session_events (session_id, ((payload->>'_idempotency_key'))) + WHERE event_type = 'user.message' AND payload ? '_idempotency_key' + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS uq_cse_session_user_message_idempotency_key") diff --git a/backend/app/MODEL.md b/backend/app/MODEL.md deleted file mode 100644 index 88794a092..000000000 --- a/backend/app/MODEL.md +++ /dev/null @@ -1,301 +0,0 @@ -# Model 系统总体文档 - -## 概述 - -Model 系统是 Agent Platform 的核心组件之一，提供了完整的模型管理功能，包括模型供应商管理、模型实例配置、模型凭据管理等。系统采用分层架构设计，实现了模型与上下游的解耦，方便开发者进行横向扩展。 - -## 系统架构 - -### 分层架构 - -``` -┌─────────────────────────────────────────┐ -│ API 层 (api/v1/) │ -│ - models.py │ -│ - model_providers.py │ -│ - model_credentials.py │ -└─────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ Service 层 (services/) │ -│ - model_service.py │ -│ - model_provider_service.py │ -│ - model_credential_service.py │ -└─────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ Repository 层 (repositories/) │ -│ - model_provider.py │ -│ - model_instance.py │ -│ - model_credential.py │ -└─────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ Model 层 (models/) │ -│ - model_provider.py │ -│ - model_instance.py │ -│ - model_credential.py │ -└─────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ Core 层 (core/model/) │ -│ - factory.py (模型工厂) │ -│ - providers/ (供应商实现) │ -│ - models/ (模型包装器) │ -│ - utils/ (工具函数) │ -└─────────────────────────────────────────┘ -``` - -### 核心组件 - -1. **API 层**: 提供 HTTP 接口，处理请求和响应 -2. **Service 层**: 实现业务逻辑，协调各层交互 -3. **Repository 层**: 数据访问层，封装数据库操作 -4. **Model 层**: 数据库模型定义（ORM） -5. **Core 层**: 模型工厂和供应商实现 - -## 目录结构 - -``` -backend/app/ -├── api/v1/ -│ ├── models.py # 模型实例 API -│ ├── model_providers.py # 模型供应商 API -│ └── model_credentials.py # 模型凭据 API -│ -├── services/ -│ ├── model_service.py # 模型服务 -│ ├── model_provider_service.py # 模型供应商服务 -│ └── model_credential_service.py # 模型凭据服务 -│ -├── repositories/ -│ ├── model_provider.py # 模型供应商 Repository -│ ├── model_instance.py # 模型实例 Repository -│ └── model_credential.py # 模型凭据 Repository -│ -├── models/ -│ ├── model_provider.py # 模型供应商模型 -│ ├── model_instance.py # 模型实例模型 -│ └── model_credential.py # 模型凭据模型 -│ -└── core/model/ - ├── factory.py # 模型工厂 - ├── providers/ # 供应商实现 - │ ├── base.py # 供应商基类 - │ └── OpenaiApiCompatible.py # OpenAI 兼容供应商 - ├── models/ # 模型包装器 - │ ├── base.py # 模型包装器基类 - │ ├── chat_model.py # 聊天模型包装器 - │ ├── embedding_model.py # 嵌入模型包装器 - │ └── rerank_model.py # 重排序模型包装器 - └── utils/ # 工具函数 - └── encryption.py # 加密工具 -``` - -## 数据模型 - -### 1. ModelProvider (模型供应商) - -存储供应商的基本信息和配置规则。 - -**主要字段**: -- `name`: 供应商唯一标识 -- `display_name`: 显示名称 -- `supported_model_types`: 支持的模型类型列表 -- `credential_schema`: 凭据表单规则 -- `config_schema`: 模型参数配置规则 -- `is_enabled`: 是否启用 - -### 2. ModelInstance (模型实例) - -存储用户配置的模型实例信息。 - -**主要字段**: -- `provider_id`: 供应商ID -- `model_name`: 模型名称 -- `model_parameters`: 模型参数配置 -- `is_default`: 是否为默认模型 -- `user_id`: 用户ID（NULL 表示全局） -- `workspace_id`: 工作空间ID（NULL 表示用户级） - -### 3. ModelCredential (模型凭据) - -存储加密后的供应商凭据信息。 - -**主要字段**: -- `provider_id`: 供应商ID -- `credentials`: 加密存储的凭据 -- `is_valid`: 凭据是否有效 -- `last_validated_at`: 最后验证时间 -- `validation_error`: 验证错误信息 -- `user_id`: 用户ID（NULL 表示全局） -- `workspace_id`: 工作空间ID（NULL 表示用户级） - -## 核心功能 - -### 1. 模型供应商管理 - -- **注册供应商**: 通过工厂模式注册新的供应商 -- **同步供应商**: 从代码同步供应商信息到数据库 -- **查询供应商**: 获取所有供应商列表和详情 -- **启用/禁用**: 控制供应商的可用性 - -### 2. 模型实例管理 - -- **创建配置**: 创建模型实例配置 -- **查询列表**: 获取可用模型列表 -- **默认模型**: 设置和管理默认模型 -- **模型测试**: 测试模型输出 - -### 3. 模型凭据管理 - -- **创建/更新**: 创建或更新供应商凭据 -- **加密存储**: 所有凭据加密存储 -- **验证凭据**: 验证凭据的有效性 -- **查询凭据**: 获取凭据列表和详情 -- **删除凭据**: 删除不再需要的凭据 - -### 4. 模型调用 - -- **创建实例**: 通过工厂创建 LangChain 模型实例 -- **统一接口**: 提供统一的模型调用接口 -- **参数配置**: 支持模型参数的动态配置 - -## 工作流程 - -### 1. 初始化流程 - -``` -1. 系统启动 - ↓ -2. 调用 sync_all() 同步供应商、模型和凭据 - ↓ -3. 从工厂获取供应商信息 - ↓ -4. 同步到数据库（model_provider 表） - ↓ -5. 从工厂获取模型列表 - ↓ -6. 同步到数据库（model_instance 表，全局记录） - ↓ -7. 从 .env 读取凭据 - ↓ -8. 加密并同步到数据库（model_credential 表，全局记录） -``` - -### 2. 使用模型流程 - -``` -1. 用户请求使用模型 - ↓ -2. API 层接收请求 - ↓ -3. Service 层处理业务逻辑 - ↓ -4. Repository 层查询数据库 - ↓ -5. Service 层获取解密后的凭据 - ↓ -6. Core 层工厂创建模型实例 - ↓ -7. 返回模型实例供使用 -``` - -### 3. 创建凭据流程 - -``` -1. 用户提交凭据 - ↓ -2. API 层接收请求 - ↓ -3. Service 层验证供应商是否存在 - ↓ -4. Service 层验证凭据（可选） - ↓ -5. Service 层加密凭据 - ↓ -6. Repository 层保存到数据库 - ↓ -7. 返回创建结果 -``` - -## API 端点总览 - -### 模型实例 API - -- `GET /api/v1/models` - 获取可用模型列表 -- `POST /api/v1/models/instances` - 创建模型实例配置 -- `GET /api/v1/models/instances` - 获取模型实例配置列表 -- `POST /api/v1/models/test-output` - 测试模型输出 - -### 模型供应商 API - -- `GET /api/v1/model-providers` - 获取所有供应商列表 -- `GET /api/v1/model-providers/{provider_name}` - 获取单个供应商详情 -- `POST /api/v1/model-providers/sync` - 同步供应商、模型和认证信息 - -### 模型凭据 API - -- `POST /api/v1/model-credentials` - 创建或更新凭据 -- `GET /api/v1/model-credentials` - 获取凭据列表 -- `GET /api/v1/model-credentials/{credential_id}` - 获取凭据详情 -- `POST /api/v1/model-credentials/{credential_id}/validate` - 验证凭据 -- `DELETE /api/v1/model-credentials/{credential_id}` - 删除凭据 - -## 扩展指南 - -### 添加新供应商 - -1. 在 `core/model/providers/` 创建新的供应商类 -2. 继承 `BaseProvider` 并实现所有抽象方法 -3. 在工厂中注册新供应商 -4. 运行同步接口更新数据库 - -### 添加新模型类型 - -1. 在 `ModelType` 枚举中添加新类型 -2. 在供应商中实现该类型的支持 -3. 创建对应的模型包装器（如需要） - -### 添加新 API 端点 - -1. 在相应的 API 文件中添加路由 -2. 在 Service 层添加业务逻辑 -3. 在 Repository 层添加数据访问方法（如需要） - -## 安全考虑 - -1. **凭据加密**: 所有凭据在存储前都进行加密 -2. **访问控制**: API 层应该实现认证和授权（当前为临时实现） -3. **验证机制**: 提供凭据验证功能，确保凭据有效 -4. **错误处理**: 不暴露敏感信息 in 错误消息 - -## 性能优化 - -1. **索引优化**: 数据库表添加了适当的索引 -2. **预加载**: 使用 `selectinload` 预加载关联对象 -3. **缓存**: 可以考虑添加缓存层（未来优化） - -## 注意事项 - -1. **全局可见性**: 当前实现中，模型实例和凭据对所有用户和工作空间可见（user_id 和 workspace_id 为 NULL 的记录） -2. **认证**: 当前代码中用户认证部分被注释，使用匿名用户ID，后续需要恢复认证机制 -3. **同步机制**: 供应商和模型信息通过工厂模式从代码同步到数据库 -4. **凭据管理**: 凭据加密存储，解密操作只在 Service 层进行 - -## 相关文档 - -- [API 层文档](./api/v1/MODEL.md) -- [Service 层文档](./services/MODEL.md) -- [Repository 层文档](./repositories/MODEL.md) -- [Model 层文档](./models/MODEL.md) -- [Core 层文档](./core/model/MODEL.md) -- [Core 层详细文档](./core/model/README_CN.md) - -## 未来改进 - -1. **多租户支持**: 完善用户和工作空间的隔离机制 -2. **缓存机制**: 添加缓存层提高性能 -3. **监控和日志**: 添加模型调用的监控和日志 -4. **限流和配额**: 实现模型调用的限流和配额管理 -5. **更多供应商**: 支持更多模型供应商 diff --git a/backend/app/SKILL.md b/backend/app/SKILL.md deleted file mode 100644 index 36b46ff9e..000000000 --- a/backend/app/SKILL.md +++ /dev/null @@ -1,445 +0,0 @@ -# SKILL 后端功能说明文档 - -## 概述 - -SKILL（技能）模块是平台的核心功能之一，用于管理和组织可复用的技能资源。每个技能可以包含描述、内容、标签、文件等丰富的信息，支持公开分享和私有管理。 - -## 架构设计 - -### 分层架构 - -SKILL 模块采用经典的分层架构设计： - -``` -API Layer (api/v1/skills.py) - ↓ -Service Layer (services/skill_service.py) - ↓ -Repository Layer (repositories/skill.py) - ↓ -Model Layer (models/skill.py) -``` - -### 核心组件 - -1. **模型层 (Models)** - - `Skill`: 技能主表模型 - - `SkillFile`: 技能文件关联表模型 - -2. **仓库层 (Repositories)** - - `SkillRepository`: 技能数据访问层 - - `SkillFileRepository`: 技能文件数据访问层 - -3. **服务层 (Services)** - - `SkillService`: 技能业务逻辑和权限校验 - -4. **API层 (API)** - - RESTful API 端点，提供完整的 CRUD 操作 - -## 数据模型 - -### Skill 模型 - -技能主表，存储技能的核心信息： - -| 字段 | 类型 | 说明 | -|------|------|------| -| `id` | UUID | 主键，自动生成 | -| `name` | String(255) | 技能名称，必填 | -| `description` | Text | 技能描述，必填 | -| `content` | Text | 技能内容，必填 | -| `tags` | JSONB | 标签列表，默认为空列表 | -| `source_type` | String(50) | 来源类型，默认为 "local" | -| `source_url` | String(1024) | 来源 URL，可选 | -| `root_path` | String(512) | 根路径，可选 | -| `owner_id` | String(255) | 拥有者 ID，外键关联 user.id | -| `created_by_id` | String(255) | 创建者 ID，外键关联 user.id，必填 | -| `is_public` | Boolean | 是否公开，默认为 False | -| `license` | String(100) | 许可证信息，可选 | -| `created_at` | DateTime | 创建时间 | -| `updated_at` | DateTime | 更新时间 | - -**约束和索引：** -- 唯一约束：`(owner_id, name)` - 同一拥有者的技能名称必须唯一 -- 索引： - - `skills_owner_idx`: 拥有者索引 - - `skills_created_by_idx`: 创建者索引 - - `skills_public_idx`: 公开状态索引 - - `skills_tags_idx`: 标签 GIN 索引（支持 JSONB 查询） - -### SkillFile 模型 - -技能文件关联表，存储技能关联的文件信息： - -| 字段 | 类型 | 说明 | -|------|------|------| -| `id` | UUID | 主键，自动生成 | -| `skill_id` | UUID | 技能 ID，外键关联 skills.id | -| `path` | String(512) | 文件路径，必填 | -| `file_name` | String(255) | 文件名，必填 | -| `file_type` | String(50) | 文件类型，必填 | -| `content` | Text | 文件内容，可选 | -| `storage_type` | String(20) | 存储类型，默认为 "database" | -| `storage_key` | String(512) | 存储键（如对象存储的 key），可选 | -| `size` | Integer | 文件大小（字节），默认为 0 | -| `created_at` | DateTime | 创建时间 | -| `updated_at` | DateTime | 更新时间 | - -**索引：** -- `skill_files_skill_idx`: 技能 ID 索引 -- `skill_files_path_idx`: 技能 ID + 路径复合索引 - -## 核心功能 - -### 1. 技能列表查询 - -**功能描述：** 获取技能列表，支持按用户、公开状态、标签过滤。 - -**权限控制：** -- 未登录用户：只能查看公开技能 -- 已登录用户：可查看自己的技能 + 公开技能 - -**查询参数：** -- `include_public` (bool): 是否包含公开技能，默认 True -- `tags` (List[str]): 标签过滤，支持多标签筛选 - -**实现位置：** -- API: `GET /v1/skills` -- Service: `SkillService.list_skills()` -- Repository: `SkillRepository.list_by_user()` - -**查询逻辑：** -1. 如果 `user_id` 存在且 `include_public=True`：返回 `owner_id == user_id` 或 `is_public == True` 或 `owner_id == None`（系统级公共技能） -2. 如果 `user_id` 存在且 `include_public=False`：只返回 `owner_id == user_id` 的技能 -3. 如果 `user_id` 不存在且 `include_public=True`：返回所有公开技能 -4. 如果 `user_id` 不存在且 `include_public=False`：不返回任何结果 -5. 如果指定了 `tags`，使用 JSONB 的 `contains` 操作符进行标签过滤 - -### 2. 技能详情查询 - -**功能描述：** 根据技能 ID 获取技能详情，包含关联的文件列表。 - -**权限控制：** -- 拥有者：可以访问自己的技能 -- 其他用户：只能访问公开的技能 -- 未登录用户：只能访问公开的技能 - -**实现位置：** -- API: `GET /v1/skills/{skill_id}` -- Service: `SkillService.get_skill()` -- Repository: `SkillRepository.get_with_files()` - -**权限检查逻辑：** -```python -if skill.owner_id and skill.owner_id != current_user_id and not skill.is_public: - raise ForbiddenException("You don't have permission to access this skill") -``` - -### 3. 创建技能 - -**功能描述：** 创建新的技能，支持同时创建关联的文件。 - -**权限要求：** 需要登录 - -**实现位置：** -- API: `POST /v1/skills` -- Service: `SkillService.create_skill()` - -**业务规则：** -1. 如果未指定 `owner_id`，则使用 `created_by_id` 作为拥有者 -2. 检查同一拥有者下是否存在同名技能，如果存在则抛出异常 -3. 支持在创建时同时添加文件列表 -4. 文件通过 `files` 参数传入，每个文件包含： - - `path`: 文件路径 - - `file_name`: 文件名 - - `file_type`: 文件类型 - - `content`: 文件内容（可选） - - `storage_type`: 存储类型（默认 "database"） - - `storage_key`: 存储键（可选） - - `size`: 文件大小（默认 0） - -**请求体示例：** -```json -{ - "name": "Python 数据分析", - "description": "使用 Python 进行数据分析的技能", - "content": "详细的技能内容...", - "tags": ["python", "data-analysis"], - "source_type": "local", - "is_public": false, - "files": [ - { - "path": "/examples", - "file_name": "example.py", - "file_type": "python", - "content": "print('Hello World')", - "storage_type": "database", - "size": 20 - } - ] -} -``` - -### 4. 更新技能 - -**功能描述：** 更新已存在的技能信息。 - -**权限要求：** 只有拥有者可以更新自己的技能 - -**实现位置：** -- API: `PUT /v1/skills/{skill_id}` -- Service: `SkillService.update_skill()` - -**业务规则：** -1. 权限检查：只有拥有者可以更新 -2. 如果更新名称，需要检查新名称在同一拥有者下是否已存在 -3. 支持部分更新，所有字段都是可选的 -4. 更新后自动刷新技能信息 - -**可更新字段：** -- `name`: 技能名称 -- `description`: 技能描述 -- `content`: 技能内容 -- `tags`: 标签列表 -- `source_type`: 来源类型 -- `source_url`: 来源 URL -- `root_path`: 根路径 -- `owner_id`: 拥有者 ID -- `is_public`: 是否公开 -- `license`: 许可证 - -### 5. 删除技能 - -**功能描述：** 删除技能及其关联的所有文件。 - -**权限要求：** 只有拥有者可以删除自己的技能 - -**实现位置：** -- API: `DELETE /v1/skills/{skill_id}` -- Service: `SkillService.delete_skill()` - -**业务规则：** -1. 权限检查：只有拥有者可以删除 -2. 级联删除：删除技能时自动删除所有关联的文件（通过外键约束） -3. 使用 `SkillFileRepository.delete_by_skill()` 显式删除文件记录 - -### 6. 添加文件 - -**功能描述：** 向已存在的技能添加文件。 - -**权限要求：** 只有拥有者可以向自己的技能添加文件 - -**实现位置：** -- API: `POST /v1/skills/{skill_id}/files` -- Service: `SkillService.add_file()` - -**业务规则：** -1. 权限检查：只有拥有者可以添加文件 -2. 文件信息包括路径、文件名、类型、内容等 -3. 支持多种存储类型（默认 "database"） - -### 7. 删除文件 - -**功能描述：** 删除技能关联的特定文件。 - -**权限要求：** 只有拥有者可以删除自己技能的文件 - -**实现位置：** -- API: `DELETE /v1/skills/files/{file_id}` -- Service: `SkillService.delete_file()` - -**业务规则：** -1. 权限检查：通过文件的 `skill_id` 找到对应的技能，检查拥有者权限 -2. 删除指定的文件记录 - -## 权限控制 - -### 访问权限 - -| 操作 | 拥有者 | 其他用户 | 未登录用户 | -|------|--------|----------|------------| -| 查看自己的技能 | ✅ | ❌ | ❌ | -| 查看公开技能 | ✅ | ✅ | ✅ | -| 创建技能 | ✅ | ❌ | ❌ | -| 更新自己的技能 | ✅ | ❌ | ❌ | -| 删除自己的技能 | ✅ | ❌ | ❌ | -| 添加文件 | ✅ | ❌ | ❌ | -| 删除文件 | ✅ | ❌ | ❌ | - -### 权限检查实现 - -所有权限检查都在 `SkillService` 层实现： - -1. **查看权限：** 在 `get_skill()` 方法中检查 -2. **更新权限：** 在 `update_skill()` 方法中检查 -3. **删除权限：** 在 `delete_skill()` 方法中检查 -4. **文件操作权限：** 在 `add_file()` 和 `delete_file()` 方法中检查 - -## 数据访问层 - -### SkillRepository - -提供技能数据访问方法： - -- `list_by_user()`: 根据用户 ID、公开状态、标签查询技能列表 -- `get_with_files()`: 获取技能及其关联的文件（使用 `selectinload` 预加载） -- `count_by_user()`: 统计用户拥有的技能数量 -- `get_by_name_and_owner()`: 根据名称和拥有者查询技能（用于唯一性检查） - -### SkillFileRepository - -提供技能文件数据访问方法： - -- `list_by_skill()`: 获取技能的所有文件 -- `delete_by_skill()`: 删除技能的所有文件（批量删除） - -## API 端点 - -### 基础路径 - -所有技能相关的 API 端点都在 `/v1/skills` 路径下。 - -### 端点列表 - -| 方法 | 路径 | 说明 | 需要认证 | -|------|------|------|----------| -| GET | `/v1/skills` | 获取技能列表 | 可选 | -| POST | `/v1/skills` | 创建技能 | ✅ | -| GET | `/v1/skills/{skill_id}` | 获取技能详情 | 可选 | -| PUT | `/v1/skills/{skill_id}` | 更新技能 | ✅ | -| DELETE | `/v1/skills/{skill_id}` | 删除技能 | ✅ | -| POST | `/v1/skills/{skill_id}/files` | 添加文件 | ✅ | -| DELETE | `/v1/skills/files/{file_id}` | 删除文件 | ✅ | - -### 响应格式 - -所有 API 响应都遵循统一的格式： - -**成功响应：** -```json -{ - "success": true, - "data": { ... } -} -``` - -**错误响应：** -```json -{ - "success": false, - "error": "错误信息" -} -``` - -## 异常处理 - -技能模块使用以下自定义异常： - -- `NotFoundException`: 资源不存在（如技能或文件不存在） -- `ForbiddenException`: 权限不足（如非拥有者尝试修改技能） -- `BadRequestException`: 请求参数错误（如同名技能已存在） - -所有异常都在 `SkillService` 层抛出，由 API 层的全局异常处理器统一处理。 - -## 数据库关系 - -### 外键关系 - -1. **Skill → AuthUser (owner)** - - `owner_id` → `user.id` - - `ondelete="SET NULL"`: 用户删除时，拥有者设为 NULL - -2. **Skill → AuthUser (created_by)** - - `created_by_id` → `user.id` - - `ondelete="CASCADE"`: 用户删除时，删除其创建的所有技能 - -3. **SkillFile → Skill** - - `skill_id` → `skills.id` - - `ondelete="CASCADE"`: 技能删除时，级联删除所有关联文件 - -### 关系加载策略 - -- `Skill.owner`: `lazy="selectin"` - 使用 selectin 加载 -- `Skill.created_by`: `lazy="selectin"` - 使用 selectin 加载 -- `Skill.files`: `lazy="selectin"` - 使用 selectin 加载，支持级联删除 -- `SkillFile.skill`: `lazy="selectin"` - 使用 selectin 加载 - -## 使用示例 - -### 创建技能 - -```python -from app.services.skill_service import SkillService - -service = SkillService(db) -skill = await service.create_skill( - created_by_id="user123", - name="Python 爬虫", - description="使用 Python 进行网页爬取的技能", - content="详细的技能内容...", - tags=["python", "web-scraping"], - is_public=True, - files=[ - { - "path": "/examples", - "file_name": "scraper.py", - "file_type": "python", - "content": "import requests\n...", - "storage_type": "database", - "size": 1024 - } - ] -) -``` - -### 查询技能列表 - -```python -# 获取当前用户的所有技能（包括公开的） -skills = await service.list_skills( - current_user_id="user123", - include_public=True, - tags=["python"] -) -``` - -### 更新技能 - -```python -skill = await service.update_skill( - skill_id=skill.id, - current_user_id="user123", - description="更新后的描述", - is_public=False -) -``` - -## 扩展性考虑 - -### 未来可能的扩展 - -1. **版本管理：** 支持技能版本控制，记录历史版本 -2. **评分系统：** 允许用户对公开技能进行评分和评论 -3. **使用统计：** 记录技能的使用次数和频率 -4. **导入导出：** 支持技能的批量导入和导出 -5. **模板系统：** 支持从模板创建技能 -6. **协作功能：** 支持多人协作编辑技能 -7. **搜索优化：** 使用全文搜索提升搜索性能 -8. **文件存储优化：** 支持大文件使用对象存储（S3、OSS 等） - -## 注意事项 - -1. **唯一性约束：** 同一拥有者的技能名称必须唯一，创建和更新时都会检查 -2. **权限边界：** 所有权限检查都在服务层实现，确保数据安全 -3. **级联删除：** 删除技能时会自动删除所有关联文件，注意数据备份 -4. **标签查询：** 使用 PostgreSQL 的 JSONB GIN 索引，支持高效的标签查询 -5. **关系加载：** 使用 `selectinload` 策略，避免 N+1 查询问题 -6. **事务管理：** 所有写操作都在事务中执行，确保数据一致性 - -## 相关文件 - -- 模型定义：`app/models/skill.py` -- 数据访问：`app/repositories/skill.py` -- 业务逻辑：`app/services/skill_service.py` -- API 端点：`app/api/v1/skills.py` -- 数据库迁移：`alembic/versions/` diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py deleted file mode 100644 index 4c26d35e5..000000000 --- a/backend/app/api/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -API route aggregation - -- /api/v1/... versioned endpoints (including auth, workspaces, graphs, users, environment, etc.) -""" - -from fastapi import APIRouter - -from .v1 import api_router as api_v1_router - -api_router = APIRouter() -api_router.include_router(api_v1_router) - -__all__ = ["api_router"] diff --git a/backend/app/api/schemas.py b/backend/app/api/schemas.py deleted file mode 100644 index 6a00aac60..000000000 --- a/backend/app/api/schemas.py +++ /dev/null @@ -1,84 +0,0 @@ -from enum import Enum -from typing import Generic, List, Optional, TypeVar - -from pydantic import BaseModel, ConfigDict, Field - - -class BadRequestResponse(BaseModel): - model_config = ConfigDict(json_schema_extra={"example": {"detail": "Bad request", "error_code": "BAD_REQUEST"}}) - - detail: str = Field(..., description="Error detail message") - error_code: Optional[str] = Field(None, description="Error code for categorization") - - -class NotFoundResponse(BaseModel): - model_config = ConfigDict(json_schema_extra={"example": {"detail": "Not found", "error_code": "NOT_FOUND"}}) - - detail: str = Field(..., description="Error detail message") - error_code: Optional[str] = Field(None, description="Error code for categorization") - - -class UnauthorizedResponse(BaseModel): - model_config = ConfigDict( - json_schema_extra={"example": {"detail": "Unauthorized access", "error_code": "UNAUTHORIZED"}} - ) - - detail: str = Field(..., description="Error detail message") - error_code: Optional[str] = Field(None, description="Error code for categorization") - - -class UnauthenticatedResponse(BaseModel): - model_config = ConfigDict( - json_schema_extra={"example": {"detail": "Unauthenticated access", "error_code": "UNAUTHENTICATED"}} - ) - - detail: str = Field(..., description="Error detail message") - error_code: Optional[str] = Field(None, description="Error code for categorization") - - -class ValidationErrorResponse(BaseModel): - model_config = ConfigDict( - json_schema_extra={"example": {"detail": "Validation error", "error_code": "VALIDATION_ERROR"}} - ) - - detail: str = Field(..., description="Error detail message") - error_code: Optional[str] = Field(None, description="Error code for categorization") - - -class InternalServerErrorResponse(BaseModel): - model_config = ConfigDict( - json_schema_extra={"example": {"detail": "Internal server error", "error_code": "INTERNAL_SERVER_ERROR"}} - ) - - detail: str = Field(..., description="Error detail message") - error_code: Optional[str] = Field(None, description="Error code for categorization") - - -class HealthResponse(BaseModel): - model_config = ConfigDict(json_schema_extra={"example": {"status": "ok", "instantiated_at": "1760169236.778903"}}) - - status: str = Field(..., description="Health status of the service") - instantiated_at: str = Field(..., description="Unix timestamp when service was instantiated") - - -T = TypeVar("T") - - -class SortOrder(str, Enum): - ASC = "asc" - DESC = "desc" - - -class PaginationInfo(BaseModel): - page: int = Field(0, description="Current page number (0-indexed)", ge=0) - limit: int = Field(20, description="Number of items per page", ge=1, le=100) - total_pages: int = Field(0, description="Total number of pages", ge=0) - total_count: int = Field(0, description="Total count of items", ge=0) - search_time_ms: float = Field(0, description="Search execution time in milliseconds", ge=0) - - -class PaginatedResponse(BaseModel, Generic[T]): - """Wrapper to add pagination info to classes used as response models""" - - data: List[T] = Field(..., description="List of items for the current page") - meta: PaginationInfo = Field(..., description="Pagination metadata") diff --git a/backend/app/api/v1/MODEL.md b/backend/app/api/v1/MODEL.md deleted file mode 100644 index 157f0195f..000000000 --- a/backend/app/api/v1/MODEL.md +++ /dev/null @@ -1,155 +0,0 @@ -# Model API 文档 - -## 概述 - -`backend/app/api/v1` 目录下的模型相关 API 提供了模型管理的 HTTP 接口，包括模型实例、模型供应商和模型凭据的管理功能。 - -## 文件结构 - -- `models.py` - 模型实例管理 API -- `model_providers.py` - 模型供应商管理 API -- `model_credentials.py` - 模型凭据管理 API - -## API 端点 - -### 模型实例 API (`models.py`) - -#### 1. 获取可用模型列表 -- **端点**: `GET /api/v1/models` -- **功能**: 获取指定类型的可用模型列表 -- **参数**: - - `model_type` (query): 模型类型，如 "chat", "embedding" 等 - - `workspaceId` (query, optional): 工作空间ID -- **返回**: 可用模型列表，包含供应商信息、模型名称、是否可用等 - -#### 2. 创建模型实例配置 -- **端点**: `POST /api/v1/models/instances` -- **功能**: 创建新的模型实例配置 -- **请求体**: - - `provider_name`: 供应商名称 - - `model_name`: 模型名称 - - `model_type`: 模型类型 - - `model_parameters`: 模型参数配置（可选） - - `workspaceId`: 工作空间ID（可选） - - `is_default`: 是否为默认模型 -- **返回**: 创建的模型实例配置 - -#### 3. 获取模型实例配置列表 -- **端点**: `GET /api/v1/models/instances` -- **功能**: 获取所有模型实例配置 -- **参数**: - - `workspaceId` (query, optional): 工作空间ID -- **返回**: 模型实例配置列表 - -#### 4. 测试模型输出 -- **端点**: `POST /api/v1/models/test-output` -- **功能**: 测试指定模型的输出 -- **请求体**: - - `model_name`: 模型名称 - - `input`: 输入文本 - - `workspaceId`: 工作空间ID（可选） -- **返回**: 模型输出结果 - -### 模型供应商 API (`model_providers.py`) - -#### 1. 获取所有供应商列表 -- **端点**: `GET /api/v1/model-providers` -- **功能**: 获取所有已注册的模型供应商信息 -- **返回**: 供应商列表，包含： - - `provider_name`: 供应商名称 - - `display_name`: 显示名称 - - `supported_model_types`: 支持的模型类型列表 - - `credential_schema`: 凭据表单规则 - - `config_schemas`: 配置规则（按模型类型） - - `is_enabled`: 是否启用 - -#### 2. 获取单个供应商详情 -- **端点**: `GET /api/v1/model-providers/{provider_name}` -- **功能**: 获取指定供应商的详细信息 -- **参数**: - - `provider_name` (path): 供应商名称 -- **返回**: 供应商详情 - -#### 3. 同步供应商、模型和认证信息 -- **端点**: `POST /api/v1/model-providers/sync` -- **功能**: 统一同步接口，将供应商、模型和认证信息同步到数据库 -- **说明**: - - 同步供应商信息（从工厂同步） - - 同步模型信息（从工厂同步到 model_instance 表，全局记录） - - 同步认证信息（从 .env 读取并同步到 model_credential 表，全局记录） -- **返回**: 同步结果统计 - -### 模型凭据 API (`model_credentials.py`) - -#### 1. 创建或更新凭据 -- **端点**: `POST /api/v1/model-credentials` -- **功能**: 创建或更新模型供应商的凭据 -- **请求体**: - - `provider_name`: 供应商名称 - - `credentials`: 凭据字典（明文） - - `workspaceId`: 工作空间ID（可选） - - `validate`: 是否验证凭据（默认 true） -- **返回**: 创建的凭据信息（不包含解密后的凭据） - -#### 2. 获取凭据列表 -- **端点**: `GET /api/v1/model-credentials` -- **功能**: 获取所有凭据列表 -- **参数**: - - `workspaceId` (query, optional): 工作空间ID -- **返回**: 凭据列表 - -#### 3. 获取凭据详情 -- **端点**: `GET /api/v1/model-credentials/{credential_id}` -- **功能**: 获取指定凭据的详细信息 -- **参数**: - - `credential_id` (path): 凭据ID -- **返回**: 凭据详情（不包含解密后的凭据） - -#### 4. 验证凭据 -- **端点**: `POST /api/v1/model-credentials/{credential_id}/validate` -- **功能**: 验证指定凭据的有效性 -- **参数**: - - `credential_id` (path): 凭据ID -- **返回**: 验证结果，包含 `is_valid`、`error`、`last_validated_at` - -#### 5. 删除凭据 -- **端点**: `DELETE /api/v1/model-credentials/{credential_id}` -- **功能**: 删除指定凭据 -- **参数**: - - `credential_id` (path): 凭据ID - -## 数据流 - -``` -客户端请求 - ↓ -API 路由层 (models.py, model_providers.py, model_credentials.py) - ↓ -Service 层 (ModelService, ModelProviderService, ModelCredentialService) - ↓ -Repository 层 (ModelInstanceRepository, ModelProviderRepository, ModelCredentialRepository) - ↓ -数据库 (model_instance, model_provider, model_credential 表) -``` - -## 依赖关系 - -- **Service 层**: 依赖 `ModelService`、`ModelProviderService`、`ModelCredentialService` -- **Core 层**: 依赖 `app.core.model` 模块（工厂、模型类型等） -- **数据库**: 使用 SQLAlchemy 异步会话 - -## 注意事项 - -1. **认证**: 当前代码中用户认证部分被注释，使用匿名用户ID，后续需要恢复认证机制 -2. **全局可见性**: 模型实例和凭据对所有用户和工作空间可见（user_id 和 workspace_id 为 NULL 的记录） -3. **凭据加密**: 所有凭据在存储前都会进行加密处理 -4. **同步机制**: 供应商和模型信息通过工厂模式从代码同步到数据库 - -## 扩展指南 - -要添加新的 API 端点： - -1. 在相应的文件中添加新的路由函数 -2. 使用 `@router.get/post/put/delete` 装饰器 -3. 通过 Service 层处理业务逻辑 -4. 返回统一的响应格式（使用 `success_response`） diff --git a/backend/app/api/v1/__init__.py b/backend/app/api/v1/__init__.py deleted file mode 100644 index 8bd3e1831..000000000 --- a/backend/app/api/v1/__init__.py +++ /dev/null @@ -1,88 +0,0 @@ -"""API v1 route aggregation. - -This module composes all v1 sub-routers into a single `api_router`. -Each sub-router is expected to declare its own `prefix` and `tags`. -""" - -from fastapi import APIRouter - -from .artifacts import router as artifacts_router -from .auth import router as auth_router -from .conversations import router as conversations_router -from .custom_tools import router as custom_tools_router -from .environment import router as environment_router -from .files import router as files_router -from .graph_code import router as graph_code_router -from .graph_deployments import router as graph_deployments_router -from .graphs import router as graphs_router -from .mcp import router as mcp_router -from .memory import router as memory_router -from .model_credentials import router as model_credentials_router -from .model_providers import router as model_providers_router -from .model_usage import router as model_usage_router -from .models import router as models_router -from .oauth import router as oauth_router -from .openapi_graph import router as openapi_graph_router -from .openclaw_chat import router as openclaw_chat_router -from .openclaw_devices import router as openclaw_devices_router -from .openclaw_instances import router as openclaw_instances_router -from .openclaw_proxy import router as openclaw_proxy_router -from .organizations import router as organizations_router -from .runs import router as runs_router -from .sandboxes import router as sandboxes_router -from .skill_collaborators import router as skill_collaborators_router -from .skill_versions import router as skill_versions_router -from .skills import router as skills_router -from .tokens import router as tokens_router -from .tools import router as tools_router -from .traces import router as traces_router -from .users import router as users_router -from .version import router as version_router -from .workspace_files import router as workspace_files_router -from .workspace_folders import router as workspace_folders_router -from .workspaces import router as workspaces_router - -ROUTERS = [ - sandboxes_router, - auth_router, - artifacts_router, - conversations_router, - files_router, - memory_router, - oauth_router, - organizations_router, - runs_router, - workspaces_router, - workspace_folders_router, - workspace_files_router, - custom_tools_router, - tools_router, - mcp_router, - model_providers_router, - model_credentials_router, - models_router, - model_usage_router, - graph_code_router, - graph_deployments_router, - skills_router, - skill_versions_router, - skill_collaborators_router, - tokens_router, - graphs_router, - traces_router, - users_router, - environment_router, - openclaw_instances_router, - openclaw_chat_router, - openclaw_devices_router, - openclaw_proxy_router, - openapi_graph_router, - version_router, -] - - -api_router = APIRouter() -for router in ROUTERS: - api_router.include_router(router) - -__all__ = ["api_router"] diff --git a/backend/app/api/v1/artifacts.py b/backend/app/api/v1/artifacts.py deleted file mode 100644 index ebb532e2e..000000000 --- a/backend/app/api/v1/artifacts.py +++ /dev/null @@ -1,166 +0,0 @@ -""" -Agent run artifacts API: list runs, list files, download, delete. - -All paths are scoped by current user (user_id from CurrentUser). -""" - -import mimetypes -from functools import lru_cache -from typing import Any - -from fastapi import APIRouter, Depends -from fastapi.responses import FileResponse, PlainTextResponse -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import CurrentUser -from app.common.exceptions import AppException, BadRequestException, InternalServerException, NotFoundException -from app.common.response import success_response -from app.core.agent.artifacts import ArtifactResolver, FileInfo, RunInfo -from app.core.database import get_db - -router = APIRouter(prefix="/v1/artifacts", tags=["Artifacts"]) - - -@lru_cache -def get_resolver() -> ArtifactResolver: - return ArtifactResolver() - - -def _run_info_to_dict(r: RunInfo) -> dict: - return { - "run_id": r.run_id, - "thread_id": r.thread_id, - "user_id": r.user_id, - "path": r.path, - "started_at": r.started_at, - "completed_at": r.completed_at, - "status": r.status, - "agent_type": r.agent_type, - "graph_id": r.graph_id, - "file_count": r.file_count, - } - - -def _file_info_to_dict(f: FileInfo) -> dict: - d: dict[str, Any] = { - "name": f.name, - "path": f.path, - "type": f.type, - "size": f.size, - "content_type": f.content_type, - } - if f.children is not None: - d["children"] = [_file_info_to_dict(c) for c in f.children] - return d - - -@router.get("/{thread_id}/runs") -async def list_artifact_runs( - thread_id: str, - current_user: CurrentUser, - resolver: ArtifactResolver = Depends(get_resolver), -): - """List all runs for the given thread (current user's artifacts).""" - runs = resolver.list_runs(str(current_user.id), thread_id) - data = [_run_info_to_dict(r) for r in runs] - return {**success_response(data=data, message="Fetched runs"), "runs": data} - - -@router.get("/{thread_id}/{run_id}/files") -async def list_artifact_files( - thread_id: str, - run_id: str, - current_user: CurrentUser, - resolver: ArtifactResolver = Depends(get_resolver), -): - """List files (tree) for the given run.""" - files = resolver.list_files_tree(str(current_user.id), thread_id, run_id) - data = [_file_info_to_dict(f) for f in files] - return {**success_response(data=data, message="Fetched files"), "files": data} - - -@router.get("/{thread_id}/{run_id}/download/{file_path:path}") -async def download_artifact_file( - thread_id: str, - run_id: str, - file_path: str, - current_user: CurrentUser, - resolver: ArtifactResolver = Depends(get_resolver), -): - """Download or preview a file from the run. Returns file with appropriate Content-Type.""" - path = resolver.get_file_path(str(current_user.id), thread_id, run_id, file_path) - if path is None: - raise NotFoundException("File not found or path invalid") - filename = path.name - media_type, _ = mimetypes.guess_type(str(path)) - return FileResponse( - path=path, - media_type=media_type or "application/octet-stream", - filename=filename, - ) - - -@router.get("/{thread_id}/live/{file_path:path}") -async def live_read_file( - thread_id: str, - file_path: str, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -): - """Read a file directly from the running sandbox container (live preview during execution).""" - from app.services.sandbox_manager import SandboxManagerService, _sandbox_pool - - service = SandboxManagerService(db) - user_id = str(current_user.id) - record = await service.get_user_sandbox_record(user_id) - if not record: - raise NotFoundException("No sandbox found") - - handle = None - adapter = await _sandbox_pool.get(record.id) - if not adapter or not adapter.is_started(): - if adapter: - await _sandbox_pool.release(record.id) - adapter = None - # Prefer reconnect over 404 to tolerate transient pool restarts. - try: - handle = await service.ensure_sandbox_running(user_id) - adapter = handle.adapter - except Exception as e: - logger.warning(f"Sandbox reconnect failed for user {user_id}: {e}", exc_info=True) - raise NotFoundException("Sandbox not running") - - try: - raw_read = getattr(adapter, "raw_read", None) - if callable(raw_read): - content = raw_read(file_path) - else: - content = adapter.read(file_path) - if content.startswith("[Error:") or content.startswith("Error:"): - raise NotFoundException(content) - return PlainTextResponse(content) - except AppException: - raise - except Exception as e: - logger.warning(f"Live read failed for {file_path}: {e}") - raise InternalServerException(f"Failed to read file: {e}") - finally: - if handle: - await handle.release() - else: - await _sandbox_pool.release(record.id) - - -@router.delete("/{thread_id}/{run_id}") -async def delete_artifact_run( - thread_id: str, - run_id: str, - current_user: CurrentUser, - resolver: ArtifactResolver = Depends(get_resolver), -): - """Delete all artifacts for the given run.""" - ok = resolver.delete_run(str(current_user.id), thread_id, run_id) - if not ok: - raise BadRequestException("Delete failed or path invalid") - return success_response(message="Run artifacts deleted", data={"run_id": run_id}) diff --git a/backend/app/api/v1/auth.py b/backend/app/api/v1/auth.py deleted file mode 100644 index a82f35946..000000000 --- a/backend/app/api/v1/auth.py +++ /dev/null @@ -1,497 +0,0 @@ -"""Auth controller endpoints (reworked for auth.user & auth.session).""" - -import uuid -from datetime import datetime, timedelta, timezone -from typing import Literal, Optional, cast - -from fastapi import APIRouter, BackgroundTasks, Body, Depends, Header, Request, Response -from fastapi.security import OAuth2PasswordRequestForm -from loguru import logger -from pydantic import BaseModel, ConfigDict, EmailStr, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.exceptions import AppException, UnauthorizedException -from app.common.response import success_response -from app.core.database import AsyncSessionLocal, get_db -from app.core.rate_limit import auth_rate_limit, strict_rate_limit -from app.core.security import Token, create_access_token, decode_token -from app.core.settings import settings -from app.models.auth import AuthUser -from app.services.auth_service import AuthService -from app.services.auth_session_service import AuthSessionService -from app.services.sandbox_manager import SandboxManagerService - -router = APIRouter(prefix="/v1/auth", tags=["Auth"]) - -# Type alias for SameSite cookie attribute -SameSiteType = Literal["lax", "strict", "none"] - - -def _get_samesite_value(value: str) -> Optional[SameSiteType]: - """Convert string to SameSite literal type for cookie operations.""" - normalized = value.lower().strip() - if normalized in ("lax", "strict", "none"): - return cast(SameSiteType, normalized) - return None - - -# Schemas - - -class RegisterRequest(BaseModel): - email: EmailStr - name: str = Field(..., min_length=1, max_length=255) - password: str = Field(..., min_length=6, max_length=100) - image: Optional[str] = None - is_super_user: bool = False - - -class LoginRequest(BaseModel): - email: EmailStr - password: str - - -class ForgotPasswordRequest(BaseModel): - email: EmailStr - - -class ResetPasswordRequest(BaseModel): - token: str - new_password: str = Field(..., min_length=6, max_length=100) - - -class SearchUsersResponse(BaseModel): - id: str - email: str - name: str - image: Optional[str] - email_verified: bool - is_super_user: bool - - -class UserResponse(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: str - email: str - name: str - image: Optional[str] - email_verified: bool - is_super_user: bool - - -async def _warm_up_sandbox(user_id: str): - """Background task to warm up user sandbox. Does not increment active_count.""" - try: - async with AsyncSessionLocal() as session: - sandbox_service = SandboxManagerService(session) - await sandbox_service.warm_up_sandbox(user_id) - logger.info(f"Sandbox pre-warming triggered for user {user_id}") - except Exception as e: - logger.error(f"Failed to pre-warm sandbox for user {user_id}: {e}") - - -# Endpoints - - -@router.post("/sign-up/email") -@auth_rate_limit() -async def sign_up_with_email( - http_request: Request, - body: RegisterRequest, - response: Response, - db: AsyncSession = Depends(get_db), -): - """Email registration endpoint.""" - service = AuthService(db) - data = await service.register( - email=body.email, - name=body.name, - password=body.password, - image=body.image, - is_super_user=body.is_super_user, - ) - - # Do not auto-login after signup; no Cookie is set - # User must sign in manually - # Return user info only, no token - user_data = data.get("user", {}) - - return success_response(data={"user": user_data}, message="Registration successful. Please sign in to continue.") - - -@router.post("/sign-in/email") -@auth_rate_limit() -async def sign_in_with_email( - http_request: Request, - body: LoginRequest, - response: Response, - background_tasks: BackgroundTasks, - db: AsyncSession = Depends(get_db), -): - """Email login endpoint.""" - service = AuthService(db) - result = await service.login(email=body.email, password=body.password) - - access_token = result.get("access_token") - refresh_token = result.get("refresh_token") - csrf_token = result.get("csrf_token") - expires_in = result.get("expires_in", settings.cookie_max_age) - - if access_token: - response.set_cookie( - key=settings.cookie_name, - value=access_token, - max_age=expires_in, - httponly=True, - secure=settings.cookie_secure_effective, - samesite=_get_samesite_value(settings.cookie_samesite), - domain=settings.cookie_domain, - path="/", - ) - - if refresh_token: - refresh_expires = settings.refresh_token_expire_days * 24 * 60 * 60 - response.set_cookie( - key="refresh_token", - value=refresh_token, - max_age=refresh_expires, - httponly=True, - secure=settings.cookie_secure_effective, - samesite=_get_samesite_value(settings.cookie_samesite), - domain=settings.cookie_domain, - path="/", - ) - - # Return CSRF token in response body, not via non-HttpOnly Cookie - # Frontend stores it in memory and sends via X-CSRF-Token header - # This avoids XSS stealing CSRF tokens - if csrf_token: - result["csrf_token"] = csrf_token - - # Trigger sandbox pre-warming - if result.get("user"): - user_id = result["user"].get("id") - if user_id: - background_tasks.add_task(_warm_up_sandbox, str(user_id)) - - return success_response(data=result, message="Login successful") - - -@router.post("/login/form", response_model=Token) -async def login_form( - background_tasks: BackgroundTasks, # BackgroundTasks must be required for injection - form_data: OAuth2PasswordRequestForm = Depends(), - db: AsyncSession = Depends(get_db), -): - """Login via OAuth2 password form (for Swagger UI compatibility).""" - service = AuthService(db) - result = await service.login(email=form_data.username, password=form_data.password) - - # Trigger sandbox pre-warming if background_tasks is available - if background_tasks and result.get("user"): - user_id = result["user"].get("id") - if user_id: - background_tasks.add_task(_warm_up_sandbox, str(user_id)) - - return Token( - access_token=result["access_token"], - token_type=result["token_type"], - expires_in=result["expires_in"], - ) - - -@router.post("/logout") -async def logout( - request: Request, - response: Response, - token: Optional[str] = Header(None, alias="Authorization"), - db: AsyncSession = Depends(get_db), -): - """Logout current user by invalidating tokens and clearing cookies.""" - try: - service = AuthService(db) - refresh_token = request.cookies.get("refresh_token") - - user_id = None - try: - current_user = await _get_current_auth_user(token, db, request) - user_id = current_user.id - except AppException: - logger.debug("Failed to resolve current user during logout", exc_info=True) - - if refresh_token and user_id: - try: - await service._delete_refresh_token(refresh_token, user_id) - except Exception: - logger.debug("Failed to delete refresh token during logout", exc_info=True) - - response.delete_cookie( - key=settings.cookie_name, - domain=settings.cookie_domain, - path="/", - samesite=_get_samesite_value(settings.cookie_samesite), - ) - response.delete_cookie( - key="refresh_token", - domain=settings.cookie_domain, - path="/", - samesite=_get_samesite_value(settings.cookie_samesite), - ) - response.delete_cookie( - key="csrf_token", - domain=settings.cookie_domain, - path="/", - samesite=_get_samesite_value(settings.cookie_samesite), - ) - - return success_response(message="Logout successful") - - except Exception: - logger.debug("Failed to perform full logout, clearing cookies anyway", exc_info=True) - response.delete_cookie( - key=settings.cookie_name, - domain=settings.cookie_domain, - path="/", - ) - response.delete_cookie( - key="refresh_token", - domain=settings.cookie_domain, - path="/", - ) - response.delete_cookie( - key="csrf_token", - domain=settings.cookie_domain, - path="/", - ) - - return success_response(message="Logout successful") - - -@router.post("/forgot-password") -@strict_rate_limit() -async def forgot_password( - http_request: Request, - body: ForgotPasswordRequest, - db: AsyncSession = Depends(get_db), -): - """Request a password reset email (silent even if email is unknown).""" - service = AuthService(db) - await service.request_password_reset(body.email) - - return success_response(message="If your email is registered, you will receive a password reset link shortly.") - - -@router.post("/reset-password") -@strict_rate_limit() -async def reset_password( - http_request: Request, - body: ResetPasswordRequest, - db: AsyncSession = Depends(get_db), -): - """Reset password using a one-time token.""" - service = AuthService(db) - await service.reset_password( - token=body.token, - new_password=body.new_password, - ) - - return success_response(message="Password reset successful") - - -class ResetPasswordForCurrentUserRequest(BaseModel): - new_password: str = Field(..., min_length=6, max_length=100) - - -@router.post("/me/reset-password") -async def reset_password_for_current_user( - http_request: Request, - body: ResetPasswordForCurrentUserRequest, - db: AsyncSession = Depends(get_db), - token: Optional[str] = Header(None, alias="Authorization"), -): - """Reset password for the current logged-in user (no old password required).""" - current_user = await _get_current_auth_user(token, db, http_request) - service = AuthService(db) - await service.reset_password_for_current_user( - user=current_user, - new_password=body.new_password, - ) - - return success_response(message="Password reset successful") - - -@router.post("/verify-email") -async def verify_email( - token: str = Body(..., embed=True), - db: AsyncSession = Depends(get_db), -): - """Verify email ownership using the provided token.""" - service = AuthService(db) - await service.verify_email(token) - - return success_response(message="Email verified successfully") - - -@router.post("/resend-verification") -async def resend_verification( - db: AsyncSession = Depends(get_db), - token: Optional[str] = Header(None, alias="Authorization"), -): - """Resend a verification email to the current user.""" - current_user = await _get_current_auth_user(token, db) - service = AuthService(db) - await service.resend_verification_email(current_user) - - return success_response(message="Verification email sent") - - -@router.get("/session") -async def get_session( - request: Request, - db: AsyncSession = Depends(get_db), - token: Optional[str] = Header(None, alias="Authorization"), -): - """Get current user session (JWT mode: returns user info from token).""" - try: - # Pass request to read token from Cookie - current_user = await _get_current_auth_user(token, db, request) - return success_response(data={"user": _user_to_response(current_user)}) - except AppException: - # Return null user when unauthenticated - return success_response(data={"user": None}) - - -@router.get("/ws-token") -async def get_ws_token( - request: Request, - db: AsyncSession = Depends(get_db), - token: Optional[str] = Header(None, alias="Authorization"), -): - """Return a short-lived token for WebSocket authentication (60 s).""" - current_user = await _get_current_auth_user(token, db, request) - ws_token = create_access_token(str(current_user.id), expires_delta=timedelta(seconds=60)) - return success_response(data={"token": ws_token}) - - -@router.post("/refresh") -async def refresh_token( - request: Request, - db: AsyncSession = Depends(get_db), - token: Optional[str] = Header(None, alias="Authorization"), -): - """Refresh access token using refresh token from Cookie or Authorization header.""" - - service = AuthService(db) - - # Try to read refresh token from Cookie - refresh_token_value = None - try: - refresh_token_value = request.cookies.get("refresh_token") - except Exception: - logger.debug("Failed to read refresh_token from cookies", exc_info=True) - - if not refresh_token_value and token: - try: - bearer_token = token.replace("Bearer ", "") if token.startswith("Bearer ") else token - payload = decode_token(bearer_token) - if payload: - user_id = payload.sub - user = await service.get_user_by_id(str(user_id)) - if user and user.is_active: - ( - access_token, - new_refresh_token, - csrf_token, - access_expires, - refresh_expires, - ) = await service._issue_jwt_tokens(user.id) - return success_response( - data={ - "access_token": access_token, - "token_type": "bearer", - "expires_in": int((access_expires - datetime.now(timezone.utc)).total_seconds()), - } - ) - except Exception: - logger.debug("Failed to refresh token via Authorization header", exc_info=True) - - if refresh_token_value: - try: - result = await service.refresh_token(refresh_token_value) - return success_response( - data={ - "access_token": result["access_token"], - "token_type": result["token_type"], - "expires_in": result["expires_in"], - } - ) - except Exception: - logger.debug("Failed to refresh token via cookie refresh_token", exc_info=True) - - raise UnauthorizedException("Invalid or expired refresh token") - - -# Helpers - - -def _extract_bearer(auth_header: Optional[str]) -> str: - if not auth_header or not auth_header.lower().startswith("bearer "): - raise UnauthorizedException("Missing bearer token") - return auth_header.split(" ", 1)[1] - - -async def _get_current_auth_user( - auth_header: Optional[str], db: AsyncSession, request: Optional[Request] = None -) -> AuthUser: - """Validate and return AuthUser from Bearer token or Cookie (JWT or session token).""" - token = None - - if auth_header: - try: - token = _extract_bearer(auth_header) - except UnauthorizedException: - logger.debug("Failed to extract bearer token from Authorization header", exc_info=True) - - if not token and request: - try: - from app.common.cookie_auth import extract_token_from_cookies - - token = extract_token_from_cookies(request.cookies) - except Exception: - logger.debug("Failed to read token from cookies", exc_info=True) - - if not token: - raise UnauthorizedException("Missing credentials") - - user_service = AuthService(db) - - payload = decode_token(token) - if payload: - user_id = payload.sub - user = await user_service.get_user_by_id(str(user_id)) - if user and user.is_active: - return user - raise UnauthorizedException("User not found or inactive") - - session_service = AuthSessionService(db) - session = await session_service.get_session_by_token(token) - if session: - user = await user_service.user_repo.get(uuid.UUID(session.user_id)) - if user and user.is_active: - return user - raise UnauthorizedException("User not found or inactive") - - raise UnauthorizedException("Invalid or expired token") - - -def _user_to_response(user: AuthUser) -> UserResponse: - """Serialize AuthUser into response-friendly dict.""" - return UserResponse( - id=str(user.id), - email=user.email, - name=user.name, - image=user.image, - email_verified=user.email_verified, - is_super_user=user.is_super_user, - ) diff --git a/backend/app/api/v1/chat.py b/backend/app/api/v1/chat.py deleted file mode 100644 index f7770f06b..000000000 --- a/backend/app/api/v1/chat.py +++ /dev/null @@ -1,537 +0,0 @@ -""" -Module: Chat API (Production Ready) - -Overview: -- Streaming helper module reused by the chat WebSocket handler -- Provide LangGraph event dispatch, state queries, message persistence, and result archival -- No longer exposes a `/v1/chat` HTTP endpoint - -Dependencies: -- Database: async SQLAlchemy session -- LangGraph: v2 event stream processing -- WebSocket chat handler: `app.websocket.chat_ws_handler` -""" - -import asyncio -import uuid -from typing import Any, AsyncGenerator, Dict - -from langchain.messages import AIMessage -from langchain_core.messages.base import BaseMessage -from langchain_core.runnables import RunnableConfig -from loguru import logger -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.core.database import AsyncSessionLocal -from app.core.settings import settings -from app.models import Conversation, Message -from app.utils.datetime import utc_now -from app.utils.file_event_emitter import FileEventEmitter -from app.utils.stream_event_handler import StreamEventHandler, StreamState - -# LangGraph control-flow exception: do not mark trace as FAILED -try: - from langgraph.errors import GraphBubbleUp -except ImportError: - GraphBubbleUp = None # type: ignore[misc, assignment] - - -async def safe_get_state( - graph: Any, config: RunnableConfig, max_retries: int = 3, initial_delay: float = 0.1, log: Any = None -) -> Any: - """ - Safely retrieve graph state with retry logic to avoid connection conflicts. - - Args: - graph: LangGraph graph instance - config: RunnableConfig configuration - max_retries: maximum number of retries - initial_delay: initial delay in seconds, doubled on each retry - log: optional logger - - Returns: - Graph state snapshot - - Raises: - Exception: if all retries are exhausted - """ - if log is None: - log = logger - - last_error = None - delay = initial_delay - - for attempt in range(max_retries): - try: - snap = await graph.aget_state(config) - return snap - except Exception as e: - last_error = e - error_msg = str(e) - - # check for connection conflict error - is_connection_error = ( - "another command is already in progress" in error_msg.lower() or "connection" in error_msg.lower() - ) - - # last attempt — stop retrying - if attempt >= max_retries - 1: - break - - # connection error — wait and retry - if is_connection_error: - log.debug( - f"Connection conflict detected (attempt {attempt + 1}/{max_retries}), " - f"retrying after {delay:.2f}s delay" - ) - await asyncio.sleep(delay) - delay *= 2 # exponential backoff - else: - # non-connection error — log warning but still retry (may be transient) - log.warning(f"Failed to get state (attempt {attempt + 1}/{max_retries}): {e}") - await asyncio.sleep(delay) - delay *= 2 - - # all retries exhausted - log.error(f"Failed to get state after {max_retries} attempts: {last_error}") - if last_error is not None: - raise last_error - raise RuntimeError("Failed to get state after all retries") - - -# ==================== Persistence Logic ==================== - - -async def save_run_result( - thread_id: str, - state: StreamState, - log, - *, - graph_id: str | None = None, - workspace_id: str | None = None, - user_id: str | None = None, - graph_name: str | None = None, -) -> None: - """ - Persist run results. - - Use a fresh DB session to ensure the connection is available even when - called from a finally block. Also batch-persist Trace + Observations. - """ - # --- 1. persist messages --- - if state.assistant_content or state.all_messages: - if not state.all_messages and state.assistant_content: - log.warning(f"Using fallback content accumulation for thread {thread_id}") - state.all_messages = [AIMessage(content=state.assistant_content)] - - if state.all_messages: - try: - async with AsyncSessionLocal() as session: - await save_assistant_message(thread_id, state.all_messages, session, update_conversation=True) - log.info(f"Persisted messages for thread {thread_id}") - except asyncio.CancelledError: - log.warning(f"Save run result cancelled for thread {thread_id}") - except Exception as e: - log.error(f"Failed to persist messages for thread {thread_id}: {e}") - - # --- 2. persist Trace + Observations (transaction-safe) --- - all_observations = state.get_all_observations() - if all_observations: - try: - await _persist_trace_data( - state, - log, - observations=all_observations, - graph_id=graph_id, - workspace_id=workspace_id, - user_id=user_id, - graph_name=graph_name, - ) - except asyncio.CancelledError: - log.debug(f"Trace persistence cancelled for thread {thread_id}") - except Exception as e: - log.warning(f"Failed to persist trace data for thread {thread_id}: {e}") - - -async def _persist_trace_data( - state: StreamState, - log, - *, - observations: list | None = None, - graph_id: str | None = None, - workspace_id: str | None = None, - user_id: str | None = None, - graph_name: str | None = None, -) -> None: - """ - Batch-write accumulated Observation data from StreamState to the database. - - Transaction-safe: uses session.begin() for atomicity. - Incomplete observations are marked INTERRUPTED by state.get_all_observations(). - """ - from datetime import datetime, timezone - - from app.models.execution_trace import ( - ExecutionObservation, - ExecutionTrace, - ObservationLevel, - ObservationStatus, - ObservationType, - TraceStatus, - ) - from app.utils.stream_event_handler import ObsLevel, ObsStatus, ObsType - - all_obs = observations if observations is not None else state.get_all_observations() - if not all_obs: - return - - # determine trace status - if state.has_error: - trace_status = TraceStatus.FAILED - elif state.interrupted: - trace_status = TraceStatus.INTERRUPTED - elif state.stopped: - trace_status = TraceStatus.FAILED - else: - trace_status = TraceStatus.COMPLETED - - now = datetime.now(timezone.utc) - trace_start = datetime.fromtimestamp(state.trace_start_time / 1000, tz=timezone.utc) - duration_ms = int(now.timestamp() * 1000 - state.trace_start_time) - - # aggregate token statistics - total_tokens = 0 - for obs_rec in all_obs: - if obs_rec.type == ObsType.GENERATION and obs_rec.total_tokens: - total_tokens += obs_rec.total_tokens - - # build ExecutionTrace ORM object - trace_uuid = uuid.UUID(state.trace_id) - trace = ExecutionTrace( - id=trace_uuid, - workspace_id=uuid.UUID(workspace_id) if workspace_id else None, - graph_id=uuid.UUID(graph_id) if graph_id else None, - thread_id=state.thread_id, - user_id=user_id, - name=graph_name or "graph_execution", - status=trace_status, - start_time=trace_start, - end_time=now, - duration_ms=duration_ms, - total_tokens=total_tokens or None, - ) - - # enum mapping - type_map = { - ObsType.SPAN: ObservationType.SPAN, - ObsType.GENERATION: ObservationType.GENERATION, - ObsType.TOOL: ObservationType.TOOL, - ObsType.EVENT: ObservationType.EVENT, - } - level_map = { - ObsLevel.DEBUG: ObservationLevel.DEBUG, - ObsLevel.DEFAULT: ObservationLevel.DEFAULT, - ObsLevel.WARNING: ObservationLevel.WARNING, - ObsLevel.ERROR: ObservationLevel.ERROR, - } - status_map = { - ObsStatus.RUNNING: ObservationStatus.RUNNING, - ObsStatus.COMPLETED: ObservationStatus.COMPLETED, - ObsStatus.FAILED: ObservationStatus.FAILED, - ObsStatus.INTERRUPTED: ObservationStatus.INTERRUPTED, - } - - # build ExecutionObservation ORM objects - db_observations = [] - for rec in all_obs: - obs = ExecutionObservation( - id=uuid.UUID(rec.id), - trace_id=trace_uuid, - parent_observation_id=uuid.UUID(rec.parent_observation_id) if rec.parent_observation_id else None, - type=type_map.get(rec.type, ObservationType.EVENT), - name=rec.name, - level=level_map.get(rec.level, ObservationLevel.DEFAULT), - status=status_map.get(rec.status, ObservationStatus.COMPLETED), - status_message=rec.status_message, - start_time=datetime.fromtimestamp(rec.start_time / 1000, tz=timezone.utc), - end_time=datetime.fromtimestamp(rec.end_time / 1000, tz=timezone.utc) if rec.end_time else None, - duration_ms=rec.duration_ms, - completion_start_time=( - datetime.fromtimestamp(rec.completion_start_time / 1000, tz=timezone.utc) - if rec.completion_start_time - else None - ), - input=rec.input_data, - output=rec.output_data, - model_name=rec.model_name, - model_provider=rec.model_provider, - model_parameters=rec.model_parameters, - prompt_tokens=rec.prompt_tokens, - completion_tokens=rec.completion_tokens, - total_tokens=rec.total_tokens, - metadata_=rec.metadata, - version=rec.version, - ) - db_observations.append(obs) - - # transaction-safe batch insert - async with AsyncSessionLocal() as session: - async with session.begin(): - session.add(trace) - session.add_all(db_observations) - # commit is automatic when begin() context exits - log.info(f"Persisted trace {state.trace_id} with {len(db_observations)} observations | thread={state.thread_id}") - - -# ==================== Database Operations ==================== - - -async def get_or_create_conversation( - thread_id: str | None, - message: str, - user_id: str, - metadata: dict | None, - db: AsyncSession, -) -> tuple[str, Conversation]: - if not thread_id: - # No thread_id provided, create new conversation - thread_id = str(uuid.uuid4()) - conversation = Conversation( - thread_id=thread_id, - user_id=user_id, - title=message[:50] if len(message) > 50 else message, - meta_data=metadata or {}, - ) - db.add(conversation) - await db.commit() - return thread_id, conversation - else: - # Thread_id provided, try to find existing conversation - result = await db.execute( - select(Conversation).where(Conversation.thread_id == thread_id, Conversation.user_id == user_id) - ) - conv = result.scalar_one_or_none() - if not conv: - # Conversation not found - create new one with the provided thread_id - # This allows frontend to generate thread_id and let backend create conversation on first message - conversation = Conversation( - thread_id=thread_id, - user_id=user_id, - title=message[:50] if len(message) > 50 else message, - meta_data=metadata or {}, - ) - db.add(conversation) - await db.commit() - await db.refresh(conversation) - return thread_id, conversation - return thread_id, conv - - -async def get_user_config(user_id: str, thread_id: str): - """Retrieve user configuration (RunnableConfig for LangGraph).""" - from app.core.agent.langfuse_callback import get_langfuse_callbacks - from app.core.trace_context import get_trace_id - - config: RunnableConfig = { - "configurable": {"thread_id": thread_id, "user_id": str(user_id), "trace_id": get_trace_id()}, - "recursion_limit": 300, - "callbacks": get_langfuse_callbacks(enabled=settings.langfuse_enabled), - } - - return config, {} - - -async def save_user_message(thread_id: str, message: str, metadata: dict | None, db: AsyncSession): - user_message = Message( - thread_id=thread_id, - role="user", - content=message, - meta_data=metadata or {}, - ) - db.add(user_message) - await db.commit() - - -async def save_assistant_message( - thread_id: str, messages: list[BaseMessage], db: AsyncSession, update_conversation: bool = True -): - """Save assistant message, extracting tool calls if present.""" - # find the last AI message - ai_msg = next((m for m in reversed(messages) if isinstance(m, AIMessage)), None) - if not ai_msg: - return - - meta_data = dict(ai_msg.additional_kwargs) if ai_msg.additional_kwargs else {} - - # extract tool calls (simplified — a strict implementation would match subsequent ToolMessages by ID) - if hasattr(ai_msg, "tool_calls") and ai_msg.tool_calls: - tool_calls_data = [] - for tc in ai_msg.tool_calls: - tool_calls_data.append({"name": tc.get("name"), "arguments": tc.get("args"), "id": tc.get("id")}) - meta_data["tool_calls"] = tool_calls_data - - message = Message( - thread_id=thread_id, - role="assistant", - content=str(ai_msg.content) if ai_msg.content else "", - meta_data=meta_data, - ) - db.add(message) - - if update_conversation: - result = await db.execute(select(Conversation).where(Conversation.thread_id == thread_id)) - if conv := result.scalar_one_or_none(): - conv.updated_at = utc_now() - await db.commit() - - -async def _clear_interrupt_marker(thread_id: str, log: Any) -> None: - """Clear the interrupted_graph_id marker from Conversation metadata.""" - try: - async with AsyncSessionLocal() as session: - result_query = await session.execute(select(Conversation).where(Conversation.thread_id == thread_id)) - if conv := result_query.scalar_one_or_none(): - if conv.meta_data and "interrupted_graph_id" in conv.meta_data: - del conv.meta_data["interrupted_graph_id"] - await session.commit() - log.debug(f"Cleared interrupt marker from conversation | thread_id={thread_id}") - except asyncio.CancelledError: - log.debug(f"Clear interrupt marker cancelled for thread {thread_id} (connection closing)") - except Exception as e: - log.warning(f"Failed to clear interrupt marker for conversation | thread_id={thread_id} | error={e}") - - -# ==================== Message Enrichment ==================== - - -def _enrich_message(message: str, metadata: dict, *, is_new_thread: bool, log, endpoint: str) -> str: - """Append edit_skill_id context (first message only) and file info to user message.""" - enriched = message - - # Only inject editing context on the first message of a new thread - edit_skill_id = metadata.get("edit_skill_id") - if edit_skill_id and is_new_thread: - log.info(f"[{endpoint}] edit-skill mode: edit_skill_id={edit_skill_id}") - enriched += ( - f"\n\n[Editing Mode] The user wants to modify an existing skill (ID: {edit_skill_id}). " - f"The skill files have been pre-loaded into the sandbox. " - f"Read the existing files first, then apply the user's requested changes." - ) - - files = metadata.get("files", []) - if files: - log.info(f"[{endpoint}] found {len(files)} attached file(s): {files}") - file_lines = "\n".join([f"- {f['filename']}: {f['path']}" for f in files]) - enriched += f"\n\nAttached files:\n{file_lines}\nUse the read_file tool to read the content of these files." - log.info(f"[{endpoint}] message enriched with file paths, length={len(enriched)}") - - return enriched - - -# ==================== Event Dispatch Helpers ==================== - - -def _extract_run_ids(event_dict: dict) -> tuple[str, str | None]: - """ - Extract run_id and parent_run_id from a LangGraph v2 event. - - Each LangGraph v2 astream_events event contains: - - run_id: unique identifier for the event (UUID or str) - - parent_ids: list ordered from root to immediate parent - - All values are normalised to str to avoid UUID-as-dict-key issues. - """ - raw_run_id = event_dict.get("run_id") - run_id = str(raw_run_id) if raw_run_id else "" - parent_ids = event_dict.get("parent_ids", []) - parent_run_id = str(parent_ids[-1]) if parent_ids else None - return run_id, parent_run_id - - -async def _dispatch_stream_event( - event: Any, - handler: StreamEventHandler, - state: StreamState, - file_emitter: FileEventEmitter | None = None, -) -> AsyncGenerator[str, None]: - """ - Translate a single LangGraph v2 astream_events event into SSE strings. - - Yields zero or more SSE strings. Callers: ``async for sse in _dispatch_stream_event(...): yield sse``. - file_emitter is only passed by chat_stream (not chat_resume). - """ - event_dict: dict[str, Any] - if isinstance(event, dict): - event_dict = event # type: ignore[assignment] - else: - event_dict = {"event": str(type(event).__name__), "data": event} if event else {} - - event_type = event_dict.get("event") - event_name = event_dict.get("name", "") - metadata = event_dict.get("metadata", {}) if isinstance(event_dict.get("metadata"), dict) else {} - langgraph_node = metadata.get("langgraph_node") - - is_node_event = langgraph_node is not None or ( - event_name - and "node" in event_name.lower() - and "tool" not in event_name.lower() - and "model" not in event_name.lower() - and "llm" not in event_name.lower() - and "chat" not in event_name.lower() - ) - - run_id, parent_run_id = _extract_run_ids(event_dict) - - if event_type == "on_chat_model_start": - yield await handler.handle_chat_model_start(event_dict, state, run_id, parent_run_id) - - elif event_type == "on_chat_model_stream": - if sse := await handler.handle_chat_model_stream(event_dict, state, run_id, parent_run_id): - yield sse - - elif event_type == "on_chat_model_end": - yield await handler.handle_chat_model_end(event_dict, state, run_id, parent_run_id) - - elif event_type == "on_tool_start": - yield await handler.handle_tool_start(event_dict, state, run_id, parent_run_id) - - elif event_type == "on_tool_end": - yield await handler.handle_tool_end(event_dict, state, run_id, parent_run_id) - - elif event_type == "on_chain_start" and is_node_event: - yield await handler.handle_node_start(event_dict, state, run_id, parent_run_id) - - elif event_type == "on_chain_end": - if is_node_event: - result = await handler.handle_node_end(event_dict, state, run_id, parent_run_id) - if isinstance(result, list): - for event_str in result: - if event_str and event_str.strip(): - yield event_str.strip() + "\n\n" - elif isinstance(result, str) and result.strip(): - yield result - - data_raw: Any = event_dict.get("data", {}) - data: Dict[str, Any] = data_raw if isinstance(data_raw, dict) else {} # type: ignore[assignment] - output = data.get("output") if isinstance(data, dict) else None - if output and isinstance(output, dict) and "messages" in output: - msgs = output["messages"] - from langgraph.types import Overwrite - - state.all_messages = msgs.value if isinstance(msgs, Overwrite) else msgs - - # Drain file events (chat_stream only) - if file_emitter is not None: - for file_evt in file_emitter.drain(): - yield handler.format_sse( - "file_event", - { - "action": file_evt.action, - "path": file_evt.path, - "size": file_evt.size, - "timestamp": file_evt.timestamp, - }, - state.thread_id, - state, - ) diff --git a/backend/app/api/v1/conversations.py b/backend/app/api/v1/conversations.py deleted file mode 100644 index 44357d34b..000000000 --- a/backend/app/api/v1/conversations.py +++ /dev/null @@ -1,927 +0,0 @@ -""" -Module: Conversations API - -Overview: -- Provides conversation create, query, update, delete (soft/hard delete) -- Provides message pagination, checkpoint retrieval, and conversation reset -- Supports conversation data export/import and full-text search -- Provides per-user conversation statistics - -Routes: -- POST /conversations: Create conversation -- GET /conversations: Get conversation list (paginated) -- DELETE /conversations/all: Delete all historical conversations (soft/hard) -- GET /conversations/{thread_id}: Get conversation details -- PATCH /conversations/{thread_id}: Update conversation -- DELETE /conversations/{thread_id}: Delete conversation (soft/hard) -- POST /conversations/{thread_id}/reset: Reset conversation (clear messages and checkpoints) -- GET /conversations/{thread_id}/messages: Get conversation messages (paginated) -- GET /conversations/{thread_id}/checkpoints: Get conversation checkpoints -- GET /conversations/{thread_id}/export: Export conversation (hidden from OpenAPI) -- POST /conversations/import: Import conversation (hidden from OpenAPI) -- POST /conversations/search: Search conversations and messages -- GET /conversations/users/stats: Get current user's conversation statistics - -Dependencies: -- Auth: CurrentUser -- Database: AsyncSession (Depends(get_db)) -- Graph: LangGraph checkpoints via checkpointer -- Utilities: utc_now, SQLAlchemy select/func, etc. - -Requests/Responses: -- Pagination: PaginationParams, PageResult[T] -- Conversation/Message models: ConversationCreate/Update/Response/DetailResponse, MessageResponse -- Others: CheckpointResponse, ConversationExportResponse, ConversationImportRequest, SearchRequest/Response, UserStatsResponse -- Unified response: BaseResponse[T] - -Error codes: -- 404: Conversation not found or not owned by current user -- 400: Invalid parameters or import/export failure -- 500: Internal server error -""" - -import uuid - -from fastapi import APIRouter, Depends, Query -from langchain_core.runnables import RunnableConfig -from loguru import logger -from sqlalchemy import delete, func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import CurrentUser -from app.common.exceptions import InternalServerException, raise_not_found_error -from app.common.pagination import ConversationMessagesPaginationParams, PageResult, PaginationParams, Paginator -from app.core.agent.checkpointer.checkpointer import get_checkpointer -from app.core.database import get_db -from app.models import Conversation, Message -from app.schemas import ( - BaseResponse, - CheckpointResponse, - ConversationCreate, - ConversationDetailResponse, - ConversationExportResponse, - ConversationImportRequest, - ConversationMessageResponse, - ConversationResponse, - ConversationUpdate, - SearchRequest, - SearchResponse, - UserStatsResponse, -) -from app.utils.datetime import utc_now - -router = APIRouter(prefix="/v1/conversations", tags=["Conversations"]) - - -# ==================== Helper functions ==================== - - -async def verify_conversation_ownership(thread_id: str, user_id: str, db: AsyncSession) -> Conversation: - """Verify conversation ownership""" - result = await db.execute( - select(Conversation).where(Conversation.thread_id == thread_id, Conversation.user_id == user_id) - ) - conversation = result.scalar_one_or_none() - if not conversation: - raise_not_found_error("Conversation") - # At this point, conversation is guaranteed to be non-None - assert conversation is not None - return conversation - - -# ==================== Conversation management endpoints ==================== - - -@router.post( - "", - response_model=BaseResponse[ConversationResponse], - summary="Create conversation", - description="Create a new conversation for the current user.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Internal server error"}, - }, -) -async def create_conversation( - conv: ConversationCreate, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[ConversationResponse]: - """ - Create a new conversation - - Args: - conv: Conversation creation request - current_user: Current user - db: Database session - - Returns: - BaseResponse[ConversationResponse]: Conversation response - """ - conversation = Conversation( - thread_id=str(uuid.uuid4()), - user_id=current_user.id, # Use current user ID - title=conv.title, - meta_data=conv.metadata or {}, - ) - db.add(conversation) - await db.commit() - await db.refresh(conversation) - - return BaseResponse( - success=True, - code=201, - msg="Conversation created successfully", - data=ConversationResponse( - id=conversation.id, - thread_id=conversation.thread_id, - user_id=conversation.user_id, - title=conversation.title, - metadata=conversation.meta_data or {}, - created_at=conversation.created_at, - updated_at=conversation.updated_at, - message_count=0, - ), - ) - - -@router.get( - "", - response_model=BaseResponse[PageResult[ConversationResponse]], - summary="List conversations", - description="List the current user's conversations with pagination.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Internal server error"}, - }, -) -async def list_conversations( - current_user: CurrentUser, - page: int = Query(default=1, ge=1, description="Page number"), - page_size: int = Query(default=20, ge=1, le=100, description="Items per page"), - db: AsyncSession = Depends(get_db), -) -> BaseResponse[PageResult[ConversationResponse]]: - """ - Get the current user's conversation list - - Args: - current_user: Current user - page: Page number (starting from 1) - page_size: Number of items per page - db: Database session - - Returns: - BaseResponse[PageResult[ConversationResponse]]: Paginated conversation list - """ - # Create PaginationParams from query parameters - page_query = PaginationParams(page=page, page_size=page_size) - - paginator = Paginator(db) - page_result = await paginator.paginate( - select(Conversation) - .where(Conversation.user_id == current_user.id, Conversation.is_active == 1) - .order_by(Conversation.updated_at.desc()), - page_query, - ) - conversations = page_result.items - - response_list = [] - for conv in conversations: - # Get message count - count_result = await db.execute(select(func.count(Message.id)).where(Message.thread_id == conv.thread_id)) - message_count = count_result.scalar() or 0 - - response_list.append( - ConversationResponse( - id=conv.id, - thread_id=conv.thread_id, - user_id=conv.user_id, - title=conv.title, - metadata=conv.meta_data or {}, - created_at=conv.created_at, - updated_at=conv.updated_at, - message_count=message_count, - ) - ) - - return BaseResponse( - success=True, - code=200, - msg="Fetched conversation list successfully", - data=PageResult( - items=response_list, - total=page_result.total, - page=page_result.page, - page_size=page_result.page_size, - pages=page_result.pages, - ), - ) - - -@router.delete( - "/all", - summary="Delete all historical conversations", - description="Delete all conversations for the current user. Supports soft delete or hard delete.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Internal server error"}, - }, -) -async def delete_all_conversations( - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), - hard_delete: bool = True, -) -> BaseResponse: - """Delete all historical conversations for the current user - - Args: - current_user: Current authenticated user - db: Database session - hard_delete: Whether to hard delete (permanent), defaults to True - - Returns: - BaseResponse: Delete result - """ - # Get all conversations for the current user - result = await db.execute( - select(Conversation).where(Conversation.user_id == current_user.id, Conversation.is_active == 1) - ) - conversations = result.scalars().all() - - if not conversations: - return BaseResponse( - success=True, - code=200, - msg="No conversations to delete", - data={"deleted_count": 0}, - ) - - deleted_count = 0 - - if hard_delete: - # Hard delete: remove all conversations and related data - from app.core.agent.checkpointer.checkpointer import delete_thread_checkpoints - - for conversation in conversations: - try: - # delete checkpoints - await delete_thread_checkpoints(conversation.thread_id) - except Exception as e: - logger.warning(f"Failed to delete checkpoints for {conversation.thread_id}: {e}") - - # delete conversation (messages are cascade-deleted) - await db.delete(conversation) - deleted_count += 1 - else: - # Soft delete: mark all conversations as inactive - for conversation in conversations: - conversation.is_active = 0 - deleted_count += 1 - - await db.commit() - - return BaseResponse( - success=True, - code=200, - msg=f"Deleted {deleted_count} conversations successfully", - data={ - "deleted_count": deleted_count, - "hard_delete": hard_delete, - }, - ) - - -@router.get( - "/{thread_id}", - response_model=BaseResponse[ConversationDetailResponse], - summary="Get conversation details", - description="Get conversation details by thread_id for the current user.", - responses={ - 401: {"description": "Unauthorized"}, - 404: {"description": "Conversation not found"}, - 500: {"description": "Internal server error"}, - }, -) -async def get_conversation( - thread_id: str, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[ConversationDetailResponse]: - """ - Get a single conversation's details - - Args: - thread_id: Thread ID - current_user: Current user - db: Database session - - Returns: - BaseResponse[ConversationDetailResponse]: Conversation details - """ - # Verify conversation ownership - conversation = await verify_conversation_ownership(thread_id, current_user.id, db) - - messages_result = await db.execute( - select(Message).where(Message.thread_id == thread_id).order_by(Message.created_at) - ) - messages = messages_result.scalars().all() - - conv_response = ConversationResponse( - id=conversation.id, - thread_id=conversation.thread_id, - user_id=conversation.user_id, - title=conversation.title, - metadata=conversation.meta_data or {}, - created_at=conversation.created_at, - updated_at=conversation.updated_at, - message_count=len(messages), - ) - - messages_data = [ - { - "id": msg.id, - "role": msg.role, - "content": msg.content, - "metadata": msg.meta_data or {}, - "created_at": msg.created_at.isoformat(), - } - for msg in messages - ] - - return BaseResponse( - success=True, - code=200, - msg="Fetched conversation details successfully", - data=ConversationDetailResponse(conversation=conv_response, messages=messages_data), - ) - - -@router.patch( - "/{thread_id}", - response_model=BaseResponse[dict], - summary="Update conversation", - description="Update conversation title and/or metadata.", - responses={ - 401: {"description": "Unauthorized"}, - 404: {"description": "Conversation not found"}, - 500: {"description": "Internal server error"}, - }, -) -async def update_conversation( - thread_id: str, - update: ConversationUpdate, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[dict]: - """ - Update conversation information - - Args: - thread_id: Thread ID - update: Update payload - current_user: Current user - db: Database session - - Returns: - BaseResponse[dict]: Update status - """ - # Verify conversation ownership - conversation = await verify_conversation_ownership(thread_id, current_user.id, db) - - if update.title is not None: - conversation.title = update.title - if update.metadata is not None: - conversation.meta_data = update.metadata - - conversation.updated_at = utc_now() - await db.commit() - - return BaseResponse( - success=True, - code=200, - msg="Conversation updated successfully", - data={"status": "updated", "thread_id": thread_id}, - ) - - -@router.delete( - "/{thread_id}", - response_model=BaseResponse[dict], - summary="Delete conversation", - description="Delete a conversation (soft delete or hard delete). Hard delete removes all related data.", - responses={ - 401: {"description": "Unauthorized"}, - 404: {"description": "Conversation not found"}, - 500: {"description": "Internal server error"}, - }, -) -async def delete_conversation( - thread_id: str, - current_user: CurrentUser, - hard_delete: bool = True, - db: AsyncSession = Depends(get_db), -): - """ - Delete conversation (soft or hard delete), hard delete by default - - Args: - thread_id: Thread ID - hard_delete: Whether to hard delete - current_user: Current user - db: Database session - - Returns: - BaseResponse[dict]: Delete status - """ - # Verify conversation ownership - conversation = await verify_conversation_ownership(thread_id, current_user.id, db) - - if hard_delete: - # Hard delete: remove all related data - # Delete checkpoints first - from app.core.agent.checkpointer.checkpointer import delete_thread_checkpoints - - try: - await delete_thread_checkpoints(thread_id) - except Exception as e: - logger.warning(f"Failed to delete checkpoints: {e}") - - # Delete conversation (messages are cascade-deleted) - await db.delete(conversation) - else: - # Soft delete - conversation.is_active = 0 - - await db.commit() - return BaseResponse( - success=True, - code=200, - msg="Conversation deleted successfully", - data={"status": "deleted", "thread_id": thread_id}, - ) - - -@router.post( - "/{thread_id}/reset", - response_model=BaseResponse[dict], - summary="Reset conversation", - description="Clear all checkpoints and messages, but keep the conversation record.", - responses={ - 401: {"description": "Unauthorized"}, - 404: {"description": "Conversation not found"}, - 500: {"description": "Internal server error"}, - }, -) -async def reset_conversation( - thread_id: str, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[dict]: - """ - Reset conversation: clear all checkpoints and messages, but keep the conversation record - - After reset, the conversation returns to the initial state and can start over. - - Args: - thread_id: Thread ID - current_user: Current user - db: Database session - - Returns: - BaseResponse[dict]: Reset status - """ - # Verify conversation ownership - conversation = await verify_conversation_ownership(thread_id, current_user.id, db) - - try: - # 1. Delete LangGraph checkpoints - from app.core.agent.checkpointer.checkpointer import delete_thread_checkpoints - - await delete_thread_checkpoints(thread_id) - logger.info(f"✅ Deleted LangGraph checkpoints for thread: {thread_id}") - - # 2. Delete all message records - result = await db.execute(delete(Message).where(Message.thread_id == thread_id)) - # get deleted row count (SQLAlchemy 2.0+ Result has rowcount attribute) - deleted_count = getattr(result, "rowcount", 0) - logger.info(f"✅ Deleted {deleted_count} messages for thread: {thread_id}") - - # 3. Update conversation timestamp - conversation.updated_at = utc_now() - - await db.commit() - - return BaseResponse( - success=True, - code=200, - msg=f"Conversation reset; deleted {deleted_count} messages", - data={ - "status": "reset", - "thread_id": thread_id, - "deleted_count": deleted_count, - }, - ) - - except Exception as e: - await db.rollback() - logger.error(f"Failed to reset conversation {thread_id}: {e}") - raise InternalServerException("Failed to reset conversation") from e - - -# ==================== Message management endpoints ==================== - - -@router.get( - "/{thread_id}/messages", - response_model=BaseResponse[PageResult[ConversationMessageResponse]], - summary="List conversation messages", - description="Get a paginated list of messages in the conversation.", - responses={ - 401: {"description": "Unauthorized"}, - 404: {"description": "Conversation not found"}, - 500: {"description": "Internal server error"}, - }, -) -async def get_messages( - thread_id: str, - current_user: CurrentUser, - page_query: ConversationMessagesPaginationParams = Depends(), - db: AsyncSession = Depends(get_db), -) -> BaseResponse[PageResult[ConversationMessageResponse]]: - """ - Get conversation message history - - Args: - thread_id: Thread ID - current_user: Current user - page_query: Pagination parameters (page, page_size) - db: Database session - - Returns: - BaseResponse[PageResult[ConversationMessageResponse]]: Paginated message list - """ - # Verify conversation ownership - await verify_conversation_ownership(thread_id, current_user.id, db) - - paginator = Paginator(db) - page_result = await paginator.paginate( - select(Message).where(Message.thread_id == thread_id).order_by(Message.created_at.desc()), - page_query, - ) - messages = page_result.items - - message_list = [ - ConversationMessageResponse( - id=msg.id, - role=msg.role, - content=msg.content, - metadata=msg.meta_data or {}, - created_at=msg.created_at, - ) - for msg in reversed(list(messages)) - ] - - logger.debug(f"Loaded {len(message_list)} messages for thread {thread_id}") - - return BaseResponse( - success=True, - code=200, - msg="Fetched message list successfully", - data=PageResult( - items=message_list, - total=page_result.total, - page=page_result.page, - page_size=page_result.page_size, - pages=page_result.pages, - ), - ) - - -@router.get( - "/{thread_id}/checkpoints", - response_model=BaseResponse[CheckpointResponse], - summary="Get conversation checkpoints", - description="Retrieve checkpoints from LangGraph state history.", - responses={ - 401: {"description": "Unauthorized"}, - 404: {"description": "Conversation not found"}, - 500: {"description": "Internal server error"}, - }, -) -async def get_checkpoints( - thread_id: str, - current_user: CurrentUser, - limit: int = 10, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[CheckpointResponse]: - """ - Get all conversation checkpoints - - Args: - thread_id: Thread ID - limit: Number of checkpoints to return - - Returns: - BaseResponse[CheckpointResponse]: Checkpoints response - """ - # Verify conversation ownership - await verify_conversation_ownership(thread_id, current_user.id, db) - - config: RunnableConfig = {"configurable": {"thread_id": thread_id, "user_id": str(current_user.id)}} - try: - checkpointer = get_checkpointer() - if not checkpointer: - raise RuntimeError("Checkpointer not initialized") - checkpoints = [] - async for checkpoint_tuple in checkpointer.alist(config): - cp_config = checkpoint_tuple.config or {} - cp = checkpoint_tuple.checkpoint or {} - checkpoints.append( - { - "checkpoint_id": cp_config.get("configurable", {}).get("checkpoint_id"), - "values": cp.get("channel_values", {}), - "next": [], - "metadata": checkpoint_tuple.metadata, - "created_at": checkpoint_tuple.metadata.get("created_at") if checkpoint_tuple.metadata else None, - } - ) - if len(checkpoints) >= limit: - break - - return BaseResponse( - success=True, - code=200, - msg="Fetched checkpoints successfully", - data=CheckpointResponse(thread_id=thread_id, checkpoints=checkpoints), - ) - except Exception as e: - logger.error(f"Get checkpoints error: {e}") - raise InternalServerException("Failed to fetch checkpoints") from e - - -# ==================== Export/Import endpoints ==================== - - -@router.get("/{thread_id}/export", response_model=BaseResponse[ConversationExportResponse], include_in_schema=False) -async def export_conversation( - thread_id: str, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -): - """ - Export conversation data - - Args: - thread_id: Thread ID - current_user: Current user - db: Database session - - Returns: - BaseResponse[ConversationExportResponse]: Exported data - """ - # Verify conversation ownership - conversation = await verify_conversation_ownership(thread_id, current_user.id, db) - - messages_result = await db.execute( - select(Message).where(Message.thread_id == thread_id).order_by(Message.created_at) - ) - messages = messages_result.scalars().all() - - # Get LangGraph state - config: RunnableConfig = {"configurable": {"thread_id": thread_id, "user_id": str(current_user.id)}} - try: - checkpointer = get_checkpointer() - if checkpointer: - checkpoint_tuple = await checkpointer.aget_tuple(config) - if checkpoint_tuple and checkpoint_tuple.checkpoint: - state_values = checkpoint_tuple.checkpoint.get("channel_values", {}) - else: - state_values = None - else: - state_values = None - except Exception: - state_values = None - - return BaseResponse( - success=True, - code=200, - msg="Conversation exported successfully", - data=ConversationExportResponse( - conversation={ - "thread_id": conversation.thread_id, - "user_id": conversation.user_id, - "title": conversation.title, - "metadata": conversation.meta_data or {}, - "created_at": conversation.created_at.isoformat(), - "updated_at": conversation.updated_at.isoformat(), - }, - messages=[ - { - "role": msg.role, - "content": msg.content, - "metadata": msg.meta_data or {}, - "created_at": msg.created_at.isoformat(), - } - for msg in messages - ], - state=state_values, - ), - ) - - -@router.post("/import", include_in_schema=False) -async def import_conversation( - request: ConversationImportRequest, - current_user: CurrentUser, - db: AsyncSession = Depends( - get_db, - ), -): - """ - Import conversation data - - Args: - request: Import request - current_user: Current user - db: Database session - - Returns: - BaseResponse[dict]: Import status - """ - data = request.data - thread_id = str(uuid.uuid4()) - - # Create conversation - conversation = Conversation( - thread_id=thread_id, - user_id=current_user.id, - title=data["conversation"]["title"], - meta_data=data["conversation"].get("metadata", {}), - ) - db.add(conversation) - - # Import messages - for msg_data in data["messages"]: - message = Message( - thread_id=thread_id, - role=msg_data["role"], - content=msg_data["content"], - meta_data=msg_data.get("metadata", {}), - ) - db.add(message) - - await db.commit() - - # Restore LangGraph state (best-effort) - if "state" in data and data["state"]: - config: RunnableConfig = {"configurable": {"thread_id": thread_id, "user_id": str(current_user.id)}} - try: - checkpointer = get_checkpointer() - if checkpointer: - import uuid as _uuid - - from langgraph.checkpoint.base import empty_checkpoint - - checkpoint = empty_checkpoint() - checkpoint["id"] = str(_uuid.uuid4()) - checkpoint["channel_values"] = data["state"] - await checkpointer.aput(config, checkpoint, {"source": "input"}, {}) - except Exception as e: - logger.warning(f"Could not restore state: {e}") - - return BaseResponse( - success=True, - code=200, - msg="Conversation imported successfully", - data={"thread_id": thread_id, "status": "imported"}, - ) - - -# ==================== Search endpoints ==================== - - -@router.post( - "/search", - response_model=BaseResponse[SearchResponse], - summary="Search conversations and messages", - description="Search messages content and related conversation titles for the current user.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Internal server error"}, - }, -) -async def search_conversations( - request: SearchRequest, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[SearchResponse]: - """ - Search conversations and messages - - Args: - request: Search request - current_user: Current user - db: Database session - - Returns: - BaseResponse[SearchResponse]: Search results - """ - # Use SQLite LIKE search - result = await db.execute( - select(Message) - .join(Conversation, Message.thread_id == Conversation.thread_id) - .where(Message.content.like(f"%{request.query}%"), Conversation.user_id == current_user.id) - .order_by(Message.created_at.desc()) - .offset(request.skip) - .limit(request.limit) - ) - messages = result.scalars().all() - - results = [] - for msg in messages: - conv_result = await db.execute(select(Conversation).where(Conversation.thread_id == msg.thread_id)) - conversation = conv_result.scalar_one_or_none() - - results.append( - { - "message_id": msg.id, - "thread_id": msg.thread_id, - "conversation_title": conversation.title if conversation else "", - "role": msg.role, - "content": msg.content, - "created_at": msg.created_at.isoformat(), - } - ) - - return BaseResponse( - success=True, - code=200, - msg="Search completed", - data=SearchResponse(query=request.query, results=results), - ) - - -# ==================== Statistics endpoints ==================== - - -@router.get( - "/users/stats", - response_model=BaseResponse[UserStatsResponse], - summary="Get user statistics", - description="Get statistics about the current user's conversations and messages.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Internal server error"}, - }, -) -async def get_user_stats( - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[UserStatsResponse]: - """ - Get user statistics - - Args: - current_user: Current user - db: Database session - - Returns: - BaseResponse[UserStatsResponse]: User statistics - """ - # total conversations - conv_result = await db.execute( - select(func.count(Conversation.id)).where(Conversation.user_id == current_user.id, Conversation.is_active == 1) - ) - total_conversations = conv_result.scalar() or 0 - - # total messages - msg_result = await db.execute( - select(func.count(Message.id)) - .join(Conversation, Message.thread_id == Conversation.thread_id) - .where(Conversation.user_id == current_user.id) - ) - total_messages = msg_result.scalar() or 0 - - # recent conversations - recent_result = await db.execute( - select(Conversation) - .where(Conversation.user_id == current_user.id, Conversation.is_active == 1) - .order_by(Conversation.updated_at.desc()) - .limit(5) - ) - recent_conversations = recent_result.scalars().all() - - return BaseResponse( - success=True, - code=200, - msg="Fetched statistics successfully", - data=UserStatsResponse( - user_id=str(current_user.id), - total_conversations=total_conversations, - total_messages=total_messages, - recent_conversations=[ - {"thread_id": conv.thread_id, "title": conv.title, "updated_at": conv.updated_at.isoformat()} - for conv in recent_conversations - ], - ), - ) diff --git a/backend/app/api/v1/custom_tools.py b/backend/app/api/v1/custom_tools.py deleted file mode 100644 index 96d24371e..000000000 --- a/backend/app/api/v1/custom_tools.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -Custom Tool CRUD API (User-level) -- Read/write: based on user ownership -- User-level quota limit (default 100) -""" - -from __future__ import annotations - -import uuid -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, ConfigDict, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.custom_tool_service import CustomToolService - -router = APIRouter(prefix="/custom-tools", tags=["CustomTools"]) - - -class CustomToolCreate(BaseModel): - name: str = Field(..., max_length=255) - code: str - json_schema: Dict[str, Any] = Field(default_factory=dict, alias="schema") - runtime: str = Field(default="python", max_length=50) - enabled: bool = True - - model_config = ConfigDict(populate_by_name=True) - - -class CustomToolUpdate(BaseModel): - name: Optional[str] = Field(None, max_length=255) - code: Optional[str] = None - json_schema: Optional[Dict[str, Any]] = Field(None, alias="schema") - runtime: Optional[str] = Field(None, max_length=50) - enabled: Optional[bool] = None - - model_config = ConfigDict(populate_by_name=True) - - -def _serialize(tool) -> Dict[str, Any]: - return { - "id": str(tool.id), - "ownerId": str(tool.owner_id), - "name": tool.name, - "code": tool.code, - "schema": tool.schema, - "runtime": tool.runtime, - "enabled": tool.enabled, - "createdAt": tool.created_at, - "updatedAt": tool.updated_at, - } - - -@router.get("") -async def list_custom_tools( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List all tools for the current user.""" - service = CustomToolService(db) - tools = await service.list_tools(current_user.id) - return {"success": True, "data": [_serialize(t) for t in tools]} - - -@router.post("") -async def create_custom_tool( - payload: CustomToolCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create a tool (user-level).""" - service = CustomToolService(db) - tool = await service.create_tool( - owner_id=current_user.id, - name=payload.name, - code=payload.code, - schema=payload.json_schema, - runtime=payload.runtime, - enabled=payload.enabled, - ) - return {"success": True, "data": _serialize(tool)} - - -@router.get("/{tool_id}") -async def get_custom_tool( - tool_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get tool details (owner only).""" - service = CustomToolService(db) - tool = await service.repo.get(tool_id) - if not tool: - return {"success": False, "error": "Not found"} - # verify ownership - if tool.owner_id != current_user.id: - return {"success": False, "error": "Forbidden"} - return {"success": True, "data": _serialize(tool)} - - -@router.put("/{tool_id}") -async def update_custom_tool( - tool_id: uuid.UUID, - payload: CustomToolUpdate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update a tool (owner only).""" - service = CustomToolService(db) - tool = await service.update_tool( - tool_id, - current_user.id, - name=payload.name, - code=payload.code, - schema=payload.json_schema, - runtime=payload.runtime, - enabled=payload.enabled, - ) - return {"success": True, "data": _serialize(tool)} - - -@router.delete("/{tool_id}") -async def delete_custom_tool( - tool_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = CustomToolService(db) - await service.delete_tool(tool_id, current_user.id) - return {"success": True} diff --git a/backend/app/api/v1/environment.py b/backend/app/api/v1/environment.py deleted file mode 100644 index 379587acf..000000000 --- a/backend/app/api/v1/environment.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Environment variable management API (path: /api/v1/environment) - -- /api/v1/environment/user Get current user's environment variables (keys only, values masked) -- /api/v1/environment/user (PUT) Update current user's environment variables -- /api/v1/environment/workspaces/{id} Get workspace environment variables (admin+ required, masked) -- /api/v1/environment/workspaces/{id} (PUT)Update workspace environment variables (admin+ required) -""" - -from __future__ import annotations - -import uuid -from typing import Dict - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user, require_workspace_role -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.workspace import WorkspaceMemberRole -from app.services.environment_service import EnvironmentService - -router = APIRouter(prefix="/v1/environment", tags=["Environment"]) - - -class EnvPayload(BaseModel): - variables: Dict[str, str] = Field(default_factory=dict) - - -@router.get("/user") -async def get_user_environment( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = EnvironmentService(db) - # Note: EnvironmentService expects uuid.UUID but user.id and Environment.user_id are both strings. - # Converting str to UUID for compatibility with service signature - import uuid as uuid_lib - - user_id = uuid_lib.UUID(current_user.id) if isinstance(current_user.id, str) else current_user.id - variables = await service.get_user_env(user_id) - return {"success": True, "variables": service.mask_variables(variables)} - - -@router.put("/user") -async def update_user_environment( - payload: EnvPayload, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = EnvironmentService(db) - # Note: EnvironmentService expects uuid.UUID but user.id and Environment.user_id are both strings. - # Converting str to UUID for compatibility with service signature - import uuid as uuid_lib - - user_id = uuid_lib.UUID(current_user.id) if isinstance(current_user.id, str) else current_user.id - variables = await service.upsert_user_env(user_id, payload.variables) - return {"success": True, "variables": service.mask_variables(variables)} - - -@router.get("/workspaces/{workspace_id}") -async def get_workspace_environment( - workspace_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.admin), -): - service = EnvironmentService(db) - variables = await service.get_workspace_env(workspace_id) - return {"success": True, "variables": service.mask_variables(variables)} - - -@router.put("/workspaces/{workspace_id}") -async def update_workspace_environment( - workspace_id: uuid.UUID, - payload: EnvPayload, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.admin), -): - service = EnvironmentService(db) - variables = await service.upsert_workspace_env(workspace_id, payload.variables) - return {"success": True, "variables": service.mask_variables(variables)} diff --git a/backend/app/api/v1/files.py b/backend/app/api/v1/files.py deleted file mode 100644 index bbd6558b7..000000000 --- a/backend/app/api/v1/files.py +++ /dev/null @@ -1,455 +0,0 @@ -""" -Module: Files API - -Overview: -- Provides file upload, read, delete, clear, and list operations within a user's sandbox -- All operations go through PydanticSandboxAdapter (Docker sandbox API) -- Files are stored at /workspace/uploads/ inside the container and accessible to the Agent - via FilesystemMiddleware - -Routes: -- POST /files/upload: Upload a file -- GET /files/list: List files -- GET /files/read/{filename}: Read file content -- DELETE /files/{filename}: Delete specified file -- DELETE /files: Clear all files in upload directory - -Dependencies: -- Auth: CurrentUser -- Storage: PydanticSandboxAdapter (Docker sandbox) -- Unified response: BaseResponse[T] - -Security notes: -- Always use sanitize_filename() to avoid path traversal -- Upload directory is scoped to /workspace/uploads/ inside the container - -Error codes: -- 404: File not found -- 500: File upload/read/delete failed -""" - -import asyncio -import base64 -import mimetypes -from pathlib import Path - -from fastapi import APIRouter, File, Request, UploadFile -from loguru import logger -from pydantic import BaseModel - -from app.common.dependencies import CurrentUser -from app.common.exceptions import AppException, BadRequestException, InternalServerException, NotFoundException -from app.core.agent.backends.constants import ( - DEFAULT_WORKING_DIR, - SANDBOX_UPLOADS_SUBDIR, -) -from app.core.rate_limit import get_client_ip, rate_limit -from app.schemas import BaseResponse -from app.utils.path_utils import sanitize_filename - -# Container-side path for uploaded files (what the Agent sees) -CONTAINER_UPLOADS_PATH = f"{DEFAULT_WORKING_DIR}/{SANDBOX_UPLOADS_SUBDIR}" - -router = APIRouter(prefix="/v1/files", tags=["Files"]) - -# File upload security limits (matching frontend) -MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 # 50MB -MAX_STORAGE_PER_USER = 5 * 1024 * 1024 * 1024 # 5GB per user -ALLOWED_EXTENSIONS = { - ".pdf", - ".doc", - ".docx", - ".xls", - ".xlsx", - ".ppt", - ".pptx", - ".odt", - ".ods", - ".odp", - ".rtf", - ".epub", - ".txt", - ".csv", - ".md", - ".html", - ".css", - ".js", - ".ts", - ".py", - ".java", - ".c", - ".cpp", - ".h", - ".hpp", - ".cs", - ".go", - ".rs", - ".rb", - ".php", - ".swift", - ".kt", - ".scala", - ".sh", - ".sql", - ".yaml", - ".yml", - ".toml", - ".xml", - ".json", - ".jsx", - ".tsx", - ".vue", - ".svelte", - ".jpeg", - ".jpg", - ".png", - ".gif", - ".webp", - ".zip", - ".tar", - ".gz", - ".7z", - ".rar", - ".apk", -} - - -class FileInfo(BaseModel): - filename: str - size: int - path: str - - -class FileListResponse(BaseModel): - files: list[FileInfo] - total: int - - -class UploadResponse(BaseModel): - filename: str - path: str - size: int - message: str - - -# Magic number signatures for file type validation -MAGIC_NUMBERS: dict[str, list[bytes]] = { - ".pdf": [b"%PDF"], - ".zip": [b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"], - ".png": [b"\x89PNG\r\n\x1a\n"], - ".jpg": [b"\xff\xd8\xff"], - ".jpeg": [b"\xff\xd8\xff"], - ".gif": [b"GIF87a", b"GIF89a"], - ".webp": [b"RIFF", b"WEBP"], - ".tar": [b"ustar", b"GNUtar"], - ".gz": [b"\x1f\x8b"], - ".7z": [b"7z\xbc\xaf\x27\x1c"], - ".rar": [b"Rar!\x1a\x07", b"Rar!\x1a\x07\x00"], - ".doc": [b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"], - ".docx": [b"PK\x03\x04"], - ".xls": [b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"], - ".xlsx": [b"PK\x03\x04"], - ".ppt": [b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"], - ".pptx": [b"PK\x03\x04"], - ".apk": [b"PK\x03\x04"], -} - - -def _validate_file_content(filename: str, content: bytes) -> None: - """Validate file content using magic number check.""" - if len(content) == 0: - return - file_ext = Path(filename).suffix.lower() - if file_ext not in MAGIC_NUMBERS: - return - expected_signatures = MAGIC_NUMBERS[file_ext] - content_start = content[: max(len(sig) for sig in expected_signatures)] - if not any(content_start.startswith(sig) for sig in expected_signatures): - logger.warning(f"File content validation failed for {filename}: got {content_start[:16].hex()}") - raise BadRequestException( - f"File content does not match declared type: {file_ext}", - ) - - -def _validate_file_type(filename: str, content_type: str | None) -> None: - """Validate file type (extension and MIME type).""" - file_ext = Path(filename).suffix.lower() - if file_ext and file_ext not in ALLOWED_EXTENSIONS: - raise BadRequestException(f"File type {file_ext} is not supported") - if content_type: - inferred_type, _ = mimetypes.guess_type(filename) - if inferred_type and content_type != inferred_type: - logger.warning(f"MIME type mismatch for {filename}: expected {inferred_type}, got {content_type}") - - -def get_container_path(filename: str) -> str: - """Get the container-side path for a file (what the Agent sees).""" - return f"{CONTAINER_UPLOADS_PATH}/{filename}" - - -async def _get_sandbox_handle(user_id: str): - """Acquire a SandboxHandle for the user. Caller MUST release it.""" - from app.services.sandbox_manager import get_sandbox_handle - - return await get_sandbox_handle(user_id) - - -def _validate_file_upload( - filename: str, - content: bytes, - content_type: str | None, -) -> tuple[str, None] | tuple[None, BadRequestException]: - """Validate file upload (size, type, content). Returns (safe_filename, None) or (None, error).""" - if len(content) == 0: - return None, BadRequestException("File cannot be empty") - - if len(content) > MAX_FILE_SIZE_BYTES: - return None, BadRequestException( - f"File size exceeds maximum allowed size ({MAX_FILE_SIZE_BYTES / 1024 / 1024}MB)" - ) - - safe_filename = sanitize_filename(filename) - - try: - _validate_file_type(safe_filename, content_type) - except BadRequestException as e: - return None, e - - try: - _validate_file_content(safe_filename, content) - except BadRequestException as e: - return None, e - - return safe_filename, None - - -@router.post( - "/upload", - response_model=BaseResponse[UploadResponse], - summary="Upload file", - description="Upload a file to the user's sandbox at /workspace/uploads/.", - responses={ - 400: {"description": "Invalid file type"}, - 413: {"description": "File size exceeds limit"}, - 401: {"description": "Unauthorized"}, - 429: {"description": "Rate limit exceeded"}, - 500: {"description": "Failed to upload file"}, - }, -) -@rate_limit(max_requests=10, window_seconds=60) -async def upload_file( - request: Request, - current_user: CurrentUser, - file: UploadFile = File(..., description="File to upload"), -) -> BaseResponse[UploadResponse]: - """Upload a file to the user's sandbox via adapter API.""" - client_ip = get_client_ip(request) - original_filename = file.filename or "unnamed" - - try: - content = await file.read() - - safe_filename, err = _validate_file_upload(original_filename, content, file.content_type) - if err: - logger.warning( - f"File upload rejected: user={current_user.id}, filename={original_filename}, ip={client_ip}" - ) - raise err - - assert safe_filename is not None - - container_path = get_container_path(safe_filename) - - async with await _get_sandbox_handle(str(current_user.id)) as handle: - await asyncio.to_thread(handle.adapter.mkdir, CONTAINER_UPLOADS_PATH) - result = await asyncio.to_thread(handle.adapter.write_overwrite, container_path, content) - if getattr(result, "error", None): - raise InternalServerException(f"Failed to write file: {result.error}") - - logger.info( - f"File uploaded to sandbox: user={current_user.id}, " - f"filename={safe_filename}, size={len(content)}, path={container_path}, ip={client_ip}" - ) - - return BaseResponse( - success=True, - code=200, - msg="File uploaded successfully", - data=UploadResponse( - filename=safe_filename, - path=container_path, - size=len(content), - message=f"File {safe_filename} has been uploaded to your working directory", - ), - ) - except AppException: - raise - except Exception as e: - logger.error( - f"Failed to upload file: user={current_user.id}, filename={original_filename}, ip={client_ip}, error={e}", - exc_info=True, - ) - raise InternalServerException("Failed to upload file, please try again later") from e - - -@router.get( - "/list", - response_model=BaseResponse[FileListResponse], - summary="List files", - description="List all files in the user's sandbox upload directory.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Failed to list files"}, - }, -) -async def list_files(current_user: CurrentUser) -> BaseResponse[FileListResponse]: - """List all files in the user's sandbox upload directory via adapter API.""" - try: - async with await _get_sandbox_handle(str(current_user.id)) as handle: - infos = await asyncio.to_thread(handle.adapter.ls_info, CONTAINER_UPLOADS_PATH) - - files = [ - FileInfo( - filename=Path(info["path"]).name, - size=info.get("size", 0), - path=info["path"], - ) - for info in infos - if not info.get("is_dir", False) - ] - - return BaseResponse( - success=True, - code=200, - msg="Fetched file list successfully", - data=FileListResponse(files=files, total=len(files)), - ) - except Exception as e: - logger.error(f"Failed to list files: {e}", exc_info=True) - raise InternalServerException("Failed to list files, please try again later") from e - - -@router.get( - "/read/{filename}", - response_model=BaseResponse[dict], - summary="Read file content", - description="Read the content of a file in the user's sandbox upload directory.", - responses={ - 404: {"description": "File not found"}, - 500: {"description": "Failed to read file"}, - }, -) -async def read_file(request: Request, filename: str, current_user: CurrentUser) -> BaseResponse[dict]: - """Read file content from the user's sandbox via adapter API.""" - client_ip = get_client_ip(request) - - try: - safe_filename = sanitize_filename(filename) - container_path = get_container_path(safe_filename) - - async with await _get_sandbox_handle(str(current_user.id)) as handle: - content = await asyncio.to_thread(handle.adapter.raw_read, container_path) - - if content.startswith("[Error:") or content.startswith("Error:"): - raise NotFoundException("File not found") - - # raw_read returns text; for binary files it may be garbled - is_binary = False - try: - content.encode("utf-8") - except UnicodeEncodeError: - content = base64.b64encode(content.encode("latin-1")).decode("ascii") - is_binary = True - - logger.info(f"File read: user={current_user.id}, filename={safe_filename}, ip={client_ip}") - - return BaseResponse( - success=True, - code=200, - msg="Read file successfully", - data={"filename": safe_filename, "content": content, "is_binary": is_binary}, - ) - except AppException: - raise - except Exception as e: - logger.error( - f"Failed to read file: user={current_user.id}, filename={filename}, ip={client_ip}, error={e}", - exc_info=True, - ) - raise InternalServerException("Failed to read file, please try again later") from e - - -@router.delete( - "/{filename}", - response_model=BaseResponse[dict], - summary="Delete file", - description="Delete a file from the user's sandbox upload directory.", - responses={ - 404: {"description": "File not found"}, - 500: {"description": "Failed to delete file"}, - }, -) -async def delete_file(request: Request, filename: str, current_user: CurrentUser) -> BaseResponse[dict]: - """Delete a file from the user's sandbox via adapter API.""" - client_ip = get_client_ip(request) - - try: - safe_filename = sanitize_filename(filename) - container_path = get_container_path(safe_filename) - - async with await _get_sandbox_handle(str(current_user.id)) as handle: - ok = await asyncio.to_thread(handle.adapter.delete, container_path) - - if not ok: - raise NotFoundException(f"File not found: {filename}") - - logger.info(f"File deleted: user={current_user.id}, filename={safe_filename}, ip={client_ip}") - - return BaseResponse( - success=True, - code=200, - msg="File deleted successfully", - data={"filename": safe_filename, "message": f"File {safe_filename} has been deleted"}, - ) - except AppException: - raise - except Exception as e: - logger.error( - f"Failed to delete file: user={current_user.id}, filename={filename}, ip={client_ip}, error={e}", - exc_info=True, - ) - raise InternalServerException("Failed to delete file, please try again later") from e - - -@router.delete( - "", - response_model=BaseResponse[dict], - summary="Clear all files", - description="Clear all files in the user's sandbox upload directory.", - responses={ - 401: {"description": "Unauthorized"}, - 500: {"description": "Failed to clear files"}, - }, -) -async def clear_all_files(request: Request, current_user: CurrentUser) -> BaseResponse[dict]: - """Clear all files in the user's sandbox upload directory via adapter API.""" - client_ip = get_client_ip(request) - - try: - async with await _get_sandbox_handle(str(current_user.id)) as handle: - await asyncio.to_thread(handle.adapter.execute, f"rm -rf {CONTAINER_UPLOADS_PATH}/*") - await asyncio.to_thread(handle.adapter.mkdir, CONTAINER_UPLOADS_PATH) - - logger.info(f"All files cleared: user={current_user.id}, ip={client_ip}") - - return BaseResponse( - success=True, - code=200, - msg="Cleared files successfully", - data={"message": "Cleared working directory"}, - ) - except AppException: - raise - except Exception as e: - logger.error(f"Failed to clear files: user={current_user.id}, ip={client_ip}, error={e}", exc_info=True) - raise InternalServerException("Failed to clear files, please try again later") from e diff --git a/backend/app/api/v1/graph_code.py b/backend/app/api/v1/graph_code.py deleted file mode 100644 index 9500b9af8..000000000 --- a/backend/app/api/v1/graph_code.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Graph Code API — save and run user LangGraph code. - -Routes are nested under ``/api/v1/graphs`` and add code-specific -operations as sub-resources of an existing graph: - -- ``POST /api/v1/graphs/{graph_id}/code/save`` — persist code -- ``POST /api/v1/graphs/{graph_id}/code/run`` — execute code and return result -""" - -import asyncio -import re -import uuid -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends -from loguru import logger -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import NotFoundException -from app.core.code_executor import execute_code -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.workspace import WorkspaceMemberRole -from app.services.graph_service import GraphService - -router = APIRouter(prefix="/v1/graphs", tags=["Graph Code"]) - -# Execution timeout for ainvoke (seconds) -RUN_TIMEOUT = 30.0 - - -# --------------------------------------------------------------------------- -# Request / response models -# --------------------------------------------------------------------------- - - -class CodeSaveRequest(BaseModel): - code: str = Field(..., description="Python code to save") - name: Optional[str] = Field(default=None, description="Optional graph name update") - - -class CodeRunRequest(BaseModel): - input: Optional[Dict[str, Any]] = Field( - default=None, - description="Initial state input for the graph", - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _sanitize_error(msg: str) -> str: - """Remove server file paths from error messages.""" - return re.sub(r"/[^\s\"']+/", "/", msg) - - -# --------------------------------------------------------------------------- -# Endpoints -# --------------------------------------------------------------------------- - - -@router.post("/{graph_id}/code/save") -async def save_code( - graph_id: uuid.UUID, - payload: CodeSaveRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Persist user code to ``graph.variables``. - - Requires member (write) permission. - """ - service = GraphService(db) - graph = await service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException(f"Graph {graph_id} not found") - - # Permission check: need member role to save - await service._ensure_access(graph, current_user, WorkspaceMemberRole.member) - - variables = dict(graph.variables or {}) - variables["graph_mode"] = "code" - variables["code_content"] = payload.code - graph.variables = variables - - if payload.name is not None: - graph.name = payload.name - - await db.commit() - - logger.info( - f"[GraphCodeAPI] Saved code | graph_id={graph_id} | code_len={len(payload.code)} | user={current_user.id}" - ) - return {"success": True} - - -@router.post("/{graph_id}/code/run") -async def run_code( - graph_id: uuid.UUID, - payload: CodeRunRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Execute user code: exec → StateGraph → compile → invoke. - - Requires viewer permission. Execution has a 30s timeout. - """ - service = GraphService(db) - graph = await service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException(f"Graph {graph_id} not found") - - # Permission check: need viewer role to run - await service._ensure_access(graph, current_user, WorkspaceMemberRole.viewer) - - code = (graph.variables or {}).get("code_content", "") - if not code.strip(): - return { - "success": False, - "message": "No code to execute. Save your code first.", - } - - try: - # Step 1: exec user code → get StateGraph (has its own 10s timeout) - state_graph = execute_code(code) - - # Step 2: compile - compiled = state_graph.compile() - - # Step 3: invoke with timeout - initial_state = payload.input or {} - result = await asyncio.wait_for( - compiled.ainvoke(initial_state), # type: ignore[arg-type] - timeout=RUN_TIMEOUT, - ) - - logger.info(f"[GraphCodeAPI] Code run success | graph_id={graph_id}") - return { - "success": True, - "data": { - "result": _serialize_result(result), - }, - } - - except SyntaxError as e: - return { - "success": False, - "message": f"Syntax error at line {e.lineno}: {e.msg}", - } - except ImportError as e: - return { - "success": False, - "message": str(e), - } - except TimeoutError: - return { - "success": False, - "message": "Execution timed out. Check for infinite loops or long-running operations.", - } - except ValueError as e: - return { - "success": False, - "message": str(e), - } - except Exception as e: - logger.error(f"[GraphCodeAPI] Code run failed | graph_id={graph_id} | error={e}") - return { - "success": False, - "message": _sanitize_error(f"Runtime error: {type(e).__name__}: {e}"), - } - - -def _serialize_result(result: Any) -> Any: - """Best-effort serialization of graph execution result.""" - if result is None: - return None - if isinstance(result, dict): - return {k: _serialize_result(v) for k, v in result.items()} - if isinstance(result, (list, tuple)): - return [_serialize_result(item) for item in result] - if isinstance(result, (str, int, float, bool)): - return result - try: - return str(result) - except Exception: - return f"<{type(result).__name__}>" diff --git a/backend/app/api/v1/graph_deployments.py b/backend/app/api/v1/graph_deployments.py deleted file mode 100644 index 6d0e1c797..000000000 --- a/backend/app/api/v1/graph_deployments.py +++ /dev/null @@ -1,205 +0,0 @@ -""" -Graph deployment version API -""" - -import uuid -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends, Request -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.schemas.graph_deployment_version import ( - GraphDeploymentVersionListResponse, - GraphDeploymentVersionResponseCamel, - GraphDeploymentVersionStateResponse, - GraphDeployRequest, - GraphRenameVersionRequest, - GraphRevertResponse, -) -from app.services.graph_deployment_version_service import GraphDeploymentVersionService - -router = APIRouter(prefix="/v1/graphs", tags=["Graph Deployments"]) - - -def _bind_log(request: Request, **kwargs): - trace_id = getattr(request.state, "trace_id", "-") - return logger.bind(trace_id=trace_id, **kwargs) - - -@router.get("/{graph_id}/deploy", response_model=Dict[str, Any]) -async def get_deployment_status( - graph_id: uuid.UUID, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Get deployment status.""" - log = _bind_log(request, graph_id=str(graph_id)) - log.info("Getting deployment status for graph: {}", graph_id) - - service = GraphDeploymentVersionService(db) - return await service.get_deployment_status(graph_id, current_user) - - -@router.post("/{graph_id}/deploy", response_model=Dict[str, Any]) -async def deploy_graph( - graph_id: uuid.UUID, - request: Request, - body: Optional[GraphDeployRequest] = None, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Deploy a graph.""" - log = _bind_log(request, graph_id=str(graph_id)) - log.info("Deploying graph: {}", graph_id) - - service = GraphDeploymentVersionService(db) - name = body.name if body else None - - result = await service.deploy(graph_id, current_user, name) - - return { - "success": result.success, - "message": result.message, - "version": result.version, - "isActive": result.isActive, - "needsRedeployment": result.needsRedeployment, - } - - -@router.delete("/{graph_id}/deploy", response_model=Dict[str, Any]) -async def undeploy_graph( - graph_id: uuid.UUID, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Undeploy a graph.""" - log = _bind_log(request, graph_id=str(graph_id)) - log.info("Undeploying graph: {}", graph_id) - - service = GraphDeploymentVersionService(db) - return await service.undeploy(graph_id, current_user) - - -@router.get("/{graph_id}/deployments", response_model=GraphDeploymentVersionListResponse) -async def list_deployment_versions( - graph_id: uuid.UUID, - request: Request, - page: int = 1, - page_size: int = 10, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """List all deployment versions (paginated).""" - log = _bind_log(request, graph_id=str(graph_id)) - log.info("Listing deployment versions for graph: {} (page={}, page_size={})", graph_id, page, page_size) - - service = GraphDeploymentVersionService(db) - return await service.list_versions(graph_id, current_user, page=page, page_size=page_size) - - -@router.get("/{graph_id}/deployments/{version}", response_model=GraphDeploymentVersionResponseCamel) -async def get_deployment_version( - graph_id: uuid.UUID, - version: int, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Get a specific deployment version.""" - log = _bind_log(request, graph_id=str(graph_id), version=version) - log.info("Getting deployment version: {} v{}", graph_id, version) - - service = GraphDeploymentVersionService(db) - return await service.get_version(graph_id, version, current_user) - - -@router.get("/{graph_id}/deployments/{version}/state", response_model=GraphDeploymentVersionStateResponse) -async def get_deployment_version_state( - graph_id: uuid.UUID, - version: int, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Get a specific deployment version's full state (for preview).""" - log = _bind_log(request, graph_id=str(graph_id), version=version) - log.info("Getting deployment version state: {} v{}", graph_id, version) - - service = GraphDeploymentVersionService(db) - return await service.get_version_state(graph_id, version, current_user) - - -@router.patch("/{graph_id}/deployments/{version}", response_model=GraphDeploymentVersionResponseCamel) -async def rename_deployment_version( - graph_id: uuid.UUID, - version: int, - body: GraphRenameVersionRequest, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Rename a deployment version.""" - log = _bind_log(request, graph_id=str(graph_id), version=version) - log.info("Renaming deployment version: {} v{} to '{}'", graph_id, version, body.name) - - service = GraphDeploymentVersionService(db) - return await service.rename_version(graph_id, version, body.name, current_user) - - -@router.post("/{graph_id}/deployments/{version}/activate", response_model=Dict[str, Any]) -async def activate_deployment_version( - graph_id: uuid.UUID, - version: int, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Activate a deployment version.""" - log = _bind_log(request, graph_id=str(graph_id), version=version) - log.info("Activating deployment version: {} v{}", graph_id, version) - - service = GraphDeploymentVersionService(db) - activated = await service.activate_version(graph_id, version, current_user) - - return { - "success": True, - "deployedAt": activated.createdAt, - } - - -@router.post("/{graph_id}/deployments/{version}/revert", response_model=GraphRevertResponse) -async def revert_to_deployment_version( - graph_id: uuid.UUID, - version: int, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Revert to a specific deployment version.""" - log = _bind_log(request, graph_id=str(graph_id), version=version) - log.info("Reverting to deployment version: {} v{}", graph_id, version) - - service = GraphDeploymentVersionService(db) - return await service.revert_to_version(graph_id, version, current_user) - - -@router.delete("/{graph_id}/deployments/{version}", response_model=Dict[str, Any]) -async def delete_deployment_version( - graph_id: uuid.UUID, - version: int, - request: Request, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Delete a deployment version.""" - log = _bind_log(request, graph_id=str(graph_id), version=version) - log.info("Deleting deployment version: {} v{}", graph_id, version) - - service = GraphDeploymentVersionService(db) - return await service.delete_version(graph_id, version, current_user) diff --git a/backend/app/api/v1/graphs.py b/backend/app/api/v1/graphs.py deleted file mode 100644 index 215538b04..000000000 --- a/backend/app/api/v1/graphs.py +++ /dev/null @@ -1,639 +0,0 @@ -""" -Graph API (path: /api/v1/graphs) -""" - -import time -import uuid -from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, Depends, Query, Request -from loguru import logger -from pydantic import BaseModel, Field -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import ForbiddenException, NotFoundException -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.graph import AgentGraph, GraphNode -from app.models.workspace import WorkspaceMemberRole -from app.repositories.agent_run import AgentRunRepository -from app.repositories.workspace import WorkspaceRepository -from app.services.graph_service import GraphService - -router = APIRouter(prefix="/v1/graphs", tags=["Graphs"]) - - -def _bind_log(request: Request, **kwargs): - trace_id = getattr(request.state, "trace_id", "-") - return logger.bind(trace_id=trace_id, **kwargs) - - -class GraphStatePayload(BaseModel): - """Graph state payload.""" - - nodes: List[Dict[str, Any]] = Field(default_factory=list, description="Node list") - edges: List[Dict[str, Any]] = Field(default_factory=list, description="Edge list") - viewport: Optional[Dict[str, Any]] = Field(default=None, description="Viewport info") - variables: Optional[Dict[str, Any]] = Field(default=None, description="Graph variables (e.g. context variables)") - # optional graph creation params (for upsert mode) - name: Optional[str] = Field(default=None, max_length=200, description="Graph name (for creating a new graph)") - workspaceId: Optional[uuid.UUID] = Field(default=None, description="Workspace ID (for creating a new graph)") - - -class CreateGraphRequest(BaseModel): - """Create graph request.""" - - name: str = Field(..., min_length=1, max_length=200, description="Graph name") - description: Optional[str] = Field(default=None, max_length=2000, description="Graph description") - color: Optional[str] = Field(default=None, max_length=2000, description="Color") - workspaceId: Optional[uuid.UUID] = Field(default=None, description="Workspace ID") - folderId: Optional[uuid.UUID] = Field(default=None, description="Folder ID") - parentId: Optional[uuid.UUID] = Field(default=None, description="Parent graph ID") - variables: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Variables") - - -class UpdateGraphRequest(BaseModel): - """Update graph request.""" - - name: Optional[str] = Field(default=None, min_length=1, max_length=200, description="Graph name") - description: Optional[str] = Field(default=None, max_length=2000, description="Graph description") - color: Optional[str] = Field(default=None, max_length=2000, description="Color") - folderId: Optional[uuid.UUID] = Field(default=None, description="Folder ID") - parentId: Optional[uuid.UUID] = Field(default=None, description="Parent graph ID") - isDeployed: Optional[bool] = Field(default=None, description="Whether deployed") - - -async def _ensure_workspace_member( - *, - db: AsyncSession, - workspace_id: uuid.UUID, - current_user: User, - min_role: WorkspaceMemberRole, -) -> None: - """ - Ensure the user is a workspace member with sufficient permissions. - - Args: - db: database session - workspace_id: workspace ID - current_user: current user - min_role: minimum required role - - Raises: - NotFoundException: if the workspace does not exist - ForbiddenException: if the user is not a member or lacks permission - """ - from app.services.workspace_permission import check_workspace_access - - # check workspace existence - workspace_repo = WorkspaceRepository(db) - workspace = await workspace_repo.get(workspace_id) - if not workspace: - raise NotFoundException("Workspace not found") - - # check access permission - has_access = await check_workspace_access(db, workspace_id, current_user, min_role) - if not has_access: - raise ForbiddenException("No access to workspace or insufficient permission") - - -def _serialize_graph_row(graph: AgentGraph, node_count: int = 0) -> Dict[str, Any]: - """ - Serialize a graph object to a dict. - - Args: - graph: graph object - node_count: node count (optional) - - Returns: - Serialized dict - """ - return { - "id": str(graph.id), - "userId": str(graph.user_id), - "workspaceId": str(graph.workspace_id) if graph.workspace_id else None, - "folderId": str(graph.folder_id) if graph.folder_id else None, - "parentId": str(graph.parent_id) if graph.parent_id else None, - "name": graph.name, - "description": graph.description, - "color": graph.color, - "isDeployed": graph.is_deployed, - "variables": graph.variables or {}, - "createdAt": graph.created_at.isoformat() if graph.created_at else None, - "updatedAt": graph.updated_at.isoformat() if graph.updated_at else None, - "nodeCount": node_count, - } - - -@router.get("") -async def list_graphs( - request: Request, - workspace_id: Optional[uuid.UUID] = Query(default=None, alias="workspaceId"), - parent_id: Optional[uuid.UUID] = Query(default=None, alias="parentId"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - List graphs. - - Filtering logic: - - Default (no workspace_id): list all graphs owned by the current user (personal graphs) - - If workspace_id is provided: - - Check that the user has access to the workspace (at least viewer) - - If authorized, return all graphs in the workspace (not limited to user-created ones) - - If unauthorized, return an empty list - - If parentId is provided: list sub-graphs under the specified parent graph - """ - service = GraphService(db) - - log = _bind_log(request, user_id=str(current_user.id)) - - log.info(f"graph.list start workspace_id={workspace_id} parent_id={parent_id}") - - # if workspace_id is provided, check permission and return all workspace graphs - if workspace_id: - # check that the user has access (at least viewer) - await _ensure_workspace_member( - db=db, - workspace_id=workspace_id, - current_user=current_user, - min_role=WorkspaceMemberRole.viewer, - ) - # return all graphs in the workspace (not limited to user) - query = select(AgentGraph).where( - AgentGraph.workspace_id == workspace_id, - AgentGraph.deleted_at.is_(None), - ) - if parent_id is not None: - query = query.where(AgentGraph.parent_id == parent_id) - query = query.order_by(AgentGraph.created_at.desc(), AgentGraph.id.desc()) - result = await db.execute(query) - graphs = list(result.scalars().all()) - else: - # no workspace_id — return user-owned graphs (personal graphs) - graphs = await service.graph_repo.list_by_user_with_filters( - user_id=current_user.id, - parent_id=parent_id, - workspace_id=None, - ) - - # batch-query node counts for each graph - graph_ids = [graph.id for graph in graphs] - node_counts: Dict[Any, int] = {} - if graph_ids: - # use GROUP BY to query all node counts in one shot - count_query = ( - select(GraphNode.graph_id, func.count(GraphNode.id).label("count")) - .where(GraphNode.graph_id.in_(graph_ids)) - .group_by(GraphNode.graph_id) - ) - result = await db.execute(count_query) - for row in result: - count_val = getattr(row, "count", 0) - node_counts[row.graph_id] = int(count_val) if not callable(count_val) else count_val() # type: ignore[call-overload] - - log.info(f"graph.list success count={len(graphs)}") - return {"data": [_serialize_graph_row(graph, node_counts.get(graph.id, 0)) for graph in graphs]} - - -@router.get("/deployed") -async def list_deployed_graphs( - request: Request, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - List deployed graphs accessible to the current user. - - Includes: - 1. Deployed graphs created by the user - 2. Deployed graphs in workspaces the user has access to (at least viewer) - - Filters: - - is_deployed = True - - User is the graph owner, or user has workspace access - """ - from sqlalchemy import or_ - - from app.models.workspace import WorkspaceMemberRole - from app.repositories.workspace import WorkspaceRepository - from app.services.workspace_permission import check_workspace_access - - log = _bind_log(request, user_id=str(current_user.id)) - log.info("graph.list_deployed start") - - workspace_repo = WorkspaceRepository(db) - user_workspaces = await workspace_repo.list_for_user(current_user.id) - accessible_workspace_ids = [ws.id for ws in user_workspaces] - - conditions = [AgentGraph.is_deployed, AgentGraph.deleted_at.is_(None)] - - user_owned_condition = AgentGraph.user_id == str(current_user.id) - - if accessible_workspace_ids: - workspace_condition = AgentGraph.workspace_id.in_(accessible_workspace_ids) - graph_condition = or_(user_owned_condition, workspace_condition) - else: - graph_condition = user_owned_condition - - conditions.append(graph_condition) # type: ignore[arg-type] - - query = select(AgentGraph).where(*conditions).order_by(AgentGraph.created_at.desc()) - result = await db.execute(query) - all_graphs = list(result.scalars().all()) - - filtered_graphs = [] - for graph in all_graphs: - if graph.user_id == str(current_user.id): - filtered_graphs.append(graph) - elif graph.workspace_id: - has_access = await check_workspace_access( - db, - graph.workspace_id, - current_user, - WorkspaceMemberRole.viewer, - ) - if has_access: - filtered_graphs.append(graph) - - graphs = filtered_graphs - - graph_ids = [graph.id for graph in graphs] - node_counts: Dict[Any, int] = {} - if graph_ids: - count_query = ( - select(GraphNode.graph_id, func.count(GraphNode.id).label("count")) - .where(GraphNode.graph_id.in_(graph_ids)) - .group_by(GraphNode.graph_id) - ) - result = await db.execute(count_query) - for row in result: - count_val = getattr(row, "count", 0) - node_counts[row.graph_id] = int(count_val) if not callable(count_val) else count_val() # type: ignore[call-overload] - - log.info(f"graph.list_deployed success count={len(graphs)}") - return {"data": [_serialize_graph_row(graph, node_counts.get(graph.id, 0)) for graph in graphs]} - - -@router.post("") -async def create_graph( - request: Request, - payload: CreateGraphRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Create a new graph. - - - personal graph: created by the current user - - workspace graph: requires workspace write permission (member+) - """ - log = _bind_log(request, user_id=str(current_user.id)) - parent_id = payload.parentId - workspace_id = payload.workspaceId - - # workspace_id may be None (personal graph) - - if workspace_id: - await _ensure_workspace_member( - db=db, - workspace_id=workspace_id, - current_user=current_user, - min_role=WorkspaceMemberRole.member, - ) - - service = GraphService(db) - graph = await service.create_graph( - name=payload.name.strip(), - user_id=current_user.id, - workspace_id=workspace_id, - folder_id=payload.folderId, - parent_id=parent_id, - description=payload.description.strip() if payload.description else None, - color=payload.color, - variables=payload.variables, - ) - await db.commit() - log.info(f"graph.create success graph_id={graph.id} workspace_id={workspace_id} parent_id={parent_id}") - return {"data": _serialize_graph_row(graph)} - - -@router.get("/{graph_id}") -async def get_graph( - graph_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Get graph details including nodes and edges. - - Returns: - { - "data": { - "id": "...", - "name": "...", - "nodes": [...], - "edges": [...], - "viewport": {...}, - ... - } - } - """ - service = GraphService(db) - data = await service.get_graph_detail(graph_id, current_user) - return {"data": data} - - -@router.put("/{graph_id}") -async def update_graph( - graph_id: uuid.UUID, - payload: UpdateGraphRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update graph metadata (name/description/color/folderId/parentId/isDeployed).""" - service = GraphService(db) - graph = await service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException("Graph not found") - - # permission check - await service._ensure_access(graph, current_user, WorkspaceMemberRole.member) - - update_data: Dict[str, Any] = {} - fields_set: set[str] = getattr(payload, "model_fields_set", set()) - - if payload.name is not None: - update_data["name"] = payload.name.strip() - if "description" in fields_set: - update_data["description"] = payload.description - if payload.color is not None: - update_data["color"] = payload.color - if "folderId" in fields_set: - # if folderId is provided, verify it exists and belongs to the current workspace - if payload.folderId is not None: - from app.repositories.workspace_folder import WorkflowFolderRepository - - folder_repo = WorkflowFolderRepository(db) - folder = await folder_repo.get(payload.folderId) - if not folder: - raise NotFoundException(f"Folder with id {payload.folderId} not found") - # ensure folder belongs to the graph's workspace - if graph.workspace_id and folder.workspace_id != graph.workspace_id: - from app.common.exceptions import BadRequestException - - raise BadRequestException( - f"Folder {payload.folderId} does not belong to workspace {graph.workspace_id}" - ) - # allow setting to None to clear the folder association - update_data["folder_id"] = payload.folderId - if "parentId" in fields_set: - # if parentId is provided, verify it exists (allow None to clear the parent relationship) - if payload.parentId is not None: - parent_graph = await service.graph_repo.get(payload.parentId) - if not parent_graph: - raise NotFoundException(f"Parent graph with id {payload.parentId} not found") - # allow setting to None to clear the parent graph relationship - update_data["parent_id"] = payload.parentId - if payload.isDeployed is not None: - update_data["is_deployed"] = payload.isDeployed - - if update_data: - await service.graph_repo.update(graph_id, update_data) - await db.commit() - - graph2 = await service.graph_repo.get(graph_id) - if not graph2: - raise NotFoundException("Graph not found") - return {"data": _serialize_graph_row(graph2)} - - -@router.delete("/{graph_id}") -async def delete_graph( - graph_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete a graph (requires member permission, i.e. write access).""" - service = GraphService(db) - graph = await service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException("Graph not found") - - # permission check: - # - personal graph: only the owner can delete - # - workspace graph: requires at least member permission (write access) - if graph.workspace_id: - # workspace graph: requires member permission to delete - await service._ensure_access( - graph, - current_user, - required_role=WorkspaceMemberRole.member, - ) - else: - # personal graph: only the owner can delete - if graph.user_id != current_user.id: - raise ForbiddenException("Only graph owner can delete personal graph") - - await service.graph_repo.soft_delete(graph_id) - await db.commit() - return {"success": True} - - -@router.get("/{graph_id}/state") -async def load_graph_state( - graph_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Load graph state (nodes and edges). - - Returns: - { - "success": true, - "data": { - "nodes": [...], - "edges": [...], - "viewport": {...} - } - } - """ - service = GraphService(db) - state = await service.load_graph_state(graph_id, current_user) - return {"success": True, "data": state} - - -@router.post("/{graph_id}/state") -async def save_graph_state( - request: Request, - graph_id: uuid.UUID, - payload: GraphStatePayload, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Save graph state (nodes and edges) — supports upsert mode. - - If the graph does not exist, automatically create a new one (requires name parameter). - - Accepts frontend format: - { - "nodes": [...], - "edges": [...], - "viewport": {...}, - "name": "optional, for creating a new graph", - "workspaceId": "optional, for creating a new graph" - } - """ - log = _bind_log(request, user_id=str(current_user.id), graph_id=str(graph_id)) - service = GraphService(db) - - # workspace_id may be None (personal graph) - workspace_id = payload.workspaceId - - result = await service.save_graph_state( - graph_id=graph_id, - nodes=payload.nodes, - edges=payload.edges, - viewport=payload.viewport, - variables=payload.variables, - current_user=current_user, - # upsert parameters - name=payload.name, - workspace_id=workspace_id, - ) - - # explicitly commit the transaction to persist data - # note: get_db() does not auto-commit; an explicit commit() is required - await db.commit() - - log.info(f"graph.state.save success nodes={len(payload.nodes)} edges={len(payload.edges)}") - return {"success": True, **result} - - -@router.put("/{graph_id}/state") -async def save_graph_state_put( - request: Request, - graph_id: uuid.UUID, - payload: GraphStatePayload, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """PUT alias for saving graph state.""" - return await save_graph_state(request, graph_id, payload, db, current_user) - - -# ==================== Compile (pre-build + cache warm) ==================== - - -@router.post("/{graph_id}/compile") -async def compile_graph( - request: Request, - graph_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> Dict[str, Any]: - """Compile a graph and warm the cache; return build time. Execution uses the cache if not expired.""" - service = GraphService(db) - graph = await service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException("Graph not found") - await service._ensure_access(graph, current_user, WorkspaceMemberRole.viewer) - start = time.time() - try: - await service.create_graph_by_graph_id( - graph_id=graph_id, - user_id=current_user.id, - current_user=current_user, - ) - build_time_ms = (time.time() - start) * 1000 - return {"ok": True, "build_time_ms": round(build_time_ms, 2)} - except Exception as e: - log = _bind_log(request, user_id=str(current_user.id), graph_id=str(graph_id)) - log.error(f"graph.compile failed: {e}") - raise - - -# ==================== Copilot Endpoints ==================== - - -@router.get("/{graph_id}/copilot/history") -async def get_copilot_history( - request: Request, - graph_id: uuid.UUID, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - log = _bind_log(request, user_id=str(current_user.id), graph_id=str(graph_id)) - log.info("copilot.history.get start") - - graph_service = GraphService(db) - graph = await graph_service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException("Graph not found") - await graph_service._ensure_access(graph, current_user, WorkspaceMemberRole.viewer) - - repo = AgentRunRepository(db) - runs = await repo.list_recent_runs_for_user( - user_id=str(current_user.id), - agent_name="copilot", - graph_id=graph_id, - limit=100, - ) - - messages = [] - for run in reversed(list(runs)): # oldest first - snapshot = await repo.get_snapshot(run.id) - if not snapshot or not snapshot.projection: - continue - p = snapshot.projection - messages.append( - { - "role": "user", - "content": run.title or "", - "created_at": run.created_at.isoformat() if run.created_at else None, - } - ) - messages.append( - { - "role": "assistant", - "content": p.get("result_message") or p.get("content", ""), - "created_at": run.updated_at.isoformat() if run.updated_at else None, - "actions": p.get("result_actions", []), - "thought_steps": p.get("thought_steps", []), - "tool_calls": p.get("tool_calls", []), - } - ) - - log.info(f"copilot.history.get success messages_count={len(messages)}") - return {"data": {"graph_id": str(graph_id), "messages": messages}} - - -@router.delete("/{graph_id}/copilot/history") -async def clear_copilot_history( - request: Request, - graph_id: uuid.UUID, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - log = _bind_log(request, user_id=str(current_user.id), graph_id=str(graph_id)) - log.info("copilot.history.clear start") - - graph_service = GraphService(db) - graph = await graph_service.graph_repo.get(graph_id) - if not graph: - raise NotFoundException("Graph not found") - await graph_service._ensure_access(graph, current_user, WorkspaceMemberRole.member) - - repo = AgentRunRepository(db) - deleted = await repo.delete_runs_for_graph( - user_id=str(current_user.id), - agent_name="copilot", - graph_id=graph_id, - ) - - log.info(f"copilot.history.clear success deleted={deleted}") - return {"success": True} diff --git a/backend/app/api/v1/mcp.py b/backend/app/api/v1/mcp.py deleted file mode 100644 index cba65c8e1..000000000 --- a/backend/app/api/v1/mcp.py +++ /dev/null @@ -1,408 +0,0 @@ -""" -MCP Server API - MCP server management - -Follows project conventions: -- Query params: snake_case (alias supports camelCase) -- Response body: camelCase (frontend compatible) -- Return format: {"success": True, "data": ...} -""" - -from typing import Any, Dict, Optional -from uuid import UUID - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import BadRequestException, NotFoundException -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.mcp_client_service import McpConnectionConfig, get_mcp_client -from app.services.tool_service import ToolService - -router = APIRouter(prefix="/v1/mcp", tags=["MCP Servers"]) - - -# ==================== Request Schemas ==================== - - -class McpServerCreateRequest(BaseModel): - """Create MCP server.""" - - model_config = {"populate_by_name": True} - - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - transport: str = "streamable-http" - url: Optional[str] = None - headers: Dict[str, str] = Field(default_factory=dict) - timeout: int = Field(default=30000, ge=1000, le=300000) - retries: int = Field(default=3, ge=0, le=10) - enabled: bool = True - - -class McpServerUpdateRequest(BaseModel): - """Update MCP server.""" - - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - transport: Optional[str] = None - url: Optional[str] = None - headers: Optional[Dict[str, str]] = None - timeout: Optional[int] = Field(None, ge=1000, le=300000) - retries: Optional[int] = Field(None, ge=0, le=10) - enabled: Optional[bool] = None - - -class ToggleRequest(BaseModel): - enabled: bool - - -class McpTestRequest(BaseModel): - """Connection test request.""" - - transport: str = "streamable-http" - url: Optional[str] = None - headers: Optional[Dict[str, str]] = None - timeout: int = 30000 - - -class McpToolExecuteRequest(BaseModel): - """Tool execution request. - - Uses serverName to look up the server (unique per user). - """ - - serverName: str = Field(..., description="Server name (unique per user)") - toolName: str - arguments: Dict[str, Any] = Field(default_factory=dict) - - -# ==================== Serialization (camelCase for frontend) ==================== - - -def _serialize_server(server) -> Dict[str, Any]: - """Serialize a server to a camelCase response.""" - return { - "id": str(server.id), - "name": server.name, - "description": server.description, - "transport": server.transport, - "url": server.url, - "headers": server.headers or {}, - "timeout": server.timeout or 30000, - "retries": server.retries or 3, - "enabled": server.enabled, - "connectionStatus": server.connection_status, - "lastConnected": server.last_connected.isoformat() if server.last_connected else None, - "lastError": server.last_error, - "toolCount": server.tool_count or 0, - "createdAt": server.created_at.isoformat() if server.created_at else None, - "updatedAt": server.updated_at.isoformat() if server.updated_at else None, - } - - -def _serialize_tool(tool_info) -> Dict[str, Any]: - """Serialize a tool to a camelCase response.""" - display_name = tool_info.label_name or tool_info.name - return { - "id": tool_info.id, - "name": tool_info.name, - "labelName": display_name, - "label": display_name.replace("_", " ").title(), - "description": tool_info.description, - "toolType": tool_info.tool_type, - "category": tool_info.category, - "tags": tool_info.tags, - "mcpServer": tool_info.mcp_server, - "mcpToolName": tool_info.mcp_tool_name, - "enabled": tool_info.enabled, - } - - -# ==================== Server CRUD ==================== - - -@router.get("/servers") -async def list_mcp_servers( - enabled_only: bool = Query(False, alias="enabledOnly"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List MCP servers for the current user (user-level).""" - service = ToolService(db) - servers = await service.list_mcp_servers( - user_id=current_user.id, - enabled_only=enabled_only, - ) - - return {"success": True, "data": {"servers": [_serialize_server(s) for s in servers]}} - - -@router.post("/servers") -async def create_mcp_server( - request: McpServerCreateRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create an MCP server.""" - from loguru import logger - - from app.schemas.mcp import McpServerCreate - - logger.info(f"[MCP] Creating server - user_id={current_user.id}, name={request.name}") - - service = ToolService(db) - - server = await service.create_mcp_server( - user_id=current_user.id, - data=McpServerCreate( - name=request.name, - description=request.description, - transport=request.transport, - url=request.url, - headers=request.headers, - timeout=request.timeout, - retries=request.retries, - enabled=request.enabled, - ), - ) - - logger.info(f"[MCP] Server created - id={server.id}, name={server.name}") - return {"success": True, "data": {"serverId": str(server.id)}} - - -@router.get("/servers/{server_id}") -async def get_mcp_server( - server_id: UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get MCP server details.""" - service = ToolService(db) - server = await service.get_mcp_server(server_id=server_id, user_id=current_user.id) - - return {"success": True, "data": _serialize_server(server)} - - -@router.put("/servers/{server_id}") -async def update_mcp_server( - server_id: UUID, - request: McpServerUpdateRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update an MCP server.""" - from app.schemas.mcp import McpServerUpdate - - service = ToolService(db) - server = await service.update_mcp_server( - server_id=server_id, - user_id=current_user.id, - data=McpServerUpdate( - name=request.name, - description=request.description, - transport=request.transport, - url=request.url, - headers=request.headers, - timeout=request.timeout, - retries=request.retries, - enabled=request.enabled, - ), - ) - - return {"success": True, "data": _serialize_server(server)} - - -@router.delete("/servers/{server_id}") -async def delete_mcp_server( - server_id: UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete an MCP server.""" - service = ToolService(db) - await service.delete_mcp_server(server_id=server_id, user_id=current_user.id) - - return {"success": True, "data": None} - - -# ==================== Server Actions ==================== - - -@router.post("/servers/{server_id}/toggle") -async def toggle_mcp_server( - server_id: UUID, - request: ToggleRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Enable/disable an MCP server.""" - service = ToolService(db) - server = await service.toggle_mcp_server( - server_id=server_id, - user_id=current_user.id, - enabled=request.enabled, - ) - - return {"success": True, "data": _serialize_server(server)} - - -@router.post("/servers/{server_id}/test") -async def test_server_connection( - server_id: UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Test an existing server's connection.""" - service = ToolService(db) - result = await service.test_connection(server_id=server_id, user_id=current_user.id) - - return { - "success": True, - "data": { - "success": result.success, - "message": result.message, - "toolCount": result.tool_count, - "tools": result.tools, - "latencyMs": result.latency_ms, - }, - } - - -@router.post("/servers/{server_id}/refresh") -async def refresh_server_tools( - server_id: UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Refresh a server's tool list.""" - service = ToolService(db) - tools = await service.refresh_server_tools(server_id=server_id, user_id=current_user.id) - - return {"success": True, "data": [_serialize_tool(t) for t in tools]} - - -@router.get("/servers/{server_id}/tools") -async def list_server_tools( - server_id: UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List a server's tools.""" - service = ToolService(db) - tools = await service.get_server_tools(server_id=server_id, user_id=current_user.id) - - return {"success": True, "data": [_serialize_tool(t) for t in tools]} - - -# ==================== Connection Test & Tools ==================== - - -@router.post("/test") -async def test_connection( - request: McpTestRequest, - current_user: User = Depends(get_current_user), -): - """Test connection (before creation).""" - mcp_client = get_mcp_client() - config = McpConnectionConfig( - url=request.url or "", - transport=request.transport, - timeout_seconds=request.timeout // 1000, - headers=request.headers or {}, - ) - - # For test connection before creation, pass None (will create temporary server) - result = await mcp_client.test_connection(config, server=None) - - return { - "success": True, - "data": { - "success": result.success, - "error": result.error, - "tools": [{"name": t.name, "description": t.description or ""} for t in result.tools], - "latencyMs": result.latency_ms, - }, - } - - -@router.get("/tools") -async def discover_tools( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Discover all MCP tools for the current user (user-level).""" - service = ToolService(db) - - servers = await service.list_mcp_servers( - user_id=current_user.id, - enabled_only=True, - ) - - all_tools = [] - for server in servers: - tools = await service.get_server_tools(server_id=server.id, user_id=current_user.id) - for tool in tools: - all_tools.append( - { - "serverName": server.name, - "name": tool.name, - "labelName": tool.label_name or tool.name, - "description": tool.description, - } - ) - - return {"success": True, "data": {"tools": all_tools}} - - -@router.post("/tools/execute") -async def execute_tool( - request: McpToolExecuteRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Execute an MCP tool. - - Use serverName to find the real MCP server instance, validating permissions and status. - Ensure the real server instance and tool_name are used for execution. - """ - from loguru import logger - - from app.core.tools.mcp_tool_utils import get_mcp_tool_with_instance - - logger.debug( - f"[execute_tool] Executing MCP tool: serverName={request.serverName}, " - f"toolName={request.toolName}, user_id={current_user.id}" - ) - - # Get tool with instance validation (validates server exists, enabled, and user permissions) - tool = await get_mcp_tool_with_instance( - server_name=request.serverName, - tool_name=request.toolName, - user_id=current_user.id, - db=db, - ) - - if not tool: - # get_mcp_tool_with_instance already logs detailed warnings - raise NotFoundException( - f"MCP tool '{request.toolName}' not found on server '{request.serverName}' or server is not accessible" - ) - - try: - logger.debug( - f"[execute_tool] Invoking tool '{request.toolName}' on server '{request.serverName}' " - f"with arguments: {request.arguments}" - ) - result = await tool.ainvoke(request.arguments) - logger.debug(f"[execute_tool] Tool '{request.toolName}' executed successfully on server '{request.serverName}'") - return {"success": True, "data": result} - except Exception as e: - logger.error( - f"[execute_tool] Tool execution failed: serverName={request.serverName}, " - f"toolName={request.toolName}, error={str(e)}", - exc_info=True, - ) - raise BadRequestException(f"Tool execution failed: {str(e)}") diff --git a/backend/app/api/v1/memory/__init__.py b/backend/app/api/v1/memory/__init__.py deleted file mode 100644 index f9ccb7e0c..000000000 --- a/backend/app/api/v1/memory/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from app.api.v1.memory.memory import router - -__all__ = ["router"] diff --git a/backend/app/api/v1/memory/memory.py b/backend/app/api/v1/memory/memory.py deleted file mode 100644 index 8b62aa705..000000000 --- a/backend/app/api/v1/memory/memory.py +++ /dev/null @@ -1,476 +0,0 @@ -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional -from uuid import uuid4 - -from fastapi import Depends, Path, Query, Request -from fastapi.routing import APIRouter -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.schemas import ( - BadRequestResponse, - InternalServerErrorResponse, - NotFoundResponse, - PaginatedResponse, - PaginationInfo, - SortOrder, - UnauthenticatedResponse, - ValidationErrorResponse, -) -from app.api.v1.memory.schemas import ( - DeleteMemoriesRequest, - OptimizeMemoriesRequest, - OptimizeMemoriesResponse, - UserMemoryCreateSchema, - UserMemorySchema, -) -from app.common.dependencies import get_current_user -from app.common.exceptions import ( - AppException, - BadRequestException, - InternalServerException, - NotFoundException, - ValidationException, -) -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.schemas.memory import UserMemory -from app.services.memory_service import MemoryService - -# Create router and attach routes using MemoryService (async) -router = APIRouter( - prefix="/v1/memory", - dependencies=[Depends(get_current_user)], - tags=["memory"], - responses={ - 400: {"description": "Bad Request", "model": BadRequestResponse}, - 401: {"description": "Unauthorized", "model": UnauthenticatedResponse}, - 404: {"description": "Not Found", "model": NotFoundResponse}, - 422: {"description": "Validation Error", "model": ValidationErrorResponse}, - 500: {"description": "Internal Server Error", "model": InternalServerErrorResponse}, - }, -) - - -def _normalize_memory_dict(mem: Dict[str, Any]) -> Dict[str, Any]: - """Convert raw DB dict to API-friendly dict for UserMemorySchema.from_dict.""" - norm = dict(mem) - # Ensure updated_at is datetime - updated_at = norm.get("updated_at") - if isinstance(updated_at, (int, float)) and updated_at is not None: - norm["updated_at"] = datetime.fromtimestamp(updated_at, tz=timezone.utc) - elif isinstance(updated_at, str) and updated_at: - try: - norm["updated_at"] = datetime.fromisoformat(updated_at) - except Exception: - # Fallback: leave as-is; pydantic may try to coerce - pass - # Ensure user_id is string to satisfy schema - if "user_id" in norm and norm["user_id"] is not None: - norm["user_id"] = str(norm["user_id"]) - return norm - - -def parse_topics( - topics: Optional[str] = Query( - default=None, - description="Comma-separated list of topics to filter by", - examples=["preferences,technical,communication_style"], - ), -) -> Optional[List[str]]: - """Parse comma-separated topics into a list for filtering memories by topic.""" - if not topics: - return None - - try: - # Split by comma and strip whitespace, filter out empty strings - return [topic.strip() for topic in topics.split(",") if topic.strip()] - - except Exception as e: - raise ValidationException(f"Invalid topics format: {e}") - - -@router.post( - "/memories", - response_model=UserMemorySchema, - status_code=200, - operation_id="create_memory", - summary="Create Memory", - description=( - "Create a new user memory with content and associated topics. " - "Memories are used to store contextual information for users across conversations." - ), - responses={ - 200: { - "description": "Memory created successfully", - "content": { - "application/json": { - "example": { - "memory_id": "mem-123", - "memory": "User prefers technical explanations with code examples", - "topics": ["preferences", "communication_style", "technical"], - "user_id": "user-456", - "created_at": "2024-01-15T10:30:00Z", - "updated_at": "2024-01-15T10:30:00Z", - } - } - }, - }, - 400: {"description": "Invalid request data", "model": BadRequestResponse}, - 422: {"description": "Validation error in payload", "model": ValidationErrorResponse}, - 500: {"description": "Failed to create memory", "model": InternalServerErrorResponse}, - }, -) -async def create_memory( - request: Request, - payload: UserMemoryCreateSchema, - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> UserMemorySchema: - payload.user_id = str(current_user.id) - - db = MemoryService(db_session) - - user_memory = await db.upsert_user_memory( - memory=UserMemory( - memory_id=str(uuid4()), - memory=payload.memory, - topics=payload.topics or [], - user_id=payload.user_id, - ), - deserialize=False, - ) - - if not user_memory: - raise InternalServerException("Failed to create memory") - - return UserMemorySchema.from_dict(_normalize_memory_dict(user_memory)) # type: ignore - - -@router.delete( - "/memories/{memory_id}", - status_code=204, - operation_id="delete_memory", - summary="Delete Memory", - description="Permanently delete a specific user memory. This action cannot be undone.", - responses={ - 204: {"description": "Memory deleted successfully"}, - 404: {"description": "Memory not found", "model": NotFoundResponse}, - 500: {"description": "Failed to delete memory", "model": InternalServerErrorResponse}, - }, -) -async def delete_memory( - request: Request, - memory_id: str = Path(description="Memory ID to delete"), - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> None: - db = MemoryService(db_session) - success = await db.delete_user_memory(memory_id=memory_id, user_id=str(current_user.id)) - if not success: - raise NotFoundException(f"Memory with ID {memory_id} not found") - return None - - -@router.delete( - "/memories", - status_code=204, - operation_id="delete_memories", - summary="Delete Multiple Memories", - description=( - "Delete multiple user memories by their IDs in a single operation. " - "This action cannot be undone and all specified memories will be permanently removed." - ), - responses={ - 204: {"description": "Memories deleted successfully"}, - 400: {"description": "Invalid request - empty memory_ids list", "model": BadRequestResponse}, - 500: {"description": "Failed to delete memories", "model": InternalServerErrorResponse}, - }, -) -async def delete_memories( - request: DeleteMemoriesRequest, - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> None: - if not request.memory_ids: - raise BadRequestException("memory_ids must not be empty") - db = MemoryService(db_session) - await db.delete_user_memories(memory_ids=request.memory_ids, user_id=str(current_user.id)) - return None - - -@router.get( - "/memories", - response_model=PaginatedResponse[UserMemorySchema], - status_code=200, - operation_id="get_memories", - summary="List Memories", - description=( - "Retrieve paginated list of user memories with filtering and search capabilities. " - "Filter by user, agent, team, topics, or search within memory content." - ), -) -async def get_memories( - request: Request, - user_id: Optional[str] = Query(default=None, description="Filter memories by user ID"), - agent_id: Optional[str] = Query(default=None, description="Filter memories by agent ID"), - team_id: Optional[str] = Query(default=None, description="Filter memories by team ID"), - topics: Optional[List[str]] = Depends(parse_topics), - search_content: Optional[str] = Query(default=None, description="Fuzzy search within memory content"), - limit: Optional[int] = Query(default=20, description="Number of memories to return per page"), - page: Optional[int] = Query(default=1, description="Page number for pagination"), - sort_by: Optional[str] = Query(default="updated_at", description="Field to sort memories by"), - sort_order: Optional[SortOrder] = Query(default="desc", description="Sort order (asc or desc)"), - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> PaginatedResponse[UserMemorySchema]: - db = MemoryService(db_session) - - # restrict to current user - user_id = str(current_user.id) - - # Ensure limit/page are proper ints - limit = int(limit) if limit is not None else 20 - page = int(page) if page is not None else 1 - - user_memories_raw, total_count = await db.get_user_memories( - limit=limit, - page=page, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - topics=topics, - search_content=search_content, - sort_by=sort_by, - sort_order=sort_order, - deserialize=False, - ) - - memories = [ - UserMemorySchema.from_dict(_normalize_memory_dict(user_memory)) # type: ignore - for user_memory in user_memories_raw # type: ignore - ] - memories = [m for m in memories if m is not None] - - return PaginatedResponse( - data=memories, # type: ignore - meta=PaginationInfo( - page=page, - limit=limit, - total_count=total_count, # type: ignore - total_pages=(total_count + limit - 1) // limit if limit is not None and limit > 0 else 0, # type: ignore - ), - ) - - -@router.get( - "/memories/{memory_id}", - response_model=UserMemorySchema, - status_code=200, - operation_id="get_memory", - summary="Get Memory by ID", - description="Retrieve detailed information about a specific user memory by its ID.", - responses={ - 200: {"description": "Memory retrieved successfully"}, - 404: {"description": "Memory not found", "model": NotFoundResponse}, - }, -) -async def get_memory( - request: Request, - memory_id: str = Path(description="Memory ID to retrieve"), - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> UserMemorySchema: - db = MemoryService(db_session) - - user_memory = await db.get_user_memory(memory_id=memory_id, user_id=str(current_user.id), deserialize=False) - if not user_memory: - raise NotFoundException(f"Memory with ID {memory_id} not found") - - return UserMemorySchema.from_dict(_normalize_memory_dict(user_memory)) # type: ignore - - -@router.get( - "/memory_topics", - response_model=List[str], - status_code=200, - operation_id="get_memory_topics", - summary="Get Memory Topics", - description=( - "Retrieve all unique topics associated with memories in the system. " - "Useful for filtering and categorizing memories by topic." - ), -) -async def get_topics( - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> List[str]: - db = MemoryService(db_session) - # only return topics for the current user; for global stats, an admin check could be added - user_id = str(current_user.id) - return await db.get_all_memory_topics(user_id=user_id) - - -@router.patch( - "/memories/{memory_id}", - response_model=UserMemorySchema, - status_code=200, - operation_id="update_memory", - summary="Update Memory", - description=( - "Update an existing user memory's content and topics. " - "Replaces the entire memory content and topic list with the provided values." - ), - responses={ - 200: {"description": "Memory updated successfully"}, - 400: {"description": "Invalid request data", "model": BadRequestResponse}, - 404: {"description": "Memory not found", "model": NotFoundResponse}, - 422: {"description": "Validation error in payload", "model": ValidationErrorResponse}, - 500: {"description": "Failed to update memory", "model": InternalServerErrorResponse}, - }, -) -async def update_memory( - request: Request, - payload: UserMemoryCreateSchema, - memory_id: str = Path(description="Memory ID to update"), - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> UserMemorySchema: - # restrict to current user - payload.user_id = str(current_user.id) - - db = MemoryService(db_session) - - user_memory = await db.upsert_user_memory( - memory=UserMemory( - memory_id=memory_id, - memory=payload.memory, - topics=payload.topics or [], - user_id=payload.user_id, - ), - deserialize=False, - ) - if not user_memory: - raise InternalServerException("Failed to update memory") - - return UserMemorySchema.from_dict(_normalize_memory_dict(user_memory)) # type: ignore - - -@router.post( - "/optimize-memories", - response_model=OptimizeMemoriesResponse, - status_code=200, - operation_id="optimize_memories", - summary="Optimize User Memories", - description=( - "Optimize user memories using the default summarize strategy. " - "This operation combines all memories into a single comprehensive summary." - ), -) -async def optimize_memories( - request: OptimizeMemoriesRequest, - db_session: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -) -> OptimizeMemoriesResponse: - """Optimize user memories using the default summarize strategy.""" - from app.core.agent.memory.manager import MemoryManager - from app.core.agent.memory.strategies.summarize import SummarizeStrategy - from app.core.agent.memory.strategies.types import MemoryOptimizationStrategyType - - try: - # Resolve model for memory optimization via ModelService - from typing import cast - - from langchain_core.language_models.chat_models import BaseChatModel - - from app.core.model.utils.model_ref import parse_model_ref - from app.services.model_service import ModelService - - if not request.model: - raise BadRequestException( - "Model is required. Specify 'model' in format 'provider:model_name' (e.g., 'openai:gpt-4o-mini').", - ) - - provider_name, model_name = parse_model_ref(request.model) - if not model_name: - raise BadRequestException( - "Invalid model format. Specify 'model' in format 'provider:model_name' (e.g., 'openai:gpt-4o-mini').", - ) - - model_service = ModelService(db_session) - - if provider_name: - memory_model = await model_service.get_model_instance( - user_id=str(current_user.id), - provider_name=provider_name, - model_name=model_name, - ) - else: - memory_model = await model_service.get_runtime_model_by_name( - model_name=model_name, - user_id=str(current_user.id), - ) - - # Create memory manager with MemoryService and explicit model - db = MemoryService(db_session) - memory_manager = MemoryManager(model=cast(BaseChatModel, memory_model), db=db) - - # Get current memories to count tokens before optimization - user_id = request.user_id or str(current_user.id) - memories_before = await memory_manager.aget_user_memories(user_id=user_id) - if not memories_before: - raise NotFoundException(f"No memories found for user {user_id}") - - # Count tokens before optimization - strategy = SummarizeStrategy() - tokens_before = strategy.count_tokens(memories_before) - memories_before_count = len(memories_before) - - # Optimize memories with default SUMMARIZE strategy - optimized_memories = await memory_manager.aoptimize_memories( - user_id=user_id, - strategy=MemoryOptimizationStrategyType.SUMMARIZE, - apply=request.apply, - ) - - # Count tokens after optimization - tokens_after = strategy.count_tokens(optimized_memories) - memories_after_count = len(optimized_memories) - - # Calculate statistics (clamp to 0 when summarization increases tokens) - tokens_saved = max(0, tokens_before - tokens_after) - reduction_percentage = ( - max(0.0, (tokens_before - tokens_after) / tokens_before * 100.0) if tokens_before > 0 else 0.0 - ) - - # Convert to schema objects - optimized_memory_schemas = [ - UserMemorySchema( - memory_id=mem.memory_id or "", - memory=mem.memory or "", - topics=mem.topics, - agent_id=mem.agent_id, - team_id=mem.team_id, - user_id=mem.user_id, - updated_at=datetime.fromtimestamp(mem.updated_at, tz=timezone.utc) - if isinstance(mem.updated_at, (int, float)) - else mem.updated_at, # type: ignore - ) - for mem in optimized_memories - ] - - return OptimizeMemoriesResponse( - memories=optimized_memory_schemas, - memories_before=memories_before_count, - memories_after=memories_after_count, - tokens_before=tokens_before, - tokens_after=tokens_after, - tokens_saved=tokens_saved, - reduction_percentage=reduction_percentage, - ) - - except AppException: - raise - except Exception as e: - logger.error(f"Failed to optimize memories for user {request.user_id}: {str(e)}") - raise InternalServerException(f"Failed to optimize memories: {str(e)}") diff --git a/backend/app/api/v1/memory/schemas.py b/backend/app/api/v1/memory/schemas.py deleted file mode 100644 index eb1297915..000000000 --- a/backend/app/api/v1/memory/schemas.py +++ /dev/null @@ -1,91 +0,0 @@ -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - -from pydantic import BaseModel, Field - - -class DeleteMemoriesRequest(BaseModel): - memory_ids: List[str] = Field(..., description="List of memory IDs to delete", min_length=1) - user_id: Optional[str] = Field(None, description="User ID to filter memories for deletion") - - -class UserMemorySchema(BaseModel): - memory_id: str = Field(..., description="Unique identifier for the memory") - memory: str = Field(..., description="Memory content text") - topics: Optional[List[str]] = Field(None, description="Topics or tags associated with the memory") - - agent_id: Optional[str] = Field(None, description="Agent ID associated with this memory") - team_id: Optional[str] = Field(None, description="Team ID associated with this memory") - user_id: Optional[str] = Field(None, description="User ID who owns this memory") - - updated_at: Optional[datetime] = Field(None, description="Timestamp when memory was last updated") - - @classmethod - def from_dict(cls, memory_dict: Dict[str, Any]) -> Optional["UserMemorySchema"]: - if memory_dict["memory"] == "": - return None - - return cls( - memory_id=memory_dict["memory_id"], - user_id=str(memory_dict["user_id"]), - agent_id=memory_dict.get("agent_id"), - team_id=memory_dict.get("team_id"), - memory=memory_dict["memory"], - topics=memory_dict.get("topics", []), - updated_at=memory_dict["updated_at"], - ) - - -class UserMemoryCreateSchema(BaseModel): - """Define the payload expected for creating a new user memory""" - - memory: str = Field(..., description="Memory content text", min_length=1, max_length=5000) - user_id: Optional[str] = Field(None, description="User ID who owns this memory") - topics: Optional[List[str]] = Field(None, description="Topics or tags to categorize the memory") - - -class UserStatsSchema(BaseModel): - """Schema for user memory statistics""" - - user_id: str = Field(..., description="User ID") - total_memories: int = Field(..., description="Total number of memories for this user", ge=0) - last_memory_updated_at: Optional[datetime] = Field(None, description="Timestamp of the most recent memory update") - - @classmethod - def from_dict(cls, user_stats_dict: Dict[str, Any]) -> "UserStatsSchema": - updated_at = user_stats_dict.get("last_memory_updated_at") - - return cls( - user_id=str(user_stats_dict["user_id"]), - total_memories=user_stats_dict["total_memories"], - last_memory_updated_at=datetime.fromtimestamp(updated_at, tz=timezone.utc) if updated_at else None, - ) - - -class OptimizeMemoriesRequest(BaseModel): - """Schema for memory optimization request""" - - user_id: Optional[str] = Field( - default=None, - description="User ID to optimize memories for. If not provided, uses current authenticated user.", - ) - model: Optional[str] = Field( - default=None, - description="Model to use for optimization in format 'provider:model_id' (e.g., 'openai:gpt-4o-mini', 'anthropic:claude-3-5-sonnet-20241022', 'google:gemini-2.0-flash-exp'). If not specified, uses MemoryManager's default model (gpt-4o).", - ) - apply: bool = Field( - default=True, - description="If True, apply optimization changes to database. If False, return preview only without saving.", - ) - - -class OptimizeMemoriesResponse(BaseModel): - """Schema for memory optimization response""" - - memories: List[UserMemorySchema] = Field(..., description="List of optimized memory objects") - memories_before: int = Field(..., description="Number of memories before optimization", ge=0) - memories_after: int = Field(..., description="Number of memories after optimization", ge=0) - tokens_before: int = Field(..., description="Token count before optimization", ge=0) - tokens_after: int = Field(..., description="Token count after optimization", ge=0) - tokens_saved: int = Field(..., description="Number of tokens saved through optimization", ge=0) - reduction_percentage: float = Field(..., description="Percentage of token reduction achieved", ge=0.0, le=100.0) diff --git a/backend/app/api/v1/model_credentials.py b/backend/app/api/v1/model_credentials.py deleted file mode 100644 index f687c12c4..000000000 --- a/backend/app/api/v1/model_credentials.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Model credential management API -""" - -import uuid -from typing import Any, Dict - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.model_credential_service import ModelCredentialService - -router = APIRouter(prefix="/v1/model-credentials", tags=["ModelCredentials"]) - - -class CredentialCreate(BaseModel): - """Create/update credential request (builtin providers only).""" - - provider_name: str = Field(description="Provider name", examples=["openaiapicompatible"]) - credentials: Dict[str, Any] = Field(..., description="Credentials dict (plaintext)") - should_validate: bool = Field(default=True, alias="validate", description="Whether to validate credentials") - - -@router.post("") -async def create_or_update_credential( - payload: CredentialCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create or update a builtin provider's credential.""" - service = ModelCredentialService(db) - credential = await service.upsert_credential( - user_id=current_user.id, - provider_name=payload.provider_name, - credentials=payload.credentials, - validate=payload.should_validate, - ) - return success_response(data=credential, message="Credential created/updated") - - -@router.get("") -async def list_credentials( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - List credentials (global, workspace-independent). - """ - service = ModelCredentialService(db) - credentials = await service.list_credentials() - return success_response(data=credentials, message="Credential list retrieved") - - -@router.get("/{credential_id}") -async def get_credential( - credential_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Get credential details. - - Args: - credential_id: credential ID - - Returns: - Credential details (without decrypted credentials) - """ - service = ModelCredentialService(db) - credential = await service.get_credential(credential_id, include_credentials=True) - return success_response(data=credential, message="Credential details retrieved") - - -@router.post("/{credential_id}/validate") -async def validate_credential( - credential_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Validate a credential. - - Args: - credential_id: credential ID - - Returns: - Validation result - """ - service = ModelCredentialService(db) - result = await service.validate_credential(credential_id) - return success_response(data=result, message="Credential validation completed") - - -@router.delete("/{credential_id}") -async def delete_credential( - credential_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Delete a credential. - - Args: - credential_id: credential ID - """ - service = ModelCredentialService(db) - await service.delete_credential(credential_id) - return success_response(message="Credential deleted") diff --git a/backend/app/api/v1/model_providers.py b/backend/app/api/v1/model_providers.py deleted file mode 100644 index 1df3a48b9..000000000 --- a/backend/app/api/v1/model_providers.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Model provider management API""" - -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.model_provider_service import ModelProviderService - -router = APIRouter(prefix="/v1/model-providers", tags=["ModelProviders"]) - - -class ProviderDefaultsUpdate(BaseModel): - """Update provider default parameters request.""" - - default_parameters: Dict[str, Any] = Field( - description="Provider-level default parameters, e.g. {temperature: 0.7, max_tokens: 2000}" - ) - - -@router.get("") -async def list_providers( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List all providers.""" - service = ModelProviderService(db) - providers = await service.get_all_providers() - return success_response(data=providers, message="Provider list retrieved") - - -class CustomProviderCreate(BaseModel): - """Add custom provider request.""" - - model_name: str = Field(description="Model name", examples=["gpt-4o"]) - credentials: Dict[str, Any] = Field(description="Credentials dict (plaintext)") - display_name: Optional[str] = Field(default=None, description="Custom display name") - model_parameters: Optional[Dict[str, Any]] = Field(default=None, description="Model parameters") - validate_credentials: bool = Field(default=True, description="Whether to validate credentials") - - -@router.post("/custom") -async def add_custom_provider( - payload: CustomProviderCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Add a custom provider (one-step creation of provider + credential + model_instance).""" - service = ModelProviderService(db) - result = await service.add_custom_provider( - user_id=current_user.id, - credentials=payload.credentials, - model_name=payload.model_name, - display_name=payload.display_name, - model_parameters=payload.model_parameters, - validate=payload.validate_credentials, - ) - return success_response(data=result, message="Custom provider added") - - -@router.get("/{provider_name}") -async def get_provider( - provider_name: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get a single provider's details.""" - service = ModelProviderService(db) - provider = await service.get_provider(provider_name) - - if not provider: - from app.common.exceptions import NotFoundException - - raise NotFoundException(f"Provider not found: {provider_name}") - - return success_response(data=provider, message="Provider details retrieved") - - -@router.patch("/{provider_name}/defaults") -async def update_provider_defaults( - provider_name: str, - payload: ProviderDefaultsUpdate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update provider-level default parameters.""" - service = ModelProviderService(db) - provider = await service.update_provider_defaults(provider_name, payload.default_parameters) - return success_response(data=provider, message="Provider defaults updated") - - -@router.post("/sync") -async def sync_providers( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Sync providers and model info to the database.""" - service = ModelProviderService(db) - result = await service.sync_all() - return success_response(data=result, message="Sync completed") - - -@router.delete("/{provider_name}") -async def delete_provider( - provider_name: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete a provider (custom providers only).""" - service = ModelProviderService(db) - await service.delete_provider(provider_name) - return success_response(message=f"Provider {provider_name} deleted") diff --git a/backend/app/api/v1/model_usage.py b/backend/app/api/v1/model_usage.py deleted file mode 100644 index 69da53f63..000000000 --- a/backend/app/api/v1/model_usage.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Model usage statistics API -""" - -from typing import Optional - -from fastapi import APIRouter, Depends, Query -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.model_usage_service import ModelUsageService - -router = APIRouter(prefix="/v1/models", tags=["Model Usage"]) - - -@router.get("/usage/stats") -async def get_usage_stats( - period: str = Query(default="24h", description="Time range: 24h/7d/30d"), - granularity: str = Query(default="hour", description="Time granularity: hour/day"), - provider_name: Optional[str] = Query(default=None, description="Filter by provider"), - model_name: Optional[str] = Query(default=None, description="Filter by model"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get model usage statistics.""" - service = ModelUsageService(db) - stats = await service.get_stats( - period=period, - granularity=granularity, - provider_name=provider_name, - model_name=model_name, - ) - return success_response(data=stats, message="Usage statistics retrieved") diff --git a/backend/app/api/v1/models.py b/backend/app/api/v1/models.py deleted file mode 100644 index b2177c9e8..000000000 --- a/backend/app/api/v1/models.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Model management API (global, workspace-independent) -""" - -import uuid -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends, Query -from fastapi.responses import StreamingResponse -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.response import success_response -from app.core.database import get_db -from app.core.model import ModelType -from app.models.auth import AuthUser as User -from app.services.model_service import ModelService - -router = APIRouter(prefix="/v1/models", tags=["Models"]) - - -class ModelInstanceCreate(BaseModel): - """Create model instance configuration request.""" - - provider_name: str = Field(description="Provider name", examples=["openaiapicompatible"]) - model_name: str = Field(description="Model name", examples=["gpt-4o"]) - model_type: str = Field(default="chat", description="Model type: chat, llm, embedding, etc.", examples=["chat"]) - model_parameters: Optional[Dict[str, Any]] = Field( - default=None, description="Model parameter configuration", examples=[{}] - ) - - -class ModelInstanceUpdate(BaseModel): - """Update model instance request.""" - - model_parameters: Optional[Dict[str, Any]] = Field( - default=None, description="Model parameter overrides (only user-specified fields)" - ) - - -class ModelTestRequest(BaseModel): - """Test model output request.""" - - model_name: str = Field(description="Model name", examples=["gpt-4o"]) - input: str = Field(description="Input text", examples=["Hello, please introduce yourself"]) - - -@router.get("/overview") -async def get_models_overview( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get global model overview: provider health summary, recent credential failures.""" - service = ModelService(db) - overview = await service.get_overview() - return success_response(data=overview, message="Model overview retrieved") - - -@router.get("") -async def list_available_models( - model_type: str = Query(default="chat", description="Model type: chat, llm, embedding, etc."), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List available models (includes unavailable_reason).""" - try: - model_type_enum = ModelType(model_type) - except ValueError: - from app.common.exceptions import BadRequestException - - raise BadRequestException(f"Unsupported model type: {model_type}") - - service = ModelService(db) - models = await service.get_available_models(model_type=model_type_enum, user_id=current_user.id) - return success_response(data=models, message="Model list retrieved") - - -@router.post("/instances") -async def create_model_instance( - payload: ModelInstanceCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create a model instance configuration.""" - try: - model_type_enum = ModelType(payload.model_type) - except ValueError: - from app.common.exceptions import BadRequestException - - raise BadRequestException(f"Unsupported model type: {payload.model_type}") - - service = ModelService(db) - instance = await service.create_model_instance_config( - user_id=current_user.id, - provider_name=payload.provider_name, - model_name=payload.model_name, - model_type=model_type_enum, - model_parameters=payload.model_parameters, - ) - return success_response(data=instance, message="Model instance configuration created") - - -@router.get("/instances") -async def list_model_instances( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List model instance configurations (global).""" - service = ModelService(db) - instances = await service.list_model_instances() - return success_response(data=instances, message="Model instance configurations retrieved") - - -@router.patch("/instances/{instance_id}") -async def update_model_instance( - instance_id: uuid.UUID, - payload: ModelInstanceUpdate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update model instance parameters.""" - service = ModelService(db) - instance = await service.update_model_instance( - instance_id=instance_id, - model_parameters=payload.model_parameters, - ) - return success_response(data=instance, message="Model instance updated") - - -class ModelTestStreamRequest(BaseModel): - """Streaming model output test request.""" - - model_name: str = Field(description="Model name") - input: str = Field(description="Input text") - model_parameters: Optional[Dict[str, Any]] = Field(default=None, description="Temporary parameter overrides") - - -@router.post("/test-output-stream") -async def test_output_stream( - payload: ModelTestStreamRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Test model output via SSE streaming.""" - service = ModelService(db) - - async def event_generator(): - async for event in service.test_output_stream( - user_id=current_user.id, - model_name=payload.model_name, - input_text=payload.input, - model_parameters=payload.model_parameters, - ): - yield event - - return StreamingResponse( - event_generator(), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "X-Accel-Buffering": "no", - }, - ) - - -@router.post("/test-output") -async def test_output( - payload: ModelTestRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Test model output.""" - service = ModelService(db) - output = await service.test_output( - user_id=current_user.id, - model_name=payload.model_name, - input_text=payload.input, - ) - return success_response(data={"output": output}, message="Model output test completed") diff --git a/backend/app/api/v1/oauth.py b/backend/app/api/v1/oauth.py deleted file mode 100644 index 78f4b830b..000000000 --- a/backend/app/api/v1/oauth.py +++ /dev/null @@ -1,419 +0,0 @@ -""" -OAuth/OIDC auth API endpoints. - -Provides OAuth login flow APIs: -- GET /oauth/providers - list enabled providers -- GET /oauth/{provider} - start OAuth authorization -- GET /oauth/{provider}/callback - handle OAuth callback - -Multi-protocol support: -- oauth2 (standard): GitHub, Google, Microsoft, GitLab, etc. -- jd_sso (JD SSA): JD enterprise login -""" - -import json -import secrets -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, Depends, Query, Request -from fastapi.responses import RedirectResponse -from loguru import logger -from pydantic import BaseModel -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_db -from app.common.exceptions import BadRequestException -from app.core.oauth import get_oauth_config, get_protocol_handler -from app.core.redis import RedisClient -from app.core.security import create_access_token, create_csrf_token, generate_refresh_token -from app.core.settings import settings -from app.services.oauth_service import OAuthService - -LOG_PREFIX = "[OAuthAPI]" -router = APIRouter(prefix="/v1/auth/oauth", tags=["OAuth"]) - - -# ==================== Response Models ==================== - - -class OAuthProviderInfo(BaseModel): - """OAuth provider info (no sensitive fields).""" - - id: str - display_name: str - icon: str - - -class OAuthProvidersResponse(BaseModel): - """OAuth provider list response.""" - - providers: List[OAuthProviderInfo] - - -# ==================== API Endpoints ==================== - - -@router.get("/providers", response_model=OAuthProvidersResponse) -async def list_oauth_providers() -> OAuthProvidersResponse: - """ - List enabled OAuth providers. - - Used by frontend to render SSO buttons. - """ - oauth_config = get_oauth_config() - providers = oauth_config.list_providers() - - return OAuthProvidersResponse(providers=[OAuthProviderInfo(**p) for p in providers]) - - -@router.get("/{provider}") -async def oauth_authorize( - provider: str, - request: Request, - callback_url: Optional[str] = Query(None, description="Redirect URL after successful login"), - db: AsyncSession = Depends(get_db), -) -> RedirectResponse: - """ - Start OAuth authorization flow. - - Redirect users to the provider's authorization page. - - Args: - provider: Provider key (e.g. "github", "google", "jd") - callback_url: Redirect URL after login (optional) - """ - oauth_config = get_oauth_config() - oauth_service = OAuthService(db) - - # Build callback URL - base_url = _get_base_url(request) - redirect_uri = f"{base_url}/api/v1/auth/oauth/{provider}/callback" - - # Generate state (includes callback_url) - state = secrets.token_urlsafe(32) - state_data = { - "provider": provider, - "redirect_uri": redirect_uri, - "callback_url": callback_url or oauth_config.settings.default_redirect_url, - "created_at": datetime.now(timezone.utc).isoformat(), - } - - # Store state in Redis - if RedisClient.is_available(): - try: - await RedisClient.set(f"oauth_state:{state}", json.dumps(state_data), expire=600) - except Exception as e: - logger.warning(f"{LOG_PREFIX} Failed to store state in Redis: {e}") - - # Generate authorization URL - try: - authorization_url, _ = await oauth_service.generate_authorization_url( - provider_name=provider, - redirect_uri=redirect_uri, - state=state, - ) - except Exception as e: - logger.error(f"{LOG_PREFIX} Failed to generate authorization URL: {e}") - raise BadRequestException(f"Failed to initiate OAuth flow: {str(e)}") - - logger.info(f"{LOG_PREFIX} Redirecting to {provider} authorization") - return RedirectResponse(url=authorization_url, status_code=302) - - -@router.get("/{provider}/callback") -async def oauth_callback( - provider: str, - request: Request, - code: Optional[str] = Query(None, description="Auth code (required for OAuth2, optional for JD SSO)"), - state: Optional[str] = Query(None, description="State parameter (optional for JD SSO)"), - error: Optional[str] = Query(None, description="Error message"), - error_description: Optional[str] = Query(None, description="Error description"), - db: AsyncSession = Depends(get_db), -) -> RedirectResponse: - """ - Handle OAuth callback. - - Validate authorization, fetch user info, create/link user, issue JWT tokens. - - Multi-protocol support (by provider protocol field): - - oauth2 (standard): exchange code for token, then userinfo - - jd_sso (JD SSA): use Cookie + verifyTicket for userinfo - """ - oauth_config = get_oauth_config() - frontend_url = settings.frontend_url.rstrip("/") - - # Handle user denial - if error: - logger.warning(f"{LOG_PREFIX} OAuth error: {error} - {error_description}") - return _redirect_with_error(frontend_url, "oauth_denied", error_description or error) - - # 2. Load provider config (needed to detect protocol) - provider_config = oauth_config.get_provider(provider) - if not provider_config: - logger.error(f"{LOG_PREFIX} Provider not found: {provider}") - return _redirect_with_error(frontend_url, "provider_not_found") - - # 1. Validate state (JD SSO can skip; it relies on Cookie, not auth code) - callback_url = oauth_config.settings.default_redirect_url - state_data: dict[Any, Any] | None = {} - - if state: - # Validate when state is present - state_data, callback_url = await _validate_state(state, oauth_config) - if state_data is None: - return _redirect_with_error(frontend_url, "invalid_state") - - # Validate provider match - if state_data.get("provider") != provider: - logger.warning(f"{LOG_PREFIX} Provider mismatch: expected {state_data.get('provider')}, got {provider}") - return _redirect_with_error(frontend_url, "provider_mismatch") - elif provider_config.protocol != "jd_sso": - # Non-JD SSO protocols require state - logger.warning(f"{LOG_PREFIX} Missing state parameter for {provider_config.protocol}") - return _redirect_with_error(frontend_url, "missing_state") - - try: - # 3. Use protocol handler to fetch user info - handler = get_protocol_handler(provider_config.protocol) - redirect_uri = (state_data or {}).get( - "redirect_uri" - ) or f"{_get_base_url(request)}/api/v1/auth/oauth/{provider}/callback" - - logger.info(f"{LOG_PREFIX} Processing {provider_config.protocol} callback for {provider}") - - user_info = await handler.get_user_info( - request=request, - provider_config=provider_config, - code=code, - redirect_uri=redirect_uri, - ) - - # 4. Find or create user - oauth_service = OAuthService(db) - user, is_new_user = await oauth_service.find_or_create_user( - provider_name=provider, - provider_account_id=user_info.provider_id, - email=user_info.email, - name=user_info.name, - avatar=user_info.avatar, - tokens={}, # Tokens handled by protocol handler - raw_userinfo=user_info.raw, - ) - - # 5. Commit transaction & post-login init - await db.commit() - ip_address = _get_client_ip(request) - - from app.services.login_init import run_post_login_init - - await run_post_login_init(db, user, ip_address) - - # 6. Issue JWT tokens - jwt_access_token = create_access_token( - subject=user.id, - expires_delta=timedelta(minutes=settings.access_token_expire_minutes), - ) - jwt_refresh_token = generate_refresh_token() - csrf_token = create_csrf_token(user.id) - - # Store refresh token in Redis - await _store_refresh_token(jwt_refresh_token, user.id) - - # 7. Set cookies and redirect - response = _create_auth_response( - frontend_url=frontend_url, - callback_url=callback_url, - access_token=jwt_access_token, - refresh_token=jwt_refresh_token, - csrf_token=csrf_token, - ) - - logger.info( - f"{LOG_PREFIX} OAuth login successful", - extra={"provider": provider, "user_id": user.id, "is_new_user": is_new_user}, - ) - - return response - - except BadRequestException: - raise - except ValueError as e: - # Validation error raised by protocol handler - logger.error(f"{LOG_PREFIX} OAuth callback validation error: {e}") - await db.rollback() - return _redirect_with_error(frontend_url, "oauth_failed", str(e)) - except Exception as e: - logger.error(f"{LOG_PREFIX} OAuth callback error: {e}", exc_info=True) - await db.rollback() - return _redirect_with_error(frontend_url, "oauth_failed", str(e)) - - -# ==================== User OAuth Account Management ==================== - - -class UserOAuthAccount(BaseModel): - """User OAuth account info.""" - - id: str - provider: str - provider_account_id: str - email: Optional[str] - created_at: datetime - - -class UserOAuthAccountsResponse(BaseModel): - """User OAuth account list response.""" - - accounts: List[UserOAuthAccount] - - -@router.get("/accounts/me", response_model=UserOAuthAccountsResponse) -async def get_my_oauth_accounts( - request: Request, - db: AsyncSession = Depends(get_db), -) -> UserOAuthAccountsResponse: - """Get OAuth account bindings for current user.""" - from app.common.dependencies import get_current_user - - current_user = await get_current_user(None, request, db) - oauth_service = OAuthService(db) - accounts = await oauth_service.get_user_oauth_accounts(current_user.id) - - return UserOAuthAccountsResponse( - accounts=[ - UserOAuthAccount( - id=acc.id, - provider=acc.provider, - provider_account_id=acc.provider_account_id, - email=acc.email, - created_at=acc.created_at, - ) - for acc in accounts - ] - ) - - -@router.delete("/accounts/{provider}") -async def unlink_oauth_account( - provider: str, - request: Request, - db: AsyncSession = Depends(get_db), -) -> Dict[str, Any]: - """Unlink OAuth account.""" - from app.common.dependencies import get_current_user - - current_user = await get_current_user(None, request, db) - oauth_service = OAuthService(db) - success = await oauth_service.unlink_oauth_account(current_user.id, provider) - - if success: - await db.commit() - - return {"success": success, "provider": provider} - - -# ==================== Helpers ==================== - - -def _get_base_url(request: Request) -> str: - """Get base URL, with proxy support.""" - base_url = str(request.base_url).rstrip("/") - forwarded_proto = request.headers.get("x-forwarded-proto") - forwarded_host = request.headers.get("x-forwarded-host") - if forwarded_host: - proto = forwarded_proto or "https" - base_url = f"{proto}://{forwarded_host}" - return base_url - - -def _get_client_ip(request: Request) -> str: - """Get client IP, with proxy support.""" - ip = request.client.host if request.client else "unknown" - forwarded_for = request.headers.get("X-Forwarded-For") - if forwarded_for: - ip = forwarded_for.split(",")[0].strip() - return ip - - -def _redirect_with_error(frontend_url: str, error: str, description: Optional[str] = None) -> RedirectResponse: - """Build error redirect response.""" - error_url = f"{frontend_url}/signin?error={error}" - if description: - error_url += f"&error_description={description}" - return RedirectResponse(url=error_url, status_code=302) - - -async def _validate_state(state: str, oauth_config) -> tuple[Optional[Dict], str]: - """Validate state and return state_data and callback_url.""" - callback_url = oauth_config.settings.default_redirect_url - - if not RedisClient.is_available(): - return {}, callback_url - - try: - state_key = f"oauth_state:{state}" - state_data_str = await RedisClient.get(state_key) - if state_data_str: - state_data = json.loads(state_data_str) - callback_url = state_data.get("callback_url", callback_url) - await RedisClient.delete(state_key) - return state_data, callback_url - else: - logger.warning(f"{LOG_PREFIX} Invalid or expired state: {state[:20]}...") - return None, callback_url - except Exception as e: - logger.warning(f"{LOG_PREFIX} Failed to validate state: {e}") - return {}, callback_url - - -async def _store_refresh_token(refresh_token: str, user_id: str) -> None: - """Store refresh token in Redis.""" - if not RedisClient.is_available(): - return - - try: - expire_seconds = settings.refresh_token_expire_days * 24 * 60 * 60 - await RedisClient.set(f"refresh_token:{refresh_token}", user_id, expire=expire_seconds) - await RedisClient.set(f"account_refresh_token:{user_id}", refresh_token, expire=expire_seconds) - except Exception as e: - logger.warning(f"{LOG_PREFIX} Failed to store refresh token: {e}") - - -def _create_auth_response( - frontend_url: str, - callback_url: str, - access_token: str, - refresh_token: str, - csrf_token: str, -) -> RedirectResponse: - """Create redirect response with auth cookies.""" - if not callback_url.startswith("/"): - callback_url = f"/{callback_url}" - final_url = f"{frontend_url}{callback_url}" - - response = RedirectResponse(url=final_url, status_code=302) - - # Cookie defaults - cookie_kwargs: Dict[str, Any] = { - "httponly": True, - "samesite": settings.cookie_samesite, - "secure": settings.cookie_secure_effective, - "path": "/", - } - if settings.cookie_domain: - cookie_kwargs["domain"] = settings.cookie_domain - - # Access token - access_expires = datetime.now(timezone.utc) + timedelta(minutes=settings.access_token_expire_minutes) - response.set_cookie(key=settings.cookie_name, value=access_token, expires=access_expires, **cookie_kwargs) - - # Refresh token - refresh_expires = datetime.now(timezone.utc) + timedelta(days=settings.refresh_token_expire_days) - response.set_cookie(key="refresh_token", value=refresh_token, expires=refresh_expires, **cookie_kwargs) - - # CSRF token (not httponly) - csrf_kwargs = {**cookie_kwargs, "httponly": False} - response.set_cookie(key="csrf_token", value=csrf_token, expires=access_expires, **csrf_kwargs) - - return response diff --git a/backend/app/api/v1/openapi_graph.py b/backend/app/api/v1/openapi_graph.py deleted file mode 100644 index 1c2b585a2..000000000 --- a/backend/app/api/v1/openapi_graph.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -OpenAPI Graph routes — trigger Graph execution via PlatformToken auth - -Endpoints: -- POST /v1/openapi/graph/{graphId}/run Start execution -- GET /v1/openapi/graph/{executionId}/status Query status -- POST /v1/openapi/graph/{executionId}/abort Abort execution -- GET /v1/openapi/graph/{executionId}/result Get result -""" - -from __future__ import annotations - -import uuid -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends, Request -from loguru import logger -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.auth_dependency import AuthContext, get_current_user_or_token -from app.common.exceptions import ForbiddenException -from app.common.permissions import check_token_permission -from app.core.database import get_db -from app.services.openapi_graph_service import OpenApiGraphService - -router = APIRouter(prefix="/v1/openapi/graph", tags=["OpenAPI Graph"]) - - -# ─── Request / Response Models ───────────────────────────────── - - -class RunGraphRequest(BaseModel): - """Request body for starting a Graph execution.""" - - variables: Optional[Dict[str, Any]] = Field( - default=None, - description="Runtime variables. message/query is used as the user message; the rest are context variables.", - ) - - -class OpenApiResponse(BaseModel): - """Unified response format.""" - - success: bool = True - data: Optional[Dict[str, Any]] = None - errCode: Optional[str] = None - errMsg: Optional[str] = None - - -# ─── Helper ───────────────────────────────────── - - -def _bind_log(request: Request, **kwargs): - trace_id = getattr(request.state, "trace_id", "-") - return logger.bind(trace_id=trace_id, **kwargs) - - -def _require_graph_execute(auth: AuthContext, graph_id: uuid.UUID) -> None: - """Require graphs:execute scope if using token auth.""" - if not auth.is_token_auth: - return - has_perm = check_token_permission( - token_scopes=auth.token_scopes or [], - required_scope="graphs:execute", - resource_type="graph", - resource_id=str(graph_id), - token_resource_type=auth.token_resource_type, - token_resource_id=auth.token_resource_id, - ) - if not has_perm: - raise ForbiddenException("Token missing required scope: graphs:execute") - - -# ─── Endpoints ───────────────────────────────────── - - -@router.post("/{graph_id}/run") -async def run_graph( - request: Request, - graph_id: uuid.UUID, - payload: RunGraphRequest = RunGraphRequest(), - auth: AuthContext = Depends(get_current_user_or_token), - db: AsyncSession = Depends(get_db), -): - """ - Start a Graph execution. - - Authenticate via PlatformToken and start an async Graph execution. - Returns an executionId for subsequent status queries and result retrieval. - """ - _require_graph_execute(auth, graph_id) - user = auth.user - log = _bind_log(request, user_id=str(user.id), graph_id=str(graph_id)) - log.info("openapi.graph.run start") - - service = OpenApiGraphService(db) - result = await service.run_graph( - graph_id=graph_id, - user_id=user.id, - variables=payload.variables, - ) - - log.info(f"openapi.graph.run success execution_id={result['executionId']}") - return {"success": True, "data": result} - - -@router.get("/{execution_id}/status") -async def get_execution_status( - request: Request, - execution_id: uuid.UUID, - auth: AuthContext = Depends(get_current_user_or_token), - db: AsyncSession = Depends(get_db), -): - """ - Query execution status. - - Return the current status (init / executing / finish / failed). - """ - user = auth.user - log = _bind_log(request, user_id=str(user.id), execution_id=str(execution_id)) - log.info("openapi.graph.status start") - - service = OpenApiGraphService(db) - result = await service.get_status(execution_id, user.id) - - _require_graph_execute(auth, uuid.UUID(result["graphId"])) - - log.info(f"openapi.graph.status success status={result['status']}") - return {"success": True, "data": result} - - -@router.post("/{execution_id}/abort") -async def abort_execution( - request: Request, - execution_id: uuid.UUID, - auth: AuthContext = Depends(get_current_user_or_token), - db: AsyncSession = Depends(get_db), -): - """ - Abort execution. - - Abort a running Graph execution. - """ - user = auth.user - log = _bind_log(request, user_id=str(user.id), execution_id=str(execution_id)) - log.info("openapi.graph.abort start") - - service = OpenApiGraphService(db) - result = await service.abort_execution(execution_id, user.id) - - _require_graph_execute(auth, uuid.UUID(result["graphId"])) - - log.info(f"openapi.graph.abort success status={result['status']}") - return {"success": True, "data": result} - - -@router.get("/{execution_id}/result") -async def get_execution_result( - request: Request, - execution_id: uuid.UUID, - auth: AuthContext = Depends(get_current_user_or_token), - db: AsyncSession = Depends(get_db), -): - """ - Get execution result. - - Retrieve the output of a Graph execution. - If execution is not yet complete, output is null. - """ - user = auth.user - log = _bind_log(request, user_id=str(user.id), execution_id=str(execution_id)) - log.info("openapi.graph.result start") - - service = OpenApiGraphService(db) - result = await service.get_result(execution_id, user.id) - - _require_graph_execute(auth, uuid.UUID(result["graphId"])) - - log.info(f"openapi.graph.result success status={result['status']}") - return {"success": True, "data": result} diff --git a/backend/app/api/v1/openclaw_chat.py b/backend/app/api/v1/openclaw_chat.py deleted file mode 100644 index b6dc4242f..000000000 --- a/backend/app/api/v1/openclaw_chat.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -OpenClaw Chat API — proxy to the user's OpenClaw /v1/chat/completions -and /tools/invoke endpoints. - -Automatically ensures the user's instance is running before forwarding. -Supports SSE streaming for chat completions. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -import httpx -from fastapi import APIRouter, Depends -from fastapi.responses import StreamingResponse -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.openclaw_instance_service import OpenClawInstanceService - -router = APIRouter(prefix="/v1/openclaw/chat", tags=["OpenClaw Chat"]) - - -class ChatMessage(BaseModel): - role: str = "user" - content: str - - -class ChatRequest(BaseModel): - messages: List[ChatMessage] - model: str = Field(default="openclaw:main") - stream: bool = Field(default=True) - temperature: Optional[float] = None - max_tokens: Optional[int] = None - - -class ToolInvokeRequest(BaseModel): - tool: str - input: Dict[str, Any] = Field(default_factory=dict) - - -@router.post("") -async def chat_completions( - payload: ChatRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Proxy chat completions to the user's OpenClaw instance. - Returns SSE stream if stream=True, JSON otherwise.""" - service = OpenClawInstanceService(db) - instance = await service.ensure_instance_running(str(current_user.id)) - url = f"{service.get_gateway_url(instance)}/v1/chat/completions" - headers = { - "Authorization": f"Bearer {instance.gateway_token}", - "Content-Type": "application/json", - } - - body = payload.model_dump(exclude_none=True) - - if payload.stream: - return StreamingResponse( - _stream_sse(url, headers, body), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "X-Accel-Buffering": "no", - }, - ) - else: - async with httpx.AsyncClient(timeout=300) as client: - resp = await client.post(url, json=body, headers=headers) - resp.raise_for_status() - return resp.json() - - -async def _stream_sse(url: str, headers: dict, body: dict): - """Forward SSE stream from OpenClaw gateway.""" - try: - async with httpx.AsyncClient(timeout=300) as client: - async with client.stream("POST", url, json=body, headers=headers) as resp: - resp.raise_for_status() - async for line in resp.aiter_lines(): - if line: - yield f"{line}\n\n" - else: - yield "\n" - except Exception as e: - yield f"data: {{'error': '{str(e)}'}}\n\n" - - -@router.post("/tools/invoke") -async def invoke_tool( - payload: ToolInvokeRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Proxy tool invocation to the user's OpenClaw instance.""" - service = OpenClawInstanceService(db) - instance = await service.ensure_instance_running(str(current_user.id)) - url = f"{service.get_gateway_url(instance)}/tools/invoke" - headers = { - "Authorization": f"Bearer {instance.gateway_token}", - "Content-Type": "application/json", - } - - async with httpx.AsyncClient(timeout=120) as client: - resp = await client.post(url, json=payload.model_dump(), headers=headers) - resp.raise_for_status() - return resp.json() diff --git a/backend/app/api/v1/openclaw_devices.py b/backend/app/api/v1/openclaw_devices.py deleted file mode 100644 index e5910c2e1..000000000 --- a/backend/app/api/v1/openclaw_devices.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -OpenClaw Device pairing management API. - -- List paired devices -- Approve individual device pairing requests -- Approve all pending requests -""" - -from __future__ import annotations - -import asyncio -import json - -import docker -from fastapi import APIRouter, Depends -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.core.agent.backends.docker_check import get_docker_client -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.enums import InstanceStatus -from app.services.openclaw_instance_service import OpenClawInstanceService - -router = APIRouter(prefix="/v1/openclaw/devices", tags=["OpenClaw Devices"]) - - -async def _get_running_instance(db: AsyncSession, user_id: str): - service = OpenClawInstanceService(db) - instance = await service.get_instance_by_user(user_id) - if not instance or instance.status != InstanceStatus.RUNNING or not instance.container_id: - return None - return instance - - -async def _docker_exec(container_id: str, cmd: list[str]) -> str: - """Run a command inside the user's OpenClaw container.""" - try: - client = get_docker_client() - container = await asyncio.to_thread(client.containers.get, container_id) - exit_code, output = await asyncio.to_thread(container.exec_run, cmd=cmd) - output_str = output.decode("utf-8") if isinstance(output, bytes) else str(output) - if exit_code != 0: - raise RuntimeError(f"Command failed with exit code {exit_code}: {output_str.strip()}") - return output_str.strip() - except docker.errors.NotFound: - raise RuntimeError(f"Container {container_id} not found") - - -@router.get("") -async def list_devices( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List devices paired with the user's OpenClaw instance.""" - instance = await _get_running_instance(db, str(current_user.id)) - if not instance: - return {"success": True, "data": []} - - try: - output = await _docker_exec(instance.container_id, ["openclaw", "devices", "list", "--json"]) - devices = json.loads(output) if output else [] - return {"success": True, "data": devices} - except Exception as e: - logger.warning(f"Failed to list devices for user {current_user.id}: {e}") - return {"success": True, "data": [], "warning": str(e)} - - -@router.post("/{device_id}/approve") -async def approve_device( - device_id: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Approve a specific device pairing request.""" - instance = await _get_running_instance(db, str(current_user.id)) - if not instance: - return {"success": False, "error": "No running instance"} - - try: - await _docker_exec(instance.container_id, ["openclaw", "devices", "approve", device_id]) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} - - -@router.post("/approve-all") -async def approve_all_devices( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Approve all pending device pairing requests.""" - instance = await _get_running_instance(db, str(current_user.id)) - if not instance: - return {"success": False, "error": "No running instance"} - - try: - output = await _docker_exec(instance.container_id, ["openclaw", "devices", "list", "--json"]) - devices = json.loads(output) if output else {} - pending = devices.get("pending", []) - for p in pending: - device_id = p.get("deviceId") - if device_id: - await _docker_exec(instance.container_id, ["openclaw", "devices", "approve", device_id]) - return {"success": True} - except Exception as e: - return {"success": False, "error": str(e)} diff --git a/backend/app/api/v1/openclaw_instances.py b/backend/app/api/v1/openclaw_instances.py deleted file mode 100644 index ae18a0698..000000000 --- a/backend/app/api/v1/openclaw_instances.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -OpenClaw Instance management API. - -Per-user OpenClaw instance lifecycle: create, get status, stop, restart, delete. -""" - -from __future__ import annotations - -from typing import Any, Dict, Optional - -from fastapi import APIRouter, Depends -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.enums import InstanceStatus -from app.services.openclaw_instance_service import OpenClawInstanceService - -router = APIRouter(prefix="/v1/openclaw/instances", tags=["OpenClaw Instances"]) - - -class InstanceConfigRequest(BaseModel): - config_json: Optional[Dict[str, Any]] = Field(default=None, description="OpenClaw config overrides") - - -def _serialize_instance(inst) -> Dict[str, Any]: - return { - "id": inst.id, - "userId": inst.user_id, - "name": inst.name, - "status": inst.status, - "containerId": inst.container_id, - "gatewayPort": inst.gateway_port, - "gatewayToken": inst.gateway_token, - "lastActiveAt": inst.last_active_at.isoformat() if inst.last_active_at else None, - "errorMessage": inst.error_message, - "createdAt": inst.created_at.isoformat() if inst.created_at else None, - "updatedAt": inst.updated_at.isoformat() if inst.updated_at else None, - } - - -@router.get("") -async def get_my_instance( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get the current user's OpenClaw instance status.""" - service = OpenClawInstanceService(db) - status_data = await service.get_instance_status(str(current_user.id)) - return {"success": True, "data": status_data} - - -@router.post("") -async def start_instance( - payload: Optional[InstanceConfigRequest] = None, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Start or ensure the current user's OpenClaw instance is running.""" - service = OpenClawInstanceService(db) - - if payload and payload.config_json: - inst = await service.get_instance_by_user(str(current_user.id)) - if inst: - inst.config_json = payload.config_json - await db.commit() - - try: - instance = await service.ensure_instance_running(str(current_user.id)) - return {"success": True, "data": _serialize_instance(instance)} - except RuntimeError as exc: - return {"success": False, "error": str(exc)} - - -@router.post("/stop") -async def stop_instance( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Stop the current user's OpenClaw instance.""" - service = OpenClawInstanceService(db) - instance = await service.stop_instance(str(current_user.id)) - if not instance: - return {"success": False, "error": "No instance found"} - return {"success": True, "data": _serialize_instance(instance)} - - -@router.post("/restart") -async def restart_instance( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Restart the current user's OpenClaw instance.""" - service = OpenClawInstanceService(db) - try: - instance = await service.restart_instance(str(current_user.id)) - return {"success": True, "data": _serialize_instance(instance)} - except RuntimeError as exc: - return {"success": False, "error": str(exc)} - - -@router.delete("") -async def delete_instance( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete the current user's OpenClaw instance and container.""" - service = OpenClawInstanceService(db) - deleted = await service.delete_instance(str(current_user.id)) - if not deleted: - return {"success": False, "error": "No instance found"} - return {"success": True} - - -@router.post("/sync-skills") -async def sync_skills( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Sync the current user's skills to their OpenClaw container.""" - service = OpenClawInstanceService(db) - instance = await service.get_instance_by_user(str(current_user.id)) - - if not instance or not instance.container_id or instance.status != InstanceStatus.RUNNING: - return {"success": False, "error": "Instance is not running"} - - synced_count = await service.sync_skills_to_container(str(current_user.id), instance.container_id) - - if synced_count < 0: - return {"success": False, "error": "Failed to sync skills"} - - return {"success": True, "data": {"syncedCount": synced_count}} diff --git a/backend/app/api/v1/openclaw_proxy.py b/backend/app/api/v1/openclaw_proxy.py deleted file mode 100644 index b2b36ca66..000000000 --- a/backend/app/api/v1/openclaw_proxy.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -OpenClaw HTTP reverse proxy — enables iframe embedding of the native -OpenClaw Web UI (Control Dashboard + WebChat). - -All requests to /api/v1/openclaw/proxy/{path} are forwarded to the user's -OpenClaw Gateway running on its allocated port. -""" - -from __future__ import annotations - -import asyncio -import json -import re -from urllib.parse import parse_qs, urlencode, urlparse - -import docker -import httpx -from fastapi import APIRouter, Depends, Request, Response -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.core.agent.backends.docker_check import get_docker_client -from app.core.database import get_db -from app.core.settings import settings -from app.models.auth import AuthUser as User -from app.models.enums import InstanceStatus -from app.services.openclaw_instance_service import OpenClawInstanceService - -router = APIRouter(prefix="/v1/openclaw/proxy", tags=["OpenClaw Proxy"]) - -PROXY_TIMEOUT = 300 - - -def _make_inject_script(ws_url: str, token: str) -> str: - """Script to set Control UI connection config in localStorage (runs before app).""" - ws_js = json.dumps(ws_url) - tok_js = json.dumps(token) - return f"""""" - - -def _inject_script_into_html(html: str, script: str) -> str: - """Inject script as early as possible (after or at start).""" - if "]*>)", r"\1" + script, html, count=1, flags=re.I) - if "]*>)", r"\1" + script, html, count=1, flags=re.I) - return script + html - - -async def _poll_approve_devices(container_id: str) -> None: - """Wait and poll to approve devices shortly after UI triggers websocket connect.""" - if not container_id: - return - - try: - client = get_docker_client() - container = await asyncio.to_thread(client.containers.get, container_id) - except Exception as e: - logger.warning(f"[Auto-Pair] Failed to get container {container_id}: {e}") - return - - for _ in range(8): # Poll every 1.5s for up to 12s - await asyncio.sleep(1.5) - try: - exit_code, output = await asyncio.to_thread( - container.exec_run, cmd=["openclaw", "devices", "list", "--json"] - ) - if exit_code != 0: - continue - - output_str = output.decode("utf-8") if isinstance(output, bytes) else output - devices = json.loads(output_str) if output_str else {} - pending = devices.get("pending", []) - - for p in pending: - request_id = p.get("requestId") - if request_id: - await asyncio.to_thread(container.exec_run, cmd=["openclaw", "devices", "approve", request_id]) - logger.info( - f"[Auto-Pair] Approved device request {request_id} for openclaw container {container_id}" - ) - if pending: - break # We found and approved pending devices, done polling. - except docker.errors.NotFound: - break - except Exception as e: - logger.warning(f"[Auto-Pair] Background approve devices failed: {e}") - - -SKIP_RESPONSE_HEADERS = { - "content-encoding", - "transfer-encoding", - "connection", - "keep-alive", - "content-security-policy", - "x-frame-options", - "content-length", -} - - -@router.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"]) -async def proxy_to_openclaw( - path: str, - request: Request, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Reverse proxy any request to the user's OpenClaw gateway.""" - service = OpenClawInstanceService(db) - instance = await service.get_instance_by_user(str(current_user.id)) - - if not instance or instance.status != InstanceStatus.RUNNING: - return Response(content="OpenClaw instance not running", status_code=503) - - base_url = service.get_gateway_url(instance) - target_url = f"{base_url}/{path}" - - # For Control UI entry pages, inject gatewayUrl + token so the dashboard can auto-connect - entry_paths = ("", "overview", "index.html") - path_normalized = (path or "").rstrip("/") or "" - is_entry = path_normalized in entry_paths - - # Auto device pair: wait for the client to connect WebSocket and then approve - if is_entry and instance.container_id: - asyncio.create_task(_poll_approve_devices(instance.container_id)) - - query_params = {k: v for k, v in parse_qs(request.url.query).items()} - if is_entry: - scheme = "wss" if request.url.scheme == "https" else "ws" - host = request.url.hostname or "localhost" - ws_url = f"{scheme}://{host}:{instance.gateway_port}" - query_params["gatewayUrl"] = [ws_url] - query_params["token"] = [instance.gateway_token] - - if query_params: - target_url += "?" + urlencode(query_params, doseq=True) - - body = await request.body() - - upstream_headers = {k: v for k, v in request.headers.items()} - upstream_headers.pop("host", None) - upstream_headers["Authorization"] = f"Bearer {instance.gateway_token}" - - try: - async with httpx.AsyncClient(timeout=PROXY_TIMEOUT, follow_redirects=True) as client: - resp = await client.request( - method=request.method, - url=target_url, - content=body if body else None, - headers=upstream_headers, - ) - - response_headers = {} - for k, v in resp.headers.multi_items(): - if k.lower() not in SKIP_RESPONSE_HEADERS: - response_headers[k] = v - - parsed = urlparse(settings.frontend_url) - frontend_origin = f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else "" - if frontend_origin: - response_headers["Content-Security-Policy"] = f"frame-ancestors 'self' {frontend_origin}" - response_headers["Access-Control-Allow-Origin"] = "*" - - content = resp.content - # Iframe: Control UI may ignore URL params. Inject localStorage for auto-connect. - if is_entry and resp.status_code == 200: - ct = (resp.headers.get("content-type") or "").lower() - if "text/html" in ct: - scheme = "wss" if request.url.scheme == "https" else "ws" - host = request.url.hostname or "localhost" - ws_url = f"{scheme}://{host}:{instance.gateway_port}" - inj = _make_inject_script(ws_url, instance.gateway_token) - try: - html = content.decode("utf-8", errors="replace") - content = _inject_script_into_html(html, inj).encode("utf-8") - except Exception: - pass - - return Response( - content=content, - status_code=resp.status_code, - headers=response_headers, - media_type=resp.headers.get("content-type"), - ) - except httpx.ConnectError: - return Response(content="Cannot reach OpenClaw gateway", status_code=502) - except httpx.TimeoutException: - return Response(content="Gateway timeout", status_code=504) diff --git a/backend/app/api/v1/organizations.py b/backend/app/api/v1/organizations.py deleted file mode 100644 index c24efcb68..000000000 --- a/backend/app/api/v1/organizations.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Organization and member API -""" - -import uuid -from typing import Optional - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, EmailStr, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user, require_org_role -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.enums import OrgRole -from app.services.organization_service import OrganizationService - -router = APIRouter(prefix="/v1/organizations", tags=["Organizations"]) - - -# --------------------------------------------------------------------------- # -# Schemas -# --------------------------------------------------------------------------- # -class UpdateOrganizationRequest(BaseModel): - name: Optional[str] = Field(None, min_length=1, max_length=255) - slug: Optional[str] = Field(None, min_length=3, max_length=255) - logo: Optional[str] = Field(None, max_length=500) - - -class UpdateSeatsRequest(BaseModel): - seats: int = Field(..., ge=1, le=50) - - -class InviteMemberRequest(BaseModel): - email: EmailStr - role: Optional[str] = Field(default=OrgRole.MEMBER) - workspaceInvitations: Optional[list] = None # compatible with frontend multi-workspace invitation params - - -class UpdateMemberRoleRequest(BaseModel): - role: str = Field(..., description="owner/admin/member") - - -class CreateOrganizationRequest(BaseModel): - name: str = Field(..., min_length=1, max_length=255) - slug: Optional[str] = Field(None, min_length=3, max_length=255) - logo: Optional[str] = Field(None, max_length=500) - - -# --------------------------------------------------------------------------- # -# Routes - Organizations -# --------------------------------------------------------------------------- # -@router.post("") -async def create_organization( - payload: CreateOrganizationRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create an organization and set the current user as owner.""" - service = OrganizationService(db) - data = await service.create_organization( - name=payload.name, - slug=payload.slug or payload.name.lower().replace(" ", "-"), - logo=payload.logo, - current_user=current_user, - ) - return success_response(data=data, message="Organization created") - - -@router.post("/{organization_id}/activate") -async def set_active_organization( - organization_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Set active organization (currently only validates membership and returns org info).""" - service = OrganizationService(db) - data = await service.set_active_organization(organization_id, current_user) - return success_response(data=data, message="Organization set active") - - -@router.get("/{organization_id}") -async def get_organization( - organization_id: uuid.UUID, - include: Optional[str] = Query(None, description="Optional: seats"), - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.MEMBER), -): - """Get organization details.""" - service = OrganizationService(db) - include_seats = "seats" in _parse_include(include) - data = await service.get_organization(organization_id, include_seats, current_user) - return success_response(data=data) - - -@router.put("/{organization_id}") -async def update_organization( - organization_id: uuid.UUID, - payload: UpdateOrganizationRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.ADMIN), -): - """Update organization settings.""" - service = OrganizationService(db) - data = await service.update_organization( - organization_id, - name=payload.name, - slug=payload.slug, - logo=payload.logo, - current_user=current_user, - ) - return success_response(data=data, message="Organization updated") - - -@router.put("/{organization_id}/seats") -async def update_seats( - organization_id: uuid.UUID, - payload: UpdateSeatsRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.ADMIN), -): - """Update seats.""" - service = OrganizationService(db) - data = await service.update_seats( - organization_id, - seats=payload.seats, - current_user=current_user, - ) - return success_response(data=data, message="Seats updated") - - -# --------------------------------------------------------------------------- # -# Routes - Members -# --------------------------------------------------------------------------- # -@router.get("/{organization_id}/members") -async def list_members( - organization_id: uuid.UUID, - include: Optional[str] = Query(None, description="Optional: usage"), - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.MEMBER), -): - """List members.""" - service = OrganizationService(db) - include_usage = "usage" in _parse_include(include) - data = await service.list_members( - organization_id, - include_usage=include_usage, - current_user=current_user, - ) - return success_response(data=data) - - -@router.post("/{organization_id}/members") -async def invite_member( - organization_id: uuid.UUID, - payload: InviteMemberRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.ADMIN), -): - """Invite a new member.""" - service = OrganizationService(db) - data = await service.invite_member( - organization_id, - email=payload.email, - role=payload.role or OrgRole.MEMBER, - current_user=current_user, - ) - return success_response(data=data, message="Member invited") - - -@router.get("/{organization_id}/members/{member_id}") -async def get_member( - organization_id: uuid.UUID, - member_id: uuid.UUID, - include: Optional[str] = Query(None, description="Optional: usage"), - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.MEMBER), -): - """Get member details.""" - service = OrganizationService(db) - include_usage = "usage" in _parse_include(include) - data = await service.get_member( - organization_id, - member_id, - include_usage=include_usage, - current_user=current_user, - ) - return success_response(data=data) - - -@router.put("/{organization_id}/members/{member_id}") -async def update_member_role( - organization_id: uuid.UUID, - member_id: uuid.UUID, - payload: UpdateMemberRoleRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_org_role(OrgRole.ADMIN), -): - """Update member role.""" - service = OrganizationService(db) - data = await service.update_member_role( - organization_id, - member_id, - role=payload.role, - current_user=current_user, - ) - return success_response(data=data, message="Member role updated") - - -@router.delete("/{organization_id}/members/{member_id}") -async def remove_member( - organization_id: uuid.UUID, - member_id: uuid.UUID, - shouldReduceSeats: bool = Query(False, description="Whether to reduce seats when removing a member"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Remove a member.""" - service = OrganizationService(db) - await service.remove_member( - organization_id, - member_id, - current_user=current_user, - should_reduce_seats=bool(shouldReduceSeats), - ) - return success_response(message="Member removed") - - -# --------------------------------------------------------------------------- # -# Helpers -# --------------------------------------------------------------------------- # -def _parse_include(include: Optional[str]) -> set[str]: - if not include: - return set() - return {part.strip() for part in include.split(",") if part.strip()} diff --git a/backend/app/api/v1/runs.py b/backend/app/api/v1/runs.py deleted file mode 100644 index 388f23cc2..000000000 --- a/backend/app/api/v1/runs.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Runs API.""" - -from __future__ import annotations - -import uuid - -from fastapi import APIRouter, Depends, Query -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import CurrentUser -from app.common.exceptions import BadRequestException -from app.core.database import get_db -from app.models.agent_run import AgentRun, AgentRunStatus -from app.schemas import BaseResponse -from app.schemas.runs import ( - AgentDefinitionResponse, - AgentListResponse, - CreateRunRequest, - CreateRunResponse, - CreateSkillCreatorRunRequest, - RunEventResponse, - RunEventsPageResponse, - RunListResponse, - RunSnapshotResponse, - RunSummary, -) -from app.services.run_reducers import agent_registry -from app.services.run_service import RunService -from app.utils.task_manager import task_manager - -router = APIRouter(prefix="/v1/runs", tags=["Runs"]) - - -def _to_run_summary(run: AgentRun) -> RunSummary: - definition = agent_registry.find(run.agent_name) - return RunSummary( - run_id=run.id, - status=run.status.value if hasattr(run.status, "value") else str(run.status), - run_type=run.run_type, - agent_name=run.agent_name, - agent_display_name=definition.display_name if definition else run.agent_name, - source=run.source, - thread_id=run.thread_id, - graph_id=run.graph_id, - title=run.title, - started_at=run.started_at, - finished_at=run.finished_at, - last_seq=run.last_seq, - error_code=run.error_code, - error_message=run.error_message, - last_heartbeat_at=run.last_heartbeat_at, - updated_at=run.updated_at, - ) - - -@router.get("", response_model=BaseResponse[RunListResponse]) -async def list_runs( - current_user: CurrentUser, - run_type: str | None = Query(None), - agent_name: str | None = Query(None), - status: str | None = Query(None), - search: str | None = Query(None, max_length=200), - limit: int = Query(50, ge=1, le=200), - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunListResponse]: - service = RunService(db) - runs = await service.list_recent_runs( - user_id=str(current_user.id), - run_type=run_type, - agent_name=agent_name, - status=status, - search=search, - limit=limit, - ) - return BaseResponse( - success=True, - code=200, - msg="ok", - data=RunListResponse(items=[_to_run_summary(run) for run in runs]), - ) - - -@router.get("/agents", response_model=BaseResponse[AgentListResponse]) -async def list_agents( - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[AgentListResponse]: - service = RunService(db) - return BaseResponse( - success=True, - code=200, - msg="ok", - data=AgentListResponse( - items=[ - AgentDefinitionResponse(agent_name=definition.agent_name, display_name=definition.display_name) - for definition in await service.list_agents() - ] - ), - ) - - -@router.get("/active", response_model=BaseResponse[RunSummary | None]) -async def get_active_run( - current_user: CurrentUser, - agent_name: str = Query(..., min_length=1), - graph_id: uuid.UUID | None = Query(None), - thread_id: str | None = Query(None), - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunSummary | None]: - service = RunService(db) - run = await service.find_latest_active_run( - user_id=str(current_user.id), - agent_name=agent_name, - graph_id=graph_id, - thread_id=thread_id, - ) - return BaseResponse( - success=True, - code=200, - msg="ok", - data=_to_run_summary(run) if run else None, - ) - - -@router.get("/active/skill-creator", response_model=BaseResponse[RunSummary | None]) -async def get_active_skill_creator_run( - current_user: CurrentUser, - graph_id: uuid.UUID, - thread_id: str | None = Query(None), - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunSummary | None]: - return await get_active_run( - current_user=current_user, - agent_name="skill_creator", - graph_id=graph_id, - thread_id=thread_id, - db=db, - ) - - -@router.post("", response_model=BaseResponse[CreateRunResponse]) -async def create_run( - request: CreateRunRequest, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[CreateRunResponse]: - service = RunService(db) - try: - run = await service.create_run( - user_id=str(current_user.id), - agent_name=request.agent_name, - graph_id=request.graph_id, - thread_id=request.thread_id, - message=request.message, - input=request.input, - ) - except ValueError as exc: - raise BadRequestException(str(exc)) - return BaseResponse( - success=True, - code=200, - msg="Run created", - data=CreateRunResponse( - run_id=run.id, - thread_id=run.thread_id or "", - status=run.status.value if hasattr(run.status, "value") else str(run.status), - ), - ) - - -@router.post("/skill-creator", response_model=BaseResponse[CreateRunResponse]) -async def create_skill_creator_run( - request: CreateSkillCreatorRunRequest, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[CreateRunResponse]: - return await create_run( - request=CreateRunRequest( - agent_name="skill_creator", - graph_id=request.graph_id, - message=request.message, - thread_id=request.thread_id, - input={"edit_skill_id": request.edit_skill_id}, - ), - current_user=current_user, - db=db, - ) - - -@router.get("/{run_id}", response_model=BaseResponse[RunSummary]) -async def get_run( - run_id: uuid.UUID, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunSummary]: - service = RunService(db) - run = await service.get_run(run_id, str(current_user.id)) - if run is None: - return BaseResponse(success=False, code=404, msg="Run not found", data=None) - return BaseResponse(success=True, code=200, msg="ok", data=_to_run_summary(run)) - - -@router.get("/{run_id}/snapshot", response_model=BaseResponse[RunSnapshotResponse]) -async def get_run_snapshot( - run_id: uuid.UUID, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunSnapshotResponse]: - service = RunService(db) - snapshot = await service.get_snapshot(run_id, str(current_user.id)) - if snapshot is None: - return BaseResponse(success=False, code=404, msg="Snapshot not found", data=None) - return BaseResponse( - success=True, - code=200, - msg="ok", - data=RunSnapshotResponse( - run_id=run_id, - status=snapshot.status, - last_seq=snapshot.last_seq, - projection=snapshot.projection or {}, - ), - ) - - -@router.get("/{run_id}/events", response_model=BaseResponse[RunEventsPageResponse]) -async def get_run_events( - current_user: CurrentUser, - run_id: uuid.UUID, - after_seq: int = Query(0, ge=0), - limit: int = Query(500, ge=1, le=1000), - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunEventsPageResponse]: - service = RunService(db) - events = await service.list_events_after(run_id, str(current_user.id), after_seq=after_seq, limit=limit) - return BaseResponse( - success=True, - code=200, - msg="ok", - data=RunEventsPageResponse( - run_id=run_id, - events=[ - RunEventResponse( - seq=event.seq, - event_type=event.event_type, - payload=event.payload or {}, - trace_id=event.trace_id, - observation_id=event.observation_id, - parent_observation_id=event.parent_observation_id, - created_at=event.created_at, - ) - for event in events - ], - next_after_seq=events[-1].seq if events else after_seq, - ), - ) - - -@router.post("/{run_id}/cancel", response_model=BaseResponse[RunSummary]) -async def cancel_run( - run_id: uuid.UUID, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -) -> BaseResponse[RunSummary]: - service = RunService(db) - run = await service.get_run(run_id, str(current_user.id)) - if run is None: - return BaseResponse(success=False, code=404, msg="Run not found", data=None) - - if run.thread_id and run.status in { - AgentRunStatus.QUEUED, - AgentRunStatus.RUNNING, - AgentRunStatus.INTERRUPT_WAIT, - }: - try: - await task_manager.stop_task(run.thread_id) - except Exception: - logger.debug("Failed to stop task for thread_id=%s during run cancellation", run.thread_id, exc_info=True) - - run = await service.mark_status( - run_id=run_id, - user_id=str(current_user.id), - status=AgentRunStatus.CANCELLED, - error_code="cancelled", - error_message="Cancelled by user", - ) - if run is None: - return BaseResponse(success=False, code=404, msg="Run not found", data=None) - return BaseResponse(success=True, code=200, msg="Run cancelled", data=_to_run_summary(run)) diff --git a/backend/app/api/v1/sandboxes.py b/backend/app/api/v1/sandboxes.py deleted file mode 100644 index 7cce489d2..000000000 --- a/backend/app/api/v1/sandboxes.py +++ /dev/null @@ -1,264 +0,0 @@ -""" -Admin Sandbox Management API -""" - -from datetime import datetime -from typing import List, Optional - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, ConfigDict -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import BadRequestException, ForbiddenException, NotFoundException -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.user_sandbox import UserSandbox -from app.services.sandbox_manager import SandboxManagerService - -router = APIRouter(prefix="/v1/sandboxes", tags=["Sandboxes"]) - - -# Dependencies - - -async def _verify_sandbox_ownership(sandbox_id: str, current_user: User, db: AsyncSession): - """Validate that the current user owns the sandbox or is a super user.""" - if current_user.is_super_user: - return - result = await db.execute(select(UserSandbox).where(UserSandbox.id == sandbox_id)) - sb = result.scalar_one_or_none() - if not sb: - raise NotFoundException("Sandbox not found") - if sb.user_id != str(current_user.id): - raise ForbiddenException("Access denied") - - -# Schemas - - -class SandboxResponse(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: str - user_id: str - container_id: Optional[str] = None - status: str - image: str - runtime: Optional[str] = None - last_active_at: Optional[datetime] = None - error_message: Optional[str] = None - cpu_limit: Optional[float] = None - memory_limit: Optional[int] = None - idle_timeout: int - created_at: datetime - updated_at: datetime - - # User info (joined) - user_name: Optional[str] = None - user_email: Optional[str] = None - - -class SandboxListResponse(BaseModel): - items: List[SandboxResponse] - total: int - page: int - size: int - - -class SandboxUpdateBody(BaseModel): - """Body for PATCH /sandboxes/{id}: update sandbox config. New image takes effect on next rebuild.""" - - image: Optional[str] = None - - -# Endpoints - - -@router.get("", response_model=SandboxListResponse) -async def list_sandboxes( - page: int = Query(1, ge=1), - size: int = Query(20, ge=1, le=100), - status: Optional[str] = Query(None), - user_id: Optional[str] = Query(None), - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """List all user sandboxes with pagination and filtering.""" - # Build query - query = select(UserSandbox).join(UserSandbox.user) - - if not current_user.is_super_user: - user_id = str(current_user.id) - - if status: - query = query.where(UserSandbox.status == status) - if user_id: - query = query.where(UserSandbox.user_id == user_id) - - # Count total - count_query = select(func.count()).select_from(query.subquery()) - total = (await db.execute(count_query)).scalar_one() - - # Pagination - query = query.offset((page - 1) * size).limit(size).order_by(UserSandbox.updated_at.desc()) - - # Execute - result = await db.execute(query) - sandboxes = result.scalars().all() - - # Serialize - items = [] - for sb in sandboxes: - # Since we joined, we can access user info if eager loaded, - # but here we rely on lazy loading (which triggers individual queries) - # or we should optins join load. For admin list, N+1 is acceptable for small page size, - # but let's be efficient if we can. - # actually, UserSandbox.user is relationship. - - # Pydantic conversion needs help with lazy loaded relationships usually - # unless we use response_model config from_attributes=True and it handles it - # But let's build dict manually for control - item = SandboxResponse( - id=sb.id, - user_id=sb.user_id, - container_id=sb.container_id, - status=sb.status, - image=sb.image, - runtime=sb.runtime, - last_active_at=sb.last_active_at, - error_message=sb.error_message, - cpu_limit=sb.cpu_limit, - memory_limit=sb.memory_limit, - idle_timeout=sb.idle_timeout, - created_at=sb.created_at, - updated_at=sb.updated_at, - user_name=sb.user.name if sb.user else "Unknown", - user_email=sb.user.email if sb.user else "Unknown", - ) - items.append(item) - - return SandboxListResponse(items=items, total=total, page=page, size=size) - - -@router.get("/{sandbox_id}", response_model=SandboxResponse) -async def get_sandbox( - sandbox_id: str, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Get sandbox details.""" - await _verify_sandbox_ownership(sandbox_id, current_user, db) - - result = await db.execute(select(UserSandbox).where(UserSandbox.id == sandbox_id)) - sb = result.scalar_one_or_none() - - if not sb: - raise NotFoundException("Sandbox not found") - - return SandboxResponse( - id=sb.id, - user_id=sb.user_id, - container_id=sb.container_id, - status=sb.status, - image=sb.image, - runtime=sb.runtime, - last_active_at=sb.last_active_at, - error_message=sb.error_message, - cpu_limit=sb.cpu_limit, - memory_limit=sb.memory_limit, - idle_timeout=sb.idle_timeout, - created_at=sb.created_at, - updated_at=sb.updated_at, - user_name=sb.user.name if sb.user else "Unknown", - user_email=sb.user.email if sb.user else "Unknown", - ) - - -@router.patch("/{sandbox_id}", response_model=None) -async def update_sandbox( - sandbox_id: str, - body: SandboxUpdateBody, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Update sandbox config (e.g. image). New image takes effect on next rebuild or container create.""" - await _verify_sandbox_ownership(sandbox_id, current_user, db) - image_value = None - if body.image is not None: - s = body.image.strip() - if not s: - raise BadRequestException("image cannot be empty") - if len(s) > 255: - raise BadRequestException("image must be at most 255 characters") - image_value = s - service = SandboxManagerService(db) - success = await service.update_sandbox_config(sandbox_id, image=image_value) - if not success: - raise NotFoundException("Sandbox not found") - return success_response(message="Sandbox config updated") - - -@router.post("/{sandbox_id}/stop") -async def stop_sandbox( - sandbox_id: str, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Stop a sandbox.""" - await _verify_sandbox_ownership(sandbox_id, current_user, db) - service = SandboxManagerService(db) - success = await service.stop_sandbox(sandbox_id) - if not success: - raise NotFoundException("Sandbox not found or already stopped") - - return success_response(message="Sandbox stopped successfully") - - -@router.post("/{sandbox_id}/restart") -async def restart_sandbox( - sandbox_id: str, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Restart a sandbox.""" - await _verify_sandbox_ownership(sandbox_id, current_user, db) - service = SandboxManagerService(db) - success = await service.restart_sandbox(sandbox_id) - if not success: - raise NotFoundException("Sandbox not found") - - return success_response(message="Sandbox scheduled for restart") - - -@router.post("/{sandbox_id}/rebuild") -async def rebuild_sandbox( - sandbox_id: str, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Rebuild a sandbox (remove old container and start new one).""" - await _verify_sandbox_ownership(sandbox_id, current_user, db) - service = SandboxManagerService(db) - success = await service.rebuild_sandbox(sandbox_id) - if not success: - raise NotFoundException("Sandbox not found") - return success_response(message="Sandbox rebuilt successfully") - - -@router.delete("/{sandbox_id}") -async def delete_sandbox( - sandbox_id: str, - current_user: User = Depends(get_current_user), - db: AsyncSession = Depends(get_db), -): - """Delete a sandbox permanently.""" - await _verify_sandbox_ownership(sandbox_id, current_user, db) - service = SandboxManagerService(db) - success = await service.delete_sandbox(sandbox_id) - if not success: - raise NotFoundException("Sandbox not found") - - return success_response(message="Sandbox deleted successfully") diff --git a/backend/app/api/v1/sessions.py b/backend/app/api/v1/sessions.py deleted file mode 100644 index 7e056d574..000000000 --- a/backend/app/api/v1/sessions.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -Module: Sessions API (based on SessionService) - -Overview: -- Provides session CRUD (create, list, get, update title, delete) -- Provides listing messages for a specific session -- Managed via SessionService - -Routes: -- POST /sessions/new_session: Create a session -- GET /sessions: List sessions for the current user -- GET /sessions/{session_id}: Get a specific session -- PATCH /sessions/{session_id}: Update session title -- DELETE /sessions/{session_id}: Delete a session -- GET /sessions/{session_id}/messages: Get messages for a session - -Dependencies: -- Session service: SessionService (Depends(get_session_service)) -- Database session: Session (Depends(get_db)) - -Requests/Responses: -- Request model: SessionCreate -- Response models: SessionResponse, MessageResponse -- Unified errors: HTTPException - -Error codes: -- 404: Session not found -- 400: Invalid parameters or business rule failure -""" - -from typing import List - -from fastapi import APIRouter, Depends -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import CurrentUser -from app.common.exceptions import AppException, InternalServerException, NotFoundException -from app.core.database import get_db -from app.schemas.common import SessionCreate, SessionMessageResponse, SessionResponse -from app.services.session_service import SessionService - -router = APIRouter() - -# --- Session endpoints (based on SessionService) --- -# ----- Create ----- -# ----- Read ----- -# ----- Update ----- -# ----- Delete ----- -# ----- Messages ----- - - -def get_session_service(db: AsyncSession = Depends(get_db)) -> SessionService: - return SessionService(db) - - -@router.post("/new_session", response_model=SessionResponse) -async def create_session( - session_data: SessionCreate, - current_user: CurrentUser, - session_service: SessionService = Depends(get_session_service), -): - """Create a new session.""" - try: - return await session_service.create_session(session_data, user_id=current_user.id) - except AppException: - raise - except Exception as e: - raise InternalServerException("Failed to create session") from e - - -@router.get("/", response_model=List[SessionResponse]) -async def get_sessions( - current_user: CurrentUser, - session_service: SessionService = Depends(get_session_service), -): - """Get all sessions for the current user.""" - try: - return await session_service.get_user_sessions(user_id=current_user.id) - except AppException: - raise - except Exception as e: - raise InternalServerException("Failed to get sessions") from e - - -@router.get("/{session_id}", response_model=SessionResponse) -async def get_session( - session_id: str, - current_user: CurrentUser, - session_service: SessionService = Depends(get_session_service), -): - """Get a specific session.""" - session = await session_service.get_session_for_user(session_id, user_id=current_user.id) - if not session: - raise NotFoundException("Session not found") - return session - - -@router.patch("/{session_id}", response_model=SessionResponse) -async def update_session_title( - session_id: str, - title: str, - current_user: CurrentUser, - session_service: SessionService = Depends(get_session_service), -): - """Update session title.""" - try: - updated_session = await session_service.update_session_title(session_id, title, user_id=current_user.id) - if not updated_session: - raise NotFoundException("Session not found") - return updated_session - except AppException: - raise - except Exception as e: - raise InternalServerException("Failed to update session title") from e - - -@router.delete("/{session_id}") -async def delete_session( - session_id: str, - current_user: CurrentUser, - session_service: SessionService = Depends(get_session_service), -): - """Delete a session.""" - try: - success = await session_service.delete_session(session_id, user_id=current_user.id) - if not success: - raise NotFoundException("Session not found") - return {"success": True, "message": "Session deleted successfully"} - except AppException: - raise - except Exception as e: - raise InternalServerException("Failed to delete session") from e - - -@router.get("/{session_id}/messages", response_model=List[SessionMessageResponse]) -async def get_session_messages( - session_id: str, - current_user: CurrentUser, - limit: int = 100, - session_service: SessionService = Depends(get_session_service), -): - """Get messages for a session.""" - try: - messages = await session_service.get_session_messages(session_id, limit, user_id=current_user.id) - return [ - SessionMessageResponse( - id=msg.id, - session_id=msg.thread_id, - content=msg.content, - role=msg.role, - metadata=msg.meta_data, - created_at=msg.created_at, - ) - for msg in messages - ] - except AppException: - raise - except Exception as e: - raise InternalServerException("Failed to get session messages") from e diff --git a/backend/app/api/v1/skill_collaborators.py b/backend/app/api/v1/skill_collaborators.py deleted file mode 100644 index c0946a176..000000000 --- a/backend/app/api/v1/skill_collaborators.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Skill Collaborator API routes.""" - -from __future__ import annotations - -import uuid - -from fastapi import APIRouter, Depends -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import NotFoundException -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.schemas.skill_collaborator import ( - CollaboratorCreate, - CollaboratorSchema, - CollaboratorUpdate, - TransferOwnershipRequest, -) -from app.services.skill_collaborator_service import SkillCollaboratorService -from app.services.user_service import UserService - -router = APIRouter(prefix="/v1/skills", tags=["Skill Collaborators"]) - - -def _serialize_collaborator(c) -> dict: - """Serialize a SkillCollaborator with related user name/email.""" - data = CollaboratorSchema.model_validate(c).model_dump() - data["user_name"] = c.user.name if c.user else None - data["user_email"] = c.user.email if c.user else None - return data - - -@router.get("/{skill_id}/collaborators") -async def list_collaborators( - skill_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = SkillCollaboratorService(db) - collaborators, skill = await service.list_collaborators( - skill_id=skill_id, - current_user_id=current_user.id, - is_superuser=current_user.is_superuser, - ) - - user_service = UserService(db) - owner = await user_service.get_user_by_id(skill.owner_id) if skill.owner_id else None - - return { - "success": True, - "data": { - "collaborators": [_serialize_collaborator(c) for c in collaborators], - "owner": { - "id": owner.id, - "name": owner.name, - "email": owner.email, - } - if owner - else {"id": skill.owner_id, "name": None, "email": None}, - }, - } - - -@router.post("/{skill_id}/collaborators") -async def add_collaborator( - skill_id: uuid.UUID, - payload: CollaboratorCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - # Resolve user_id from email if needed - target_user_id = payload.user_id - if not target_user_id and payload.email: - user_service = UserService(db) - user = await user_service.get_user_by_email(payload.email.strip()) - if not user: - raise NotFoundException("User not found") - target_user_id = user.id - - if not target_user_id: - raise NotFoundException("User not found") - - service = SkillCollaboratorService(db) - collaborator = await service.add_collaborator( - skill_id=skill_id, - current_user_id=current_user.id, - target_user_id=target_user_id, - role=payload.role, - is_superuser=current_user.is_superuser, - ) - return { - "success": True, - "data": _serialize_collaborator(collaborator), - } - - -@router.put("/{skill_id}/collaborators/{target_user_id}") -async def update_collaborator( - skill_id: uuid.UUID, - target_user_id: str, - payload: CollaboratorUpdate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = SkillCollaboratorService(db) - collaborator = await service.update_collaborator_role( - skill_id=skill_id, - current_user_id=current_user.id, - target_user_id=target_user_id, - new_role=payload.role, - is_superuser=current_user.is_superuser, - ) - return { - "success": True, - "data": _serialize_collaborator(collaborator), - } - - -@router.delete("/{skill_id}/collaborators/{target_user_id}") -async def remove_collaborator( - skill_id: uuid.UUID, - target_user_id: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = SkillCollaboratorService(db) - await service.remove_collaborator( - skill_id=skill_id, - current_user_id=current_user.id, - target_user_id=target_user_id, - is_superuser=current_user.is_superuser, - ) - return {"success": True} - - -@router.post("/{skill_id}/transfer") -async def transfer_ownership( - skill_id: uuid.UUID, - payload: TransferOwnershipRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = SkillCollaboratorService(db) - await service.transfer_ownership( - skill_id=skill_id, - current_user_id=current_user.id, - new_owner_id=payload.new_owner_id, - ) - return {"success": True} diff --git a/backend/app/api/v1/skill_versions.py b/backend/app/api/v1/skill_versions.py deleted file mode 100644 index 02bd9f523..000000000 --- a/backend/app/api/v1/skill_versions.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Skill Version API routes.""" - -from __future__ import annotations - -import uuid - -from fastapi import APIRouter, Depends -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.auth_dependency import AuthContext, get_current_user_or_token -from app.common.dependencies import get_current_user -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.schemas.skill import SkillSchema -from app.schemas.skill_version import ( - VersionPublishRequest, - VersionRestoreRequest, - VersionSchema, - VersionSummarySchema, -) -from app.services.skill_version_service import SkillVersionService - -router = APIRouter(prefix="/v1/skills", tags=["Skill Versions"]) - - -@router.post("/{skill_id}/versions") -async def publish_version( - skill_id: uuid.UUID, - payload: VersionPublishRequest, - db: AsyncSession = Depends(get_db), - auth: AuthContext = Depends(get_current_user_or_token), -): - service = SkillVersionService(db) - version = await service.publish_version( - skill_id=skill_id, - current_user_id=auth.user.id, - version_str=payload.version, - release_notes=payload.release_notes, - is_superuser=auth.user.is_superuser, - token_scopes=auth.scopes, - ) - return { - "success": True, - "data": VersionSchema.model_validate(version).model_dump(), - } - - -@router.get("/{skill_id}/versions") -async def list_versions( - skill_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - auth: AuthContext = Depends(get_current_user_or_token), -): - service = SkillVersionService(db) - versions = await service.list_versions( - skill_id=skill_id, - current_user_id=auth.user.id, - is_superuser=auth.user.is_superuser, - token_scopes=auth.scopes, - ) - return { - "success": True, - "data": [VersionSummarySchema.model_validate(v).model_dump() for v in versions], - } - - -@router.get("/{skill_id}/versions/latest") -async def get_latest_version( - skill_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - auth: AuthContext = Depends(get_current_user_or_token), -): - service = SkillVersionService(db) - version = await service.get_latest_version( - skill_id=skill_id, - current_user_id=auth.user.id, - is_superuser=auth.user.is_superuser, - token_scopes=auth.scopes, - ) - return { - "success": True, - "data": VersionSchema.model_validate(version).model_dump(), - } - - -@router.get("/{skill_id}/versions/{version}") -async def get_version( - skill_id: uuid.UUID, - version: str, - db: AsyncSession = Depends(get_db), - auth: AuthContext = Depends(get_current_user_or_token), -): - service = SkillVersionService(db) - ver = await service.get_version( - skill_id=skill_id, - version_str=version, - current_user_id=auth.user.id, - is_superuser=auth.user.is_superuser, - token_scopes=auth.scopes, - ) - return { - "success": True, - "data": VersionSchema.model_validate(ver).model_dump(), - } - - -@router.delete("/{skill_id}/versions/{version}") -async def delete_version( - skill_id: uuid.UUID, - version: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = SkillVersionService(db) - await service.delete_version( - skill_id=skill_id, - version_str=version, - current_user_id=current_user.id, - is_superuser=current_user.is_superuser, - ) - return {"success": True} - - -@router.post("/{skill_id}/restore") -async def restore_version( - skill_id: uuid.UUID, - payload: VersionRestoreRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = SkillVersionService(db) - skill = await service.restore_draft( - skill_id=skill_id, - version_str=payload.version, - current_user_id=current_user.id, - is_superuser=current_user.is_superuser, - ) - return { - "success": True, - "data": SkillSchema.model_validate(skill).model_dump(), - } diff --git a/backend/app/api/v1/skills.py b/backend/app/api/v1/skills.py deleted file mode 100644 index 4d23dd4b5..000000000 --- a/backend/app/api/v1/skills.py +++ /dev/null @@ -1,288 +0,0 @@ -""" -Skill CRUD API -""" - -from __future__ import annotations - -import uuid -from typing import List, Optional - -from fastapi import APIRouter, Depends, Query -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user, get_current_user_optional -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.enums import InstanceStatus -from app.schemas.skill import ( - SkillCreate, - SkillFileCreate, - SkillFileSchema, - SkillFileUpdate, - SkillSchema, - SkillUpdate, -) -from app.services.openclaw_instance_service import OpenClawInstanceService -from app.services.skill_service import SkillService - - -async def _trigger_openclaw_skill_sync(user_id: str, db: AsyncSession): - """Trigger a sync of skills to the user's OpenClaw container if it is running.""" - try: - instance_service = OpenClawInstanceService(db) - instance = await instance_service.get_instance_by_user(user_id) - if instance and instance.container_id and instance.status == InstanceStatus.RUNNING: - await instance_service.sync_skills_to_container(user_id, instance.container_id) - except Exception as e: - logger.error(f"Failed to trigger openclaw skill sync for user {user_id}: {e}", exc_info=True) - - -router = APIRouter(prefix="/v1/skills", tags=["Skills"]) - - -@router.get("") -async def list_skills( - include_public: bool = Query(True, description="Include public skills"), - tags: Optional[List[str]] = Query(None, description="Filter by tags"), - db: AsyncSession = Depends(get_db), - current_user: Optional[User] = Depends(get_current_user_optional), -): - """List skills.""" - service = SkillService(db) - user_id = current_user.id if current_user else None - skills = await service.list_skills( - current_user_id=user_id, - include_public=include_public, - tags=tags, - ) - return { - "success": True, - "data": [SkillSchema.model_validate(skill).model_dump() for skill in skills], - } - - -@router.post("") -async def create_skill( - payload: SkillCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create a skill.""" - service = SkillService(db) - - files_data = None - if payload.files: - files_data = [f.dict() for f in payload.files] - - skill = await service.create_skill( - created_by_id=current_user.id, - name=payload.name, - description=payload.description, - content=payload.content, - tags=payload.tags, - source_type=payload.source_type, - source_url=payload.source_url, - root_path=payload.root_path, - owner_id=payload.owner_id, - is_public=payload.is_public, - license=payload.license, - files=files_data, - ) - - # reload to include files - skill = await service.get_skill(skill.id, current_user.id) - - # Trigger sync to OpenClaw container - import asyncio - - asyncio.create_task(_trigger_openclaw_skill_sync(current_user.id, db)) - - return { - "success": True, - "data": SkillSchema.model_validate(skill).model_dump(), - } - - -@router.get("/{skill_id}") -async def get_skill( - skill_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: Optional[User] = Depends(get_current_user_optional), -): - """Get skill details.""" - service = SkillService(db) - user_id = current_user.id if current_user else None - skill = await service.get_skill(skill_id, user_id) - return { - "success": True, - "data": SkillSchema.model_validate(skill).model_dump(), - } - - -@router.put("/{skill_id}") -async def update_skill( - skill_id: uuid.UUID, - payload: SkillUpdate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update a skill.""" - service = SkillService(db) - - # Convert files to dict format if provided - files_data = None - if payload.files is not None: - files_data = [f.model_dump() for f in payload.files] - - skill = await service.update_skill( - skill_id, - current_user.id, - name=payload.name, - description=payload.description, - content=payload.content, - tags=payload.tags, - source_type=payload.source_type, - source_url=payload.source_url, - root_path=payload.root_path, - owner_id=payload.owner_id, - is_public=payload.is_public, - license=payload.license, - files=files_data, - ) - # reload to include files - skill = await service.get_skill(skill.id, current_user.id) - - # Trigger sync to OpenClaw container - import asyncio - - asyncio.create_task(_trigger_openclaw_skill_sync(current_user.id, db)) - - return { - "success": True, - "data": SkillSchema.model_validate(skill).model_dump(), - } - - -@router.delete("/{skill_id}") -async def delete_skill( - skill_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete a skill.""" - service = SkillService(db) - - # get the skill name before deletion - from app.common.exceptions import NotFoundException - - try: - skill = await service.get_skill(skill_id, current_user.id) - skill_name = skill.name - except NotFoundException: - skill_name = None - - await service.delete_skill(skill_id, current_user.id) - - if skill_name: - # Trigger incremental sync to OpenClaw container - async def _delete_from_container(): - try: - from app.services.openclaw_instance_service import OpenClawInstanceService - - instance_service = OpenClawInstanceService(db) - instance = await instance_service.get_instance_by_user(current_user.id) - if instance and instance.container_id and instance.status == InstanceStatus.RUNNING: - await instance_service.delete_skill_from_container( - current_user.id, instance.container_id, skill_name - ) - except Exception as e: - from loguru import logger - - logger.error( - f"Failed to delete skill {skill_name} from container for user {current_user.id}: {e}", exc_info=True - ) - - import asyncio - - asyncio.create_task(_delete_from_container()) - - return {"success": True} - - -@router.post("/{skill_id}/files") -async def add_file( - skill_id: uuid.UUID, - file: SkillFileCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Add a file to a skill.""" - service = SkillService(db) - file_obj = await service.add_file( - skill_id=skill_id, - current_user_id=current_user.id, - path=file.path, - file_name=file.file_name, - file_type=file.file_type, - content=file.content, - storage_type=file.storage_type, - storage_key=file.storage_key, - size=file.size, - ) - - # Trigger sync to OpenClaw container - import asyncio - - asyncio.create_task(_trigger_openclaw_skill_sync(current_user.id, db)) - - return { - "success": True, - "data": SkillFileSchema.model_validate(file_obj).model_dump(), - } - - -@router.delete("/files/{file_id}") -async def delete_file( - file_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete a file.""" - service = SkillService(db) - await service.delete_file(file_id, current_user.id) - - # Trigger sync to OpenClaw container - import asyncio - - asyncio.create_task(_trigger_openclaw_skill_sync(current_user.id, db)) - - return {"success": True} - - -@router.put("/files/{file_id}") -async def update_file( - file_id: uuid.UUID, - payload: SkillFileUpdate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update file content or rename a file.""" - service = SkillService(db) - file_obj = await service.update_file( - file_id=file_id, - current_user_id=current_user.id, - content=payload.content, - path=payload.path, - file_name=payload.file_name, - ) - - # Trigger sync to OpenClaw container - import asyncio - - asyncio.create_task(_trigger_openclaw_skill_sync(current_user.id, db)) - - return { - "success": True, - "data": SkillFileSchema.model_validate(file_obj).model_dump(), - } diff --git a/backend/app/api/v1/tokens.py b/backend/app/api/v1/tokens.py deleted file mode 100644 index 6a2d2757d..000000000 --- a/backend/app/api/v1/tokens.py +++ /dev/null @@ -1,94 +0,0 @@ -"""PlatformToken API routes. - -Token management is session-auth only — PlatformToken cannot manage PlatformToken. -""" - -from __future__ import annotations - -import uuid -from typing import Optional - -from fastapi import APIRouter, Depends, Query -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import BadRequestException -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.schemas.platform_token import ( - TokenCreate, - TokenCreateResponse, - TokenSchema, -) -from app.services.platform_token_service import PlatformTokenService -from app.utils.string import is_valid_uuid - -router = APIRouter(prefix="/v1/tokens", tags=["Tokens"]) - - -@router.post("") -async def create_token( - payload: TokenCreate, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = PlatformTokenService(db) - token_record, raw_token = await service.create_token( - user_id=current_user.id, - name=payload.name, - scopes=payload.scopes, - resource_type=payload.resource_type, - resource_id=payload.resource_id, - expires_at=payload.expires_at, - ) - response_data = TokenCreateResponse( - id=str(token_record.id), - name=token_record.name, - token=raw_token, - token_prefix=token_record.token_prefix, - scopes=token_record.scopes, - resource_type=token_record.resource_type, - expires_at=token_record.expires_at.isoformat() if token_record.expires_at else None, - created_at=token_record.created_at.isoformat() if token_record.created_at else None, - ) - return { - "success": True, - "data": response_data.model_dump(), - } - - -@router.get("") -async def list_tokens( - resource_type: Optional[str] = Query(None), - resource_id: Optional[str] = Query(None), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - # Validate resource_id is a valid UUID if provided - parsed_resource_id = None - if resource_id is not None: - if not is_valid_uuid(resource_id): - raise BadRequestException("Invalid resource_id: must be a valid UUID") - parsed_resource_id = uuid.UUID(resource_id) - - service = PlatformTokenService(db) - tokens = await service.list_tokens( - user_id=current_user.id, - resource_type=resource_type, - resource_id=parsed_resource_id, - ) - return { - "success": True, - "data": [TokenSchema.model_validate(t) for t in tokens], - } - - -@router.delete("/{token_id}") -async def revoke_token( - token_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = PlatformTokenService(db) - await service.revoke_token(token_id=token_id, user_id=current_user.id) - return {"success": True} diff --git a/backend/app/api/v1/tools.py b/backend/app/api/v1/tools.py deleted file mode 100644 index 2c63f0874..000000000 --- a/backend/app/api/v1/tools.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -Tools API - List available builtin, MCP, and custom tools - -Supports user-level tool queries -""" - -from typing import Optional - -from fastapi import APIRouter, Depends, Query -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.services.tool_service import ToolService - -router = APIRouter(prefix="/v1/tools", tags=["Tools"]) - - -@router.get("") -async def list_tools( - category: Optional[str] = Query(None, description="Filter by category"), - tool_type: Optional[str] = Query(None, description="Filter by tool type (builtin, mcp, custom)"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - List tools available to the user (user-level). - - Includes: - - Builtin tools - - User's MCP server tools - - User's custom tools - - Args: - category: filter by category - tool_type: filter by tool type (builtin, mcp, custom) - - Returns: - {"success": True, "data": [ToolResponse, ...]} - """ - service = ToolService(db) - - # Get tools for user scope (returns List[ToolInfo]) - tools = service.get_available_tools( - user_id=current_user.id, - tool_type=tool_type, - category=category, - ) - - # ToolInfo.to_response() provides unified conversion - return success_response( - data=[t.to_response() for t in tools], - message="Tools retrieved successfully", - ) - - -@router.get("/builtin") -async def list_builtin_tools( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - List all builtin tools. - - Returns: - {"success": True, "data": [ToolResponse, ...]} - """ - service = ToolService(db) - - tools = service.get_builtin_tools() - - return success_response( - data=[t.to_response() for t in tools], - message="Builtin tools retrieved successfully", - ) - - -@router.get("/{tool_id}") -async def get_tool( - tool_id: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Get tool details. - - Args: - tool_id: tool ID (for MCP tools: server::tool_name) - - Returns: - {"success": True, "data": ToolResponse} - """ - service = ToolService(db) - - tool = service.get_tool_by_key(tool_id) - - if not tool: - return success_response(data=None, message="Tool not found") - - return success_response( - data=tool.to_response(), - message="Tool retrieved successfully", - ) diff --git a/backend/app/api/v1/traces.py b/backend/app/api/v1/traces.py deleted file mode 100644 index e12ad1cb9..000000000 --- a/backend/app/api/v1/traces.py +++ /dev/null @@ -1,282 +0,0 @@ -""" -Traces API (path: /api/v1/traces) - -Query historical execution trace data. Supports trace listing, single trace detail, and observation listing. -""" - -import uuid -from datetime import datetime -from typing import Any, Optional - -from fastapi import APIRouter, Depends, Query, Request -from loguru import logger -from pydantic import BaseModel -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import CurrentUser -from app.common.exceptions import ForbiddenException -from app.core.database import get_db -from app.schemas import BaseResponse -from app.services.trace_service import TraceService - -router = APIRouter(prefix="/v1/traces", tags=["Traces"]) - - -def _bind_log(request: Request, **kwargs): - trace_id = getattr(request.state, "trace_id", "-") - return logger.bind(trace_id=trace_id, **kwargs) - - -# ==================== Response Schemas ==================== - - -class ObservationSchema(BaseModel): - """Observation response schema.""" - - id: str - trace_id: str - parent_observation_id: Optional[str] = None - type: str - name: Optional[str] = None - level: str = "DEFAULT" - status: str = "RUNNING" - status_message: Optional[str] = None - start_time: Optional[datetime] = None - end_time: Optional[datetime] = None - duration_ms: Optional[int] = None - input: Optional[Any] = None - output: Optional[Any] = None - model_name: Optional[str] = None - model_provider: Optional[str] = None - model_parameters: Optional[Any] = None - prompt_tokens: Optional[int] = None - completion_tokens: Optional[int] = None - total_tokens: Optional[int] = None - input_cost: Optional[float] = None - output_cost: Optional[float] = None - total_cost: Optional[float] = None - metadata: Optional[Any] = None - version: Optional[str] = None - - class Config: - from_attributes = True - - -class TraceSchema(BaseModel): - """Trace response schema.""" - - id: str - workspace_id: Optional[str] = None - graph_id: Optional[str] = None - thread_id: Optional[str] = None - user_id: Optional[str] = None - name: Optional[str] = None - status: str - input: Optional[Any] = None - output: Optional[Any] = None - start_time: Optional[datetime] = None - end_time: Optional[datetime] = None - duration_ms: Optional[int] = None - total_tokens: Optional[int] = None - total_cost: Optional[float] = None - metadata: Optional[Any] = None - tags: Optional[list] = None - created_at: Optional[datetime] = None - - class Config: - from_attributes = True - - -class TraceDetailSchema(BaseModel): - """Trace detail (with observations).""" - - trace: TraceSchema - observations: list[ObservationSchema] - - -class TraceListSchema(BaseModel): - """Trace list.""" - - traces: list[TraceSchema] - total: int - - -# ==================== Helper ==================== - - -def _trace_to_schema(trace) -> TraceSchema: - """Convert a Trace ORM object to a schema.""" - return TraceSchema( - id=str(trace.id), - workspace_id=str(trace.workspace_id) if trace.workspace_id else None, - graph_id=str(trace.graph_id) if trace.graph_id else None, - thread_id=trace.thread_id, - user_id=trace.user_id, - name=trace.name, - status=trace.status.value if hasattr(trace.status, "value") else str(trace.status), - input=trace.input, - output=trace.output, - start_time=trace.start_time, - end_time=trace.end_time, - duration_ms=trace.duration_ms, - total_tokens=trace.total_tokens, - total_cost=trace.total_cost, - metadata=trace.metadata_, - tags=trace.tags, - created_at=trace.created_at, - ) - - -def _obs_to_schema(obs) -> ObservationSchema: - """Convert an Observation ORM object to a schema.""" - return ObservationSchema( - id=str(obs.id), - trace_id=str(obs.trace_id), - parent_observation_id=str(obs.parent_observation_id) if obs.parent_observation_id else None, - type=obs.type.value if hasattr(obs.type, "value") else str(obs.type), - name=obs.name, - level=obs.level.value if hasattr(obs.level, "value") else str(obs.level), - status=obs.status.value if hasattr(obs.status, "value") else str(obs.status), - status_message=obs.status_message, - start_time=obs.start_time, - end_time=obs.end_time, - duration_ms=obs.duration_ms, - input=obs.input, - output=obs.output, - model_name=obs.model_name, - model_provider=obs.model_provider, - model_parameters=obs.model_parameters, - prompt_tokens=obs.prompt_tokens, - completion_tokens=obs.completion_tokens, - total_tokens=obs.total_tokens, - input_cost=obs.input_cost, - output_cost=obs.output_cost, - total_cost=obs.total_cost, - metadata=obs.metadata_, - version=obs.version, - ) - - -# ==================== Endpoints ==================== - - -@router.get("", response_model=BaseResponse) -async def list_traces( - request: Request, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), - graph_id: Optional[uuid.UUID] = Query(None, description="Filter by Graph ID"), - workspace_id: Optional[uuid.UUID] = Query(None, description="Filter by Workspace ID"), - thread_id: Optional[str] = Query(None, description="Filter by Thread ID"), - limit: int = Query(20, ge=1, le=100, description="Page size"), - offset: int = Query(0, ge=0, description="Offset"), -): - """List traces (paginated).""" - log = _bind_log(request, user_id=str(current_user.id)) - service = TraceService(db) - - if workspace_id: - from app.models.workspace import WorkspaceMemberRole - from app.services.workspace_permission import check_workspace_access - - has_access = await check_workspace_access(db, workspace_id, current_user, WorkspaceMemberRole.viewer) - if not has_access: - raise ForbiddenException("No access to workspace traces") - - total = await service.count_traces(graph_id=graph_id, workspace_id=workspace_id, thread_id=thread_id) - traces = await service.list_traces( - graph_id=graph_id, - workspace_id=workspace_id, - thread_id=thread_id, - limit=limit, - offset=offset, - ) - - log.debug( - f"Listed {len(traces)} traces (total={total}) | workspace_id={workspace_id} graph_id={graph_id} thread_id={thread_id}" - ) - - return BaseResponse( - success=True, - code=200, - msg="ok", - data={ - "traces": [_trace_to_schema(t).model_dump(mode="json") for t in traces], - "total": total, - }, - ) - - -@router.get("/{trace_id}", response_model=BaseResponse) -async def get_trace_detail( - trace_id: uuid.UUID, - request: Request, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -): - """Get a single trace's detail with all observations.""" - log = _bind_log(request, user_id=str(current_user.id)) - service = TraceService(db) - - trace = await service.get_trace(trace_id) - if trace is None: - return BaseResponse(success=False, code=404, msg="Trace not found", data=None) - - if trace.workspace_id: - from app.models.workspace import WorkspaceMemberRole - from app.services.workspace_permission import check_workspace_access - - has_access = await check_workspace_access(db, trace.workspace_id, current_user, WorkspaceMemberRole.viewer) - if not has_access: - raise ForbiddenException("No access to workspace traces") - - observations = await service.get_observations_for_trace(trace_id) - - log.debug(f"Fetched trace {trace_id} with {len(observations)} observations") - - return BaseResponse( - success=True, - code=200, - msg="ok", - data={ - "trace": _trace_to_schema(trace).model_dump(mode="json"), - "observations": [_obs_to_schema(o).model_dump(mode="json") for o in observations], - }, - ) - - -@router.get("/{trace_id}/observations", response_model=BaseResponse) -async def get_trace_observations( - trace_id: uuid.UUID, - request: Request, - current_user: CurrentUser, - db: AsyncSession = Depends(get_db), -): - """Get a flat list of observations for a trace (sorted by time).""" - log = _bind_log(request, user_id=str(current_user.id)) - service = TraceService(db) - - trace = await service.get_trace(trace_id) - if trace is None: - return BaseResponse(success=False, code=404, msg="Trace not found", data=None) - - if trace.workspace_id: - from app.models.workspace import WorkspaceMemberRole - from app.services.workspace_permission import check_workspace_access - - has_access = await check_workspace_access(db, trace.workspace_id, current_user, WorkspaceMemberRole.viewer) - if not has_access: - raise ForbiddenException("No access to workspace traces") - - observations = await service.get_observations_for_trace(trace_id) - - log.debug(f"Fetched {len(observations)} observations for trace {trace_id}") - - return BaseResponse( - success=True, - code=200, - msg="ok", - data={ - "observations": [_obs_to_schema(o).model_dump(mode="json") for o in observations], - }, - ) diff --git a/backend/app/api/v1/users.py b/backend/app/api/v1/users.py deleted file mode 100644 index 0a521dcdd..000000000 --- a/backend/app/api/v1/users.py +++ /dev/null @@ -1,343 +0,0 @@ -""" -User API (path: /api/v1/users) -""" - -from typing import Optional - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, ConfigDict, EmailStr, Field -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import ForbiddenException, NotFoundException -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.settings import Settings -from app.services.user_service import UserService -from app.services.workspace_file_service import WorkspaceFileService - -router = APIRouter(prefix="/v1/users", tags=["Users"]) - - -# Schemas - - -class UserCreateRequest(BaseModel): - """Create user request.""" - - email: EmailStr - name: str = Field(..., min_length=1, max_length=255) - image: Optional[str] = None - is_super_user: bool = False - email_verified: bool = False - - -class UserUpdateRequest(BaseModel): - """Update user request.""" - - name: Optional[str] = Field(None, min_length=1, max_length=255) - email: Optional[EmailStr] = None - image: Optional[str] = None - is_super_user: Optional[bool] = None - email_verified: Optional[bool] = None - stripe_customer_id: Optional[str] = None - - -class UserResponse(BaseModel): - """User response.""" - - model_config = ConfigDict(from_attributes=True) - - id: str - email: str - name: str - image: Optional[str] - email_verified: bool - is_super_user: bool - stripe_customer_id: Optional[str] - created_at: str - updated_at: str - - -class SettingsUpdateRequest(BaseModel): - """Update settings request.""" - - autoConnect: Optional[bool] = None - showTrainingControls: Optional[bool] = None - superUserModeEnabled: Optional[bool] = None - theme: Optional[str] = None - telemetryEnabled: Optional[bool] = None - billingUsageNotificationsEnabled: Optional[bool] = None - errorNotificationsEnabled: Optional[bool] = None - - -# Endpoints - - -@router.get("/me", response_model=UserResponse) -async def get_me( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get current user profile.""" - return success_response( - data=_user_to_response(current_user), - message="Fetched user profile", - ) - - -@router.put("/me", response_model=UserResponse) -async def update_me( - request: UserUpdateRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update current user profile.""" - service = UserService(db) - updated_user = await service.update_user( - current_user, - name=request.name, - email=request.email, - image=request.image, - ) - return success_response( - data=_user_to_response(updated_user), - message="Profile updated successfully", - ) - - -@router.get("/me/settings") -async def get_settings( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get current user settings.""" - result = await db.execute(select(Settings).where(Settings.user_id == current_user.id)) - settings = result.scalar_one_or_none() - - if not settings: - # if no record exists, return defaults - default_settings = { - "autoConnect": True, - "showTrainingControls": False, - "superUserModeEnabled": True, - "theme": "dark", - "telemetryEnabled": True, - "billingUsageNotificationsEnabled": True, - "errorNotificationsEnabled": True, - } - return {"success": True, "data": default_settings} - - return { - "success": True, - "data": { - "autoConnect": settings.auto_connect, - "showTrainingControls": settings.show_training_controls, - "superUserModeEnabled": settings.super_user_mode_enabled, - "theme": settings.theme, - "telemetryEnabled": settings.telemetry_enabled, - "billingUsageNotificationsEnabled": settings.billing_usage_notifications_enabled, - "errorNotificationsEnabled": settings.error_notifications_enabled, - }, - } - - -@router.patch("/me/settings") -async def update_settings( - request: SettingsUpdateRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update current user settings.""" - result = await db.execute(select(Settings).where(Settings.user_id == current_user.id)) - settings = result.scalar_one_or_none() - - if not settings: - # if no record exists, create one - settings = Settings(user_id=current_user.id) - db.add(settings) - - # update only the provided fields - if request.autoConnect is not None: - settings.auto_connect = request.autoConnect - if request.showTrainingControls is not None: - settings.show_training_controls = request.showTrainingControls - if request.superUserModeEnabled is not None: - settings.super_user_mode_enabled = request.superUserModeEnabled - if request.theme is not None: - settings.theme = request.theme - if request.telemetryEnabled is not None: - settings.telemetry_enabled = request.telemetryEnabled - if request.billingUsageNotificationsEnabled is not None: - settings.billing_usage_notifications_enabled = request.billingUsageNotificationsEnabled - if request.errorNotificationsEnabled is not None: - settings.error_notifications_enabled = request.errorNotificationsEnabled - - await db.commit() - await db.refresh(settings) - - return { - "success": True, - "data": { - "autoConnect": settings.auto_connect, - "showTrainingControls": settings.show_training_controls, - "superUserModeEnabled": settings.super_user_mode_enabled, - "theme": settings.theme, - "telemetryEnabled": settings.telemetry_enabled, - "billingUsageNotificationsEnabled": settings.billing_usage_notifications_enabled, - "errorNotificationsEnabled": settings.error_notifications_enabled, - }, - } - - -@router.get("/me/usage-limits") -async def get_usage_limits( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get current user's storage usage (workspace files).""" - service = WorkspaceFileService(db) - storage = await service.get_user_storage_usage(current_user) - usage = {"plan": "standard"} - base = success_response( - data={"storage": storage, "usage": usage}, - message="Fetched storage usage", - ) - return {**base, "storage": storage, "usage": usage} - - -@router.get("/{user_id}", response_model=UserResponse) -async def get_user( - user_id: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Get user by ID (requires superuser permission).""" - if not current_user.is_superuser: - raise ForbiddenException("Forbidden") - - service = UserService(db) - user = await service.get_user_by_id(user_id) - if not user: - raise NotFoundException("User not found") - - return success_response( - data=_user_to_response(user), - message="Fetched user", - ) - - -@router.get("", response_model=list[UserResponse]) -async def list_users( - keyword: Optional[str] = Query(None, description="Search keyword"), - limit: int = Query(20, ge=1, le=100, description="Result limit"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Search/list users (requires superuser permission).""" - if not current_user.is_superuser: - raise ForbiddenException("Forbidden") - - service = UserService(db) - if keyword: - users = await service.search_users(keyword, limit) - else: - users = await service.list_users(limit) - - return success_response( - data=[_user_to_response(user) for user in users], - message="Fetched users", - ) - - -@router.post("", response_model=UserResponse) -async def create_user( - request: UserCreateRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create a new user (requires superuser permission).""" - if not current_user.is_superuser: - raise ForbiddenException("Forbidden") - - service = UserService(db) - user = await service.create_user( - email=request.email, - name=request.name, - image=request.image, - is_super_user=request.is_super_user, - email_verified=request.email_verified, - ) - - return success_response( - data=_user_to_response(user), - message="User created successfully", - ) - - -@router.put("/{user_id}", response_model=UserResponse) -async def update_user( - user_id: str, - request: UserUpdateRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update user info (requires superuser permission).""" - if not current_user.is_superuser: - raise ForbiddenException("Forbidden") - - service = UserService(db) - user = await service.get_user_by_id(user_id) - if not user: - raise NotFoundException("User not found") - - updated_user = await service.update_user( - user, - name=request.name, - email=request.email, - image=request.image, - is_super_user=request.is_super_user, - email_verified=request.email_verified, - stripe_customer_id=request.stripe_customer_id, - ) - - return success_response( - data=_user_to_response(updated_user), - message="User updated successfully", - ) - - -@router.delete("/{user_id}") -async def delete_user( - user_id: str, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Delete a user (requires superuser permission).""" - if not current_user.is_superuser: - raise ForbiddenException("Forbidden") - - service = UserService(db) - await service.delete_user(user_id) - - return success_response(message="User deleted successfully") - - -# Helpers - - -def _user_to_response(user: User) -> dict: - """Convert a User model to response format.""" - return { - "id": user.id, - "email": user.email, - "name": user.name, - "image": user.image, - "email_verified": user.email_verified, - "is_super_user": user.is_super_user, - "stripe_customer_id": user.stripe_customer_id, - "created_at": user.created_at.isoformat() if user.created_at else None, - "updated_at": user.updated_at.isoformat() if user.updated_at else None, - } diff --git a/backend/app/api/v1/version.py b/backend/app/api/v1/version.py deleted file mode 100644 index ed17f7544..000000000 --- a/backend/app/api/v1/version.py +++ /dev/null @@ -1,46 +0,0 @@ -"""版本信息 API""" - -import os -import subprocess - -from fastapi import APIRouter - -from app.common.response import success_response -from app.core.settings import settings - -router = APIRouter(prefix="/v1/version", tags=["Version"]) - -_git_sha: str | None = None - - -def _get_git_sha() -> str: - global _git_sha - if _git_sha is not None: - return _git_sha - - sha = os.environ.get("GIT_COMMIT_SHA", "") - if not sha: - try: - sha = subprocess.check_output( - ["git", "rev-parse", "--short", "HEAD"], - stderr=subprocess.DEVNULL, - text=True, - ).strip() - except Exception: - sha = "unknown" - - _git_sha = sha - return _git_sha - - -@router.get("") -async def get_version(): - """获取应用版本信息""" - return success_response( - data={ - "version": settings.app_version, - "git_sha": _get_git_sha(), - "environment": settings.environment, - }, - message="Version retrieved successfully", - ) diff --git a/backend/app/api/v1/workspace_files.py b/backend/app/api/v1/workspace_files.py deleted file mode 100644 index 96b652616..000000000 --- a/backend/app/api/v1/workspace_files.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -Workspace file management API (versioned path: /api/v1/workspaces) -""" - -import uuid -from typing import Optional - -from fastapi import APIRouter, Depends, File, Query, Request, UploadFile -from fastapi.responses import FileResponse, JSONResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user_optional, require_workspace_role -from app.common.exceptions import AppException, ConflictException -from app.common.response import success_response -from app.core.database import get_db -from app.core.settings import settings -from app.models.auth import AuthUser as User -from app.models.workspace import WorkspaceMemberRole -from app.services.workspace_file_service import WorkspaceFileService - -router = APIRouter(prefix="/v1/workspaces", tags=["WorkspaceFiles"]) - - -@router.get("/{workspace_id}/files") -async def list_workspace_files( - workspace_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.viewer), -): - service = WorkspaceFileService(db) - files = await service.list_files(workspace_id, current_user) - # compatible with frontend reading files directly, while preserving unified response format - base = success_response(data={"files": files}, message="Fetched workspace files") - return {**base, "files": files} - - -@router.post("/{workspace_id}/files") -async def upload_workspace_file( - workspace_id: uuid.UUID, - file: UploadFile = File(..., description="File to upload"), - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.member), -): - # Duplicate file returns 409 + isDuplicate with error field for frontend - try: - service = WorkspaceFileService(db) - record = await service.upload_file(workspace_id, file, current_user) - base = success_response(data={"file": record}, message="File uploaded") - return {**base, "file": record} - except ConflictException as exc: - return JSONResponse( - status_code=exc.status_code, - content={ - "success": False, - "error": str(exc.detail), - "isDuplicate": True, - }, - ) - except AppException as exc: - # Return error field alongside unified response (success=false) for frontend compatibility - return JSONResponse( - status_code=exc.status_code, - content={ - "success": False, - "error": str(exc.detail), - }, - ) - - -@router.delete("/{workspace_id}/files/{file_id}") -async def delete_workspace_file( - workspace_id: uuid.UUID, - file_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.member), -): - service = WorkspaceFileService(db) - await service.delete_file(workspace_id, file_id, current_user) - # Include top-level success field for frontend compatibility - base = success_response(message="File deleted", data={"fileId": str(file_id)}) - return {**base, "success": True} - - -@router.post("/{workspace_id}/files/{file_id}/download") -async def generate_workspace_file_download_url( - workspace_id: uuid.UUID, - file_id: uuid.UUID, - request: Request, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.viewer), -): - service = WorkspaceFileService(db) - url = await service.generate_download_url(workspace_id, file_id, current_user) - record = await service.get_file_record(workspace_id, file_id) - - # Generate absolute downloadUrl from the request base URL - base_url = str(request.base_url).rstrip("/") - download_url = f"{base_url}{url}" - viewer_url = f"{settings.frontend_url.rstrip('/')}/workspace/{workspace_id}/files/{file_id}/view" - - payload = { - "downloadUrl": download_url, - "viewerUrl": viewer_url, - "fileName": record.original_name, - "expiresIn": None, - } - - base = success_response(data=payload, message="Download URL generated") - return {**base, "success": True, **payload} - - -@router.get("/{workspace_id}/files/{file_id}/serve") -async def serve_workspace_file( - workspace_id: uuid.UUID, - file_id: uuid.UUID, - token: Optional[str] = Query(default=None, description="Download signature token"), - db: AsyncSession = Depends(get_db), - current_user: Optional[User] = Depends(get_current_user_optional), -): - service = WorkspaceFileService(db) - await service.validate_token_or_user(workspace_id, file_id, token, current_user) - record = await service.get_file_record(workspace_id, file_id) - file_path = service.get_file_path(record) - - if not file_path.exists(): - # deferred validation: raise a consistent error if the file is missing - await service.read_file_bytes(record) - - # use FileResponse directly to reduce memory usage - return FileResponse( - path=file_path, - media_type=record.content_type or "application/octet-stream", - filename=record.original_name, - ) diff --git a/backend/app/api/v1/workspace_folders.py b/backend/app/api/v1/workspace_folders.py deleted file mode 100644 index e9c07aa16..000000000 --- a/backend/app/api/v1/workspace_folders.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Folders API (versioned path: /api/v1/folders)""" - -import uuid -from typing import Optional - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.response import success_response -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.repositories.workspace_folder import WorkflowFolderRepository -from app.services.workspace_folder_service import FolderService - -router = APIRouter(prefix="/v1/folders", tags=["Folders"]) - - -class CreateFolderRequest(BaseModel): - workspaceId: uuid.UUID - name: str = Field(..., min_length=1, max_length=255) - parentId: Optional[uuid.UUID] = None - color: Optional[str] = Field(default=None, max_length=32) - - -class UpdateFolderRequest(BaseModel): - workspaceId: Optional[uuid.UUID] = None - name: Optional[str] = Field(None, min_length=1, max_length=255) - color: Optional[str] = Field(None, max_length=32) - isExpanded: Optional[bool] = None - parentId: Optional[uuid.UUID] = None - - -class DuplicateFolderRequest(BaseModel): - name: str = Field(..., min_length=1, max_length=255) - workspaceId: Optional[uuid.UUID] = None - parentId: Optional[uuid.UUID] = None - color: Optional[str] = Field(default=None, max_length=32) - - -def _serialize_folder(folder) -> dict: - return { - "id": str(folder.id), - "name": folder.name, - "workspaceId": str(folder.workspace_id), - "parentId": str(folder.parent_id) if folder.parent_id else None, - "color": folder.color, - "isExpanded": folder.is_expanded, - "sortOrder": folder.sort_order, - "createdAt": folder.created_at, - "updatedAt": folder.updated_at, - "userId": str(folder.user_id), - } - - -@router.get("") -async def list_folders( - workspace_id: uuid.UUID = Query(..., alias="workspaceId"), - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = FolderService(db) - folders = await service.list_folders(workspace_id, current_user=current_user) - payload = [_serialize_folder(f) for f in folders] - base = success_response(data={"folders": payload}, message="Fetched folders") - return {**base, "folders": payload} - - -@router.post("") -async def create_folder( - body: CreateFolderRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - service = FolderService(db) - folder = await service.create_folder( - workspace_id=body.workspaceId, - current_user=current_user, - name=body.name, - parent_id=body.parentId, - color=body.color, - is_expanded=False, - ) - payload = _serialize_folder(folder) - base = success_response(data={"folder": payload}, message="Folder created") - return {**base, "folder": payload} - - -@router.put("/{folder_id}") -async def update_folder( - folder_id: uuid.UUID, - body: UpdateFolderRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - repo = WorkflowFolderRepository(db) - existing = await repo.get(folder_id) - if not existing: - from app.common.exceptions import NotFoundException - - raise NotFoundException("Folder not found") - - workspace_id = body.workspaceId or existing.workspace_id - service = FolderService(db) - folder = await service.update_folder( - folder_id, - workspace_id=workspace_id, - current_user=current_user, - name=body.name, - color=body.color, - is_expanded=body.isExpanded, - parent_id=body.parentId, - ) - payload = _serialize_folder(folder) - base = success_response(data={"folder": payload}, message="Folder updated") - return {**base, "folder": payload} - - -@router.delete("/{folder_id}") -async def delete_folder( - folder_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - repo = WorkflowFolderRepository(db) - existing = await repo.get(folder_id) - if not existing: - from app.common.exceptions import NotFoundException - - raise NotFoundException("Folder not found") - - service = FolderService(db) - stats = await service.delete_folder_tree( - folder_id, - workspace_id=existing.workspace_id, - current_user=current_user, - ) - base = success_response(data={"deletedItems": stats}, message="Folder deleted") - return {**base, "success": True, "deletedItems": stats} - - -@router.post("/{folder_id}/duplicate") -async def duplicate_folder( - folder_id: uuid.UUID, - body: DuplicateFolderRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - repo = WorkflowFolderRepository(db) - source = await repo.get(folder_id) - if not source: - from app.common.exceptions import NotFoundException - - raise NotFoundException("Source folder not found") - - target_workspace_id = body.workspaceId or source.workspace_id - - service = FolderService(db) - new_root = await service.duplicate_folder( - folder_id, - workspace_id=target_workspace_id, - current_user=current_user, - name=body.name, - parent_id=body.parentId, - color=body.color, - ) - - result = { - "id": str(new_root.id), - "name": new_root.name, - "color": new_root.color, - "workspaceId": str(new_root.workspace_id), - "parentId": str(new_root.parent_id) if new_root.parent_id else None, - } - base = success_response(data=result, message="Folder duplicated", code=201) - return {**base, **result} - - -@router.get("/{folder_id}/graphs") -async def list_folder_graphs( - folder_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List all graphs in a folder.""" - from sqlalchemy import func, select - - from app.models.graph import AgentGraph, GraphNode - from app.repositories.graph import GraphRepository - - # verify folder exists and get workspace_id - repo = WorkflowFolderRepository(db) - folder = await repo.get(folder_id) - if not folder: - from app.common.exceptions import NotFoundException - - raise NotFoundException("Folder not found") - - # verify user permission (read access) - service = FolderService(db) - await service._ensure_permission(folder.workspace_id, current_user, "read") - - # query all graphs in this folder - GraphRepository(db) - stmt = ( - select(AgentGraph) - .where(AgentGraph.folder_id == folder_id, AgentGraph.user_id == current_user.id) - .order_by(AgentGraph.created_at.desc()) - ) - - result = await db.execute(stmt) - graphs = list(result.scalars().all()) - - # batch-query node counts - graph_ids = [graph.id for graph in graphs] - node_counts = {} - if graph_ids: - count_query = ( - select(GraphNode.graph_id, func.count(GraphNode.id).label("count")) - .where(GraphNode.graph_id.in_(graph_ids)) - .group_by(GraphNode.graph_id) - ) - count_result = await db.execute(count_query) - for row in count_result: - node_counts[row.graph_id] = row.count - - # serialize graphs - data = [] - for graph in graphs: - data.append( - { - "id": str(graph.id), - "userId": str(graph.user_id), - "workspaceId": str(graph.workspace_id) if graph.workspace_id else None, - "folderId": str(graph.folder_id) if graph.folder_id else None, - "parentId": str(graph.parent_id) if graph.parent_id else None, - "name": graph.name, - "description": graph.description, - "color": graph.color, - "isDeployed": graph.is_deployed, - "variables": graph.variables or {}, - "createdAt": graph.created_at.isoformat() if graph.created_at else None, - "updatedAt": graph.updated_at.isoformat() if graph.updated_at else None, - "nodeCount": node_counts.get(graph.id, 0), - } - ) - - base = success_response(data={"graphs": data}, message="Fetched graphs") - return {**base, "graphs": data} diff --git a/backend/app/api/v1/workspaces.py b/backend/app/api/v1/workspaces.py deleted file mode 100644 index 2ac773a33..000000000 --- a/backend/app/api/v1/workspaces.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Workspace API.""" - -import uuid -from typing import Optional - -from fastapi import APIRouter, Body, Depends, Query -from pydantic import BaseModel, EmailStr, Field -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user, require_workspace_role -from app.common.exceptions import BadRequestException, ForbiddenException -from app.common.pagination import PaginationParams -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.workspace import WorkspaceMemberRole -from app.services.user_service import UserService -from app.services.workspace_service import WorkspaceService - -router = APIRouter(prefix="/v1/workspaces", tags=["Workspaces"]) - - -class CreateWorkspaceRequest(BaseModel): - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = Field(None, max_length=500) - type: Optional[str] = Field(default="team", description="Workspace type: personal or team") - - -class UpdateWorkspaceRequest(BaseModel): - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = Field(None, max_length=500) - settings: Optional[dict] = None - - -class DeleteWorkspaceRequest(BaseModel): - deleteTemplates: bool = Field(default=True, description="Whether to also delete template data") - - -class AddMemberRequest(BaseModel): - email: EmailStr - role: str = Field(default="member", description="Member role: admin/member/viewer") - - -class RemoveMemberRequest(BaseModel): - workspaceId: uuid.UUID - - -class UpdateMemberRoleRequest(BaseModel): - workspaceId: uuid.UUID - role: str = Field(..., description="Member role: owner/admin/member/viewer") - - -@router.get("") -async def list_workspaces( - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """List all workspaces for the current user.""" - service = WorkspaceService(db) - data = await service.list_workspaces(current_user) - return {"workspaces": data} - - -@router.post("") -async def create_workspace( - payload: CreateWorkspaceRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Create a new workspace (defaults to team workspace).""" - from app.models.workspace import WorkspaceType - - workspace_type = WorkspaceType.team - if payload.type: - try: - workspace_type = WorkspaceType(payload.type) - except ValueError: - raise BadRequestException(f"Invalid workspace type: {payload.type}. Must be 'personal' or 'team'") - - service = WorkspaceService(db) - workspace = await service.create_workspace( - name=payload.name, - description=payload.description, - current_user=current_user, - workspace_type=workspace_type, - ) - return {"workspace": workspace} - - -# ------------------------------------------------------------------ # -# Dynamic /{workspace_id} routes -# ------------------------------------------------------------------ # - - -@router.get("/{workspace_id}") -async def get_workspace( - workspace_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.viewer), -): - """Get a single workspace's details.""" - service = WorkspaceService(db) - workspace = await service.get_workspace(workspace_id, current_user) - return {"workspace": workspace} - - -@router.patch("/{workspace_id}") -async def update_workspace( - workspace_id: uuid.UUID, - payload: UpdateWorkspaceRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.admin), -): - """Update workspace metadata.""" - service = WorkspaceService(db) - workspace = await service.update_workspace( - workspace_id, - name=payload.name, - description=payload.description, - settings=payload.settings, - current_user=current_user, - ) - return {"workspace": workspace} - - -@router.put("/{workspace_id}") -async def update_workspace_put( - workspace_id: uuid.UUID, - payload: UpdateWorkspaceRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.admin), -): - """Legacy compatibility: PUT alias.""" - return await update_workspace(workspace_id, payload, db, current_user) - - -@router.delete("/{workspace_id}") -async def delete_workspace( - workspace_id: uuid.UUID, - payload: DeleteWorkspaceRequest = Body(default_factory=DeleteWorkspaceRequest), - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.owner), -): - """Delete a workspace and all related data.""" - service = WorkspaceService(db) - await service.delete_workspace( - workspace_id, - delete_templates=payload.deleteTemplates, - current_user=current_user, - ) - return {"success": True} - - -@router.post("/{workspace_id}/duplicate") -async def duplicate_workspace( - workspace_id: uuid.UUID, - payload: dict = Body(default_factory=dict), - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.member), -): - """Duplicate a workspace.""" - service = WorkspaceService(db) - workspace = await service.duplicate_workspace( - workspace_id, - name=payload.get("name"), - current_user=current_user, - ) - return {"workspace": workspace} - - -@router.post("/{workspace_id}/members") -async def add_member( - workspace_id: uuid.UUID, - payload: AddMemberRequest, - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.admin), -): - """Add a member to the workspace directly.""" - service = WorkspaceService(db) - member = await service.add_member( - workspace_id=workspace_id, - email=payload.email, - role=payload.role, - current_user=current_user, - ) - return {"member": member} - - -@router.get("/{workspace_id}/members") -async def list_members( - workspace_id: uuid.UUID, - pagination: PaginationParams = Depends(), - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.viewer), -): - """List workspace members (paginated).""" - service = WorkspaceService(db) - result = await service.list_members_paginated(workspace_id, current_user, pagination) - return result - - -@router.get("/{workspace_id}/my-permission") -async def get_my_permission( - workspace_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """ - Get the current user's permission in the workspace (lightweight, returns only the current user's permission). - - Returns: - { - "role": "owner" | "admin" | "member" | "viewer", - "permissionType": "read" | "write" | "admin", - "isOwner": boolean - } - """ - from app.common.response import success_response - - service = WorkspaceService(db) - role = await service.get_user_role(workspace_id, current_user) - - if not role: - raise ForbiddenException("No access to workspace") - - # reuse the frontend's role-to-permission mapping for consistency - role_to_permission = { - WorkspaceMemberRole.owner: "admin", - WorkspaceMemberRole.admin: "admin", - WorkspaceMemberRole.member: "write", - WorkspaceMemberRole.viewer: "read", - } - - workspace = await service.workspace_repo.get(workspace_id) - is_owner = workspace.owner_id == current_user.id if workspace else False - - # reuse the existing success_response helper for consistent response format - return success_response( - data={ - "role": role.value, - "permissionType": role_to_permission.get(role, "read"), - "isOwner": is_owner, - }, - message="Permission retrieved successfully", - ) - - -@router.get("/{workspace_id}/search-users") -async def search_users_for_invitation( - workspace_id: uuid.UUID, - keyword: str = Query(..., min_length=1, description="Search keyword (email or name)"), - limit: int = Query(10, ge=1, le=20, description="Result limit"), - db: AsyncSession = Depends(get_db), - current_user: User = require_workspace_role(WorkspaceMemberRole.admin), -): - """Search users (for adding members, requires admin permission).""" - user_service = UserService(db) - users = await user_service.search_users(keyword, limit) - - # serialize user info - result = [] - for user in users: - result.append( - { - "id": user.id, - "email": user.email, - "name": user.name, - "image": user.image, - } - ) - - return {"users": result} - - -# ------------------------------------------------------------------ # -# Member management routes (/members/* won't be shadowed by /{workspace_id}) -# ------------------------------------------------------------------ # - - -@router.patch("/members/{user_id}") -async def update_member_role( - user_id: uuid.UUID, - payload: UpdateMemberRoleRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Update a workspace member's role (admin only).""" - from app.services.workspace_permission import check_workspace_access - - has_access = await check_workspace_access( - db, - payload.workspaceId, - current_user, - WorkspaceMemberRole.admin, - ) - if not has_access: - raise ForbiddenException("Insufficient workspace permission") - - try: - new_role = WorkspaceMemberRole(payload.role) - except ValueError: - raise BadRequestException(f"Invalid role: {payload.role}") - - service = WorkspaceService(db) - member = await service.update_member_role( - workspace_id=payload.workspaceId, - target_user_id=str(user_id), - new_role=new_role, - current_user=current_user, - ) - return {"member": member} - - -@router.delete("/members/{user_id}") -async def remove_member( - user_id: uuid.UUID, - payload: RemoveMemberRequest, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Remove a workspace member (admin can remove others, members can remove themselves).""" - # Allow self-removal without admin check; otherwise require admin role - if str(user_id) != str(current_user.id): - from app.services.workspace_permission import check_workspace_access - - has_access = await check_workspace_access( - db, - payload.workspaceId, - current_user, - WorkspaceMemberRole.admin, - ) - if not has_access: - raise ForbiddenException("Insufficient workspace permission") - - service = WorkspaceService(db) - await service.remove_member( - workspace_id=payload.workspaceId, - target_user_id=user_id, - current_user=current_user, - ) - return {"success": True} diff --git a/backend/app/common/auth_dependency.py b/backend/app/common/auth_dependency.py deleted file mode 100644 index 08b4f56a8..000000000 --- a/backend/app/common/auth_dependency.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Dual-mode authentication: session/JWT + PlatformToken (sk_ prefix).""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timezone -from typing import List, Optional - -from fastapi import Depends, Request -from fastapi.security import OAuth2PasswordBearer -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.dependencies import get_current_user -from app.common.exceptions import UnauthorizedException -from app.core.database import get_db -from app.models.auth import AuthUser as User -from app.models.platform_token import PlatformToken -from app.services.platform_token_service import TOKEN_PREFIX -from app.utils.string import hash_string_sha256 - -oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login", auto_error=False) - -# Debounce interval for last_used_at updates (5 minutes) -_LAST_USED_DEBOUNCE_SECONDS = 300 - - -@dataclass -class AuthContext: - """Result of authentication — carries user + optional token scopes.""" - - user: User - token_scopes: Optional[List[str]] = None - token_resource_type: Optional[str] = None - token_resource_id: Optional[str] = None - - @property - def is_token_auth(self) -> bool: - return self.token_scopes is not None - - @property - def scopes(self) -> Optional[List[str]]: - return self.token_scopes - - -async def get_current_user_or_token( - token: Optional[str] = Depends(oauth2_scheme), - request: Request = None, # type: ignore[assignment] - db: AsyncSession = Depends(get_db), -) -> AuthContext: - """ - Authenticate via session/JWT or PlatformToken. - - If Bearer token starts with 'sk_', route to PlatformToken path. - Otherwise, fall through to existing session/JWT auth. - """ - # Try to extract token from cookie if not in header - raw_token = token - if not raw_token and request: - from app.common.cookie_auth import extract_token_from_cookies - - raw_token = extract_token_from_cookies(request.cookies) - - # PlatformToken path - if raw_token and raw_token.startswith(TOKEN_PREFIX): - return await _authenticate_platform_token(raw_token, db) - - # Fall through to existing session/JWT auth - if request is None: - from app.common.exceptions import UnauthorizedException - - raise UnauthorizedException("Authentication required") - user = await get_current_user(token=token, request=request, db=db) - return AuthContext(user=user, token_scopes=None) - - -async def _authenticate_platform_token( - raw_token: str, - db: AsyncSession, -) -> AuthContext: - """Verify a PlatformToken and return AuthContext with scopes.""" - token_hash = hash_string_sha256(raw_token) - - result = await db.execute(select(PlatformToken).where(PlatformToken.token_hash == token_hash)) - pt = result.scalar_one_or_none() - - if not pt: - raise UnauthorizedException("Invalid API token") - - if not pt.is_active: - raise UnauthorizedException("API token has been revoked") - - if pt.expires_at and pt.expires_at < datetime.now(timezone.utc): - raise UnauthorizedException("API token has expired") - - # Debounce last_used_at update - now = datetime.now(timezone.utc) - if not pt.last_used_at or (now - pt.last_used_at).total_seconds() > _LAST_USED_DEBOUNCE_SECONDS: - pt.last_used_at = now - await db.commit() - - # Load the user — use the already-loaded relationship if available - user = pt.user if pt.user else None - if not user: - user_result = await db.execute(select(User).where(User.id == pt.user_id)) - user = user_result.scalar_one_or_none() - if not user or not user.is_active: - raise UnauthorizedException("Token owner account is inactive") - - return AuthContext( - user=user, - token_scopes=list(pt.scopes), - token_resource_type=pt.resource_type, - token_resource_id=str(pt.resource_id) if pt.resource_id else None, - ) diff --git a/backend/app/common/dependencies.py b/backend/app/common/dependencies.py deleted file mode 100644 index 9a5cf9331..000000000 --- a/backend/app/common/dependencies.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -Common dependencies. -""" - -import uuid -from typing import Annotated, Optional - -from fastapi import Depends, Request -from fastapi.security import OAuth2PasswordBearer -from loguru import logger -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.cookie_auth import extract_token_from_cookies -from app.common.exceptions import ForbiddenException, NotFoundException, UnauthorizedException -from app.core.database import get_db -from app.core.security import decode_token -from app.models.auth import AuthUser as User -from app.models.enums import OrgRole -from app.models.organization import Member as OrgMember -from app.models.workspace import WorkspaceMemberRole -from app.repositories.workspace import WorkspaceMemberRepository, WorkspaceRepository -from app.services.auth_session_service import AuthSessionService - -oauth2_scheme_optional = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login", auto_error=False) - - -async def get_current_user( - token: Annotated[Optional[str], Depends(oauth2_scheme_optional)], - request: Request, - db: AsyncSession = Depends(get_db), -) -> User: - """ - Get the current user (login required). - Support two authentication methods: - 1. JWT token (preferred): decode JWT token to obtain user ID - 2. Session token (backward-compatible): verify via auth.session table - Also support token delivery via Cookie (prefer the configured cookie_name) - """ - cookie_token = None - try: - if request: - cookie_token = extract_token_from_cookies(request.cookies) - except Exception: - logger.debug("Failed to read auth token from cookies", exc_info=True) - token = token or cookie_token - if not token: - raise UnauthorizedException("Missing credentials") - - # try JWT token first (JWT mode) - payload = decode_token(token) - if payload: - user_id = payload.sub - result = await db.execute(select(User).where(User.id == str(user_id))) - user = result.scalar_one_or_none() - if user is None: - raise UnauthorizedException("User not found") - if not user.is_active: - raise UnauthorizedException("User is inactive") - return user - - # if JWT validation fails, try as session token (backward-compatible) - session_service = AuthSessionService(db) - session = await session_service.get_session_by_token(token) - if session: - result = await db.execute(select(User).where(User.id == session.user_id)) - user = result.scalar_one_or_none() - if user is None: - raise UnauthorizedException("User not found") - if not user.is_active: - raise UnauthorizedException("User is inactive") - return user - - raise UnauthorizedException("Could not validate credentials") - - -async def get_current_user_optional( - token: Annotated[Optional[str], Depends(oauth2_scheme_optional)], - request: Request, - db: AsyncSession = Depends(get_db), -) -> Optional[User]: - """Get the current user (optional; return None if not logged in). Also support token in Cookie.""" - cookie_token = None - try: - if request: - cookie_token = extract_token_from_cookies(request.cookies) - except Exception: - logger.debug("Failed to read auth token from cookies", exc_info=True) - token = token or cookie_token - if not token: - return None - - # prefer JWT token - payload = decode_token(token) - if payload: - user_id = payload.sub - result = await db.execute(select(User).where(User.id == str(user_id))) - user = result.scalar_one_or_none() - if user and user.is_active: - return user - return None - - # fall back to session token - session_service = AuthSessionService(db) - session = await session_service.get_session_by_token(token) - if session: - result = await db.execute(select(User).where(User.id == session.user_id)) - user = result.scalar_one_or_none() - if user and user.is_active: - return user - return None - - return None - - -# --------------------------------------------------------------------------- # -# Workspace / Organization role dependency helpers -# --------------------------------------------------------------------------- # -def _role_rank(role: WorkspaceMemberRole) -> int: - order = [ - WorkspaceMemberRole.viewer, - WorkspaceMemberRole.member, - WorkspaceMemberRole.admin, - WorkspaceMemberRole.owner, - ] - try: - return order.index(role) - except ValueError: - return -1 - - -def require_workspace_role(min_role: WorkspaceMemberRole): - """ - Route-level dependency that verifies the current user's role on the given - workspace_id is >= min_role. Requires workspace_id in path/query params. - """ - - async def checker( - workspace_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), - ) -> User: - if current_user.is_superuser: - return current_user - - ws_repo = WorkspaceRepository(db) - member_repo = WorkspaceMemberRepository(db) - - workspace = await ws_repo.get(workspace_id) - if not workspace: - raise NotFoundException("Workspace not found") - - if workspace.owner_id == current_user.id: - return current_user - - member = await member_repo.get_member(workspace_id, current_user.id) - if not member: - raise ForbiddenException("No access to workspace") - - if _role_rank(member.role) < _role_rank(min_role): - raise ForbiddenException("Insufficient workspace permission") - - return current_user - - return Depends(checker) - - -def require_org_role(min_role: OrgRole): - """ - Verify the current user's role on the given organization_id - (simple string comparison: owner > admin > member). - Requires organization_id in path/query params. - """ - role_order = [OrgRole.MEMBER, OrgRole.ADMIN, OrgRole.OWNER] - - def _rank(r: str) -> int: - try: - return role_order.index(OrgRole(r)) - except ValueError: - return -1 - - async def checker( - organization_id: uuid.UUID, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), - ) -> User: - if current_user.is_superuser: - return current_user - - result = await db.execute( - select(OrgMember).where( - OrgMember.organization_id == organization_id, - OrgMember.user_id == current_user.id, - ) - ) - member = result.scalar_one_or_none() - if not member: - raise ForbiddenException("No access to organization") - if _rank(member.role) < _rank(min_role): - raise ForbiddenException("Insufficient organization permission") - return current_user - - return Depends(checker) - - -CurrentUser = Annotated[User, Depends(get_current_user)] diff --git a/backend/app/common/exceptions.py b/backend/app/common/exceptions.py deleted file mode 100644 index 5a4dfbc12..000000000 --- a/backend/app/common/exceptions.py +++ /dev/null @@ -1,305 +0,0 @@ -""" -Unified exception system (single entry point). - -- Exception classes: all inherit from `AppException(HTTPException)`, supporting separate - `status_code` (HTTP) and `code` (business/error code), with `data` for extra error details. -- Global handlers: provide FastAPI exception handler functions and a one-call registration - function `register_exception_handlers`, ensuring the unified response format defined by - `app.common.response.error_response`. -""" - -from __future__ import annotations - -from typing import Any, Dict, Iterable, List, Mapping, Optional - -from fastapi import HTTPException, Request, status -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse, Response -from pydantic import ValidationError as PydanticValidationError - -from app.common.response import error_response - - -class AppException(HTTPException): - """Base application exception (recommended for all business code).""" - - code: int - data: Any - - def __init__( - self, - status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR, - message: str = "Internal Server Error", - *, - code: int | None = None, - data: Any = None, - headers: Optional[Dict[str, str]] = None, - ): - super().__init__(status_code=status_code, detail=message, headers=headers) - self.code = status_code if code is None else code - self.data = data - - -# Common HTTP exceptions (raise directly from business code) - - -class NotFoundException(AppException): - """Resource not found (404).""" - - def __init__(self, message: str = "Resource not found", *, code: int | None = None, data: Any = None): - super().__init__(status_code=status.HTTP_404_NOT_FOUND, message=message, code=code, data=data) - - -class ModelConfigError(AppException): - """Model configuration error with structured error_code + params for frontend i18n. - - error_code: Frontend i18n key (e.g. MODEL_NOT_FOUND, MODEL_NO_CREDENTIALS) - params: Interpolation params (e.g. {model: "gpt-4o", provider: "openai"}) - message: English fallback (shown when frontend has no i18n key) - """ - - # Error code constants — shared with frontend i18n keys - MODEL_NOT_FOUND = "MODEL_NOT_FOUND" - MODEL_NO_CREDENTIALS = "MODEL_NO_CREDENTIALS" - PROVIDER_NOT_FOUND = "PROVIDER_NOT_FOUND" - MODEL_NAME_REQUIRED = "MODEL_NAME_REQUIRED" - - error_code: str - params: Dict[str, Any] - - def __init__( - self, - error_code: str, - message: str = "Model configuration error", - *, - params: Dict[str, Any] | None = None, - ): - self.error_code = error_code - self.params = params or {} - super().__init__( - status_code=status.HTTP_400_BAD_REQUEST, - message=message, - data={"error_code": error_code, "params": self.params}, - ) - - -class BadRequestException(AppException): - """Bad request (400).""" - - def __init__(self, message: str = "Bad request", *, code: int | None = None, data: Any = None): - super().__init__(status_code=status.HTTP_400_BAD_REQUEST, message=message, code=code, data=data) - - -class UnauthorizedException(AppException): - """Unauthorized (401).""" - - def __init__(self, message: str = "Unauthorized", *, code: int | None = None, data: Any = None): - super().__init__( - status_code=status.HTTP_401_UNAUTHORIZED, - message=message, - code=code, - data=data, - headers={"WWW-Authenticate": "Bearer"}, - ) - - -class ForbiddenException(AppException): - """Forbidden (403).""" - - def __init__(self, message: str = "Forbidden", *, code: int | None = None, data: Any = None): - super().__init__(status_code=status.HTTP_403_FORBIDDEN, message=message, code=code, data=data) - - -class ValidationException(AppException): - """Request validation failed (422).""" - - def __init__(self, message: str = "Validation error", *, code: int | None = None, data: Any = None): - super().__init__(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, message=message, code=code, data=data) - - -class ConflictException(AppException): - """Resource conflict (409).""" - - def __init__(self, message: str = "Resource conflict", *, code: int | None = None, data: Any = None): - super().__init__(status_code=status.HTTP_409_CONFLICT, message=message, code=code, data=data) - - -class TooManyRequestsException(AppException): - """Too many requests (429).""" - - def __init__(self, message: str = "Too many requests", *, code: int | None = None, data: Any = None): - super().__init__(status_code=status.HTTP_429_TOO_MANY_REQUESTS, message=message, code=code, data=data) - - -class InternalServerException(AppException): - """Internal server error (500).""" - - def __init__(self, message: str = "Internal Server Error", *, code: int | None = 1007, data: Any = None): - super().__init__(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, message=message, code=code, data=data) - - -class ClientClosedException(AppException): - """Client disconnected early (499).""" - - def __init__(self, message: str = "Client has closed the connection", *, code: int | None = 1008, data: Any = None): - # 499 is a non-standard HTTP status code, but some gateways/logging systems use it - super().__init__(status_code=499, message=message, code=code, data=data) - - -class BusinessLogicException(BadRequestException): - """Business logic error (default 400, business code default 1006).""" - - def __init__(self, message: str, *, code: int | None = 1006, data: Any = None): - super().__init__(message=message, code=code, data=data) - - -class ParameterValidationException(BadRequestException): - """Parameter/business validation error (default 400, business code default 1001).""" - - def __init__(self, message: str, *, code: int | None = 1001, data: Any = None): - super().__init__(message=message, code=code, data=data) - - -# Aliases - -# Authentication -> 401, Authorization -> 403 -AuthenticationException = UnauthorizedException -AuthorizationException = ForbiddenException -ResourceNotFoundException = NotFoundException -ResourceConflictException = ConflictException - - -# Unified error response construction & global exception handlers - - -def create_error_response(*, status_code: int, code: int, message: str, data: Any = None) -> Response: - """Build a unified error response (conforming to app.common.response.error_response).""" - return JSONResponse( - status_code=status_code, - content=error_response(message=message, code=code, data=data), - ) - - -async def app_exception_handler(request: Request, exc: AppException) -> Response: - """Handle application exceptions (AppException).""" - code_value = getattr(exc, "code", exc.status_code) - code = code_value if isinstance(code_value, int) else exc.status_code - return create_error_response( - status_code=exc.status_code, - code=code, - message=str(exc.detail), - data=getattr(exc, "data", None), - ) - - -async def http_exception_handler(request: Request, exc: HTTPException) -> Response: - """Handle FastAPI/Starlette HTTPException (non-AppException).""" - return create_error_response( - status_code=exc.status_code, - code=exc.status_code, - message=str(exc.detail), - data=getattr(exc, "data", None), - ) - - -def _format_validation_errors(errors: Iterable[Mapping[str, Any]]) -> List[dict[str, Any]]: - formatted: List[dict[str, Any]] = [] - for err in errors: - loc = err.get("loc", ()) - field_path = ".".join(str(x) for x in loc) - formatted.append( - { - "field": field_path, - "message": err.get("msg"), - "type": err.get("type"), - } - ) - return formatted - - -async def request_validation_exception_handler(request: Request, exc: Exception) -> Response: - """Handle request validation exceptions (RequestValidationError / PydanticValidationError).""" - errors: List[dict[str, Any]] = [] - if isinstance(exc, (RequestValidationError, PydanticValidationError)): - errors = _format_validation_errors(exc.errors()) - - return create_error_response( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - code=status.HTTP_422_UNPROCESSABLE_ENTITY, - message="Request parameter validation failed", - data={"validation_errors": errors} if errors else None, - ) - - -async def general_exception_handler(request: Request, exc: Exception) -> Response: - """Handle uncaught exceptions (500).""" - try: - from loguru import logger - - logger.exception("Unhandled exception: {}", exc) - except Exception: - # fallback when logger is unavailable - pass - - debug = False - try: - from app.core.settings import settings - - debug = bool(getattr(settings, "debug", False)) - except Exception: - debug = False - - return create_error_response( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - code=status.HTTP_500_INTERNAL_SERVER_ERROR, - message=str(exc) if debug else "Internal Server Error", - data={"error_type": type(exc).__name__} if debug else None, - ) - - -def register_exception_handlers(app: Any) -> None: - """ - Register all exception handlers on the FastAPI app in one call. - - Note: keep this function free of hard FastAPI type dependencies to avoid circular imports. - """ - app.add_exception_handler(AppException, app_exception_handler) - app.add_exception_handler(HTTPException, http_exception_handler) - app.add_exception_handler(RequestValidationError, request_validation_exception_handler) - app.add_exception_handler(PydanticValidationError, request_validation_exception_handler) - app.add_exception_handler(Exception, general_exception_handler) - - -# Convenience raise_* helpers - - -def raise_validation_error(message: str, data: Any = None) -> None: - raise ParameterValidationException(message, code=1001, data=data) - - -def raise_auth_error(message: str = "Authentication failed, please sign in again", data: Any = None) -> None: - raise UnauthorizedException(message, code=1002, data=data) - - -def raise_permission_error(message: str = "Insufficient permissions", data: Any = None) -> None: - raise ForbiddenException(message, code=1003, data=data) - - -def raise_not_found_error(resource: str, data: Any = None) -> None: - raise NotFoundException(f"{resource} not found", code=1004, data=data) - - -def raise_conflict_error(message: str, data: Any = None) -> None: - raise ConflictException(message, code=1005, data=data) - - -def raise_client_closed_error(message: str = "Client has closed the connection", data: Any = None) -> None: - raise ClientClosedException(message, code=1008, data=data) - - -def raise_business_error(message: str, data: Any = None) -> None: - raise BusinessLogicException(message, code=1006, data=data) - - -def raise_internal_error(message: str = "Internal server error", data: Any = None) -> None: - raise InternalServerException(message, code=1007, data=data) diff --git a/backend/app/common/logging.py b/backend/app/common/logging.py deleted file mode 100644 index c208194c1..000000000 --- a/backend/app/common/logging.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Logging middleware. - -Log detailed information for each request, including method, path, duration, status code, etc. -""" -# mypy: ignore-errors - -import logging -import os -import time -from collections.abc import Callable - -from fastapi import Request, Response -from loguru import logger -from starlette.middleware.base import BaseHTTPMiddleware - -from app.core.trace_context import get_trace_id, set_trace_id - - -class InterceptHandler(logging.Handler): - """Intercept standard logging messages and route them to loguru.""" - - def emit(self, record): - try: - level = logger.level(record.levelname).name - except ValueError: - level = record.levelno - - frame, depth = logging.currentframe(), 2 - logging_file = getattr(logging, "__file__", "") - while frame and frame.f_code.co_filename == logging_file: - frame = frame.f_back - depth += 1 - - logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage()) - - -class LoggingMiddleware(BaseHTTPMiddleware): - """HTTP request logging middleware.""" - - async def dispatch(self, request: Request, call_next: Callable) -> Response: - """Process the request and log details.""" - start_time = time.time() - method = request.method - path = request.url.path - client_host = request.client.host if request.client else "unknown" - - trace_id = set_trace_id(request.headers.get("X-Request-ID") or None) - request.state.trace_id = trace_id - log = logger.bind(trace_id=trace_id, method=method, path=path, client=client_host) - - log.info("request.start") - - try: - response = await call_next(request) - - process_time = time.time() - start_time - status_code = response.status_code - message = f"request.completed status={status_code} duration={process_time:.3f}s" - - if status_code >= 500: - log.error(message) - elif status_code >= 400: - log.warning(message) - else: - log.info(message) - - response.headers["X-Process-Time"] = str(process_time) - response.headers["X-Trace-Id"] = trace_id - return response - - except Exception as e: - process_time = time.time() - start_time - log.opt(exception=True).error(f"request.failed duration={process_time:.3f}s error={type(e).__name__}") - raise - - -def setup_logging(): - """ - Configure loguru logging. - - Set up log format, level, output files, etc. - """ - try: - os.makedirs("logs", exist_ok=True) - except PermissionError: - # if unable to create logs directory (e.g. insufficient permissions in Docker), use console only - pass - logger.configure( - patcher=lambda record: record["extra"].update(trace_id=get_trace_id() or record["extra"].get("trace_id", "-")), - extra={"trace_id": "-", "method": "-", "path": "-", "client": "-"}, - ) - - # remove default handler - logger.remove() - - # add console output (with color) - logger.add( - sink=lambda msg: print(msg, end=""), - format=( - "{time:YYYY-MM-DD HH:mm:ss} | " - "{level: <8} | " - "trace_id={extra[trace_id]} | " - "{extra[method]} {extra[path]} | " - "{name}:{function}:{line} | " - "{message}" - ), - level="INFO", - colorize=True, - ) - - # add file output (all logs) - try: - logger.add( - "logs/app.log", - rotation="100 MB", # rotate when file reaches 100 MB - retention="30 days", # retain logs for 30 days - compression="zip", # compress old logs - format=( - "{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | trace_id={extra[trace_id]} | " - "{extra[method]} {extra[path]} | {name}:{function}:{line} | {message}" - ), - level="INFO", - ) - - # add error log file - logger.add( - "logs/error.log", - rotation="50 MB", - retention="30 days", - compression="zip", - format=( - "{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | trace_id={extra[trace_id]} | " - "{extra[method]} {extra[path]} | {name}:{function}:{line} | {message}" - ), - level="ERROR", - ) - except (PermissionError, OSError): - # if unable to create log files (e.g. insufficient permissions), use console only - pass - - # intercept ALL standard logging into loguru (root + named loggers) - intercept_handler = InterceptHandler() - root_logger = logging.root - root_logger.handlers = [intercept_handler] - root_logger.setLevel(logging.DEBUG) - for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access", "fastapi"): - std_logger = logging.getLogger(logger_name) - std_logger.handlers = [intercept_handler] - std_logger.propagate = False - - logger.info("Logging system initialized") diff --git a/backend/app/common/permissions.py b/backend/app/common/permissions.py deleted file mode 100644 index 112959bef..000000000 --- a/backend/app/common/permissions.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Unified permission check with hierarchical scope matching.""" - -from typing import List, Optional - -SCOPE_HIERARCHY = { - "skills": ["admin", "publish", "execute", "write", "read"], - "graphs": ["execute", "read"], - "tools": ["execute", "read"], -} - -VALID_SCOPES = [f"{resource}:{action}" for resource, actions in SCOPE_HIERARCHY.items() for action in actions] - - -def _scope_satisfies(token_scope: str, required_scope: str) -> bool: - """Check if token_scope satisfies required_scope via hierarchy.""" - if not token_scope or not required_scope: - return False - if token_scope == required_scope: - return True - - # Parse resource:action - try: - token_resource, token_action = token_scope.split(":") - required_resource, required_action = required_scope.split(":") - except ValueError: - return False - - if token_resource != required_resource: - return False - - hierarchy = SCOPE_HIERARCHY.get(token_resource, []) - try: - token_level = hierarchy.index(token_action) - required_level = hierarchy.index(required_action) - return token_level <= required_level # Lower index = higher permission - except ValueError: - return False - - -def check_token_permission( - token_scopes: List[str], - required_scope: str, - resource_type: str, - resource_id: str, - token_resource_type: Optional[str], - token_resource_id: Optional[str], -) -> bool: - """Unified permission check with hierarchical scope matching.""" - # 1. Check scope presence (with hierarchy) - has_scope = any(_scope_satisfies(ts, required_scope) for ts in token_scopes) - if not has_scope: - return False - - # 2. Check resource binding - if token_resource_type is None: - # Global token, pass - return True - - if token_resource_type == resource_type and str(token_resource_id) == str(resource_id): - # Bound to target resource, pass - return True - - return False diff --git a/backend/app/common/skill_permissions.py b/backend/app/common/skill_permissions.py deleted file mode 100644 index a9e749924..000000000 --- a/backend/app/common/skill_permissions.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Unified skill permission check — replaces hardcoded owner_id comparisons.""" - -from __future__ import annotations - -from typing import List, Optional - -from sqlalchemy import and_, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.common.exceptions import ForbiddenException -from app.common.permissions import check_token_permission -from app.models.skill import Skill -from app.models.skill_collaborator import CollaboratorRole, SkillCollaborator - - -async def _get_collaborator( - db: AsyncSession, - skill_id, - user_id: str, -) -> Optional[SkillCollaborator]: - result = await db.execute( - select(SkillCollaborator).where( - and_( - SkillCollaborator.skill_id == skill_id, - SkillCollaborator.user_id == user_id, - ) - ) - ) - return result.scalar_one_or_none() - - -async def check_skill_access( - db: AsyncSession, - skill: Skill, - user_id: str, - min_role: CollaboratorRole, - *, - is_superuser: bool = False, - token_scopes: Optional[List[str]] = None, - token_resource_type: Optional[str] = None, - token_resource_id: Optional[str] = None, - required_scope: Optional[str] = None, -) -> None: - """ - Unified permission check. - - Raises ForbiddenException if the user lacks sufficient access. - """ - # 1. Superuser bypass - if is_superuser: - _check_token_scope(token_scopes, required_scope, str(skill.id), token_resource_type, token_resource_id) - return - - # 2. Owner always passes - if skill.owner_id and skill.owner_id == user_id: - _check_token_scope(token_scopes, required_scope, str(skill.id), token_resource_type, token_resource_id) - return - - # 3. Public skill + viewer access (skip DB query) - if skill.is_public and min_role == CollaboratorRole.viewer: - _check_token_scope(token_scopes, required_scope, str(skill.id), token_resource_type, token_resource_id) - return - - # 4. Check collaborator role - collab = await _get_collaborator(db, skill.id, user_id) - if collab and collab.role >= min_role: - _check_token_scope(token_scopes, required_scope, str(skill.id), token_resource_type, token_resource_id) - return - - raise ForbiddenException("You don't have permission to access this skill") - - -def _check_token_scope( - token_scopes: Optional[List[str]], - required_scope: Optional[str], - skill_id: str, - token_resource_type: Optional[str] = None, - token_resource_id: Optional[str] = None, -) -> None: - """If request came via PlatformToken, verify scope.""" - if token_scopes is not None and required_scope is not None: - has_permission = check_token_permission( - token_scopes=token_scopes, - required_scope=required_scope, - resource_type="skill", - resource_id=str(skill_id), - token_resource_type=token_resource_type, - token_resource_id=str(token_resource_id) if token_resource_id else None, - ) - if not has_permission: - raise ForbiddenException(f"Token missing required scope or resource binding: {required_scope}") diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py deleted file mode 100644 index 1b35e9bfe..000000000 --- a/backend/app/core/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Core module — configuration and utilities.""" - -from .database import Base, engine, get_db -from .settings import settings - -__all__ = ["settings", "get_db", "Base", "engine"] diff --git a/backend/app/core/a2a/__init__.py b/backend/app/core/a2a/__init__.py deleted file mode 100644 index 772f7053c..000000000 --- a/backend/app/core/a2a/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -"""A2A (Agent-to-Agent) protocol client for calling remote A2A-compliant agents. - -Production features: -- Long-running task polling (tasks/get) -- Retry with exponential backoff -- Connection pooling -- Structured logging -""" - -from app.core.a2a.client import ( - A2AClientConfig, - A2ASendResult, - close_all_clients, - get_task, - resolve_a2a_url, - send_message, -) - -__all__ = [ - "A2AClientConfig", - "A2ASendResult", - "close_all_clients", - "get_task", - "resolve_a2a_url", - "send_message", -] diff --git a/backend/app/core/a2a/client.py b/backend/app/core/a2a/client.py deleted file mode 100644 index 86c68eba4..000000000 --- a/backend/app/core/a2a/client.py +++ /dev/null @@ -1,479 +0,0 @@ -"""A2A protocol client: message/send, tasks/get and Task parsing. - -Reference: https://google.github.io/A2A/specification/ -Example: https://raw.githubusercontent.com/langchain-samples/A2A-google-adk/refs/heads/main/test_agent_conversation.py - -Production features: -- Long-running task polling (tasks/get) -- Retry with exponential backoff -- Connection pooling -- Structured logging with context -- Optional Langfuse tracing (via trace_id in extra) -""" - -from __future__ import annotations - -import asyncio -import time -import uuid -from dataclasses import dataclass, field -from typing import Any, Optional - -import httpx -from loguru import logger - -LOG_PREFIX = "[A2A]" - -# ==================== Configuration ==================== - - -@dataclass -class A2AClientConfig: - """Global A2A client configuration.""" - - # Timeout settings - connect_timeout: float = 10.0 - read_timeout: float = 120.0 - write_timeout: float = 30.0 - - # Retry settings - max_retries: int = 3 - retry_base_delay: float = 1.0 - retry_max_delay: float = 30.0 - retry_exponential_base: float = 2.0 - - # Polling settings (for long-running tasks) - poll_interval: float = 2.0 - max_poll_attempts: int = 60 # 60 * 2s = 2 minutes max - - # Connection pool - max_connections: int = 10 - max_keepalive_connections: int = 5 - - -# Default config instance -DEFAULT_CONFIG = A2AClientConfig() - -# ==================== Connection Pool ==================== - -_client_pool: dict[str, httpx.AsyncClient] = {} - - -def _get_client(base_url: str, config: A2AClientConfig = DEFAULT_CONFIG) -> httpx.AsyncClient: - """Get or create a pooled async client for the given base URL.""" - # Normalize URL for pool key - pool_key = base_url.rstrip("/").split("://")[-1].split("/")[0] # Extract host - - if pool_key not in _client_pool: - _client_pool[pool_key] = httpx.AsyncClient( - timeout=httpx.Timeout( - connect=config.connect_timeout, - read=config.read_timeout, - write=config.write_timeout, - pool=config.connect_timeout, - ), - limits=httpx.Limits( - max_connections=config.max_connections, - max_keepalive_connections=config.max_keepalive_connections, - ), - ) - return _client_pool[pool_key] - - -async def close_all_clients(): - """Close all pooled clients. Call on shutdown.""" - for client in _client_pool.values(): - await client.aclose() - _client_pool.clear() - - -def _inject_trace_header(headers: dict[str, str]) -> None: - """Add X-Request-ID from the current trace context if available.""" - from app.core.trace_context import get_trace_id - - trace_id = get_trace_id() - if trace_id: - headers["X-Request-ID"] = trace_id - - -# ==================== Result Types ==================== - - -@dataclass -class A2ASendResult: - """Result of a message/send or tasks/get call.""" - - ok: bool - text: str - task_id: Optional[str] = None - context_id: Optional[str] = None - state: Optional[str] = None # Task state (completed, working, failed, etc.) - error: Optional[str] = None - duration_ms: int = 0 - raw_result: Optional[dict] = field(default=None, repr=False) # Full result for debugging - - -async def resolve_a2a_url( - agent_card_url: str, - auth_headers: Optional[dict[str, str]] = None, - config: A2AClientConfig = DEFAULT_CONFIG, -) -> str: - """Resolve A2A base URL from an Agent Card URL. - - Fetches the Agent Card JSON and returns the 'url' field (A2A service endpoint). - """ - client = _get_client(agent_card_url, config) - headers = dict(auth_headers or {}) - _inject_trace_header(headers) - try: - resp = await client.get(agent_card_url, headers=headers or None) - resp.raise_for_status() - card = resp.json() - except httpx.HTTPStatusError as e: - raise ValueError(f"Failed to fetch Agent Card: HTTP {e.response.status_code}") from e - except Exception as e: - raise ValueError(f"Failed to fetch Agent Card: {e}") from e - - url = card.get("url") - if not url or not isinstance(url, str): - raise ValueError(f"Agent Card missing or invalid 'url': {agent_card_url}") - result: str = url.rstrip("/") - return result - - -def _extract_text_from_task(result: dict[str, Any]) -> str: - """Extract agent reply text from A2A Task result.""" - artifacts = result.get("artifacts") or [] - if not artifacts: - # Fallback: status.message may contain text - status = result.get("status") or {} - msg = status.get("message") or {} - parts = (msg.get("parts") or []) if isinstance(msg, dict) else [] - for p in parts: - if isinstance(p, dict) and p.get("kind") == "text": - return (p.get("text") or "").strip() - return "" - first = artifacts[0] if isinstance(artifacts[0], dict) else {} - parts = first.get("parts") or [] - for p in parts: - if isinstance(p, dict) and p.get("kind") == "text": - return (p.get("text") or "").strip() - return "" - - -def _task_state_terminal(state: str) -> bool: - """Check if task state is terminal (no more updates expected).""" - terminal = ("completed", "canceled", "failed", "rejected", "unknown") - return (state or "").lower() in terminal - - -def _is_retryable_error(e: Exception) -> bool: - """Check if an error is retryable (network issues, 5xx, etc.).""" - if isinstance(e, httpx.TimeoutException): - return True - if isinstance(e, httpx.ConnectError): - return True - if isinstance(e, httpx.HTTPStatusError): - return e.response.status_code >= 500 - return False - - -async def _request_with_retry( - client: httpx.AsyncClient, - method: str, - url: str, - *, - json: Optional[dict] = None, - headers: Optional[dict] = None, - config: A2AClientConfig = DEFAULT_CONFIG, -) -> httpx.Response: - """Make HTTP request with exponential backoff retry.""" - last_error: Optional[Exception] = None - - for attempt in range(config.max_retries + 1): - try: - if method.upper() == "POST": - resp = await client.post(url, json=json, headers=headers) - else: - resp = await client.get(url, headers=headers) - resp.raise_for_status() - return resp - except Exception as e: - last_error = e - if not _is_retryable_error(e) or attempt >= config.max_retries: - raise - # Exponential backoff - delay = min( - config.retry_base_delay * (config.retry_exponential_base**attempt), - config.retry_max_delay, - ) - logger.warning( - f"{LOG_PREFIX} Request failed (attempt {attempt + 1}/{config.max_retries + 1}), " - f"retrying in {delay:.1f}s: {e}" - ) - await asyncio.sleep(delay) - - # Should not reach here, but just in case - raise last_error or RuntimeError("Request failed after retries") - - -async def get_task( - url: str, - task_id: str, - *, - auth_headers: Optional[dict[str, str]] = None, - config: A2AClientConfig = DEFAULT_CONFIG, -) -> A2ASendResult: - """Query task status via tasks/get (JSON-RPC 2.0). - - Args: - url: A2A Server base URL. - task_id: Task ID to query. - auth_headers: Optional HTTP headers. - config: Client configuration. - - Returns: - A2ASendResult with current task state and text. - """ - start_time = time.monotonic() - payload = { - "jsonrpc": "2.0", - "id": str(uuid.uuid4()), - "method": "tasks/get", - "params": {"id": task_id}, - } - - post_url = url.rstrip("/") - if not post_url.startswith(("http://", "https://")): - return A2ASendResult(ok=False, text="", error=f"Invalid A2A URL: {url}") - - headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_headers: - headers.update(auth_headers) - _inject_trace_header(headers) - - client = _get_client(url, config) - - try: - resp = await _request_with_retry(client, "POST", post_url, json=payload, headers=headers, config=config) - data = resp.json() - except httpx.HTTPStatusError as e: - err = f"HTTP {e.response.status_code}: {e.response.text[:200] if e.response.text else ''}" - logger.warning(f"{LOG_PREFIX} tasks/get failed: {err}", task_id=task_id) - return A2ASendResult(ok=False, text="", error=err, duration_ms=int((time.monotonic() - start_time) * 1000)) - except Exception as e: - logger.warning(f"{LOG_PREFIX} tasks/get error: {e}", task_id=task_id) - return A2ASendResult(ok=False, text="", error=str(e), duration_ms=int((time.monotonic() - start_time) * 1000)) - - duration_ms = int((time.monotonic() - start_time) * 1000) - - if "error" in data: - err_obj = data["error"] - err_msg = err_obj.get("message", str(err_obj)) if isinstance(err_obj, dict) else str(err_obj) - return A2ASendResult(ok=False, text="", error=err_msg, duration_ms=duration_ms) - - result = data.get("result") - if result is None: - return A2ASendResult(ok=False, text="", error="Response missing 'result'", duration_ms=duration_ms) - - task_id_out = result.get("id") if isinstance(result, dict) else None - context_id_out = result.get("contextId") if isinstance(result, dict) else None - status = result.get("status") or {} if isinstance(result, dict) else {} - state = status.get("state") if isinstance(status, dict) else "unknown" - text_out = _extract_text_from_task(result) - - return A2ASendResult( - ok=True, - text=text_out, - task_id=task_id_out, - context_id=context_id_out, - state=state, - duration_ms=duration_ms, - raw_result=result, - ) - - -async def send_message( - url: str, - text: str, - *, - context_id: Optional[str] = None, - task_id: Optional[str] = None, - auth_headers: Optional[dict[str, str]] = None, - config: A2AClientConfig = DEFAULT_CONFIG, - wait_for_completion: bool = True, -) -> A2ASendResult: - """Send a message to an A2A Server via message/send (JSON-RPC 2.0). - - Production features: - - Automatic retry with exponential backoff - - Long-running task polling (tasks/get) - - Connection pooling - - Structured logging - - Args: - url: A2A Server base URL (e.g. from Agent Card or config). - text: User message text to send. - context_id: Optional context ID for multi-turn. - task_id: Optional task ID for follow-up. - auth_headers: Optional HTTP headers (e.g. Authorization). - config: Client configuration (timeouts, retries, polling). - wait_for_completion: If True, poll until task reaches terminal state. - - Returns: - A2ASendResult with ok, text, task_id, context_id, state, or error. - """ - start_time = time.monotonic() - request_id = str(uuid.uuid4()) - message_id = str(uuid.uuid4()) - - message: dict[str, Any] = { - "role": "user", - "parts": [{"kind": "text", "text": text}], - "messageId": message_id, - "kind": "message", - } - if context_id: - message["contextId"] = context_id - if task_id: - message["taskId"] = task_id - - payload = { - "jsonrpc": "2.0", - "id": request_id, - "method": "message/send", - "params": { - "message": message, - "messageId": message_id, - }, - } - - post_url = url.rstrip("/") - if not post_url.startswith(("http://", "https://")): - return A2ASendResult(ok=False, text="", error=f"Invalid A2A URL: {url}") - - headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_headers: - headers.update(auth_headers) - _inject_trace_header(headers) - - client = _get_client(url, config) - - # Log request (structured) - compatible with Langfuse via trace_id - logger.info( - f"{LOG_PREFIX} message/send request", - extra={ - "a2a_url": post_url, - "request_id": request_id, - "context_id": context_id, - "task_id": task_id, - "text_length": len(text), - "span_type": "a2a_send", # For tracing integration - }, - ) - - try: - resp = await _request_with_retry(client, "POST", post_url, json=payload, headers=headers, config=config) - data = resp.json() - except httpx.HTTPStatusError as e: - duration_ms = int((time.monotonic() - start_time) * 1000) - err = f"HTTP {e.response.status_code}: {e.response.text[:200] if e.response.text else ''}" - logger.warning( - f"{LOG_PREFIX} message/send failed", - extra={"a2a_url": post_url, "error": err, "duration_ms": duration_ms}, - ) - return A2ASendResult(ok=False, text="", error=err, duration_ms=duration_ms) - except Exception as e: - duration_ms = int((time.monotonic() - start_time) * 1000) - logger.warning( - f"{LOG_PREFIX} message/send error", - extra={"a2a_url": post_url, "error": str(e), "duration_ms": duration_ms}, - ) - return A2ASendResult(ok=False, text="", error=str(e), duration_ms=duration_ms) - - if "error" in data: - duration_ms = int((time.monotonic() - start_time) * 1000) - err_obj = data["error"] - err_msg = err_obj.get("message", str(err_obj)) if isinstance(err_obj, dict) else str(err_obj) - logger.warning( - f"{LOG_PREFIX} message/send JSON-RPC error", - extra={"a2a_url": post_url, "error": err_msg, "duration_ms": duration_ms}, - ) - return A2ASendResult(ok=False, text="", error=err_msg, duration_ms=duration_ms) - - result = data.get("result") - if result is None: - duration_ms = int((time.monotonic() - start_time) * 1000) - return A2ASendResult(ok=False, text="", error="Response missing 'result'", duration_ms=duration_ms) - - # Parse initial result - task_id_out = result.get("id") if isinstance(result, dict) else None - context_id_out = result.get("contextId") if isinstance(result, dict) else None - status = result.get("status") or {} if isinstance(result, dict) else {} - state_raw = status.get("state") if isinstance(status, dict) else "unknown" - state: str = str(state_raw) if state_raw else "unknown" - text_out = _extract_text_from_task(result) - - # Poll for completion if task is not terminal - if wait_for_completion and not _task_state_terminal(state) and task_id_out: - logger.info( - f"{LOG_PREFIX} Task not terminal (state={state}), starting polling", - extra={"task_id": task_id_out, "a2a_url": post_url}, - ) - - for poll_attempt in range(config.max_poll_attempts): - await asyncio.sleep(config.poll_interval) - - poll_result = await get_task(url, task_id_out, auth_headers=auth_headers, config=config) - - if not poll_result.ok: - logger.warning( - f"{LOG_PREFIX} Poll attempt {poll_attempt + 1} failed: {poll_result.error}", - extra={"task_id": task_id_out}, - ) - continue - - state = poll_result.state or "unknown" - if _task_state_terminal(state): - text_out = poll_result.text - context_id_out = poll_result.context_id or context_id_out - logger.info( - f"{LOG_PREFIX} Task reached terminal state", - extra={"task_id": task_id_out, "state": state, "poll_attempts": poll_attempt + 1}, - ) - break - - logger.debug( - f"{LOG_PREFIX} Poll attempt {poll_attempt + 1}: state={state}", - extra={"task_id": task_id_out}, - ) - else: - # Max polls reached without terminal state - logger.warning( - f"{LOG_PREFIX} Max poll attempts reached, returning partial result", - extra={"task_id": task_id_out, "state": state, "max_polls": config.max_poll_attempts}, - ) - - duration_ms = int((time.monotonic() - start_time) * 1000) - - logger.info( - f"{LOG_PREFIX} message/send completed", - extra={ - "a2a_url": post_url, - "task_id": task_id_out, - "context_id": context_id_out, - "state": state, - "duration_ms": duration_ms, - "text_length": len(text_out) if text_out else 0, - }, - ) - - return A2ASendResult( - ok=True, - text=text_out or "(no text in response)", - task_id=task_id_out, - context_id=context_id_out, - state=state, - duration_ms=duration_ms, - raw_result=result, - ) diff --git a/backend/app/core/agent/__init__.py b/backend/app/core/agent/__init__.py deleted file mode 100644 index 2c194f52e..000000000 --- a/backend/app/core/agent/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Agent subpackage (core agent runtime components). - -This file exists to make `app.core.agent` a regular Python package so that -static analyzers (e.g. Pylance/Pyright) can reliably resolve imports like: -`app.core.agent.memory.strategies`. - -Keep this module side-effect free: avoid importing heavy dependencies here. -""" - -__all__: list[str] = [] diff --git a/backend/app/core/agent/artifacts/__init__.py b/backend/app/core/agent/artifacts/__init__.py deleted file mode 100644 index c8854cce2..000000000 --- a/backend/app/core/agent/artifacts/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Agent run artifacts: collection, resolution, and manifest management.""" - -from app.core.agent.artifacts.collector import ArtifactCollector -from app.core.agent.artifacts.resolver import ArtifactResolver, FileInfo, RunInfo - -__all__ = [ - "ArtifactCollector", - "ArtifactResolver", - "FileInfo", - "RunInfo", -] diff --git a/backend/app/core/agent/artifacts/collector.py b/backend/app/core/agent/artifacts/collector.py deleted file mode 100644 index 2fdee4f6a..000000000 --- a/backend/app/core/agent/artifacts/collector.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -ArtifactCollector: collect agent run outputs into a unified artifacts directory. - -When using FilesystemBackend with root_dir pointing to the run directory, -no copy is needed; this module writes _manifest.json after the run. -""" - -from __future__ import annotations - -import json -import mimetypes -import os -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from loguru import logger - -from app.utils.path_utils import sanitize_path_component - -MANIFEST_FILENAME = "_manifest.json" - - -def _default_artifacts_root() -> Path: - return Path.home() / ".agent-platform" / "agent-artifacts" - - -def resolve_artifacts_root() -> Path: - env = os.getenv("AGENT_ARTIFACTS_ROOT", "").strip() - if env: - return Path(env).expanduser().resolve() - return _default_artifacts_root().resolve() - - -class ArtifactCollector: - """Collects and indexes agent run artifacts under a unified directory layout.""" - - def __init__(self, artifacts_root: str | Path | None = None) -> None: - self.root = Path(artifacts_root) if artifacts_root else resolve_artifacts_root() - self.root = self.root.resolve() - - def get_run_dir( - self, - user_id: str, - thread_id: str, - run_id: str, - ) -> Path: - """Return the run artifacts directory path (sanitized, no fs ops).""" - uid = sanitize_path_component(user_id, default="default") - tid = sanitize_path_component(thread_id, default="default") - rid = sanitize_path_component(run_id, default="default") - return self.root / uid / tid / rid - - def ensure_run_dir( - self, - user_id: str, - thread_id: str, - run_id: str, - ) -> Path: - """Create the run directory if it does not exist; return its path.""" - run_dir = self.get_run_dir(user_id, thread_id, run_id) - run_dir.mkdir(parents=True, exist_ok=True) - return run_dir - - def _collect_file_entries(self, run_dir: Path) -> list[dict[str, Any]]: - """Scan run_dir for files (excluding _manifest.json) and return manifest file entries.""" - entries: list[dict[str, Any]] = [] - run_dir_resolved = run_dir.resolve() - for path in run_dir.rglob("*"): - if not path.is_file(): - continue - try: - rel = path.resolve().relative_to(run_dir_resolved) - rel_str = rel.as_posix() - if rel_str == MANIFEST_FILENAME: - continue - size = path.stat().st_size - guess_type, _ = mimetypes.guess_type(str(path)) - entries.append( - { - "path": rel_str, - "size": size, - "type": guess_type or "application/octet-stream", - } - ) - except (ValueError, OSError) as e: - logger.warning(f"[ArtifactCollector] Skip file {path}: {e}") - return entries - - def write_manifest( - self, - run_dir: Path, - metadata: dict[str, Any], - ) -> None: - """ - Scan run_dir for files, then write _manifest.json with metadata and file list. - - metadata may include: run_id, thread_id, user_id, agent_type, graph_id, - started_at, completed_at, status. Files list is generated by scanning. - """ - run_dir = Path(run_dir).resolve() - if not run_dir.exists() or not run_dir.is_dir(): - logger.warning(f"[ArtifactCollector] Run dir does not exist or is not a directory: {run_dir}") - return - - # Ensure run_dir is under self.root (security) - try: - run_dir.relative_to(self.root) - except ValueError: - logger.error(f"[ArtifactCollector] run_dir {run_dir} is not under artifacts root {self.root}") - return - - payload = dict(metadata) - payload.setdefault("completed_at", datetime.now(timezone.utc).isoformat()) - payload["files"] = self._collect_file_entries(run_dir) - - manifest_path = run_dir / MANIFEST_FILENAME - with manifest_path.open("w", encoding="utf-8") as f: - json.dump(payload, f, ensure_ascii=False, indent=2, default=str) - logger.debug(f"[ArtifactCollector] Wrote manifest at {manifest_path}") diff --git a/backend/app/core/agent/artifacts/resolver.py b/backend/app/core/agent/artifacts/resolver.py deleted file mode 100644 index 74778a9a8..000000000 --- a/backend/app/core/agent/artifacts/resolver.py +++ /dev/null @@ -1,232 +0,0 @@ -""" -ArtifactResolver: list runs, list files, and resolve safe file paths for download. -""" - -from __future__ import annotations - -import json -import mimetypes -import shutil -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Optional, cast - -from app.core.agent.artifacts.collector import ( - MANIFEST_FILENAME, - resolve_artifacts_root, -) -from app.utils.path_utils import sanitize_path_component - - -@dataclass -class RunInfo: - """Summary of a single run's artifacts.""" - - run_id: str - thread_id: str - user_id: str - path: str # relative path under artifacts root, e.g. user_id/thread_id/run_id - started_at: Optional[str] = None - completed_at: Optional[str] = None - status: Optional[str] = None - agent_type: Optional[str] = None - graph_id: Optional[str] = None - file_count: int = 0 - - -@dataclass -class FileInfo: - """A single file or directory in a run's artifact tree.""" - - name: str - path: str # relative path from run root - type: str # "file" | "directory" - size: Optional[int] = None - content_type: Optional[str] = None - children: Optional[list["FileInfo"]] = None - - -class ArtifactResolver: - """Resolves artifact paths and lists runs/files from the artifacts filesystem.""" - - def __init__(self, artifacts_root: str | Path | None = None) -> None: - self.root = Path(artifacts_root) if artifacts_root else resolve_artifacts_root() - self.root = self.root.resolve() - - def _run_dir(self, user_id: str, thread_id: str, run_id: str) -> Path: - """Sanitized run directory path (may not exist).""" - uid = sanitize_path_component(user_id, default="default") - tid = sanitize_path_component(thread_id, default="default") - rid = sanitize_path_component(run_id, default="default") - return self.root / uid / tid / rid - - def list_runs(self, user_id: str, thread_id: str) -> list[RunInfo]: - """List all run directories for the given user and thread.""" - uid = sanitize_path_component(user_id, default="default") - tid = sanitize_path_component(thread_id, default="default") - thread_dir = self.root / uid / tid - if not thread_dir.exists() or not thread_dir.is_dir(): - return [] - - runs: list[RunInfo] = [] - for run_path in thread_dir.iterdir(): - if not run_path.is_dir(): - continue - run_id = run_path.name - manifest = self.read_manifest(user_id, thread_id, run_id) - if manifest: - runs.append( - RunInfo( - run_id=run_id, - thread_id=thread_id, - user_id=user_id, - path=f"{uid}/{tid}/{run_id}", - started_at=manifest.get("started_at"), - completed_at=manifest.get("completed_at"), - status=manifest.get("status"), - agent_type=manifest.get("agent_type"), - graph_id=manifest.get("graph_id"), - file_count=len(manifest.get("files") or []), - ) - ) - else: - # No manifest: still list the run, scan file count - file_count = sum(1 for _ in run_path.rglob("*") if _.is_file() and _.name != MANIFEST_FILENAME) - runs.append( - RunInfo( - run_id=run_id, - thread_id=thread_id, - user_id=user_id, - path=f"{uid}/{tid}/{run_id}", - file_count=file_count, - ) - ) - - # Sort by completed_at or path, newest first - def sort_key(r: RunInfo) -> tuple: - return (r.completed_at or r.run_id or "",) - - runs.sort(key=sort_key, reverse=True) - return runs - - def read_manifest(self, user_id: str, thread_id: str, run_id: str) -> Optional[dict[str, Any]]: - """Read _manifest.json for the run if it exists.""" - run_dir = self._run_dir(user_id, thread_id, run_id) - manifest_path = run_dir / MANIFEST_FILENAME - if not manifest_path.exists(): - return None - try: - with manifest_path.open("r", encoding="utf-8") as f: - return cast(dict[str, Any], json.load(f)) - except (json.JSONDecodeError, OSError): - return None - - def list_files(self, user_id: str, thread_id: str, run_id: str) -> list[FileInfo]: - """List files and directories in the run as a tree (one level of children).""" - run_dir = self._run_dir(user_id, thread_id, run_id) - if not run_dir.exists() or not run_dir.is_dir(): - return [] - - result: list[FileInfo] = [] - for path in sorted(run_dir.iterdir()): - if path.name == MANIFEST_FILENAME: - continue - rel = path.relative_to(run_dir) - rel_str = rel.as_posix() - if path.is_dir(): - result.append( - FileInfo( - name=path.name, - path=rel_str, - type="directory", - children=None, - ) - ) - else: - try: - size = path.stat().st_size - except OSError: - size = None - ct, _ = mimetypes.guess_type(str(path)) - result.append( - FileInfo( - name=path.name, - path=rel_str, - type="file", - size=size, - content_type=ct, - ) - ) - return result - - def list_files_tree(self, user_id: str, thread_id: str, run_id: str) -> list[FileInfo]: - """List all files recursively as a tree (nested children).""" - run_dir = self._run_dir(user_id, thread_id, run_id) - if not run_dir.exists() or not run_dir.is_dir(): - return [] - - def build_node(p: Path, base: Path) -> FileInfo: - rel = p.relative_to(base) - rel_str = rel.as_posix() - if p.is_dir(): - children = [build_node(c, base) for c in sorted(p.iterdir()) if c.name != MANIFEST_FILENAME] - return FileInfo(name=p.name, path=rel_str, type="directory", children=children) - try: - size = p.stat().st_size - except OSError: - size = None - ct, _ = mimetypes.guess_type(str(p)) - return FileInfo(name=p.name, path=rel_str, type="file", size=size, content_type=ct) - - root_children: list[FileInfo] = [] - for item in sorted(run_dir.iterdir()): - if item.name == MANIFEST_FILENAME: - continue - root_children.append(build_node(item, run_dir)) - return root_children - - def get_file_path( - self, - user_id: str, - thread_id: str, - run_id: str, - file_path: str, - ) -> Optional[Path]: - """ - Resolve a relative file path to an absolute path under the run directory. - Returns None if the path escapes the run dir (security) or doesn't exist. - """ - run_dir = self._run_dir(user_id, thread_id, run_id) - if not run_dir.exists() or not run_dir.is_dir(): - return None - - # Normalize and resolve — the single resolve().relative_to() check - # is sufficient to prevent all path traversal attacks - cleaned = file_path.replace("\\", "/").strip("/") - if not cleaned: - return None - resolved = (run_dir / cleaned).resolve() - run_dir_resolved = run_dir.resolve() - try: - resolved.relative_to(run_dir_resolved) - except ValueError: - return None - if not resolved.exists() or not resolved.is_file(): - return None - return resolved - - def delete_run(self, user_id: str, thread_id: str, run_id: str) -> bool: - """Delete the entire run directory. Returns True if deleted or missing.""" - run_dir = self._run_dir(user_id, thread_id, run_id) - run_dir_resolved = run_dir.resolve() - try: - run_dir_resolved.relative_to(self.root) - except ValueError: - return False - if not run_dir.exists(): - return True - try: - shutil.rmtree(run_dir) - return True - except OSError: - return False diff --git a/backend/app/core/agent/backends/__init__.py b/backend/app/core/agent/backends/__init__.py deleted file mode 100644 index cc179004e..000000000 --- a/backend/app/core/agent/backends/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Sandbox backend implementations for agent execution environments. - -This module provides various backend implementations for sandbox environments: -- FilesystemSandboxBackend: Local filesystem-based backend -- StateSandboxBackend: In-memory state-based backend -- PydanticSandboxAdapter: Docker-based sandbox via pydantic-ai-backend - -All backends implement the SandboxBackendProtocol interface. -""" - -from app.core.agent.backends.filesystem_sandbox import FilesystemSandboxBackend -from app.core.agent.backends.state_sandbox import StateSandboxBackend - -try: - from app.core.agent.backends.pydantic_adapter import ( - BUILTIN_RUNTIMES, - PydanticSandboxAdapter, - RuntimeConfig, - get_builtin_runtime, - list_builtin_runtimes, - ) -except ImportError: - PydanticSandboxAdapter = None # type: ignore - RuntimeConfig = None # type: ignore - BUILTIN_RUNTIMES = {} # type: ignore - get_builtin_runtime = None # type: ignore - list_builtin_runtimes = None # type: ignore - -__all__ = [ - # Backend classes - "StateSandboxBackend", - "FilesystemSandboxBackend", - "PydanticSandboxAdapter", - # Runtime configuration - "RuntimeConfig", - "BUILTIN_RUNTIMES", - "get_builtin_runtime", - "list_builtin_runtimes", -] diff --git a/backend/app/core/agent/backends/constants.py b/backend/app/core/agent/backends/constants.py deleted file mode 100644 index 0de032d37..000000000 --- a/backend/app/core/agent/backends/constants.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Default constants for sandbox backends. - -This module defines default configuration values used across different -sandbox backend implementations to ensure consistency. -""" - -# Command execution defaults -DEFAULT_COMMAND_TIMEOUT = 30 # seconds -DEFAULT_MAX_OUTPUT_SIZE = 100000 # characters - -# Docker sandbox defaults -DEFAULT_DOCKER_IMAGE = "python:3.12-slim" -DEFAULT_WORKING_DIR = "/workspace" -DEFAULT_AUTO_REMOVE = True -DEFAULT_IDLE_TIMEOUT = 3600 # 1 hour in seconds - -# Sandbox host directory -DEFAULT_SANDBOX_HOST_ROOT = "/tmp/sandboxes" -SANDBOX_UPLOADS_SUBDIR = "uploads" - -# User sandbox defaults -DEFAULT_USER_SANDBOX_IMAGE = "python:3.12-slim" -DEFAULT_USER_SANDBOX_IDLE_TIMEOUT = 3600 # 1 hour in seconds -DEFAULT_USER_SANDBOX_CPU_LIMIT = 1.0 # 1 core -DEFAULT_USER_SANDBOX_MEMORY_LIMIT = 512 # 512MB -# Stop/restart keep container; only rebuild removes it -DEFAULT_USER_SANDBOX_AUTO_REMOVE = False -MAX_SANDBOX_POOL_SIZE = 100 - - -# File size defaults -DEFAULT_MAX_FILE_SIZE_MB = 10 - -# User-facing error messages (bilingual) -DOCKER_UNAVAILABLE_MSG = ( - "代码执行沙箱不可用，请确认 Docker Desktop 已启动后重试。" - " / Code execution sandbox unavailable. Please start Docker Desktop and try again." -) - -__all__ = [ - # Command execution - "DEFAULT_COMMAND_TIMEOUT", - "DEFAULT_MAX_OUTPUT_SIZE", - # Docker sandbox - "DEFAULT_DOCKER_IMAGE", - "DEFAULT_WORKING_DIR", - "DEFAULT_AUTO_REMOVE", - "DEFAULT_IDLE_TIMEOUT", - # File size - "DEFAULT_MAX_FILE_SIZE_MB", - # User sandbox - "DEFAULT_USER_SANDBOX_IMAGE", - "DEFAULT_USER_SANDBOX_IDLE_TIMEOUT", - "DEFAULT_USER_SANDBOX_CPU_LIMIT", - "DEFAULT_USER_SANDBOX_MEMORY_LIMIT", - "DEFAULT_USER_SANDBOX_AUTO_REMOVE", - "MAX_SANDBOX_POOL_SIZE", - # Sandbox host directory - "DEFAULT_SANDBOX_HOST_ROOT", - "SANDBOX_UPLOADS_SUBDIR", - # User-facing error messages - "DOCKER_UNAVAILABLE_MSG", -] diff --git a/backend/app/core/agent/backends/file_tracking_proxy.py b/backend/app/core/agent/backends/file_tracking_proxy.py deleted file mode 100644 index 396f5124d..000000000 --- a/backend/app/core/agent/backends/file_tracking_proxy.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Proxy that wraps SandboxBackendProtocol to emit file events on write operations.""" - -from __future__ import annotations - -from typing import Any - -from deepagents.backends.protocol import ( - EditResult, - ExecuteResponse, - FileDownloadResponse, - FileInfo, - FileUploadResponse, - GrepMatch, - SandboxBackendProtocol, - WriteResult, -) - -from app.utils.file_event_emitter import FileEventEmitter - - -class FileTrackingProxy(SandboxBackendProtocol): - """Transparent proxy that intercepts write operations and emits file events. - - All non-write methods are delegated directly to the wrapped backend. - Unknown methods are forwarded via __getattr__ for forward compatibility. - """ - - def __init__(self, backend: SandboxBackendProtocol, emitter: FileEventEmitter) -> None: - self._backend = backend - self._emitter = emitter - - # ── Write operations (intercepted) ────────────────────────────────── - - def write(self, file_path: str, content: str) -> WriteResult: - result = self._backend.write(file_path, content) - if not getattr(result, "error", None): - size = len(content.encode("utf-8")) - self._emitter.emit("write", file_path, size) - return result - - def write_overwrite(self, file_path: str, content: str) -> WriteResult: - result: WriteResult = getattr(self._backend, "write_overwrite")(file_path, content) - if not getattr(result, "error", None): - size = len(content.encode("utf-8")) - self._emitter.emit("write", file_path, size) - return result - - def edit( - self, - file_path: str, - old_string: str, - new_string: str, - replace_all: bool = False, - ) -> EditResult: - result = self._backend.edit(file_path, old_string, new_string, replace_all) - if not getattr(result, "error", None): - self._emitter.emit("edit", file_path) - return result - - def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: - results = self._backend.upload_files(files) - for (path, content), resp in zip(files, results): - if not getattr(resp, "error", None): - self._emitter.emit("write", path, len(content)) - return results - - # ── Read operations (delegated) ───────────────────────────────────── - - def read(self, *args: Any, **kwargs: Any) -> str: - return self._backend.read(*args, **kwargs) - - def raw_read(self, *args: Any, **kwargs: Any) -> str: - return str(getattr(self._backend, "raw_read")(*args, **kwargs)) - - def ls_info(self, path: str) -> list[FileInfo]: - return self._backend.ls_info(path) - - def execute(self, command: str) -> ExecuteResponse: - return self._backend.execute(command) - - def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[GrepMatch] | str: - return self._backend.grep_raw(pattern, path, glob) - - def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]: - return self._backend.glob_info(pattern, path) - - def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: - return self._backend.download_files(paths) - - # ── Lifecycle (delegated) ─────────────────────────────────────────── - - @property - def id(self) -> str: - return self._backend.id - - def is_started(self) -> bool: - return getattr(self._backend, "is_started")() # type: ignore[no-any-return] - - def start(self) -> None: - getattr(self._backend, "start")() - - def stop(self) -> None: - getattr(self._backend, "stop")() - - def cleanup(self) -> None: - getattr(self._backend, "cleanup")() - - # ── Forward compatibility ─────────────────────────────────────────── - - def __getattr__(self, name: str) -> Any: - return getattr(self._backend, name) diff --git a/backend/app/core/agent/backends/filesystem_sandbox.py b/backend/app/core/agent/backends/filesystem_sandbox.py deleted file mode 100644 index 707d0a8c1..000000000 --- a/backend/app/core/agent/backends/filesystem_sandbox.py +++ /dev/null @@ -1,98 +0,0 @@ -"""FilesystemSandboxBackend: FilesystemBackend with command execution support.""" - -import uuid -from pathlib import Path - -from deepagents.backends.filesystem import FilesystemBackend -from deepagents.backends.protocol import ExecuteResponse, SandboxBackendProtocol - -from app.core.agent.backends.constants import ( - DEFAULT_COMMAND_TIMEOUT, - DEFAULT_MAX_FILE_SIZE_MB, - DEFAULT_MAX_OUTPUT_SIZE, -) -from app.core.agent.backends.utils.command_executor import execute_local_command - - -class FilesystemSandboxBackend(FilesystemBackend, SandboxBackendProtocol): - """FilesystemBackend with command execution support. - - Extends FilesystemBackend to implement SandboxBackendProtocol by adding - command execution capabilities. Commands are executed in the local system - environment, with the working directory set to the backend's root_dir. - - Features: - - Real filesystem access (read/write actual files) - - Command execution in specified directory - - Configurable timeout and output limits - - Support for virtual_mode (path sandboxing) - - Warning: - This backend executes commands directly on the host system without - isolation. Use with caution and only with trusted agents. For production - use, consider using PydanticSandboxAdapter (Docker sandbox) instead. - - Example: - ```python - from app.core.agent.backends import FilesystemSandboxBackend - - # Create backend with specific root directory - backend = FilesystemSandboxBackend( - root_dir="/path/to/workspace", - virtual_mode=True, # Enable path sandboxing - ) - - # Use with Agent - agent = create_agent(model, tools=tools, middleware=[FilesystemMiddleware(backend=backend)]) - ``` - """ - - def __init__( - self, - root_dir: str | Path | None = None, - virtual_mode: bool = False, - max_file_size_mb: int = DEFAULT_MAX_FILE_SIZE_MB, - max_output_size: int = DEFAULT_MAX_OUTPUT_SIZE, - command_timeout: int = DEFAULT_COMMAND_TIMEOUT, - ): - """Initialize FilesystemSandboxBackend. - - Args: - root_dir: Root directory for file operations. If not provided, uses cwd. - virtual_mode: Enable path sandboxing (prevent access outside root_dir). - max_file_size_mb: Maximum file size in MB for read operations. - max_output_size: Maximum command output size in characters. - command_timeout: Command execution timeout in seconds. - """ - super().__init__( - root_dir=root_dir, - virtual_mode=virtual_mode, - max_file_size_mb=max_file_size_mb, - ) - self._id = str(uuid.uuid4()) - self.max_output_size = max_output_size - self.command_timeout = command_timeout - - @property - def id(self) -> str: - """Unique identifier for this backend instance.""" - return self._id - - def execute(self, command: str) -> ExecuteResponse: - """Execute a shell command in the root directory. - - Args: - command: Shell command to execute. - - Returns: - ExecuteResponse with combined stdout/stderr output and exit code. - - Note: - Commands are executed with cwd set to self.cwd (root_dir). - """ - return execute_local_command( - command=command, - cwd=str(self.cwd), - timeout=self.command_timeout, - max_output_size=self.max_output_size, - ) diff --git a/backend/app/core/agent/backends/pydantic_adapter.py b/backend/app/core/agent/backends/pydantic_adapter.py deleted file mode 100644 index ea523b370..000000000 --- a/backend/app/core/agent/backends/pydantic_adapter.py +++ /dev/null @@ -1,1028 +0,0 @@ -"""Adapter for pydantic-ai-backend DockerSandbox to SandboxBackendProtocol. - -This module provides an adapter layer that bridges pydantic-ai-backend's -DockerSandbox implementation with deepAgents' SandboxBackendProtocol interface. - -Supports advanced features from pydantic-ai-backend 0.1.5+: -- RuntimeConfig for pre-configured environments (python-datascience, python-web, etc.) -- session_id for multi-user session management -- idle_timeout for automatic container cleanup -- volumes for Docker volume mounting -""" - -import uuid -from datetime import datetime -from pathlib import Path -from typing import List - -from deepagents.backends.protocol import ( - EditResult, - ExecuteResponse, - FileDownloadResponse, - FileInfo, - FileUploadResponse, - GrepMatch, - SandboxBackendProtocol, - WriteResult, -) -from deepagents.backends.utils import format_read_response -from loguru import logger -from pydantic_ai_backends import DockerSandbox - -from app.core.agent.backends.constants import ( - DEFAULT_AUTO_REMOVE, - DEFAULT_COMMAND_TIMEOUT, - DEFAULT_DOCKER_IMAGE, - DEFAULT_IDLE_TIMEOUT, - DEFAULT_MAX_OUTPUT_SIZE, - DEFAULT_WORKING_DIR, -) -from app.core.agent.backends.runtime_config import ( - BUILTIN_RUNTIMES, - RuntimeConfig, - get_builtin_runtime, - list_builtin_runtimes, - resolve_runtime, -) -from app.utils.backend_utils import create_execute_response - -# Re-export for backward compatibility -__all__ = [ - "PydanticSandboxAdapter", - "RuntimeConfig", - "BUILTIN_RUNTIMES", - "get_builtin_runtime", - "list_builtin_runtimes", -] - - -class PydanticSandboxAdapter(SandboxBackendProtocol): - """Adapter that wraps pydantic-ai-backend's DockerSandbox for deepAgents. - - This adapter implements SandboxBackendProtocol by delegating to - pydantic-ai-backend's DockerSandbox, providing a seamless integration - with deepAgents' FilesystemMiddleware. - - **Lifecycle Management:** - - `start()` is called automatically in `__init__` to start the container - - `stop()`: stop the container only (no remove); use for stop/restart semantics - - `cleanup()`: stop and remove the container; use for rebuild/teardown only - - With auto_remove=False, stop() keeps the container so start() can restart it - - **Features:** - - Full SandboxBackendProtocol compatibility - - Delegates to pydantic-ai-backend's DockerSandbox - - Explicit lifecycle management (start/stop) - - Automatic resource cleanup - - Error handling and logging - - Idempotent start/stop operations - - RuntimeConfig support for pre-configured environments - - Session management with session_id - - Idle timeout for automatic container cleanup - - Docker volume mounting support - - **Pre-configured Runtimes:** - - python-minimal: Basic Python 3.12 environment - - python-datascience: Pandas, NumPy, Matplotlib, Scikit-learn - - python-web: FastAPI, Uvicorn, SQLAlchemy, httpx - - python-ml: PyTorch, Transformers - - node-minimal: Basic Node.js 20 environment - - node-react: TypeScript, Vite, React - - Example: - ```python - from app.core.agent.backends.pydantic_adapter import ( - PydanticSandboxAdapter, - RuntimeConfig, - ) - - # Basic usage - adapter = PydanticSandboxAdapter(image="python:3.12-slim") - - # Using pre-configured runtime - adapter = PydanticSandboxAdapter(runtime="python-datascience") - - # Using custom runtime - custom_runtime = RuntimeConfig( - name="ml-env", - base_image="python:3.12-slim", - packages=["torch", "transformers"], - ) - adapter = PydanticSandboxAdapter(runtime=custom_runtime) - - # With session management and volume mounting - adapter = PydanticSandboxAdapter( - runtime="python-web", - session_id="user-123", - idle_timeout=1800, - volumes={"/data": "/app/shared"}, - ) - - # Use as context manager - with PydanticSandboxAdapter(runtime="python-minimal") as adapter: - result = adapter.execute("python --version") - ``` - """ - - def __init__( - self, - image: str = DEFAULT_DOCKER_IMAGE, - working_dir: str = DEFAULT_WORKING_DIR, - auto_remove: bool = DEFAULT_AUTO_REMOVE, - max_output_size: int = DEFAULT_MAX_OUTPUT_SIZE, - command_timeout: int = DEFAULT_COMMAND_TIMEOUT, - runtime: RuntimeConfig | str | None = None, - session_id: str | None = None, - idle_timeout: int = DEFAULT_IDLE_TIMEOUT, - volumes: dict[str, str] | None = None, - ): - """Initialize PydanticSandboxAdapter. - - Creates and starts the Docker sandbox container following the - pydantic-ai-backend lifecycle pattern. The container is automatically - started via `start()` method. - - Args: - image: Docker image to use (default: python:3.12-slim). - Ignored if runtime is specified. - working_dir: Working directory in container - auto_remove: Auto-remove container on exit - max_output_size: Maximum command output size in characters - command_timeout: Command execution timeout in seconds - runtime: Pre-configured runtime environment. Can be: - - str: Name of builtin runtime ("python-datascience", "python-web", etc.) - - RuntimeConfig: Custom runtime configuration - - None: Use image parameter directly - session_id: Session identifier for multi-user scenarios. - Used to track and potentially reuse sandbox instances. - idle_timeout: Time in seconds before idle container is cleaned up (default: 3600) - volumes: Docker volume mappings {host_path: container_path} - - Raises: - ImportError: If pydantic-ai-backend is not installed - RuntimeError: If DockerSandbox creation fails - - Note: - The sandbox container is automatically started during initialization. - Call `cleanup()` when done to stop and remove the container. - - Example: - ```python - # Using builtin runtime - adapter = PydanticSandboxAdapter(runtime="python-datascience") - - # Using custom runtime - adapter = PydanticSandboxAdapter( - runtime=RuntimeConfig( - name="custom", - base_image="python:3.11", - packages=["requests"], - ), - ) - - # With volumes - adapter = PydanticSandboxAdapter( - runtime="python-web", - volumes={"/host/data": "/container/data"}, - ) - ``` - """ - - # Store parameters - self.runtime = runtime - self.session_id = session_id - self.idle_timeout = idle_timeout - self.volumes = volumes or {} - - # Resolve runtime to get effective image and config - effective_image, self._runtime_config = resolve_runtime(image, runtime) - - self._id = session_id or str(uuid.uuid4()) - self.image = effective_image - self.working_dir = working_dir - self.auto_remove = auto_remove - self.max_output_size = max_output_size - self.command_timeout = command_timeout - self._started = False # Track sandbox start state - self._saved_container_id: str | None = None # For restart: reuse same container after stop - - # Log initialization - runtime_info = f", runtime={self._runtime_config.name}" if self._runtime_config else "" - volumes_info = f", volumes={len(self.volumes)}" if self.volumes else "" - logger.info( - f"Initializing PydanticSandboxAdapter: id={self._id}, " - f"image={self.image}, working_dir={working_dir}{runtime_info}{volumes_info}" - ) - - # Prepare runtime parameter for pydantic-ai-backend - pydantic_runtime = None - if self._runtime_config: - pydantic_runtime = self._runtime_config.to_pydantic_runtime() - - # Create DockerSandbox with pydantic-ai-backend API - try: - sandbox_kwargs: dict = { - "image": self.image, - "work_dir": working_dir, - "auto_remove": auto_remove, - "runtime": pydantic_runtime, - "session_id": self.session_id, - "idle_timeout": self.idle_timeout, - "volumes": self.volumes if self.volumes else None, - } - self._sandbox = DockerSandbox(**sandbox_kwargs) - logger.info(f"DockerSandbox created: id={self._id}, image={self.image}") - except Exception as e: - logger.error(f"Failed to create DockerSandbox for adapter {self._id}: {e}", exc_info=True) - raise RuntimeError(f"Failed to create DockerSandbox: {e}") from e - - # Start the sandbox - logger.debug(f"Starting sandbox {self._id}...") - self.start() - - @property - def id(self) -> str: - """Unique identifier for this backend instance.""" - return self._id - - def is_started(self) -> bool: - """Check if the sandbox container is started. - - Returns: - True if the sandbox is started, False otherwise. - """ - return self._started - - def get_runtime_config(self) -> RuntimeConfig | None: - """Get the current runtime configuration. - - Returns: - RuntimeConfig instance, or None if no runtime config is in use. - """ - return self._runtime_config - - def get_container_id(self) -> str | None: - """Get the Docker container ID for persistence/reconnection. - - Returns: - Docker container ID string, or None if not available. - """ - container = getattr(self._sandbox, "_container", None) - if container is not None: - return getattr(container, "id", None) - return self._saved_container_id - - @classmethod - def from_existing_container( - cls, - container, - session_id: str, - image: str = DEFAULT_DOCKER_IMAGE, - idle_timeout: int = DEFAULT_IDLE_TIMEOUT, - working_dir: str = DEFAULT_WORKING_DIR, - max_output_size: int = DEFAULT_MAX_OUTPUT_SIZE, - command_timeout: int = DEFAULT_COMMAND_TIMEOUT, - ) -> "PydanticSandboxAdapter": - """Create an adapter that wraps an already-running Docker container. - - Used for app-restart recovery: reconnect to a container that was - created in a previous process lifetime. - - Args: - container: A docker.models.containers.Container instance (already running). - session_id: Session/sandbox ID. - image: Image name (for metadata only). - idle_timeout: Idle timeout in seconds. - working_dir: Working directory inside the container. - max_output_size: Max command output size. - command_timeout: Command timeout in seconds. - - Returns: - PydanticSandboxAdapter wrapping the existing container. - """ - # Create a minimal DockerSandbox and inject the container - sandbox = DockerSandbox( - image=image, - work_dir=working_dir, - auto_remove=False, - session_id=session_id, - idle_timeout=idle_timeout, - ) - # Inject the existing container so DockerSandbox uses it - sandbox._container = container - - # Build adapter without calling __init__ (which would start a new container) - adapter = cls.__new__(cls) - adapter.runtime = None - adapter.session_id = session_id - adapter.idle_timeout = idle_timeout - adapter.volumes = {} - adapter._runtime_config = None - adapter._id = session_id - adapter.image = image - adapter.working_dir = working_dir - adapter.auto_remove = False - adapter.max_output_size = max_output_size - adapter.command_timeout = command_timeout - adapter._started = True - adapter._saved_container_id = None - adapter._sandbox = sandbox - - logger.info( - f"PydanticSandboxAdapter.from_existing_container: " - f"id={session_id}, container={getattr(container, 'short_id', 'unknown')}" - ) - return adapter - - def start(self) -> None: - """Start the Docker sandbox container. Idempotent. Restart reuses same container if available.""" - if self._started: - return - - upstream_container = getattr(self._sandbox, "_container", None) - if self._saved_container_id and upstream_container is None: - try: - from app.core.agent.backends.docker_check import get_docker_client - - client = get_docker_client() - container = client.containers.get(self._saved_container_id) - container.start() - self._sandbox._container = container - self._started = True - self._saved_container_id = None - logger.info(f"Sandbox {self._id} restarted (same container)") - return - except Exception as e: - logger.warning(f"Failed to restart existing container {self._id}, will create new: {e}") - stale_id = self._saved_container_id - self._saved_container_id = None - if stale_id is not None: - self._force_remove_container(stale_id) - - try: - if hasattr(self._sandbox, "start"): - self._sandbox.start() - self._started = True - logger.info(f"Sandbox {self._id} started (image={self.image})") - except Exception as e: - logger.error(f"Failed to start sandbox {self._id}: {e}") - raise RuntimeError(f"Failed to start sandbox {self._id}: {e}") from e - - # Dangerous command patterns (defense-in-depth, not sole security boundary) - _DANGEROUS_PATTERNS = [ - r"rm\s+-rf\s+/\s*$", # rm -rf / - r"mkfs\.", # format disk - r"dd\s+.*of=/dev/", # write to device - r":\(\)\s*\{", # fork bomb :(){ :|:& };: - ] - _DANGEROUS_RE = None # Lazy-compiled combined regex - - @classmethod - def _get_dangerous_re(cls): - if cls._DANGEROUS_RE is None: - import re - - cls._DANGEROUS_RE = re.compile("|".join(f"(?:{p})" for p in cls._DANGEROUS_PATTERNS)) - return cls._DANGEROUS_RE - - def _exec_command(self, command: str) -> tuple[str, int]: - """Execute command in sandbox with safety checks. - - Args: - command: Shell command to execute - - Returns: - Tuple of (output, exit_code) - """ - if self._get_dangerous_re().search(command): - logger.warning(f"[{self._id}] Blocked dangerous command: {command[:100]}") - return "Error: command blocked by security policy", 1 - - logger.debug(f"[{self._id}] _exec_command START: {command[:100]}") - try: - result = self._sandbox.execute(command) - - # 1. Handle ExecuteResponse from pydantic-ai-backend (output/exit_code/truncated) - # This is the primary format returned by DockerSandbox.execute() - if hasattr(result, "output") and hasattr(result, "exit_code"): - output = result.output if isinstance(result.output, str) else str(result.output or "") - exit_code = result.exit_code - logger.debug(f"[{self._id}] _exec_command END: exit_code={exit_code}, output_len={len(output)}") - return output, exit_code - - # 2. Handle ExecutionResult format (stdout/returncode) - if hasattr(result, "stdout") and hasattr(result, "returncode"): - output = ( - result.stdout.decode("utf-8", errors="replace") - if isinstance(result.stdout, bytes) - else str(result.stdout or "") - ) - logger.debug( - f"[{self._id}] _exec_command END (ExecutionResult): exit_code={result.returncode}, output_len={len(output)}" - ) - return output, result.returncode - - # 3. Handle dict format (TypedDict or plain dict) - supports both naming conventions - if isinstance(result, dict): - output = str(result.get("output", result.get("stdout", ""))) - exit_code_raw = result.get("exit_code", result.get("returncode", 0)) - exit_code = 0 - if exit_code_raw is not None: - try: - exit_code = int(exit_code_raw) - except (TypeError, ValueError): - exit_code = 0 - logger.debug(f"[{self._id}] _exec_command END (dict): exit_code={exit_code}, output_len={len(output)}") - return output, exit_code - - # 4. Fallback - should rarely happen now - logger.warning( - f"[{self._id}] _exec_command: unexpected result type {type(result).__name__}, returning as string" - ) - return str(result) if result else "", 0 - except Exception as e: - logger.error(f"[{self._id}] _exec_command FAILED: {e}") - return f"Error: {e}", -1 - - # SandboxBackendProtocol implementation - - def ls_info(self, path: str) -> list[FileInfo]: - """List files and directories in the specified directory. - - Args: - path: Absolute path to directory. - - Returns: - List of FileInfo dicts for files and directories. - """ - try: - # Use ls command to list files - output, exit_code = self._exec_command(f"ls -la {path}") - if exit_code != 0: - return [] - - infos: list[FileInfo] = [] - lines = output.strip().split("\n")[1:] # Skip "total" line - - for line in lines: - parts = line.split() - if len(parts) < 9: - continue - - permissions = parts[0] - size = int(parts[4]) if parts[4].isdigit() else 0 - name = " ".join(parts[8:]) - - # Skip . and .. - if name in (".", ".."): - continue - - file_path = f"{path.rstrip('/')}/{name}" - is_dir = permissions.startswith("d") - - infos.append( - { - "path": file_path + ("/" if is_dir else ""), - "is_dir": is_dir, - "size": size, - "modified_at": "", - } - ) - - return infos - - except Exception as e: - logger.error(f"Failed to list directory {path}: {e}") - return [] - - def read( - self, - file_path: str, - offset: int = 0, - limit: int = 2000, - ) -> str: - """Read file content with line numbers. - - Uses DockerSandbox.read() which leverages Docker's get_archive API - for reliable file reading with intelligent encoding detection. - - Args: - file_path: Absolute file path - offset: Line offset to start reading from (0-indexed) - limit: Maximum number of lines to read - - Returns: - Formatted file content with line numbers, or error message. - """ - logger.info(f"[{self._id}] Reading file: {file_path}") - try: - content = self.raw_read(file_path, offset=0, limit=100000) - if content.startswith("[Error:") or content.startswith("Error:"): - return content - - # Format with line numbers using deepagents utility - lines = content.splitlines() - file_data = { - "content": lines, - "created_at": datetime.now().isoformat(), - "modified_at": datetime.now().isoformat(), - } - - result: str = format_read_response(file_data, offset, limit) - return result - except Exception as e: - logger.error(f"[{self._id}] Failed to read file {file_path}: {e}") - return f"Error: {str(e)}" - - def raw_read( - self, - file_path: str, - offset: int = 0, - limit: int = 100000, - ) -> str: - """Read raw file content without injecting line numbers. - - This is intended for UI/API consumers that need the original file text - and will handle presentation concerns such as line-number gutters. - """ - logger.info(f"[{self._id}] Raw reading file: {file_path}") - content_raw = self._sandbox.read(file_path, offset=offset, limit=limit) - content = content_raw if isinstance(content_raw, str) else str(content_raw) - - if content.startswith("[Error:") or content.startswith("Error:"): - return content - - # Remove upstream pagination footer to keep API consumers on raw file text. - if "\n\n[..." in content: - content = content.split("\n\n[...")[0] - - return content - - def write( - self, - file_path: str, - content: str | bytes, - ) -> WriteResult: - """Create a new file with content. - - Uses DockerSandbox.write() which leverages Docker's put_archive API - for reliable file writing without shell command length limits. - Accepts both str and bytes (upstream DockerSandbox supports both). - - Args: - file_path: Absolute file path - content: File content (text or binary) - - Returns: - WriteResult with success or error. - """ - logger.info(f"[{self._id}] Writing file: {file_path}") - try: - # Check if file already exists - check_result = self._exec_command(f"test -f {file_path}") - if check_result[1] == 0: # exit_code == 0 means file exists - return WriteResult( - error=f"Cannot write to {file_path} because it already exists. " - "Read and then make an edit, or write to a new path." - ) - - result = self._sandbox.write(file_path, content) - - if hasattr(result, "error") and result.error: - return WriteResult(error=result.error) - return WriteResult(path=file_path, files_update=None) - except Exception as e: - logger.error(f"[{self._id}] Failed to write file {file_path}: {e}") - return WriteResult(error=f"Failed to write file: {str(e)}") - - def write_overwrite( - self, - file_path: str, - content: str | bytes, - ) -> WriteResult: - """Write a file, overwriting if it already exists. - - Uses DockerSandbox.write() which leverages Docker's put_archive API - for reliable file writing without shell command length limits. - Accepts both str and bytes (upstream DockerSandbox supports both). - - Args: - file_path: Absolute file path - content: File content (text or binary) - - Returns: - WriteResult with success or error. - """ - logger.debug(f"[{self._id}] write_overwrite: {file_path}") - try: - # Use upstream DockerSandbox.write() which uses Docker put_archive API - # This handles large files and special characters reliably - result = self._sandbox.write(file_path, content) - - # Convert upstream WriteResult to deepagents WriteResult - if hasattr(result, "error") and result.error: - return WriteResult(error=result.error) - return WriteResult(path=file_path, files_update=None) - except Exception as e: - logger.error(f"[{self._id}] Failed to write file {file_path}: {e}") - return WriteResult(error=f"Failed to write file: {str(e)}") - - def delete(self, file_path: str) -> bool: - """Delete a file inside the container. - - Args: - file_path: Absolute file path inside the container. - - Returns: - True if deleted successfully, False otherwise. - """ - import shlex - - logger.debug(f"[{self._id}] Deleting file: {file_path}") - try: - _, exit_code = self._exec_command(f"rm -f {shlex.quote(file_path)}") - return exit_code == 0 - except Exception as e: - logger.error(f"[{self._id}] Failed to delete file {file_path}: {e}") - return False - - def mkdir(self, dir_path: str) -> bool: - """Create a directory (and parents) inside the container. - - Args: - dir_path: Absolute directory path inside the container. - - Returns: - True if created successfully, False otherwise. - """ - import shlex - - logger.debug(f"[{self._id}] Creating directory: {dir_path}") - try: - _, exit_code = self._exec_command(f"mkdir -p {shlex.quote(dir_path)}") - return exit_code == 0 - except Exception as e: - logger.error(f"[{self._id}] Failed to create directory {dir_path}: {e}") - return False - - def edit( - self, - file_path: str, - old_string: str, - new_string: str, - replace_all: bool = False, - ) -> EditResult: - """Edit a file by replacing string occurrences. - - Uses DockerSandbox.edit() which performs in-memory string replacement - and writes back using Docker's put_archive API. - - Args: - file_path: Absolute file path - old_string: String to replace - new_string: Replacement string - replace_all: Replace all occurrences (default: False) - - Returns: - EditResult with success or error. - """ - logger.info(f"[{self._id}] Editing file: {file_path}") - try: - # Use upstream DockerSandbox.edit() which: - # 1. Reads file using Docker get_archive API - # 2. Performs in-memory string replacement - # 3. Writes back using Docker put_archive API - result = self._sandbox.edit(file_path, old_string, new_string, replace_all) - - # Convert upstream EditResult to deepagents EditResult - if hasattr(result, "error") and result.error: - return EditResult(error=result.error) - return EditResult( - path=getattr(result, "path", file_path), - files_update=None, - occurrences=getattr(result, "occurrences", 1), - ) - except Exception as e: - logger.error(f"[{self._id}] Failed to edit file {file_path}: {e}") - return EditResult(error=f"Failed to edit file: {str(e)}") - - def grep_raw( - self, - pattern: str, - path: str | None = None, - glob: str | None = None, - ) -> list[GrepMatch] | str: - """Search for a pattern in files. - - Args: - pattern: Search pattern (literal string) - path: Directory to search in (default: working_dir) - glob: Glob pattern to filter files - - Returns: - List of GrepMatch dicts or error string. - """ - logger.info(f"[{self._id}] Grepping for pattern: {pattern}") - search_path = path or self.working_dir - grep_cmd = f"grep -rn '{pattern}' {search_path}" - - if glob: - grep_cmd += f" --include='{glob}'" - - output, exit_code = self._exec_command(grep_cmd) - - if exit_code != 0 and not output: - return [] - - matches: list[GrepMatch] = [] - for line in output.strip().split("\n"): - if not line: - continue - - parts = line.split(":", 2) - if len(parts) >= 3: - matches.append( - { - "path": parts[0], - "line": int(parts[1]) if parts[1].isdigit() else 0, - "text": parts[2], - } - ) - - return matches - - def glob_info(self, pattern: str, path: str = "/") -> list[FileInfo]: - """Find files matching a glob pattern. - - Args: - pattern: Glob pattern (e.g., "*.py", "**/*.txt") - path: Directory to search in - - Returns: - List of FileInfo dicts for matching files. - """ - logger.info(f"[{self._id}] Globbing for pattern: {pattern}") - # Use find command with pattern - find_cmd = f"find {path} -name '{pattern}'" - output, exit_code = self._exec_command(find_cmd) - - if exit_code != 0: - return [] - - infos: list[FileInfo] = [] - for file_path in output.strip().split("\n"): - if not file_path: - continue - - # Get file size - size_cmd = f"stat -c %s {file_path}" - size_output, _ = self._exec_command(size_cmd) - size = int(size_output.strip()) if size_output.strip().isdigit() else 0 - - infos.append( - { - "path": file_path, - "is_dir": False, - "size": size, - "modified_at": "", - } - ) - - return infos - - def execute(self, command: str) -> ExecuteResponse: - """Execute a shell command in the container. - - Args: - command: Shell command to execute. - - Returns: - ExecuteResponse with output, exit code, and truncation flag. - """ - logger.info(f"[{self._id}] execute() called: {command[:100]}...") - try: - # Execute command with timeout - output, exit_code = self._exec_command(command) - - # Create response with automatic truncation - response = create_execute_response( - output=output, - exit_code=exit_code, - max_output_size=self.max_output_size, - ) - - if response.truncated: - logger.debug( - f"[{self._id}] Output truncated: {len(output)} -> " - f"{len(response.output)} chars (max={self.max_output_size})" - ) - - logger.debug( - f"[{self._id}] Command execution completed: exit_code={exit_code}, truncated={response.truncated}" - ) - return response - - except Exception as e: - logger.error(f"[{self._id}] Error executing command '{command}': {e}", exc_info=True) - return ExecuteResponse( - output=f"Error executing command: {str(e)}", - exit_code=-1, - truncated=False, - ) - - def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: - """Download multiple files from the Docker sandbox using native API. - - Args: - paths: List of file paths to download. - - Returns: - List of FileDownloadResponse objects, one per input path. - """ - responses: list[FileDownloadResponse] = [] - for path in paths: - try: - result = self.execute(f"cat {path}") - if result.exit_code == 0: - responses.append(FileDownloadResponse(path=path, content=result.output.encode("utf-8"), error=None)) - else: - responses.append(FileDownloadResponse(path=path, content=None, error="file_not_found")) - except Exception as e: - logger.error(f"Failed to download file {path}: {e}") - responses.append(FileDownloadResponse(path=path, content=None, error="permission_denied")) - return responses - - def _collect_file_paths(self, dir_path: str, base: str, out: List[str]) -> None: - """Recursively collect all file paths under dir_path (relative to base).""" - infos = self.ls_info(dir_path) - for info in infos: - path = (info.get("path") or "").rstrip("/") - if not path: - continue - is_dir = info.get("is_dir", False) - if is_dir: - self._collect_file_paths(path, base, out) - else: - if path.startswith(base): - rel = path[len(base) :].lstrip("/") - if rel: - out.append(path) - - def export_working_dir_to(self, target_dir: Path) -> int: - """Export all files from the container working directory to target_dir. - - Preserves directory structure. Returns the number of files written. - """ - target_dir = Path(target_dir).resolve() - target_dir.mkdir(parents=True, exist_ok=True) - base = self.working_dir.rstrip("/") - if not base: - base = "/" - paths: List[str] = [] - self._collect_file_paths(base, base, paths) - if not paths: - return 0 - responses = self.download_files(paths) - written = 0 - for resp in responses: - if resp.error or resp.content is None: - continue - path = resp.path - if path.startswith(base): - rel = path[len(base) :].lstrip("/") - else: - rel = Path(path).name - dest = target_dir / rel - dest.parent.mkdir(parents=True, exist_ok=True) - try: - dest.write_bytes(resp.content) - written += 1 - except Exception as e: - logger.warning(f"[PydanticSandboxAdapter] Failed to write {dest}: {e}") - return written - - def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: - """Upload multiple files to the Docker sandbox using base64 encoding. - - Args: - files: List of (path, content) tuples to upload. - - Returns: - List of FileUploadResponse objects, one per input file. - """ - import base64 - - responses: list[FileUploadResponse] = [] - for path, content in files: - try: - encoded = base64.b64encode(content).decode("ascii") - _, exit_code = self._exec_command(f"echo '{encoded}' | base64 -d > {path}") - if exit_code == 0: - responses.append(FileUploadResponse(path=path, error=None)) - else: - responses.append(FileUploadResponse(path=path, error="permission_denied")) - except Exception as e: - logger.error(f"Failed to upload file {path}: {e}") - responses.append(FileUploadResponse(path=path, error="permission_denied")) - return responses - - @staticmethod - def _force_remove_container(container_id: str) -> None: - """Force-remove a Docker container by ID, ignoring all errors.""" - try: - from app.core.agent.backends.docker_check import get_docker_client - - get_docker_client().containers.get(container_id).remove(force=True) - logger.info(f"Force-removed stale container {container_id[:12]}") - except Exception as e: - logger.warning(f"Could not remove stale container {container_id[:12]}: {e}") - - def stop(self) -> None: - """Stop the Docker container without removing it. Idempotent. - - IMPORTANT: This only stops the container. It does NOT remove it. - After stop(), start() can restart the same container via _saved_container_id. - Never falls back to cleanup() which would destroy the container. - """ - if not self._started: - return - - # Save container_id BEFORE stopping so we can restart later - upstream_container = getattr(self._sandbox, "_container", None) - if upstream_container is not None: - self._saved_container_id = getattr(upstream_container, "id", None) - - try: - if hasattr(self._sandbox, "stop"): - self._sandbox.stop() - elif upstream_container is not None and hasattr(upstream_container, "stop"): - # Direct Docker API: stop container without removing - upstream_container.stop() - else: - # No stop method available; just mark as stopped - logger.warning( - f"Sandbox {self._id}: no stop() method available, marking as stopped but container may still run" - ) - logger.info(f"Sandbox {self._id} stopped (container kept, id={self._saved_container_id})") - except Exception as e: - logger.warning(f"Failed to stop sandbox {self._id}: {e}") - finally: - self._started = False - - def cleanup(self) -> None: - """Stop and remove the Docker container. Idempotent. Use for rebuild/teardown only. - - Sequence: - 1. Save container_id from live container reference - 2. Stop the container (sets _started=False, saves _saved_container_id) - 3. Remove the container using saved references - """ - # Save container reference BEFORE stop() clears it - pre_stop_container = getattr(self._sandbox, "_container", None) if getattr(self, "_sandbox", None) else None - pre_stop_container_id = getattr(pre_stop_container, "id", None) if pre_stop_container else None - - self.stop() - - if getattr(self, "_sandbox", None) is None: - self._saved_container_id = None - return - - # Determine which container ID to remove - container_id_to_remove = self._saved_container_id or pre_stop_container_id - - try: - # Try direct container reference first (may still be on _sandbox after stop) - container = getattr(self._sandbox, "container", None) or getattr(self._sandbox, "_container", None) - if container is not None and hasattr(container, "remove"): - container.remove(force=True) - logger.info(f"Sandbox {self._id} container removed (direct ref)") - elif container_id_to_remove: - from app.core.agent.backends.docker_check import get_docker_client - - client = get_docker_client() - client.containers.get(container_id_to_remove).remove(force=True) - logger.info(f"Sandbox {self._id} container removed (by id={container_id_to_remove[:12]})") - elif hasattr(self._sandbox, "remove"): - self._sandbox.remove() - logger.info(f"Sandbox {self._id} container removed (sandbox.remove)") - else: - logger.warning(f"Sandbox {self._id}: no way to remove container") - except Exception as e: - logger.warning(f"Failed to remove sandbox container {self._id}: {e}") - finally: - self._saved_container_id = None - - # NOTE: __del__ is intentionally NOT implemented. - # GC-time cleanup of Docker containers causes unpredictable behavior: - # - May delete containers still tracked by the pool - # - May run after event loop is closed - # - Interferes with the pool-based lifecycle management - # All cleanup must be explicit via cleanup() or the SandboxPool. - - def __enter__(self): - """Context manager entry.""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit.""" - self.cleanup() diff --git a/backend/app/core/agent/backends/runtime_config.py b/backend/app/core/agent/backends/runtime_config.py deleted file mode 100644 index 0188920f2..000000000 --- a/backend/app/core/agent/backends/runtime_config.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Runtime configuration for Docker sandbox environments. - -This module provides RuntimeConfig dataclass and pre-defined runtime configurations -for commonly used development environments (Python data science, web, ML, Node.js, etc.). - -Features: -- RuntimeConfig dataclass for defining custom runtime environments -- BUILTIN_RUNTIMES dictionary with pre-configured environments -- Helper functions for runtime management -- Compatibility with pydantic-ai-backend's RuntimeConfig -""" - -from dataclasses import dataclass, field - -from loguru import logger -from pydantic_ai_backends import RuntimeConfig as PydanticRuntimeConfig - - -@dataclass -class RuntimeConfig: - """Pre-configured runtime environment configuration. - - Defines Docker sandbox runtime environment including base image, - pre-installed packages, setup commands, and environment variables. - Compatible with pydantic-ai-backend's RuntimeConfig API. - - Attributes: - name: Runtime name identifier - base_image: Docker base image (default: python:3.12-slim) - packages: List of Python packages to pre-install - setup_commands: Commands to execute after container starts - env_vars: Environment variables dictionary - - Example: - ```python - # Create a custom machine learning runtime - ml_runtime = RuntimeConfig( - name="ml-env", - base_image="python:3.12-slim", - packages=["torch", "transformers", "numpy"], - setup_commands=["pip install --upgrade pip"], - env_vars={"CUDA_VISIBLE_DEVICES": "0"}, - ) - ``` - """ - - name: str - base_image: str = "python:3.12-slim" - packages: list[str] = field(default_factory=list) - setup_commands: list[str] = field(default_factory=list) - env_vars: dict[str, str] = field(default_factory=dict) - - def to_pydantic_runtime(self) -> "PydanticRuntimeConfig | RuntimeConfig": - """Convert to pydantic-ai-backend's RuntimeConfig (if available). - - Returns: - pydantic-ai-backend RuntimeConfig instance, or self if library not available - """ - return PydanticRuntimeConfig( - name=self.name, - base_image=self.base_image, - packages=self.packages, - ) - return self - - -# Pre-defined runtime configurations -BUILTIN_RUNTIMES: dict[str, RuntimeConfig] = { - "python-minimal": RuntimeConfig( - name="python-minimal", - base_image="python:3.12-slim", - packages=[], - ), - "python-datascience": RuntimeConfig( - name="python-datascience", - base_image="python:3.12-slim", - packages=["pandas", "numpy", "matplotlib", "scikit-learn", "scipy"], - ), - "python-web": RuntimeConfig( - name="python-web", - base_image="python:3.12-slim", - packages=["fastapi", "uvicorn", "sqlalchemy", "httpx", "aiohttp"], - ), - "python-ml": RuntimeConfig( - name="python-ml", - base_image="python:3.12-slim", - packages=["torch", "transformers", "numpy", "pandas"], - ), - "node-minimal": RuntimeConfig( - name="node-minimal", - base_image="node:20-slim", - packages=[], - ), - "node-react": RuntimeConfig( - name="node-react", - base_image="node:20-slim", - packages=["typescript", "vite", "react", "react-dom"], - ), -} - - -def get_builtin_runtime(name: str) -> RuntimeConfig | None: - """Get a pre-defined runtime configuration. - - Args: - name: Runtime name - - Returns: - RuntimeConfig instance, or None if not found - """ - return BUILTIN_RUNTIMES.get(name) - - -def list_builtin_runtimes() -> list[str]: - """List all available pre-defined runtime names. - - Returns: - List of pre-defined runtime names - """ - return list(BUILTIN_RUNTIMES.keys()) - - -def resolve_runtime( - default_image: str, - runtime: "RuntimeConfig | str | None", -) -> tuple[str, RuntimeConfig | None]: - """Resolve runtime configuration to effective image and config. - - Args: - default_image: Default Docker image (used when runtime is None) - runtime: Runtime configuration, can be: - - None: Use default_image - - str: Pre-defined runtime name or image name - - RuntimeConfig: Custom runtime configuration - - Returns: - Tuple of (effective_image, runtime_config) - - effective_image: Docker image to use - - runtime_config: RuntimeConfig instance or None - - Example: - >>> image, config = resolve_runtime("python:3.12-slim", "python-datascience") - >>> print(image) # "python:3.12-slim" - >>> print(config.packages) # ["pandas", "numpy", ...] - """ - if runtime is None: - logger.debug(f"No runtime specified, using default image: {default_image}") - return default_image, None - - if isinstance(runtime, str): - # First check if it's a pre-defined runtime - if runtime in BUILTIN_RUNTIMES: - config = BUILTIN_RUNTIMES[runtime] - logger.info(f"Using builtin runtime '{runtime}': image={config.base_image}") - return config.base_image, config - - # Check if it looks like an image name (contains ':' or '/') - if ":" in runtime or "/" in runtime: - logger.info(f"Using runtime string as image name: {runtime}") - return runtime, None - - # Neither pre-defined runtime nor looks like image name - available_runtimes = list(BUILTIN_RUNTIMES.keys()) - logger.warning( - f"Runtime '{runtime}' not found in builtin runtimes: {available_runtimes}. Treating as image name." - ) - return runtime, None - - # RuntimeConfig instance - if isinstance(runtime, RuntimeConfig): - logger.info(f"Using custom RuntimeConfig '{runtime.name}': image={runtime.base_image}") - return runtime.base_image, runtime - - # Try to handle pydantic-ai-backend's RuntimeConfig - if hasattr(runtime, "base_image") and hasattr(runtime, "name"): - logger.info(f"Using pydantic RuntimeConfig '{runtime.name}': image={runtime.base_image}") - # Convert to local RuntimeConfig - local_config = RuntimeConfig( - name=runtime.name, - base_image=runtime.base_image, - packages=getattr(runtime, "packages", []), - ) - return runtime.base_image, local_config - - # Unknown type, use default image - logger.warning(f"Unknown runtime type: {type(runtime)}, using default image: {default_image}") - return default_image, None - - -__all__ = [ - "RuntimeConfig", - "BUILTIN_RUNTIMES", - "get_builtin_runtime", - "list_builtin_runtimes", - "resolve_runtime", -] diff --git a/backend/app/core/agent/backends/state_sandbox.py b/backend/app/core/agent/backends/state_sandbox.py deleted file mode 100644 index 381db34d6..000000000 --- a/backend/app/core/agent/backends/state_sandbox.py +++ /dev/null @@ -1,80 +0,0 @@ -"""StateSandboxBackend: StateBackend with command execution support.""" - -import uuid -from typing import TYPE_CHECKING - -from deepagents.backends.protocol import ExecuteResponse, SandboxBackendProtocol -from deepagents.backends.state import StateBackend - -from app.core.agent.backends.constants import ( - DEFAULT_COMMAND_TIMEOUT, - DEFAULT_MAX_OUTPUT_SIZE, -) -from app.core.agent.backends.utils.command_executor import execute_local_command - -if TYPE_CHECKING: - from langchain.tools import ToolRuntime - - -class StateSandboxBackend(StateBackend, SandboxBackendProtocol): - """StateBackend with command execution support. - - Extends StateBackend to implement SandboxBackendProtocol by adding - command execution capabilities. Commands are executed in the local - system environment. - - Warning: - This backend executes commands directly on the host system without - isolation. Use with caution and only with trusted agents. For production - use, consider using a proper sandboxed backend (e.g., Docker-based). - - Example: - ```python - from app.core.agent.backends import StateSandboxBackend - - # Use as a factory function - agent = create_agent( - model, tools=tools, middleware=[FilesystemMiddleware(backend=lambda rt: StateSandboxBackend(rt))] - ) - ``` - """ - - def __init__( - self, - runtime: "ToolRuntime", - max_output_size: int = DEFAULT_MAX_OUTPUT_SIZE, - command_timeout: int = DEFAULT_COMMAND_TIMEOUT, - ): - """Initialize StateSandboxBackend. - - Args: - runtime: The tool runtime context. - max_output_size: Maximum size of command output in characters. - Output exceeding this limit will be truncated. - command_timeout: Command execution timeout in seconds (default: 30). - """ - super().__init__(runtime) - self._id = str(uuid.uuid4()) - self.max_output_size = max_output_size - self.command_timeout = command_timeout - - @property - def id(self) -> str: - """Unique identifier for this backend instance.""" - return self._id - - def execute(self, command: str) -> ExecuteResponse: - """Execute a shell command. - - Args: - command: Shell command to execute. - - Returns: - ExecuteResponse with combined stdout/stderr output and exit code. - """ - return execute_local_command( - command=command, - cwd=None, # StateBackend doesn't have a specific working directory - timeout=self.command_timeout, - max_output_size=self.max_output_size, - ) diff --git a/backend/app/core/agent/backends/utils/__init__.py b/backend/app/core/agent/backends/utils/__init__.py deleted file mode 100644 index 109451a18..000000000 --- a/backend/app/core/agent/backends/utils/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Utility modules for sandbox backends. - -This package contains shared utilities used across different sandbox backend implementations. -""" - -from app.core.agent.backends.utils.command_executor import ( - combine_stdout_stderr, - create_error_response, - create_timeout_error_response, - execute_local_command, -) - -__all__ = [ - "execute_local_command", - "combine_stdout_stderr", - "create_timeout_error_response", - "create_error_response", -] diff --git a/backend/app/core/agent/backends/utils/command_executor.py b/backend/app/core/agent/backends/utils/command_executor.py deleted file mode 100644 index 6195772db..000000000 --- a/backend/app/core/agent/backends/utils/command_executor.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Command execution utilities for sandbox backends. - -This module provides unified command execution logic that is shared across -different local sandbox backends (FilesystemSandboxBackend, StateSandboxBackend). - -Features: -- Unified subprocess execution with timeout support -- Consistent stdout/stderr handling -- Standardized error responses -- Output truncation support -""" - -import subprocess -from typing import Optional - -from deepagents.backends.protocol import ExecuteResponse - -from app.core.agent.backends.constants import ( - DEFAULT_COMMAND_TIMEOUT, - DEFAULT_MAX_OUTPUT_SIZE, -) -from app.utils.backend_utils import create_execute_response - - -def combine_stdout_stderr(stdout: Optional[str], stderr: Optional[str]) -> str: - """Combine stdout and stderr into a single output string. - - Args: - stdout: Standard output from command execution - stderr: Standard error from command execution - - Returns: - Combined output string with stderr appended after stdout (if both exist) - """ - output = "" - if stdout: - output += stdout - if stderr: - if output: - output += "\n" - output += stderr - return output - - -def create_timeout_error_response(timeout: int) -> ExecuteResponse: - """Create a standardized timeout error response. - - Args: - timeout: The timeout value in seconds that was exceeded - - Returns: - ExecuteResponse with timeout error message - """ - return ExecuteResponse( - output=f"Error: Command execution timed out ({timeout} seconds limit)", - exit_code=-1, - truncated=False, - ) - - -def create_error_response(error_message: str) -> ExecuteResponse: - """Create a standardized error response. - - Args: - error_message: The error message to include in the response - - Returns: - ExecuteResponse with error message - """ - return ExecuteResponse( - output=f"Error executing command: {error_message}", - exit_code=-1, - truncated=False, - ) - - -def execute_local_command( - command: str, - cwd: Optional[str] = None, - timeout: int = DEFAULT_COMMAND_TIMEOUT, - max_output_size: int = DEFAULT_MAX_OUTPUT_SIZE, -) -> ExecuteResponse: - """Execute a shell command locally with unified error handling. - - This function provides a standardized way to execute shell commands - across different sandbox backends. It handles: - - Subprocess execution with shell=True - - Timeout enforcement - - stdout/stderr combination - - Output truncation via create_execute_response - - Args: - command: Shell command to execute - cwd: Working directory for command execution (None uses current directory) - timeout: Maximum execution time in seconds (default: 30) - max_output_size: Maximum output size in characters for truncation (default: 100000) - - Returns: - ExecuteResponse with combined output, exit code, and truncation flag - - Example: - >>> result = execute_local_command("ls -la", cwd="/tmp", timeout=10) - >>> print(result.exit_code) - 0 - >>> print(result.output) - total 0 - ... - """ - try: - result = subprocess.run( - command, - shell=True, - capture_output=True, - text=True, - timeout=timeout, - cwd=cwd, - ) - - output = combine_stdout_stderr(result.stdout, result.stderr) - - return create_execute_response( - output=output, - exit_code=result.returncode, - max_output_size=max_output_size, - ) - - except subprocess.TimeoutExpired: - return create_timeout_error_response(timeout) - except Exception as e: - return create_error_response(str(e)) diff --git a/backend/app/core/agent/checkpointer/checkpointer.py b/backend/app/core/agent/checkpointer/checkpointer.py deleted file mode 100644 index 0ca5adeac..000000000 --- a/backend/app/core/agent/checkpointer/checkpointer.py +++ /dev/null @@ -1,240 +0,0 @@ -""" -LangGraph checkpoint management. - -Manages persistence of conversation state. -Centralize all checkpointer logic and provide a unified interface. -""" - -import os -from typing import TYPE_CHECKING, Optional - -from loguru import logger -from psycopg_pool import AsyncConnectionPool - -if TYPE_CHECKING: - from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver - - -class CheckpointerManager: - """Global manager for centralized checkpointer management. - - Responsible for: - - Global connection pool lifecycle management - - Providing checkpointer instances based on configuration - - Database table initialization - """ - - _pool: Optional[AsyncConnectionPool] = None - _initialized: bool = False - - @classmethod - def _get_db_uri(cls) -> str: - """Build the database connection URI. - - Reuse the database URL from Settings (which loads backend/.env via - pydantic-settings) so that local-dev, Docker, and script-based - launches all resolve credentials consistently. - - Returns: - str: PostgreSQL connection URI (postgresql://user:password@host:port/database). - """ - from app.core.settings import settings - - # settings.database_url uses the asyncpg driver; strip it for psycopg - return settings.database_url.replace("+asyncpg", "") - - @classmethod - async def initialize(cls) -> None: - """Initialize the connection pool at application startup. - - Should be called once at startup, typically in the lifespan handler. - This method will: - 1. Create an AsyncConnectionPool - 2. Open the pool - 3. Initialize database table schema - - Raises: - ValueError: If database connection config is invalid. - Exception: If pool initialization or table creation fails. - """ - if cls._initialized: - logger.warning("CheckpointerManager already initialized, skipping") - return - - try: - db_uri = cls._get_db_uri() - cls._pool = AsyncConnectionPool( - conninfo=db_uri, - min_size=int(os.getenv("DB_POOL_MIN_SIZE", 1)), - max_size=int(os.getenv("DB_POOL_MAX_SIZE", 10)), - kwargs={"autocommit": True, "prepare_threshold": 0}, - open=False, # do not auto-open in constructor - ) - # explicitly open the pool - await cls._pool.open() - cls._initialized = True - logger.info( - f"CheckpointerManager initialized | " - f"pool_size={os.getenv('DB_POOL_MIN_SIZE', 1)}-{os.getenv('DB_POOL_MAX_SIZE', 10)}" - ) - - # initialize database table schema - await cls._init_db() - except Exception as e: - logger.error(f"Failed to initialize CheckpointerManager: {e}") - # if initialization fails, ensure the pool is cleaned up - if cls._pool: - try: - await cls._pool.close() - except Exception: - logger.debug("Failed to close pool during initialization cleanup", exc_info=True) - cls._pool = None - raise - - @classmethod - async def _init_db(cls) -> None: - """Ensure database table schema is created. - - Use AsyncPostgresSaver to create the necessary tables and indexes. - Called automatically after pool initialization. - - Raises: - RuntimeError: If the pool is not initialized. - Exception: If table creation fails. - """ - if not cls._pool: - raise RuntimeError("Pool not initialized. Call initialize() first.") - - from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver - - checkpointer = AsyncPostgresSaver(cls._pool) # type: ignore[arg-type] - await checkpointer.setup() - logger.info("Checkpointer tables ready.") - - @classmethod - def _get_pool(cls) -> AsyncConnectionPool: - """Get the connection pool (internal method). - - Returns: - AsyncConnectionPool: The initialized pool instance. - - Raises: - RuntimeError: If CheckpointerManager is not initialized. - """ - if not cls._pool: - raise RuntimeError( - "CheckpointerManager not initialized. Call CheckpointerManager.initialize() at application startup." - ) - return cls._pool - - @classmethod - def get_checkpointer(cls) -> Optional["AsyncPostgresSaver"]: - """Get a checkpointer instance. - - Each call creates a new AsyncPostgresSaver instance to ensure - the latest pool state is used. - - Returns: - Optional[AsyncPostgresSaver]: An AsyncPostgresSaver instance, or None. - - Raises: - RuntimeError: If CheckpointerManager is not initialized. - """ - pool = cls._get_pool() - from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver - - return AsyncPostgresSaver(pool) # type: ignore[arg-type] - - @classmethod - async def close(cls) -> None: - """Close the connection pool at application shutdown. - - Should be called at shutdown, typically in the lifespan handler. - Safely closes the pool and cleans up resources. - - Note: - Resources are cleaned up even if an exception occurs during close. - """ - if cls._pool: - try: - await cls._pool.close() - logger.info("CheckpointerManager connection pool closed") - except Exception as e: - logger.error(f"Error closing CheckpointerManager pool: {e}") - finally: - cls._pool = None - cls._initialized = False - - -def get_checkpointer() -> Optional["AsyncPostgresSaver"]: - """Unified checkpointer access interface. - - Convenience function that delegates to CheckpointerManager.get_checkpointer(). - Automatically returns a checkpointer or None based on configuration. - - Returns: - Optional[AsyncPostgresSaver]: An AsyncPostgresSaver instance, or None. - - Raises: - RuntimeError: If CheckpointerManager is not initialized. - """ - return CheckpointerManager.get_checkpointer() - - -async def delete_thread_checkpoints(thread_id: str) -> None: - """ - Delete all checkpoints for the specified thread. - - Args: - thread_id: Thread ID. - - Raises: - RuntimeError: If checkpoint is not enabled or checkpointer is not initialized. - """ - checkpointer = get_checkpointer() - if checkpointer is None: - raise RuntimeError("Checkpoint is not enabled. Enable checkpoint in settings to use this function.") - - try: - await checkpointer.adelete_thread(thread_id) - logger.info(f"✅ Deleted checkpoints for thread: {thread_id}") - except Exception as e: - logger.error(f"❌ Failed to delete checkpoints for thread {thread_id}: {e}") - raise - - -async def get_thread_history(thread_id: str) -> list[dict]: - """ - Get execution history (checkpoints) for a thread. - - Returns a list of checkpoints ordered by timestamp (descending usually, depends on alist implementation). - """ - checkpointer = get_checkpointer() - if not checkpointer: - return [] - - config = {"configurable": {"thread_id": thread_id}} - history = [] - - try: - from typing import Any, cast - - async for checkpoint_tuple in checkpointer.alist(cast(Any, config)): - # checkpoint_tuple: (config, checkpoint, metadata, parent_config) - # transform to simple dict - history.append( - { - "timestamp": checkpoint_tuple.metadata.get("timestamp") if checkpoint_tuple.metadata else None, - "node_id": checkpoint_tuple.metadata.get("source") if checkpoint_tuple.metadata else None, - "state": checkpoint_tuple.checkpoint, - "config": checkpoint_tuple.config, - "metadata": checkpoint_tuple.metadata, - } - ) - except Exception as e: - logger.error(f"Failed to fetch history for thread {thread_id}: {e}") - # Return empty list or re-raise? - # For debugger, empty list handling in frontend is better than 500 - return [] - - return history diff --git a/backend/app/core/agent/code_agent/__init__.py b/backend/app/core/agent/code_agent/__init__.py deleted file mode 100644 index 0d0f5df5e..000000000 --- a/backend/app/core/agent/code_agent/__init__.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -CodeAgent - Production-ready code execution agent for DeepAgents. - -This module provides a complete implementation of a code-based agent that solves -tasks through the Thought → Code → Observation iterative pattern. - -Key Features: -- Secure Python execution via AST interpretation -- Docker-based sandboxing for unsafe code -- State persistence across execution steps -- Tool injection and compensation -- Planning capabilities for complex tasks -- Data analysis presets -- Multi-agent support with managed agents -- Comprehensive monitoring and logging -- Rate limiting and retry mechanisms -- Final answer validation -- Step callbacks for extensibility - -Usage: - >>> from app.core.agent.code_agent import CodeAgent, get_code_agent, tool - >>> - >>> # Quick start - >>> agent = get_code_agent(model_name="gpt-4") - >>> result = await agent.run("Calculate the sum of primes under 100") - >>> - >>> # With custom tools using @tool decorator - >>> @tool - ... def search(query: str) -> str: - ... '''Search the web. - ... - ... Args: - ... query: Search query - ... ''' - ... return web_search(query) - >>> - >>> agent = CodeAgent( - ... llm=my_llm_function, - ... tools={"search": search}, - ... ) - >>> - >>> # With streaming - >>> async for event in agent.run_stream("Analyze this data"): - ... print(f"{event.event_type}: {event.content}") - >>> - >>> # Multi-agent setup - >>> research_agent = CodeAgent(llm=llm, name="researcher", ...) - >>> main_agent = CodeAgent(llm=llm, managed_agents=[research_agent]) - -Architecture: -- agent.py: Main CodeAgent class -- executor/: Python execution backends (local, docker, router) -- interpreter/: AST-based Python interpreter with security -- memory.py: Execution history and state management -- parser.py: Code extraction and validation -- loop.py: Thought-Code-Observation iteration engine -- planning.py: Multi-step task planning -- utils.py: Rate limiting and retry utilities - -""" - -from .agent import CodeAgent, DataAnalysisAgent, get_code_agent -from .data_analysis import ( - ALL_DATA_ANALYSIS_MODULES, - CORE_DATA_MODULES, - ML_MODULES, - PRESET_BASIC, - PRESET_FULL, - PRESET_ML, - PRESET_VISUALIZATION, - STATISTICS_MODULES, - VISUALIZATION_MODULES, - DataAnalysisPreset, - create_data_analysis_tools, - get_preset, -) -from .executor import ( - BaseToolWrapper, - CodeOutput, - DockerPythonExecutor, - ExecutorRouter, - FinalAnswerException, - LocalPythonExecutor, - PythonExecutor, - SecurityError, - create_docker_executor, - create_local_executor, - create_router, - wrap_final_answer, -) -from .interpreter import ( - BASE_BUILTIN_MODULES, - BASE_PYTHON_TOOLS, - DANGEROUS_FUNCTIONS, - DANGEROUS_MODULES, - DATA_ANALYSIS_MODULES, - MAX_OPERATIONS, - MAX_WHILE_ITERATIONS, - NETWORK_MODULES, - InterpreterError, - PrintContainer, - check_import_authorized, - check_safer_result, - evaluate_ast, - get_allowed_imports, - is_safe_code, - validate_import_statement, -) -from .loop import ( - CODEAGENT_RESPONSE_SCHEMA, - CodeAgentLoop, - LoopConfig, - StepEvent, - create_simple_llm_call, -) -from .memory import ( - ActionStep, - AgentMemory, - ChatMessage, - MessageStep, - PlanningStep, - StepMetrics, - StepType, - ToolCallStep, -) -from .monitoring import ( - AgentLogger, - LogLevel, - Monitor, - Timing, - TokenUsage, -) -from .parser import ( - ParsingError, - clean_code, - extract_imports, - extract_thought_and_code, - fix_final_answer_code, - format_observation, - parse_code_blobs, - split_code_into_steps, - validate_python_syntax, -) -from .planning import ( - Plan, - PlanningEngine, - PlanStatus, - PlanStep, - create_planning_engine, -) -from .tools import ( - AUTHORIZED_TYPES, - FinalAnswerTool, - Tool, - create_final_answer_tool, - get_json_type, - python_type_to_json_type, - tool, - validate_tool_arguments, -) -from .utils import ( - RateLimiter, - Retrying, - is_rate_limit_error, - is_transient_error, - retry, -) - -__version__ = "1.0.0" - -__all__ = [ - # Main agent - "CodeAgent", - "DataAnalysisAgent", - "get_code_agent", - # Loop - "CodeAgentLoop", - "LoopConfig", - "StepEvent", - "create_simple_llm_call", - "CODEAGENT_RESPONSE_SCHEMA", - # Executors - "PythonExecutor", - "LocalPythonExecutor", - "DockerPythonExecutor", - "ExecutorRouter", - "CodeOutput", - "FinalAnswerException", - "SecurityError", - "BaseToolWrapper", - "wrap_final_answer", - "create_local_executor", - "create_docker_executor", - "create_router", - # Interpreter - "evaluate_ast", - "InterpreterError", - "PrintContainer", - "BASE_PYTHON_TOOLS", - "BASE_BUILTIN_MODULES", - "DATA_ANALYSIS_MODULES", - "NETWORK_MODULES", - "DANGEROUS_MODULES", - "DANGEROUS_FUNCTIONS", - "MAX_OPERATIONS", - "MAX_WHILE_ITERATIONS", - "check_import_authorized", - "check_safer_result", - "get_allowed_imports", - "is_safe_code", - "validate_import_statement", - # Memory - "AgentMemory", - "ActionStep", - "PlanningStep", - "ToolCallStep", - "MessageStep", - "ChatMessage", - "StepType", - "StepMetrics", - # Parser - "ParsingError", - "parse_code_blobs", - "extract_thought_and_code", - "fix_final_answer_code", - "clean_code", - "validate_python_syntax", - "extract_imports", - "format_observation", - "split_code_into_steps", - # Planning - "PlanningEngine", - "Plan", - "PlanStep", - "PlanStatus", - "create_planning_engine", - # Utils - "DataAnalysisPreset", - "PRESET_BASIC", - "PRESET_VISUALIZATION", - "PRESET_ML", - "PRESET_FULL", - "CORE_DATA_MODULES", - "VISUALIZATION_MODULES", - "ML_MODULES", - "STATISTICS_MODULES", - "ALL_DATA_ANALYSIS_MODULES", - "create_data_analysis_tools", - "get_preset", - # Tools - "Tool", - "tool", - "FinalAnswerTool", - "AUTHORIZED_TYPES", - "validate_tool_arguments", - "create_final_answer_tool", - "python_type_to_json_type", - "get_json_type", - # Monitoring - "Monitor", - "TokenUsage", - "Timing", - "AgentLogger", - "LogLevel", - # Utils - "RateLimiter", - "Retrying", - "retry", - "is_rate_limit_error", - "is_transient_error", -] diff --git a/backend/app/core/agent/code_agent/agent.py b/backend/app/core/agent/code_agent/agent.py deleted file mode 100644 index 93089739f..000000000 --- a/backend/app/core/agent/code_agent/agent.py +++ /dev/null @@ -1,539 +0,0 @@ -#!/usr/bin/env python -""" -CodeAgent - Main agent class for code-based task solving. - -This module implements the main CodeAgent class that uses the -Thought-Code-Observation pattern to solve tasks through code generation -and execution. -""" - -import asyncio -from collections.abc import AsyncGenerator, Callable -from typing import Any, Optional - -from .executor import ( - LocalPythonExecutor, - PythonExecutor, - create_default_final_answer, -) -from .loop import CodeAgentLoop, LoopConfig, StepEvent -from .memory import ActionStep, AgentMemory, PlanningStep -from .monitoring import AgentLogger, LogLevel, Monitor -from .tools import Tool - -# Type alias for final answer check functions -FinalAnswerCheck = Callable[[Any, AgentMemory, "CodeAgent"], bool] - - -class CodeAgent: - """ - A code-based agent that solves tasks by generating and executing Python code. - - The agent follows a Thought → Code → Observation iterative pattern: - 1. Thinks about what to do next - 2. Generates Python code to execute - 3. Observes the results - 4. Repeats until reaching a final answer - - Features: - - Secure code execution via AST interpretation - - State persistence across execution steps - - Tool injection for extended capabilities - - Optional planning mode for complex tasks - - Streaming execution events - - Example: - >>> agent = CodeAgent( - ... llm=my_llm_function, - ... tools={"search": web_search, "fetch": http_fetch}, - ... ) - >>> result = await agent.run("Calculate the sum of primes under 100") - >>> print(result) # 1060 - """ - - def __init__( - self, - llm: Callable[[str], str | AsyncGenerator[str, None]], - tools: Optional[dict[str, Callable | Tool]] = None, - executor: Optional[PythonExecutor] = None, - config: Optional[LoopConfig] = None, - name: str = "CodeAgent", - description: Optional[str] = None, - enable_data_analysis: bool = True, - additional_authorized_imports: Optional[list[str]] = None, - managed_agents: list["CodeAgent"] | None = None, - final_answer_checks: list[FinalAnswerCheck] | None = None, - step_callbacks: dict[type, list[Callable]] | None = None, - logger: AgentLogger | None = None, - verbosity: LogLevel = LogLevel.INFO, - ): - """ - Initialize the CodeAgent. - - Args: - llm: Function to call the LLM. Takes prompt string, returns response. - Can be sync or async, can return string or async generator. - tools: Dictionary of tools to inject into the executor. - executor: Custom Python executor. If None, uses LocalPythonExecutor. - config: Loop configuration options. - name: Name of the agent. - description: Description of the agent's purpose. - enable_data_analysis: Enable data analysis modules (pandas, numpy, etc.). - additional_authorized_imports: Additional Python modules to authorize. - managed_agents: List of sub-agents that this agent can call as tools. - final_answer_checks: List of validation functions for the final answer. - step_callbacks: Dict mapping step types to callback functions. - logger: Custom AgentLogger instance. - verbosity: Log level for the agent. - """ - self.name = name - self.description = description or "A code-based agent for solving tasks" - self.llm = llm - self.verbosity = verbosity - - # Initialize logger and monitor - self.logger = logger or AgentLogger(level=verbosity) - self.monitor = Monitor(logger=self.logger) - - # Prepare tools - support both Callable and Tool instances - self._tools: dict[str, Callable | Tool] = {"final_answer": create_default_final_answer()} - if tools: - for tool_name, tool_obj in tools.items(): - if isinstance(tool_obj, Tool): - self._tools[tool_obj.name] = tool_obj - else: - self._tools[tool_name] = tool_obj - - # Setup managed agents as tools - self.managed_agents: dict[str, "CodeAgent"] = {} - if managed_agents: - for agent in managed_agents: - self.managed_agents[agent.name] = agent - # Create a tool wrapper for the agent - self._tools[agent.name] = self._create_agent_tool(agent) - - # Final answer validation checks - self.final_answer_checks = final_answer_checks or [] - - # Step callbacks registry - self.step_callbacks = step_callbacks or {} - - # Initialize executor - if executor is not None: - self.executor = executor - else: - self.executor = LocalPythonExecutor( - enable_data_analysis=enable_data_analysis, - additional_authorized_imports=additional_authorized_imports, - ) - - self.executor.send_tools(self._tools) - - # Initialize loop - self.config = config or LoopConfig() - self._loop: CodeAgentLoop | None = None - - self.logger.log( - f"CodeAgent '{name}' initialized with {len(self._tools)} tools, " - f"{len(self.managed_agents)} managed agents, " - f"executor={type(self.executor).__name__}", - level=LogLevel.DEBUG, - ) - - @property - def tools(self) -> dict[str, Callable | Tool]: - """Get the available tools.""" - return self._tools - - @property - def inputs(self) -> dict[str, dict[str, str | bool]]: - """Get the input schema for this agent (when used as a managed agent).""" - return { - "task": { - "type": "string", - "description": "The task to perform.", - }, - "additional_args": { - "type": "object", - "description": "Optional additional arguments.", - "nullable": True, - }, - } - - @property - def output_type(self) -> str: - """Get the output type for this agent.""" - return "any" - - def _create_agent_tool(self, agent: "CodeAgent") -> Callable: - """ - Create a tool wrapper for a managed agent. - - Args: - agent: The managed agent to wrap. - - Returns: - A callable that invokes the agent. - """ - - async def agent_tool(task: str, additional_args: dict | None = None) -> Any: - """ - Call a managed sub-agent. - - Args: - task: The task to perform. - additional_args: Optional additional context. - - Returns: - The agent's result. - """ - # Build the full task with additional args if provided - full_task = task - if additional_args: - full_task += f"\n\nAdditional context: {additional_args}" - - return await agent.run(full_task) - - # Add metadata - agent_tool.__name__ = agent.name - agent_tool.__doc__ = agent.description - - return agent_tool - - def _validate_final_answer( - self, - final_answer: Any, - memory: AgentMemory, - ) -> tuple[bool, str]: - """ - Validate the final answer using registered checks. - - Args: - final_answer: The answer to validate. - memory: The agent's memory. - - Returns: - Tuple of (is_valid, error_message). - """ - for check_fn in self.final_answer_checks: - try: - result = check_fn(final_answer, memory, self) - if not result: - return False, f"Final answer check failed: {check_fn.__name__}" - except Exception as e: - return False, f"Final answer check error: {e}" - return True, "" - - def _trigger_callbacks( - self, - step: ActionStep | PlanningStep, - **kwargs, - ) -> None: - """ - Trigger registered callbacks for a step. - - Args: - step: The step that was completed. - **kwargs: Additional context for callbacks. - """ - step_type = type(step) - callbacks = self.step_callbacks.get(step_type, []) - - for callback in callbacks: - try: - callback(step, agent=self, **kwargs) - except Exception as e: - self.logger.log_error(f"Callback error: {e}") - - def add_tool(self, name: str, tool_obj: Callable | Tool) -> None: - """ - Add a tool to the agent. - - Args: - name: Name of the tool. - tool_obj: The tool function or Tool instance. - """ - if isinstance(tool_obj, Tool): - self._tools[tool_obj.name] = tool_obj - self.executor.send_tools({tool_obj.name: tool_obj}) - else: - self._tools[name] = tool_obj - self.executor.send_tools({name: tool_obj}) - self.logger.log(f"Added tool: {name}", level=LogLevel.DEBUG) - - def remove_tool(self, name: str) -> None: - """ - Remove a tool from the agent. - - Args: - name: Name of the tool to remove. - """ - if name in self._tools and name != "final_answer": - del self._tools[name] - self.logger.log(f"Removed tool: {name}", level=LogLevel.DEBUG) - - def add_managed_agent(self, agent: "CodeAgent") -> None: - """ - Add a managed sub-agent. - - Args: - agent: The agent to add as a managed agent. - """ - self.managed_agents[agent.name] = agent - self._tools[agent.name] = self._create_agent_tool(agent) - self.executor.send_tools({agent.name: self._tools[agent.name]}) - self.logger.log(f"Added managed agent: {agent.name}", level=LogLevel.DEBUG) - - def add_final_answer_check(self, check: FinalAnswerCheck) -> None: - """ - Add a final answer validation check. - - Args: - check: A function that takes (answer, memory, agent) and returns bool. - """ - self.final_answer_checks.append(check) - - def register_step_callback( - self, - step_type: type, - callback: Callable, - ) -> None: - """ - Register a callback for a step type. - - Args: - step_type: The step class (ActionStep, PlanningStep, etc.) - callback: The callback function. - """ - if step_type not in self.step_callbacks: - self.step_callbacks[step_type] = [] - self.step_callbacks[step_type].append(callback) - - async def run(self, task: str) -> Any: - """ - Run the agent on a task. - - Args: - task: The task description. - - Returns: - The final answer. - - Raises: - RuntimeError: If execution fails without producing a final answer. - ValueError: If final answer validation fails. - """ - self.monitor.start() - self.logger.log_task(task, f"Agent: {self.name}") - - self._loop = CodeAgentLoop( - llm_call=self.llm, - executor=self.executor, - config=self.config, - ) - - try: - result = await self._loop.run(task) - - # Validate final answer - if self.final_answer_checks: - is_valid, error_msg = self._validate_final_answer(result, self._loop.memory) - if not is_valid: - raise ValueError(error_msg) - - self.logger.log_final_answer(result) - return result - finally: - self.monitor.stop() - self._loop = None - - async def run_stream(self, task: str) -> AsyncGenerator[StepEvent, None]: - """ - Run the agent with streaming events. - - Args: - task: The task description. - - Yields: - StepEvent objects for each significant event. - """ - self.monitor.start() - self.logger.log_task(task, f"Agent: {self.name}") - - self._loop = CodeAgentLoop( - llm_call=self.llm, - executor=self.executor, - config=self.config, - ) - - try: - async for event in self._loop.run_stream(task): - # Update monitor on step completion - if event.event_type == "observation": - step = self._loop.memory.get_last_action_step() - if step: - self.monitor.update_metrics(step) - self._trigger_callbacks(step) - - # Validate final answer - if event.event_type == "final_answer" and self.final_answer_checks: - is_valid, error_msg = self._validate_final_answer(event.content, self._loop.memory) - if not is_valid: - yield StepEvent( - event_type="error", - content=error_msg, - step_number=event.step_number, - ) - return - - self.logger.log_final_answer(event.content) - - yield event - finally: - self.monitor.stop() - self._loop = None - - def run_sync(self, task: str) -> Any: - """ - Synchronous wrapper for run(). - - Args: - task: The task description. - - Returns: - The final answer. - """ - return asyncio.run(self.run(task)) - - def stop(self) -> None: - """Request the current execution to stop.""" - if self._loop: - self._loop.stop() - - @property - def is_running(self) -> bool: - """Check if the agent is currently running.""" - return self._loop is not None and self._loop.is_running - - @property - def current_step(self) -> int: - """Get the current step number.""" - return self._loop.current_step if self._loop else 0 - - @property - def memory(self) -> AgentMemory | None: - """Get the current memory state.""" - return self._loop.memory if self._loop else None - - def reset(self) -> None: - """Reset the agent state.""" - self.executor.reset() - self.monitor.reset() - self._loop = None - self.logger.log(f"Agent '{self.name}' reset", level=LogLevel.DEBUG) - - def visualize(self) -> None: - """Visualize the agent structure including tools and managed agents.""" - self.logger.visualize_agent_tree(self) - - def get_run_summary(self) -> dict: - """Get a summary of the last run's metrics.""" - return self.monitor.get_summary() - - def __repr__(self) -> str: - return f"CodeAgent(name='{self.name}', tools={len(self._tools)}, managed_agents={len(self.managed_agents)})" - - -def get_code_agent( - llm: Optional[Callable[[str], str | AsyncGenerator[str, None]]] = None, - tools: Optional[dict[str, Callable]] = None, - model_name: str = "gpt-4", - **kwargs, -) -> CodeAgent: - """ - Factory function to create a configured CodeAgent. - - Args: - llm: Custom LLM function. If None, creates one from model_name. - tools: Tools to inject. - model_name: Model name for default LLM. - **kwargs: Additional arguments for CodeAgent. - - Returns: - Configured CodeAgent instance. - """ - if llm is None: - from .loop import create_simple_llm_call - - llm = create_simple_llm_call(model_name) - - return CodeAgent(llm=llm, tools=tools, **kwargs) - - -class DataAnalysisAgent(CodeAgent): - """ - A CodeAgent specialized for data analysis tasks. - - Pre-configured with data analysis tools and optimized prompts. - """ - - def __init__( - self, - llm: Callable[[str], str | AsyncGenerator[str, None]], - **kwargs, - ): - # Default config for data analysis - config = kwargs.pop("config", None) or LoopConfig( - max_steps=30, # More steps for complex analysis - enable_planning=True, # Enable planning for multi-step analysis - max_observation_length=20000, # Larger outputs for data - ) - - super().__init__( - llm=llm, - config=config, - name="DataAnalysisAgent", - description="A specialized agent for data analysis and visualization", - enable_data_analysis=True, - **kwargs, - ) - - # Add data analysis helper tools - self._add_data_analysis_tools() - - def _add_data_analysis_tools(self) -> None: - """Add built-in data analysis tools.""" - - def describe_dataframe(df) -> str: - """Get a comprehensive description of a DataFrame.""" - import io - - buffer = io.StringIO() - buffer.write(f"Shape: {df.shape}\n\n") - buffer.write("Columns:\n") - buffer.write(df.dtypes.to_string()) - buffer.write("\n\nSample (first 5 rows):\n") - buffer.write(df.head().to_string()) - buffer.write("\n\nStatistics:\n") - buffer.write(df.describe().to_string()) - return buffer.getvalue() - - def save_plot(fig, filename: str) -> str: - """Save a matplotlib figure to file.""" - import os - - output_dir = "/tmp/plots" - os.makedirs(output_dir, exist_ok=True) - filepath = os.path.join(output_dir, filename) - fig.savefig(filepath, dpi=150, bbox_inches="tight") - return f"Plot saved to {filepath}" - - self.add_tool("describe_dataframe", describe_dataframe) - self.add_tool("save_plot", save_plot) - - -__all__ = [ - "CodeAgent", - "get_code_agent", - "DataAnalysisAgent", -] diff --git a/backend/app/core/agent/code_agent/data_analysis.py b/backend/app/core/agent/code_agent/data_analysis.py deleted file mode 100644 index 7254152ca..000000000 --- a/backend/app/core/agent/code_agent/data_analysis.py +++ /dev/null @@ -1,497 +0,0 @@ -#!/usr/bin/env python -""" -Data Analysis Presets for CodeAgent. - -This module provides pre-configured tools and helpers for data analysis -tasks, including pandas, numpy, visualization, and machine learning. -""" - -from typing import Any, Callable - -# ============================================================================ -# Data Analysis Module Lists -# ============================================================================ - -# Core data analysis modules -CORE_DATA_MODULES = [ - "pandas", - "numpy", -] - -# Visualization modules -VISUALIZATION_MODULES = [ - "matplotlib", - "matplotlib.pyplot", - "seaborn", - "plotly", - "plotly.express", - "plotly.graph_objects", -] - -# Machine learning modules -ML_MODULES = [ - "sklearn", - "sklearn.model_selection", - "sklearn.preprocessing", - "sklearn.linear_model", - "sklearn.tree", - "sklearn.ensemble", - "sklearn.cluster", - "sklearn.metrics", - "sklearn.neighbors", - "sklearn.svm", - "sklearn.naive_bayes", - "sklearn.decomposition", - "sklearn.pipeline", -] - -# Statistics modules -STATISTICS_MODULES = [ - "scipy", - "scipy.stats", - "scipy.optimize", - "scipy.signal", - "scipy.interpolate", - "statsmodels", - "statsmodels.api", - "statsmodels.formula.api", -] - -# Data I/O modules -DATA_IO_MODULES = [ - "csv", - "openpyxl", - "xlrd", - "xlwt", - "json", - "pickle", # Note: pickle is dangerous for untrusted data -] - -# All data analysis modules combined -ALL_DATA_ANALYSIS_MODULES = ( - CORE_DATA_MODULES + VISUALIZATION_MODULES + ML_MODULES + STATISTICS_MODULES + DATA_IO_MODULES -) - - -# ============================================================================ -# Built-in Helper Functions -# ============================================================================ - - -def create_data_analysis_tools() -> dict[str, Callable[..., Any]]: - """ - Create built-in helper tools for data analysis. - - Returns: - Dictionary of tool name -> function. - """ - tools: dict[str, Callable[..., Any]] = {} - - def describe_dataframe(df) -> str: - """ - Get a comprehensive description of a pandas DataFrame. - - Args: - df: A pandas DataFrame to describe. - - Returns: - Formatted string with shape, columns, dtypes, sample, and statistics. - """ - try: - import io - - buffer = io.StringIO() - - buffer.write("=== DataFrame Overview ===\n") - buffer.write(f"Shape: {df.shape} ({df.shape[0]} rows × {df.shape[1]} columns)\n\n") - - buffer.write("=== Columns and Types ===\n") - for col in df.columns: - dtype = df[col].dtype - null_count = df[col].isnull().sum() - unique_count = df[col].nunique() - buffer.write(f" {col}: {dtype} (nulls: {null_count}, unique: {unique_count})\n") - - buffer.write("\n=== Sample Data (first 5 rows) ===\n") - buffer.write(df.head().to_string()) - - buffer.write("\n\n=== Numeric Statistics ===\n") - numeric_df = df.select_dtypes(include=["number"]) - if not numeric_df.empty: - buffer.write(numeric_df.describe().to_string()) - else: - buffer.write("No numeric columns") - - return buffer.getvalue() - except Exception as e: - return f"Error describing DataFrame: {e}" - - tools["describe_dataframe"] = describe_dataframe - - def save_figure(fig, filename: str, dpi: int = 150) -> str: - """ - Save a matplotlib figure to file. - - Args: - fig: Matplotlib figure object. - filename: Name for the saved file. - dpi: Resolution in dots per inch. - - Returns: - Path to the saved file. - """ - try: - import os - - output_dir = "/tmp/plots" - os.makedirs(output_dir, exist_ok=True) - - filepath = os.path.join(output_dir, filename) - fig.savefig(filepath, dpi=dpi, bbox_inches="tight") - - return f"Figure saved to: {filepath}" - except Exception as e: - return f"Error saving figure: {e}" - - tools["save_figure"] = save_figure - - def analyze_correlation(df, method: str = "pearson") -> str: - """ - Analyze correlations between numeric columns. - - Args: - df: A pandas DataFrame. - method: Correlation method ('pearson', 'spearman', 'kendall'). - - Returns: - Formatted correlation matrix and top correlations. - """ - try: - import io - - buffer = io.StringIO() - - numeric_df = df.select_dtypes(include=["number"]) - if numeric_df.empty: - return "No numeric columns for correlation analysis" - - corr = numeric_df.corr(method=method) - - buffer.write(f"=== Correlation Matrix ({method}) ===\n") - buffer.write(corr.to_string()) - - # Find top correlations - buffer.write("\n\n=== Top Correlations ===\n") - pairs = [] - for i in range(len(corr.columns)): - for j in range(i + 1, len(corr.columns)): - col1, col2 = corr.columns[i], corr.columns[j] - corr_val = corr.iloc[i, j] - pairs.append((col1, col2, corr_val)) - - pairs.sort(key=lambda x: abs(x[2]), reverse=True) - for col1, col2, corr_val in pairs[:10]: - buffer.write(f" {col1} <-> {col2}: {corr_val:.4f}\n") - - return buffer.getvalue() - except Exception as e: - return f"Error analyzing correlation: {e}" - - tools["analyze_correlation"] = analyze_correlation - - def detect_outliers(df, column: str, method: str = "iqr") -> str: - """ - Detect outliers in a numeric column. - - Args: - df: A pandas DataFrame. - column: Column name to analyze. - method: Detection method ('iqr', 'zscore'). - - Returns: - Summary of detected outliers. - """ - try: - import io - - import numpy as np - - buffer = io.StringIO() - - if column not in df.columns: - return f"Column '{column}' not found" - - data = df[column].dropna() - - if method == "iqr": - Q1 = data.quantile(0.25) - Q3 = data.quantile(0.75) - IQR = Q3 - Q1 - lower = Q1 - 1.5 * IQR - upper = Q3 + 1.5 * IQR - outliers = data[(data < lower) | (data > upper)] - elif method == "zscore": - z_scores = np.abs((data - data.mean()) / data.std()) - outliers = data[z_scores > 3] - else: - return f"Unknown method: {method}" - - buffer.write(f"=== Outlier Detection ({method}) for '{column}' ===\n") - buffer.write(f"Total values: {len(data)}\n") - buffer.write(f"Outliers found: {len(outliers)} ({len(outliers) / len(data) * 100:.2f}%)\n") - - if len(outliers) > 0: - buffer.write("\nOutlier statistics:\n") - buffer.write(f" Min outlier: {outliers.min():.4f}\n") - buffer.write(f" Max outlier: {outliers.max():.4f}\n") - buffer.write(f" Mean outlier: {outliers.mean():.4f}\n") - - if method == "iqr": - buffer.write(f"\nIQR bounds: [{lower:.4f}, {upper:.4f}]\n") - - return buffer.getvalue() - except Exception as e: - return f"Error detecting outliers: {e}" - - tools["detect_outliers"] = detect_outliers - - def quick_eda(df, max_cols: int = 20) -> str: - """ - Perform quick exploratory data analysis on a DataFrame. - - Args: - df: A pandas DataFrame. - max_cols: Maximum columns to analyze in detail. - - Returns: - Comprehensive EDA report. - """ - try: - import io - - buffer = io.StringIO() - - buffer.write("=" * 60 + "\n") - buffer.write(" EXPLORATORY DATA ANALYSIS REPORT\n") - buffer.write("=" * 60 + "\n\n") - - # Basic info - buffer.write("=== Basic Information ===\n") - buffer.write(f"Rows: {df.shape[0]}\n") - buffer.write(f"Columns: {df.shape[1]}\n") - buffer.write(f"Memory usage: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\n") - buffer.write(f"Duplicate rows: {df.duplicated().sum()}\n\n") - - # Missing values - buffer.write("=== Missing Values ===\n") - missing = df.isnull().sum() - missing_pct = (missing / len(df) * 100).round(2) - for col in missing[missing > 0].index: - buffer.write(f" {col}: {missing[col]} ({missing_pct[col]}%)\n") - if missing.sum() == 0: - buffer.write(" No missing values!\n") - buffer.write("\n") - - # Data types - buffer.write("=== Data Types ===\n") - for dtype, count in df.dtypes.value_counts().items(): - buffer.write(f" {dtype}: {count} columns\n") - buffer.write("\n") - - # Numeric summary - numeric_cols = df.select_dtypes(include=["number"]).columns[:max_cols] - if len(numeric_cols) > 0: - buffer.write("=== Numeric Columns Summary ===\n") - buffer.write(df[numeric_cols].describe().to_string()) - buffer.write("\n\n") - - # Categorical summary - cat_cols = df.select_dtypes(include=["object", "category"]).columns[:max_cols] - if len(cat_cols) > 0: - buffer.write("=== Categorical Columns Summary ===\n") - for col in cat_cols: - n_unique = df[col].nunique() - top_values = df[col].value_counts().head(5) - buffer.write(f"\n{col} (unique: {n_unique}):\n") - for val, count in top_values.items(): - buffer.write(f" {val}: {count} ({count / len(df) * 100:.1f}%)\n") - - return buffer.getvalue() - except Exception as e: - return f"Error performing EDA: {e}" - - tools["quick_eda"] = quick_eda - - def create_train_test_split( - df, - target_column: str, - test_size: float = 0.2, - random_state: int = 42, - ) -> dict: - """ - Split DataFrame into train and test sets. - - Args: - df: A pandas DataFrame. - target_column: Name of the target column. - test_size: Fraction of data for testing. - random_state: Random seed for reproducibility. - - Returns: - Dictionary with X_train, X_test, y_train, y_test. - """ - try: - from sklearn.model_selection import train_test_split - - if target_column not in df.columns: - raise ValueError(f"Target column '{target_column}' not found") - - X = df.drop(columns=[target_column]) - y = df[target_column] - - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state) - - return { - "X_train": X_train, - "X_test": X_test, - "y_train": y_train, - "y_test": y_test, - "info": f"Train: {len(X_train)} samples, Test: {len(X_test)} samples", - } - except Exception as e: - return {"error": str(e)} - - tools["create_train_test_split"] = create_train_test_split - - return tools - - -# ============================================================================ -# Preset Configurations -# ============================================================================ - - -class DataAnalysisPreset: - """ - Preset configuration for data analysis CodeAgent. - """ - - def __init__( - self, - enable_visualization: bool = True, - enable_ml: bool = True, - enable_statistics: bool = True, - ): - """ - Initialize data analysis preset. - - Args: - enable_visualization: Enable visualization modules. - enable_ml: Enable machine learning modules. - enable_statistics: Enable statistics modules. - """ - self.enable_visualization = enable_visualization - self.enable_ml = enable_ml - self.enable_statistics = enable_statistics - - def get_authorized_imports(self) -> list[str]: - """Get list of authorized imports for this preset.""" - imports = list(CORE_DATA_MODULES) - - if self.enable_visualization: - imports.extend(VISUALIZATION_MODULES) - - if self.enable_ml: - imports.extend(ML_MODULES) - - if self.enable_statistics: - imports.extend(STATISTICS_MODULES) - - imports.extend(DATA_IO_MODULES) - - return imports - - def get_tools(self) -> dict[str, Callable]: - """Get helper tools for this preset.""" - return create_data_analysis_tools() - - def get_install_packages(self) -> list[str]: - """Get pip packages to install for Docker executor.""" - packages = ["pandas", "numpy"] - - if self.enable_visualization: - packages.extend(["matplotlib", "seaborn", "plotly"]) - - if self.enable_ml: - packages.append("scikit-learn") - - if self.enable_statistics: - packages.extend(["scipy", "statsmodels"]) - - return packages - - -# Common presets -PRESET_BASIC = DataAnalysisPreset( - enable_visualization=False, - enable_ml=False, - enable_statistics=False, -) - -PRESET_VISUALIZATION = DataAnalysisPreset( - enable_visualization=True, - enable_ml=False, - enable_statistics=False, -) - -PRESET_ML = DataAnalysisPreset( - enable_visualization=True, - enable_ml=True, - enable_statistics=False, -) - -PRESET_FULL = DataAnalysisPreset( - enable_visualization=True, - enable_ml=True, - enable_statistics=True, -) - - -def get_preset(name: str) -> DataAnalysisPreset: - """ - Get a data analysis preset by name. - - Args: - name: Preset name ('basic', 'visualization', 'ml', 'full'). - - Returns: - DataAnalysisPreset instance. - """ - presets = { - "basic": PRESET_BASIC, - "visualization": PRESET_VISUALIZATION, - "ml": PRESET_ML, - "full": PRESET_FULL, - } - - return presets.get(name.lower(), PRESET_FULL) - - -__all__ = [ - # Module lists - "CORE_DATA_MODULES", - "VISUALIZATION_MODULES", - "ML_MODULES", - "STATISTICS_MODULES", - "DATA_IO_MODULES", - "ALL_DATA_ANALYSIS_MODULES", - # Tools - "create_data_analysis_tools", - # Presets - "DataAnalysisPreset", - "PRESET_BASIC", - "PRESET_VISUALIZATION", - "PRESET_ML", - "PRESET_FULL", - "get_preset", -] diff --git a/backend/app/core/agent/code_agent/executor/__init__.py b/backend/app/core/agent/code_agent/executor/__init__.py deleted file mode 100644 index 6de126043..000000000 --- a/backend/app/core/agent/code_agent/executor/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -CodeAgent Python Executors. - -This module provides different execution backends for running Python code. -""" - -from .backend_executor import ( - BackendPythonExecutor, -) -from .base import ( - BaseToolWrapper, - CodeOutput, - FinalAnswerException, - PythonExecutor, - wrap_final_answer, -) -from .docker_executor import ( - DockerPythonExecutor, - create_docker_executor, -) -from .local_executor import ( - LocalPythonExecutor, - create_default_final_answer, - create_local_executor, - truncate_text, -) -from .router import ( - DANGEROUS_PATTERNS, - DATA_ANALYSIS_PATTERNS, - ExecutorRouter, - SecurityError, - create_router, -) - -__all__ = [ - # Base types - "CodeOutput", - "FinalAnswerException", - "PythonExecutor", - "BaseToolWrapper", - "wrap_final_answer", - # Local executor - "LocalPythonExecutor", - "create_local_executor", - "create_default_final_answer", - "truncate_text", - # Docker executor - "DockerPythonExecutor", - "create_docker_executor", - # Backend executor - "BackendPythonExecutor", - # Router - "ExecutorRouter", - "SecurityError", - "create_router", - "DANGEROUS_PATTERNS", - "DATA_ANALYSIS_PATTERNS", -] diff --git a/backend/app/core/agent/code_agent/executor/backend_executor.py b/backend/app/core/agent/code_agent/executor/backend_executor.py deleted file mode 100644 index b13b5e4db..000000000 --- a/backend/app/core/agent/code_agent/executor/backend_executor.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python -""" -Backend-based Python Executor for CodeAgent. - -This module provides a Python executor that uses an existing backend -(e.g., PydanticSandboxAdapter) instead of creating a new container, -allowing code execution to share the same environment as skills and other tools. -""" - -import json -import time -from typing import Any, Callable, Optional - -from loguru import logger - -from .base import CodeOutput, PythonExecutor - - -class BackendPythonExecutor(PythonExecutor): - """ - Python executor that uses an existing backend (e.g., PydanticSandboxAdapter). - - This executor reuses an existing backend instead of creating a new container, - allowing code execution to share the same environment as skills and other tools. - - **Key Benefits:** - - Resource efficiency: Reuses existing Docker container instead of creating new ones - - Data persistence: Code execution results persist in the same environment as skills - - Data sharing: Code can access files created by other nodes or pre-loaded skills - - Unified lifecycle: Backend lifecycle managed by Graph builder, not executor - - **Lifecycle:** - - The backend is created by DeepAgentsGraphBuilder and shared across all nodes - - This executor does NOT manage backend lifecycle (cleanup is handled externally) - - Code files are written to the shared backend's working directory - - Temporary code files are cleaned up after execution - - **Features:** - - ✅ Uses existing backend (no new container created) - - ✅ Shares file system with skills and other tools - - ✅ Supports variable injection via send_variables() - - ✅ Supports final_answer() function for early termination - - ✅ Proper error handling and logging - - ✅ Idempotent cleanup (safe to call multiple times) - - Example: - >>> from app.core.agent.backends.pydantic_adapter import PydanticSandboxAdapter - >>> backend = PydanticSandboxAdapter(image="python:3.12-slim") - >>> executor = BackendPythonExecutor(backend=backend) - >>> executor.send_variables({"x": 42}) - >>> result = executor("print(f'x = {x}')") - >>> print(result.logs) # "x = 42\\n" - """ - - def __init__(self, backend: Any, working_dir: str = "/workspace"): - """ - Initialize with an existing backend. - - Args: - backend: Backend instance implementing SandboxBackendProtocol - working_dir: Working directory in the backend - """ - self._backend = backend - self.working_dir = working_dir - self._variables: dict[str, Any] = {} - self._tools: dict[str, Callable] = {} - self.FINAL_ANSWER_MARKER = "__FINAL_ANSWER_MARKER__:" - - backend_id = getattr(backend, "id", "unknown") - logger.debug(f"BackendPythonExecutor initialized with backend {backend_id}, working_dir={working_dir}") - - def send_tools(self, tools: dict[str, Callable]) -> None: - """ - Register tools for use in code execution. - - Note: In backend executor, tools are made available as JSON-serializable - functions that communicate via stdin/stdout. - - Args: - tools: Dictionary mapping tool names to callable functions. - """ - self._tools = tools - logger.debug(f"Registered {len(tools)} tools for BackendPythonExecutor") - - def send_variables(self, variables: dict[str, Any]) -> None: - """ - Send variables to be available in code execution. - - Note: Variables are serialized to JSON and injected into the code. - - Args: - variables: Dictionary mapping variable names to values. - """ - self._variables = variables - logger.debug(f"Registered {len(variables)} variables for BackendPythonExecutor") - - def _prepare_code(self, code: str) -> str: - """ - Prepare code for execution in backend. - - Adds: - - Variable injection - - final_answer wrapper - - Output capture - - Args: - code: Original Python code - - Returns: - Prepared code with wrapper - """ - # Use shared utility method from base class - return PythonExecutor.prepare_code_with_wrapper( - code=code, - variables=self._variables, - final_answer_marker=self.FINAL_ANSWER_MARKER, - ) - - def __call__( - self, - code: str, - additional_tools: Optional[dict[str, Callable]] = None, - ) -> CodeOutput: - """ - Execute Python code using the shared backend. - - Args: - code: The Python code to execute. - additional_tools: Optional additional tools (not fully supported in backend). - - Returns: - CodeOutput containing the result, logs, and status. - """ - start_time = time.time() - - try: - # Prepare code with wrapper - prepared_code = self._prepare_code(code) - - # Write code to backend - code_path = f"{self.working_dir}/code_{int(time.time() * 1000)}.py" - write_result = self._backend.write(code_path, prepared_code) - - # Unified WriteResult error checking (supports both dict and object formats) - def get_write_error(wr) -> str | None: - """Extract error from WriteResult (supports dict or object).""" - if not wr: - return None - if isinstance(wr, dict): - error = wr.get("error") - return str(error) if error is not None else None - elif hasattr(wr, "error"): - error = wr.error - return str(error) if error is not None else None - return None - - write_error = get_write_error(write_result) - if write_error: - # File might exist, try a new name - code_path = f"{self.working_dir}/code_{int(time.time() * 1000000)}.py" - try: - self._backend.execute(f"rm -f {code_path}") - except Exception: - pass # Ignore cleanup errors - write_result = self._backend.write(code_path, prepared_code) - write_error = get_write_error(write_result) - if write_error: - return CodeOutput( - error=f"Failed to write code: {write_error}", - execution_time=time.time() - start_time, - ) - - # Execute code - result = self._backend.execute(f"python {code_path}") - - # Handle different result formats (ExecuteResponse, dict, or object) - # ExecuteResponse has output, exit_code, and truncated attributes - if isinstance(result, dict): - output = result.get("output", "") - exit_code = result.get("exit_code", -1) - elif hasattr(result, "output") and hasattr(result, "exit_code"): - # ExecuteResponse object or similar - output = result.output if result.output else "" - exit_code = result.exit_code if result.exit_code is not None else -1 - else: - # Fallback: treat as string output - output = str(result) if result else "" - exit_code = 0 - logger.warning(f"Unexpected result format from backend.execute(): {type(result)}") - - # Check for final answer marker - is_final_answer = False - final_answer_value = None - - if self.FINAL_ANSWER_MARKER in output: - is_final_answer = True - try: - # Parse final answer from output - marker_pos = output.find(self.FINAL_ANSWER_MARKER) - answer_json = output[marker_pos + len(self.FINAL_ANSWER_MARKER) :].strip() - # Find the JSON part - answer_data = json.loads(answer_json.split("\n")[0]) - final_answer_value = answer_data.get("answer") - # Remove marker from logs - output = output[:marker_pos].strip() - except Exception as e: - logger.warning(f"Failed to parse final answer: {e}") - - # Cleanup code file - try: - self._backend.execute(f"rm -f {code_path}") - except Exception as e: - logger.debug(f"Failed to cleanup code file {code_path}: {e}") - - if exit_code != 0 and not is_final_answer: - return CodeOutput( - error=output if output else f"Execution failed with exit code {exit_code}", - logs=output, - execution_time=time.time() - start_time, - ) - - return CodeOutput( - output=final_answer_value if is_final_answer else None, - logs=output, - is_final_answer=is_final_answer, - execution_time=time.time() - start_time, - ) - - except Exception as e: - logger.exception(f"Backend execution error: {e}") - return CodeOutput( - error=str(e), - execution_time=time.time() - start_time, - ) - - def reset(self) -> None: - """ - Reset the executor by cleaning up workspace. - - Note: This only cleans the workspace, not the backend itself. - """ - if self._backend is not None: - try: - # Clean workspace (but keep the container running) - self._backend.execute(f"rm -rf {self.working_dir}/*") - except Exception as e: - logger.warning(f"Failed to reset BackendPythonExecutor workspace: {e}") - - self._variables = {} - logger.debug("BackendPythonExecutor reset") - - def cleanup(self) -> None: - """ - Cleanup executor resources. - - Note: This is a no-op because the backend is managed externally. - The backend lifecycle is managed by the Graph builder (DeepAgentsGraphBuilder). - - The shared backend is cleaned up when: - - Graph execution completes (in API endpoint finally block) - - Graph building fails (in build() exception handler) - - Graph is no longer needed (via attached cleanup function) - - This method exists for interface compatibility with PythonExecutor, - but does not perform any cleanup to avoid interfering with shared backend. - """ - pass - - def __repr__(self) -> str: - backend_id = getattr(self._backend, "id", "unknown") - return f"BackendPythonExecutor(backend_id={backend_id}, working_dir={self.working_dir})" - - -__all__ = ["BackendPythonExecutor"] diff --git a/backend/app/core/agent/code_agent/executor/base.py b/backend/app/core/agent/code_agent/executor/base.py deleted file mode 100644 index e7bfa1bf7..000000000 --- a/backend/app/core/agent/code_agent/executor/base.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python -""" -Base classes and protocols for CodeAgent Python executors. - -This module defines the abstract base class and output types for all -Python code executors used by CodeAgent. -""" - -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from typing import Any, Callable, Optional - -from loguru import logger - - -@dataclass -class CodeOutput: - """Output from executing Python code.""" - - # The result of the last expression evaluated - output: Any = None - - # Captured print output - logs: str = "" - - # Whether this execution returned a final answer - is_final_answer: bool = False - - # Error message if execution failed - error: str | None = None - - # Execution duration in seconds - execution_time: float = 0.0 - - # Additional metadata - metadata: dict = field(default_factory=dict) - - @property - def success(self) -> bool: - """Check if execution was successful.""" - return self.error is None - - def __str__(self) -> str: - if self.error: - return f"Error: {self.error}" - if self.logs: - return self.logs.strip() - return str(self.output) if self.output is not None else "" - - -class FinalAnswerException(BaseException): - """ - Exception raised when a final answer is produced. - - This inherits from BaseException (not Exception) so it cannot be - caught by `except Exception:` blocks in the evaluated code. - """ - - def __init__(self, value: Any): - self.value = value - super().__init__(f"FinalAnswer: {value}") - - -class PythonExecutor(ABC): - """ - Abstract base class for Python code executors. - - All executors must implement the __call__ method to execute code - and return a CodeOutput object. - """ - - @abstractmethod - def send_tools(self, tools: dict[str, Callable]) -> None: - """ - Send tools to the executor for use in code execution. - - Args: - tools: Dictionary mapping tool names to callable functions. - """ - pass - - @abstractmethod - def send_variables(self, variables: dict[str, Any]) -> None: - """ - Send variables to the executor's state. - - Args: - variables: Dictionary mapping variable names to values. - """ - pass - - @abstractmethod - def __call__(self, code: str, additional_tools: Optional[dict[str, Callable]] = None) -> CodeOutput: - """ - Execute Python code and return the output. - - Args: - code: The Python code to execute. - additional_tools: Optional additional tools for this execution only. - - Returns: - CodeOutput containing the result, logs, and status. - """ - pass - - def reset(self) -> None: - """Reset the executor state. Override if needed.""" - pass - - def cleanup(self) -> None: - """Clean up resources. Override if needed.""" - pass - - @staticmethod - def prepare_code_with_wrapper( - code: str, - variables: dict[str, Any], - final_answer_marker: str = "__FINAL_ANSWER_MARKER__:", - ) -> str: - """ - Prepare code for execution by adding variable injection and final_answer wrapper. - - This is a shared utility method used by multiple executor implementations. - - Args: - code: Original Python code to execute - variables: Variables to inject into the code execution context - final_answer_marker: Marker string for final_answer detection - - Returns: - Prepared code with wrapper that injects variables and defines final_answer() - """ - import json - - # Serialize variables to JSON - try: - variables_json = json.dumps(variables, default=str) - except Exception as e: - logger.warning(f"Failed to serialize variables: {e}") - variables_json = "{}" - - # Create wrapper code - wrapper = f''' -import json -import sys - -# Inject variables -__injected_vars = json.loads('{variables_json}') -for __k, __v in __injected_vars.items(): - globals()[__k] = __v - -# Define final_answer function -def final_answer(answer): - """Return a final answer and terminate execution.""" - print(f"\\n{final_answer_marker} " + json.dumps({{"answer": answer}}, default=str)) - sys.exit(0) - -# User code starts here -{code} -''' - return wrapper - - -class BaseToolWrapper: - """Wrapper for making tools compatible with the executor.""" - - def __init__(self, tool: Callable, name: Optional[str] = None, description: Optional[str] = None): - self.tool = tool - self.name = name or getattr(tool, "__name__", "unknown_tool") - self.description = description or getattr(tool, "__doc__", "") or "" - - def __call__(self, *args, **kwargs): - return self.tool(*args, **kwargs) - - def __repr__(self): - return f"Tool({self.name})" - - -def wrap_final_answer(original_final_answer: Callable) -> Callable: - """ - Wrap a final_answer function to raise FinalAnswerException. - - This ensures that when the agent calls final_answer(), execution - immediately terminates and returns the answer. - - Args: - original_final_answer: The original final_answer function. - - Returns: - Wrapped function that raises FinalAnswerException. - """ - - def wrapped_final_answer(*args, **kwargs) -> Any: - result = original_final_answer(*args, **kwargs) - raise FinalAnswerException(result) - - wrapped_final_answer.__name__ = "final_answer" - wrapped_final_answer.__doc__ = original_final_answer.__doc__ - - return wrapped_final_answer - - -__all__ = [ - "CodeOutput", - "FinalAnswerException", - "PythonExecutor", - "BaseToolWrapper", - "wrap_final_answer", -] diff --git a/backend/app/core/agent/code_agent/executor/docker_executor.py b/backend/app/core/agent/code_agent/executor/docker_executor.py deleted file mode 100644 index 29b5525cd..000000000 --- a/backend/app/core/agent/code_agent/executor/docker_executor.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python -""" -Docker Python Executor for CodeAgent. - -This module implements a secure Docker-based Python executor that runs -code in an isolated container environment, providing enhanced security -for untrusted code execution. -""" - -import json -import time -from typing import Any, Callable, Optional - -from loguru import logger - -from .base import CodeOutput, PythonExecutor - - -class DockerPythonExecutor(PythonExecutor): - """ - Docker-based Python executor for secure code execution. - - This executor runs Python code inside a Docker container, providing: - - Full filesystem isolation - - Network isolation (optional) - - Resource limits (CPU, memory) - - Support for packages not available in AST interpreter - - Note: This executor does NOT maintain state across executions like - LocalPythonExecutor. Each execution is independent. - - Example: - >>> executor = DockerPythonExecutor( - ... image="python:3.12-slim", - ... memory_limit="1g", - ... network_mode="none", - ... ) - >>> result = executor("print('Hello from Docker!')") - >>> print(result.logs) # "Hello from Docker!\\n" - """ - - def __init__( - self, - image: str = "python:3.12-slim", - memory_limit: str = "1g", - cpu_quota: int = 100000, - network_mode: str = "none", - working_dir: str = "/workspace", - command_timeout: int = 60, - max_output_size: int = 100000, - install_packages: Optional[list[str]] = None, - ): - """ - Initialize the Docker Python executor. - - Args: - image: Docker image to use. - memory_limit: Memory limit (e.g., "512m", "1g"). - cpu_quota: CPU quota in microseconds. - network_mode: Network mode ("none" for isolation). - working_dir: Working directory in container. - command_timeout: Command execution timeout in seconds. - max_output_size: Maximum output size in characters. - install_packages: Python packages to install on init. - """ - self.image = image - self.memory_limit = memory_limit - self.cpu_quota = cpu_quota - self.network_mode = network_mode - self.working_dir = working_dir - self.command_timeout = command_timeout - self.max_output_size = max_output_size - self.install_packages = install_packages or [] - - # Backend will be lazily initialized - self._backend = None - self._tools: dict[str, Callable] = {} - self._variables: dict[str, Any] = {} - - # Special marker for final answer detection - self.FINAL_ANSWER_MARKER = "__FINAL_ANSWER_MARKER__:" - - logger.info( - f"DockerPythonExecutor initialized with image={image}, memory={memory_limit}, network={network_mode}" - ) - - def _get_backend(self): - """Lazily initialize Docker backend.""" - if self._backend is None: - try: - from app.core.agent.backends.pydantic_adapter import PydanticSandboxAdapter - - self._backend = PydanticSandboxAdapter( - image=self.image, - working_dir=self.working_dir, - command_timeout=self.command_timeout, - max_output_size=self.max_output_size, - ) - - # Install packages if specified - if self.install_packages: - self._install_packages() - - except ImportError as e: - logger.error(f"Failed to import PydanticSandboxAdapter: {e}") - raise RuntimeError( - "PydanticSandboxAdapter is required for DockerPythonExecutor. " - "Please ensure pydantic-ai-backend[docker] is installed." - ) from e - except Exception as e: - logger.error(f"Failed to create PydanticSandboxAdapter: {e}") - raise RuntimeError(f"Failed to initialize Docker backend: {e}") from e - - return self._backend - - def _install_packages(self) -> None: - """Install Python packages in the container.""" - if not self.install_packages: - return - - packages = " ".join(self.install_packages) - logger.info(f"Installing packages in container: {packages}") - - result = self._get_backend().execute(f"pip install -q {packages}") - exit_code = ( - result.exit_code - if hasattr(result, "exit_code") - else result.get("exit_code", -1) - if isinstance(result, dict) - else -1 - ) - if exit_code != 0: - output = ( - result.output - if hasattr(result, "output") - else result.get("output", "") - if isinstance(result, dict) - else str(result) - ) - logger.warning(f"Failed to install packages: {output}") - - def send_tools(self, tools: dict[str, Callable]) -> None: - """ - Register tools for use in code execution. - - Note: In Docker executor, tools are made available as JSON-serializable - functions that communicate via stdin/stdout. - - Args: - tools: Dictionary mapping tool names to callable functions. - """ - self._tools = tools - logger.debug(f"Registered {len(tools)} tools for Docker executor") - - def send_variables(self, variables: dict[str, Any]) -> None: - """ - Send variables to be available in code execution. - - Note: Variables are serialized to JSON and injected into the code. - - Args: - variables: Dictionary mapping variable names to values. - """ - self._variables = variables - logger.debug(f"Registered {len(variables)} variables for Docker executor") - - def _prepare_code(self, code: str) -> str: - """ - Prepare code for execution in Docker. - - Adds: - - Variable injection - - final_answer wrapper - - Output capture - - Args: - code: Original Python code - - Returns: - Prepared code with wrapper - """ - # Use shared utility method from base class - return PythonExecutor.prepare_code_with_wrapper( - code=code, - variables=self._variables, - final_answer_marker=self.FINAL_ANSWER_MARKER, - ) - - def __call__( - self, - code: str, - additional_tools: Optional[dict[str, Callable]] = None, - ) -> CodeOutput: - """ - Execute Python code in Docker container. - - Args: - code: The Python code to execute. - additional_tools: Optional additional tools (not supported in Docker). - - Returns: - CodeOutput containing the result, logs, and status. - """ - start_time = time.time() - - try: - backend = self._get_backend() - - # Prepare code with wrapper - prepared_code = self._prepare_code(code) - - # Write code to container - code_path = f"{self.working_dir}/code_{int(time.time())}.py" - write_result = backend.write(code_path, prepared_code) - - # Unified WriteResult error checking (supports both dict and object formats) - def _get_write_error(wr) -> str | None: - if not wr: - return None - if isinstance(wr, dict): - error = wr.get("error") - return str(error) if error is not None else None - if hasattr(wr, "error"): - error = wr.error - return str(error) if error is not None else None - return None - - if _get_write_error(write_result): - # File might exist, try a new name - code_path = f"{self.working_dir}/code_{int(time.time() * 1000)}.py" - backend.execute(f"rm -f {code_path}") # Force cleanup - write_result = backend.write(code_path, prepared_code) - if _get_write_error(write_result): - return CodeOutput( - error=f"Failed to write code: {_get_write_error(write_result)}", - execution_time=time.time() - start_time, - ) - - # Execute code - result = backend.execute(f"python {code_path}") - - # Handle different result formats (ExecuteResponse, dict, or object) - if isinstance(result, dict): - output = result.get("output", "") - exit_code = result.get("exit_code", -1) - elif hasattr(result, "output") and hasattr(result, "exit_code"): - output = result.output if result.output else "" - exit_code = result.exit_code if result.exit_code is not None else -1 - else: - output = str(result) if result else "" - exit_code = 0 - - # Check for final answer marker - is_final_answer = False - final_answer_value = None - - if self.FINAL_ANSWER_MARKER in output: - is_final_answer = True - try: - # Parse final answer from output - marker_pos = output.find(self.FINAL_ANSWER_MARKER) - answer_json = output[marker_pos + len(self.FINAL_ANSWER_MARKER) :].strip() - # Find the JSON part - answer_data = json.loads(answer_json.split("\n")[0]) - final_answer_value = answer_data.get("answer") - # Remove marker from logs - output = output[:marker_pos].strip() - except Exception as e: - logger.warning(f"Failed to parse final answer: {e}") - - # Cleanup code file - backend.execute(f"rm -f {code_path}") - - if exit_code != 0 and not is_final_answer: - return CodeOutput( - error=output if output else f"Execution failed with exit code {exit_code}", - logs=output, - execution_time=time.time() - start_time, - ) - - return CodeOutput( - output=final_answer_value if is_final_answer else None, - logs=output, - is_final_answer=is_final_answer, - execution_time=time.time() - start_time, - ) - - except Exception as e: - logger.exception(f"Docker execution error: {e}") - return CodeOutput( - error=str(e), - execution_time=time.time() - start_time, - ) - - def reset(self) -> None: - """Reset the executor by cleaning up the container.""" - if self._backend is not None: - try: - # Clean workspace - self._backend.execute(f"rm -rf {self.working_dir}/*") - except Exception as e: - logger.warning(f"Failed to reset Docker executor: {e}") - - self._variables = {} - logger.debug("Docker executor reset") - - def cleanup(self) -> None: - """Clean up Docker resources.""" - if self._backend is not None: - try: - self._backend.cleanup() - except Exception as e: - logger.warning(f"Failed to cleanup Docker backend: {e}") - finally: - self._backend = None - logger.debug("Docker executor cleaned up") - - def __del__(self): - """Cleanup on garbage collection.""" - self.cleanup() - - def __repr__(self) -> str: - return f"DockerPythonExecutor(image={self.image}, memory={self.memory_limit}, network={self.network_mode})" - - -def create_docker_executor( - install_packages: Optional[list[str]] = None, - enable_network: bool = False, - **kwargs, -) -> DockerPythonExecutor: - """ - Factory function to create a configured DockerPythonExecutor. - - Args: - install_packages: Python packages to install. - enable_network: Enable network access in container. - **kwargs: Additional arguments for DockerPythonExecutor. - - Returns: - Configured DockerPythonExecutor instance. - """ - network_mode = "bridge" if enable_network else "none" - - return DockerPythonExecutor( - install_packages=install_packages, - network_mode=network_mode, - **kwargs, - ) - - -__all__ = [ - "DockerPythonExecutor", - "create_docker_executor", -] diff --git a/backend/app/core/agent/code_agent/executor/local_executor.py b/backend/app/core/agent/code_agent/executor/local_executor.py deleted file mode 100644 index 9eb8ec1ef..000000000 --- a/backend/app/core/agent/code_agent/executor/local_executor.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python -""" -Local Python Executor for CodeAgent. - -This module implements a secure local Python executor using AST interpretation. -It maintains state across executions and supports tool injection. -""" - -import ast -import time -from typing import Any, Callable, Optional - -from loguru import logger - -from ..interpreter import ( - BASE_PYTHON_TOOLS, - InterpreterError, - PrintContainer, - evaluate_ast, - get_allowed_imports, -) -from .base import CodeOutput, FinalAnswerException, PythonExecutor, wrap_final_answer - - -def truncate_text(text: str, max_length: int = 50000) -> str: - """Truncate text if it exceeds max length.""" - if len(text) <= max_length: - return text - return text[:max_length] + f"\n... [{len(text) - max_length} more characters truncated]" - - -class LocalPythonExecutor(PythonExecutor): - """ - Secure local Python executor using AST interpretation. - - This executor: - - Maintains state across multiple code executions - - Injects tools as callable functions - - Captures print output - - Enforces security restrictions - - Handles FinalAnswerException for termination - - Example: - >>> executor = LocalPythonExecutor() - >>> executor.send_tools({"add": lambda a, b: a + b}) - >>> result = executor("x = add(1, 2); print(x)") - >>> print(result.logs) # "3\\n" - """ - - def __init__( - self, - authorized_imports: Optional[list[str]] = None, - additional_authorized_imports: Optional[list[str]] = None, - enable_data_analysis: bool = True, - max_print_output_length: int = 50000, - ): - """ - Initialize the local Python executor. - - Args: - authorized_imports: Custom list of authorized imports. If None, uses defaults. - additional_authorized_imports: Additional imports to authorize. - enable_data_analysis: Enable data analysis modules (pandas, numpy, etc.). - max_print_output_length: Maximum length of print output before truncation. - """ - # Build authorized imports list - if authorized_imports is not None: - self.authorized_imports = list(authorized_imports) - else: - self.authorized_imports = get_allowed_imports( - base=True, - data_analysis=enable_data_analysis, - network=False, - ) - - if additional_authorized_imports: - self.authorized_imports.extend(additional_authorized_imports) - - # State persists across executions - self.state: dict[str, Any] = {"__name__": "__main__"} - - # Tools provided via send_tools() - self.static_tools: dict[str, Callable] = {} - - # User-defined functions (from executed code) - self.custom_tools: dict[str, Callable] = {} - - self.max_print_output_length = max_print_output_length - - # Initialize with base Python tools - self._initialize_base_tools() - - logger.info( - f"LocalPythonExecutor initialized with {len(self.authorized_imports)} authorized imports, " - f"data_analysis={enable_data_analysis}" - ) - - def _initialize_base_tools(self) -> None: - """Initialize base Python tools.""" - self.static_tools = {**BASE_PYTHON_TOOLS} # type: ignore[dict-item] - - def send_tools(self, tools: dict[str, Callable]) -> None: - """ - Send tools to the executor for use in code execution. - - Args: - tools: Dictionary mapping tool names to callable functions. - """ - # Wrap final_answer if present - for name, func in tools.items(): - if name == "final_answer": - self.static_tools[name] = wrap_final_answer(func) - else: - self.static_tools[name] = func - - logger.debug(f"Injected {len(tools)} tools: {list(tools.keys())}") - - def send_variables(self, variables: dict[str, Any]) -> None: - """ - Send variables to the executor's state. - - Args: - variables: Dictionary mapping variable names to values. - """ - self.state.update(variables) - logger.debug(f"Updated state with {len(variables)} variables") - - def __call__( - self, - code: str, - additional_tools: Optional[dict[str, Callable]] = None, - ) -> CodeOutput: - """ - Execute Python code and return the output. - - Args: - code: The Python code to execute. - additional_tools: Optional additional tools for this execution only. - - Returns: - CodeOutput containing the result, logs, and status. - """ - start_time = time.time() - - # Prepare print container - self.state["_print_outputs"] = PrintContainer() - self.state["_operations_count"] = {"counter": 0} - - # Merge additional tools - tools = {**self.static_tools} - if additional_tools: - for name, func in additional_tools.items(): - if name == "final_answer": - tools[name] = wrap_final_answer(func) - else: - tools[name] = func - - try: - # Parse the code - try: - expression = ast.parse(code) - except SyntaxError as e: - error_msg = f"SyntaxError: {e.msg} at line {e.lineno}, column {e.offset}" - return CodeOutput( - error=error_msg, - execution_time=time.time() - start_time, - ) - - # Evaluate each statement - result = None - for node in expression.body: - try: - result = evaluate_ast( - node, - self.state, - tools, - self.custom_tools, - self.authorized_imports, - ) - except FinalAnswerException as e: - # Final answer - return immediately - logs = str(self.state.get("_print_outputs", "")) - return CodeOutput( - output=e.value, - logs=truncate_text(logs, self.max_print_output_length), - is_final_answer=True, - execution_time=time.time() - start_time, - ) - - # Get print outputs - logs = str(self.state.get("_print_outputs", "")) - - return CodeOutput( - output=result, - logs=truncate_text(logs, self.max_print_output_length), - is_final_answer=False, - execution_time=time.time() - start_time, - ) - - except InterpreterError as e: - logs = str(self.state.get("_print_outputs", "")) - return CodeOutput( - error=str(e), - logs=truncate_text(logs, self.max_print_output_length), - execution_time=time.time() - start_time, - ) - - except Exception as e: - logs = str(self.state.get("_print_outputs", "")) - error_msg = f"{type(e).__name__}: {str(e)}" - logger.exception(f"Error executing code: {error_msg}") - return CodeOutput( - error=error_msg, - logs=truncate_text(logs, self.max_print_output_length), - execution_time=time.time() - start_time, - ) - - def reset(self) -> None: - """Reset the executor state.""" - self.state = {"__name__": "__main__"} - self.custom_tools = {} - self._initialize_base_tools() - logger.debug("Executor state reset") - - def get_state_variables(self) -> dict[str, Any]: - """ - Get user-defined variables from the state. - - Returns: - Dictionary of user-defined variables (excluding internal ones). - """ - return {k: v for k, v in self.state.items() if not k.startswith("_") and k != "__name__"} - - def __repr__(self) -> str: - return ( - f"LocalPythonExecutor(" - f"tools={len(self.static_tools)}, " - f"custom_tools={len(self.custom_tools)}, " - f"state_vars={len(self.get_state_variables())}, " - f"imports={len(self.authorized_imports)}" - f")" - ) - - -def create_default_final_answer() -> Callable: - """Create a default final_answer function.""" - - def final_answer(answer: Any) -> Any: - """ - Return a final answer and terminate execution. - - Args: - answer: The final answer to return. - - Returns: - The answer provided. - """ - return answer - - return final_answer - - -def create_local_executor( - tools: Optional[dict[str, Callable]] = None, - enable_data_analysis: bool = True, - additional_imports: Optional[list[str]] = None, -) -> LocalPythonExecutor: - """ - Factory function to create a configured LocalPythonExecutor. - - Args: - tools: Tools to inject into the executor. - enable_data_analysis: Enable data analysis modules. - additional_imports: Additional modules to authorize. - - Returns: - Configured LocalPythonExecutor instance. - """ - executor = LocalPythonExecutor( - enable_data_analysis=enable_data_analysis, - additional_authorized_imports=additional_imports, - ) - - # Add default final_answer if not provided - all_tools = {"final_answer": create_default_final_answer()} - if tools: - all_tools.update(tools) - - executor.send_tools(all_tools) - - return executor - - -__all__ = [ - "LocalPythonExecutor", - "create_local_executor", - "create_default_final_answer", - "truncate_text", -] diff --git a/backend/app/core/agent/code_agent/executor/router.py b/backend/app/core/agent/code_agent/executor/router.py deleted file mode 100644 index 29b6dc74a..000000000 --- a/backend/app/core/agent/code_agent/executor/router.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python -""" -Executor Router for CodeAgent. - -This module implements intelligent routing between different Python executors -based on code analysis and security requirements. -""" - -import re -from typing import Any, Callable, Optional - -from loguru import logger - -from .base import CodeOutput, PythonExecutor -from .local_executor import LocalPythonExecutor - -# Patterns that indicate potentially dangerous code -DANGEROUS_PATTERNS = [ - # System/OS access - (r"\bimport\s+os\b", "os module import"), - (r"\bimport\s+subprocess\b", "subprocess module import"), - (r"\bimport\s+sys\b", "sys module import"), - (r"\bimport\s+socket\b", "socket module import"), - (r"\bimport\s+shutil\b", "shutil module import"), - # File operations with write mode - (r"\bopen\s*\([^)]*['\"][wa]", "file write operation"), - (r"\bopen\s*\([^)]*mode\s*=\s*['\"][wa]", "file write operation"), - # Network requests - (r"\brequests\.", "requests library usage"), - (r"\burllib\.", "urllib library usage"), - (r"\bhttpx\.", "httpx library usage"), - (r"\baiohttp\.", "aiohttp library usage"), - # Code execution - (r"\beval\s*\(", "eval() call"), - (r"\bexec\s*\(", "exec() call"), - (r"\bcompile\s*\(", "compile() call"), - (r"\b__import__\s*\(", "__import__() call"), - # Shell commands - (r"\bos\.system\s*\(", "os.system() call"), - (r"\bos\.popen\s*\(", "os.popen() call"), - (r"\bsubprocess\.", "subprocess usage"), - # Dunder access - (r"__\w+__\s*(?:\[|\.)", "dunder attribute access"), - # Pickle (arbitrary code execution) - (r"\bpickle\.load", "pickle deserialization"), - (r"\bcPickle\.load", "pickle deserialization"), -] - -# Patterns that suggest data analysis (safe for local execution) -DATA_ANALYSIS_PATTERNS = [ - r"\bimport\s+pandas\b", - r"\bimport\s+numpy\b", - r"\bimport\s+matplotlib\b", - r"\bimport\s+seaborn\b", - r"\bimport\s+sklearn\b", - r"\bimport\s+scipy\b", - r"\bpd\.", - r"\bnp\.", - r"\bplt\.", -] - - -class SecurityError(Exception): - """Exception raised when dangerous code is detected.""" - - pass - - -class ExecutorRouter: - """ - Routes code execution to appropriate executor based on analysis. - - The router analyzes code to determine: - 1. Is the code safe for local AST interpretation? - 2. Does it require Docker isolation? - 3. Should it be blocked entirely? - - Features: - - Pattern-based danger detection - - Configurable routing policies - - Fallback handling - - Execution metrics - - Example: - >>> router = ExecutorRouter( - ... local=LocalPythonExecutor(), - ... docker=DockerPythonExecutor(), - ... ) - >>> result = router("import pandas; df = pd.read_csv('data.csv')") - # Routes to local executor (safe data analysis) - - >>> result = router("import os; os.system('rm -rf /')") - # Routes to Docker executor (dangerous but allowed in sandbox) - """ - - def __init__( - self, - local: Optional[PythonExecutor] = None, - docker: Optional[PythonExecutor] = None, - allow_dangerous: bool = False, - prefer_docker: bool = False, - dangerous_patterns: Optional[list[tuple[str, str]]] = None, - ): - """ - Initialize the executor router. - - Args: - local: Local Python executor (AST-based). - docker: Docker Python executor. - allow_dangerous: Allow dangerous code (routes to Docker). - prefer_docker: Always prefer Docker when available. - dangerous_patterns: Custom dangerous patterns to check. - """ - self.local = local or LocalPythonExecutor() - self.docker = docker - self.allow_dangerous = allow_dangerous - self.prefer_docker = prefer_docker - self.dangerous_patterns = dangerous_patterns or DANGEROUS_PATTERNS - - # Metrics - self._local_count = 0 - self._docker_count = 0 - self._blocked_count = 0 - - logger.info( - f"ExecutorRouter initialized: local={type(self.local).__name__}, " - f"docker={type(self.docker).__name__ if self.docker else 'None'}, " - f"allow_dangerous={allow_dangerous}" - ) - - def analyze_code(self, code: str) -> dict[str, Any]: - """ - Analyze code for routing decisions. - - Args: - code: The Python code to analyze. - - Returns: - Analysis result with danger level and details. - """ - dangers = [] - - for pattern, description in self.dangerous_patterns: - if re.search(pattern, code, re.IGNORECASE): - dangers.append(description) - - # Check for data analysis (considered safe) - is_data_analysis = any(re.search(pattern, code, re.IGNORECASE) for pattern in DATA_ANALYSIS_PATTERNS) - - return { - "is_dangerous": len(dangers) > 0, - "dangers": dangers, - "is_data_analysis": is_data_analysis, - "danger_level": len(dangers), - } - - def route(self, code: str) -> PythonExecutor: - """ - Determine which executor to use for the code. - - Args: - code: The Python code to route. - - Returns: - The appropriate executor. - - Raises: - SecurityError: If dangerous code is not allowed. - """ - # Always prefer Docker if configured - if self.prefer_docker and self.docker is not None: - logger.debug("Routing to Docker executor (prefer_docker=True)") - return self.docker # type: ignore[return-value] - - # Analyze code - analysis = self.analyze_code(code) - - if analysis["is_dangerous"]: - dangers = ", ".join(analysis["dangers"]) - logger.warning(f"Dangerous code detected: {dangers}") - - if self.docker is not None: - logger.info("Routing dangerous code to Docker executor") - return self.docker # type: ignore[return-value] - elif self.allow_dangerous: - logger.warning("No Docker available, executing dangerous code locally") - return self.local - else: - self._blocked_count += 1 - raise SecurityError( - f"Dangerous code patterns detected: {dangers}. " - "Configure a Docker executor or set allow_dangerous=True." - ) - - # Safe code - use local executor - logger.debug("Routing to local executor (safe code)") - return self.local - - def __call__( - self, - code: str, - additional_tools: Optional[dict[str, Callable]] = None, - ) -> CodeOutput: - """ - Execute code using the appropriate executor. - - Args: - code: The Python code to execute. - additional_tools: Optional additional tools. - - Returns: - CodeOutput from the execution. - """ - try: - executor = self.route(code) - - if executor is self.local: - self._local_count += 1 - else: - self._docker_count += 1 - - return executor(code, additional_tools) - - except SecurityError as e: - return CodeOutput(error=str(e)) - - def send_tools(self, tools: dict[str, Callable]) -> None: - """Send tools to both executors.""" - self.local.send_tools(tools) - if self.docker: - self.docker.send_tools(tools) - - def send_variables(self, variables: dict[str, Any]) -> None: - """Send variables to both executors.""" - self.local.send_variables(variables) - if self.docker: - self.docker.send_variables(variables) - - def reset(self) -> None: - """Reset both executors.""" - self.local.reset() - if self.docker: - self.docker.reset() - - def cleanup(self) -> None: - """Cleanup both executors.""" - self.local.cleanup() - if self.docker: - self.docker.cleanup() - - def get_metrics(self) -> dict[str, int]: - """Get routing metrics.""" - return { - "local_executions": self._local_count, - "docker_executions": self._docker_count, - "blocked_executions": self._blocked_count, - "total_executions": self._local_count + self._docker_count, - } - - def __repr__(self) -> str: - return ( - f"ExecutorRouter(" - f"local={type(self.local).__name__}, " - f"docker={type(self.docker).__name__ if self.docker else 'None'}, " - f"metrics={self.get_metrics()}" - f")" - ) - - -def create_router( - enable_docker: bool = True, - allow_dangerous: bool = False, - prefer_docker: bool = False, - docker_kwargs: Optional[dict[Any, Any]] = None, - local_kwargs: Optional[dict[Any, Any]] = None, -) -> ExecutorRouter: - """ - Factory function to create a configured ExecutorRouter. - - Args: - enable_docker: Enable Docker executor. - allow_dangerous: Allow dangerous code. - prefer_docker: Always prefer Docker. - docker_kwargs: Arguments for DockerPythonExecutor. - local_kwargs: Arguments for LocalPythonExecutor. - - Returns: - Configured ExecutorRouter instance. - """ - local = LocalPythonExecutor(**(local_kwargs or {})) - - docker = None - if enable_docker: - try: - from .docker_executor import DockerPythonExecutor - - docker = DockerPythonExecutor(**(docker_kwargs or {})) - except Exception as e: - logger.warning(f"Failed to initialize Docker executor: {e}") - - return ExecutorRouter( - local=local, - docker=docker, - allow_dangerous=allow_dangerous, - prefer_docker=prefer_docker, - ) - - -__all__ = [ - "SecurityError", - "ExecutorRouter", - "create_router", - "DANGEROUS_PATTERNS", - "DATA_ANALYSIS_PATTERNS", -] diff --git a/backend/app/core/agent/code_agent/interpreter/__init__.py b/backend/app/core/agent/code_agent/interpreter/__init__.py deleted file mode 100644 index 6ea67166d..000000000 --- a/backend/app/core/agent/code_agent/interpreter/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -CodeAgent Python Interpreter. - -This module provides a secure Python AST interpreter for executing code -within the CodeAgent framework. -""" - -from .ast_evaluator import ( - BASE_PYTHON_TOOLS, - MAX_OPERATIONS, - MAX_WHILE_ITERATIONS, - PrintContainer, - evaluate_ast, -) -from .security import ( - BASE_BUILTIN_MODULES, - DANGEROUS_FUNCTIONS, - DANGEROUS_MODULES, - DATA_ANALYSIS_MODULES, - NETWORK_MODULES, - InterpreterError, - SecurityError, - check_import_authorized, - check_safer_result, - get_allowed_imports, - is_safe_code, - validate_import_statement, -) - -__all__ = [ - # AST Evaluator - "evaluate_ast", - "BASE_PYTHON_TOOLS", - "PrintContainer", - "MAX_OPERATIONS", - "MAX_WHILE_ITERATIONS", - # Security - "InterpreterError", - "SecurityError", - "DANGEROUS_MODULES", - "DANGEROUS_FUNCTIONS", - "BASE_BUILTIN_MODULES", - "DATA_ANALYSIS_MODULES", - "NETWORK_MODULES", - "check_import_authorized", - "check_safer_result", - "get_allowed_imports", - "is_safe_code", - "validate_import_statement", -] diff --git a/backend/app/core/agent/code_agent/interpreter/ast_evaluator.py b/backend/app/core/agent/code_agent/interpreter/ast_evaluator.py deleted file mode 100644 index 93b63ca66..000000000 --- a/backend/app/core/agent/code_agent/interpreter/ast_evaluator.py +++ /dev/null @@ -1,1349 +0,0 @@ -#!/usr/bin/env python -""" -AST Evaluator - Core Python AST interpreter for CodeAgent. - -This module implements a secure Python AST interpreter that evaluates code -with restricted access to imports and built-in functions. adapted for DeepAgents. - -Key features: -- Recursive AST evaluation supporting 30+ node types -- Security controls (operation counting, import restrictions) -- State persistence across executions -- Tool injection as callable functions -""" - -import ast -import builtins -import difflib -import math -from collections.abc import Callable, Generator, Mapping -from functools import wraps -from importlib import import_module -from types import ModuleType -from typing import Any, Optional - -from loguru import logger - -from .security import ( - BASE_BUILTIN_MODULES, - InterpreterError, - check_import_authorized, - check_safer_result, -) - -# Error types that can be raised in user code -ERRORS = { - name: getattr(builtins, name) - for name in dir(builtins) - if isinstance(getattr(builtins, name), type) and issubclass(getattr(builtins, name), BaseException) -} - -# Limits -MAX_OPERATIONS = 10_000_000 -MAX_WHILE_ITERATIONS = 1_000_000 - -# Allowed dunder methods -ALLOWED_DUNDER_METHODS = ["__init__", "__str__", "__repr__"] - - -def custom_print(*args): - """Custom print that does nothing - output is captured separately.""" - return None - - -def nodunder_getattr(obj, name, default=None): - """Safe getattr that blocks dunder attributes.""" - if name.startswith("__") and name.endswith("__"): - raise InterpreterError(f"Forbidden access to dunder attribute: {name}") - return getattr(obj, name, default) - - -# Base Python tools available in the interpreter -BASE_PYTHON_TOOLS = { - "print": custom_print, - "isinstance": isinstance, - "range": range, - "float": float, - "int": int, - "bool": bool, - "str": str, - "set": set, - "list": list, - "dict": dict, - "tuple": tuple, - "round": round, - "ceil": math.ceil, - "floor": math.floor, - "log": math.log, - "exp": math.exp, - "sin": math.sin, - "cos": math.cos, - "tan": math.tan, - "asin": math.asin, - "acos": math.acos, - "atan": math.atan, - "atan2": math.atan2, - "degrees": math.degrees, - "radians": math.radians, - "pow": pow, - "sqrt": math.sqrt, - "len": len, - "sum": sum, - "max": max, - "min": min, - "abs": abs, - "enumerate": enumerate, - "zip": zip, - "reversed": reversed, - "sorted": sorted, - "all": all, - "any": any, - "map": map, - "filter": filter, - "ord": ord, - "chr": chr, - "next": next, - "iter": iter, - "divmod": divmod, - "callable": callable, - "getattr": nodunder_getattr, - "hasattr": hasattr, - "setattr": setattr, - "issubclass": issubclass, - "type": type, - "complex": complex, - "bytes": bytes, - "bytearray": bytearray, - "memoryview": memoryview, - "frozenset": frozenset, - "slice": slice, - "object": object, - "hex": hex, - "oct": oct, - "bin": bin, - "format": format, - "repr": repr, - "hash": hash, - "id": id, - "input": lambda *args: "", # Disabled for safety - "open": None, # Will be overridden if file access is allowed -} - - -class PrintContainer: - """Container for capturing print outputs.""" - - def __init__(self): - self.value = "" - - def append(self, text): - self.value += text - return self - - def __iadd__(self, other): - self.value += str(other) - return self - - def __str__(self): - return self.value - - def __repr__(self): - return f"PrintContainer({self.value})" - - def __len__(self): - return len(self.value) - - -class BreakException(Exception): - """Exception for break statement.""" - - pass - - -class ContinueException(Exception): - """Exception for continue statement.""" - - pass - - -class ReturnException(Exception): - """Exception for return statement.""" - - def __init__(self, value): - self.value = value - - -def get_iterable(obj): - """Get iterable from an object.""" - if isinstance(obj, list): - return obj - elif hasattr(obj, "__iter__"): - return list(obj) - else: - raise InterpreterError("Object is not iterable") - - -def safer_eval(func: Callable): - """Decorator to enhance security by checking return values.""" - - @wraps(func) - def _check_return( - expression, - state, - static_tools, - custom_tools, - authorized_imports=BASE_BUILTIN_MODULES, - ): - result = func(expression, state, static_tools, custom_tools, authorized_imports=authorized_imports) - check_safer_result(result, static_tools, authorized_imports) - return result - - return _check_return - - -def safer_func( - func: Callable, - static_tools: Optional[dict[str, Callable]] = None, - authorized_imports: Optional[list[str]] = None, -): - """Decorator to enhance security of function calls.""" - if static_tools is None: - static_tools = BASE_PYTHON_TOOLS # type: ignore[assignment] - if authorized_imports is None: - authorized_imports = BASE_BUILTIN_MODULES - - if isinstance(func, type): - return func - - @wraps(func) - def _check_return(*args, **kwargs): - result = func(*args, **kwargs) - check_safer_result(result, static_tools, authorized_imports) - return result - - return _check_return - - -def build_import_tree(authorized_imports: list[str]) -> dict[str, Any]: - """Build a tree structure from authorized imports for efficient lookup.""" - tree: dict[str, Any] = {} - for import_path in authorized_imports: - parts = import_path.split(".") - current = tree - for part in parts: - if part not in current: - current[part] = {} - current = current[part] - return tree - - -def get_safe_module(raw_module, authorized_imports, visited=None): - """Create a safe copy of a module.""" - if not isinstance(raw_module, ModuleType): - return raw_module - - if visited is None: - visited = set() - - module_id = id(raw_module) - if module_id in visited: - return raw_module - visited.add(module_id) - - safe_module = ModuleType(raw_module.__name__) - - for attr_name in dir(raw_module): - try: - attr_value = getattr(raw_module, attr_name) - except (ImportError, AttributeError) as e: - logger.debug(f"Skipping {raw_module.__name__}.{attr_name}: {e}") - continue - - if isinstance(attr_value, ModuleType): - attr_value = get_safe_module(attr_value, authorized_imports, visited=visited) - setattr(safe_module, attr_name, attr_value) - - return safe_module - - -# ============================================================================ -# AST Evaluation Functions -# ============================================================================ - - -def evaluate_attribute( - expression: ast.Attribute, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate attribute access.""" - if expression.attr.startswith("__") and expression.attr.endswith("__"): - raise InterpreterError(f"Forbidden access to dunder attribute: {expression.attr}") - value = evaluate_ast(expression.value, state, static_tools, custom_tools, authorized_imports) - return getattr(value, expression.attr) - - -def evaluate_unaryop( - expression: ast.UnaryOp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate unary operations.""" - operand = evaluate_ast(expression.operand, state, static_tools, custom_tools, authorized_imports) - if isinstance(expression.op, ast.USub): - return -operand - elif isinstance(expression.op, ast.UAdd): - return operand - elif isinstance(expression.op, ast.Not): - return not operand - elif isinstance(expression.op, ast.Invert): - return ~operand - else: - raise InterpreterError(f"Unary operation {expression.op.__class__.__name__} is not supported.") - - -def evaluate_lambda( - lambda_expression: ast.Lambda, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Callable: - """Evaluate lambda expressions.""" - args = [arg.arg for arg in lambda_expression.args.args] - - def lambda_func(*values: Any) -> Any: - new_state = state.copy() - for arg, value in zip(args, values): - new_state[arg] = value - return evaluate_ast( - lambda_expression.body, - new_state, - static_tools, - custom_tools, - authorized_imports, - ) - - return lambda_func - - -def evaluate_while( - while_loop: ast.While, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Evaluate while loops.""" - iterations = 0 - while evaluate_ast(while_loop.test, state, static_tools, custom_tools, authorized_imports): - for node in while_loop.body: - try: - evaluate_ast(node, state, static_tools, custom_tools, authorized_imports) - except BreakException: - return None - except ContinueException: - break - iterations += 1 - if iterations > MAX_WHILE_ITERATIONS: - raise InterpreterError(f"Maximum number of {MAX_WHILE_ITERATIONS} iterations in While loop exceeded") - return None - - -def create_function( - func_def: ast.FunctionDef, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Callable: - """Create a function from a FunctionDef AST node.""" - source_code = ast.unparse(func_def) - - def new_func(*args: Any, **kwargs: Any) -> Any: - func_state = state.copy() - arg_names = [arg.arg for arg in func_def.args.args] - default_values = [ - evaluate_ast(d, state, static_tools, custom_tools, authorized_imports) for d in func_def.args.defaults - ] - - defaults = dict(zip(arg_names[-len(default_values) :], default_values)) if default_values else {} - - for name, value in zip(arg_names, args): - func_state[name] = value - - for name, value in kwargs.items(): - func_state[name] = value - - if func_def.args.vararg: - func_state[func_def.args.vararg.arg] = args[len(arg_names) :] - - if func_def.args.kwarg: - func_state[func_def.args.kwarg.arg] = kwargs - - for name, value in defaults.items(): - if name not in func_state: - func_state[name] = value - - if func_def.args.args and func_def.args.args[0].arg == "self": - if args: - func_state["self"] = args[0] - func_state["__class__"] = args[0].__class__ - - result = None - try: - for stmt in func_def.body: - result = evaluate_ast(stmt, func_state, static_tools, custom_tools, authorized_imports) - except ReturnException as e: - result = e.value - - if func_def.name == "__init__": - return None - return result - - new_func.__ast__ = func_def # type: ignore[attr-defined] - new_func.__source__ = source_code # type: ignore[attr-defined] - new_func.__name__ = func_def.name - return new_func - - -def evaluate_function_def( - func_def: ast.FunctionDef, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Callable: - """Evaluate function definitions.""" - custom_tools[func_def.name] = create_function(func_def, state, static_tools, custom_tools, authorized_imports) - return custom_tools[func_def.name] - - -def evaluate_class_def( - class_def: ast.ClassDef, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> type: - """Evaluate class definitions.""" - class_name = class_def.name - bases = [evaluate_ast(base, state, static_tools, custom_tools, authorized_imports) for base in class_def.bases] - - metaclass: type = type - for base in bases: - base_metaclass = type(base) - if base_metaclass is not type: - metaclass = base_metaclass # type: ignore[assignment] - break - - if hasattr(metaclass, "__prepare__"): - class_dict = metaclass.__prepare__(class_name, tuple(bases)) # type: ignore[arg-type] - else: - class_dict = {} - - for stmt in class_def.body: - if isinstance(stmt, ast.FunctionDef): - class_dict[stmt.name] = evaluate_ast(stmt, state, static_tools, custom_tools, authorized_imports) - elif isinstance(stmt, ast.AnnAssign): - if stmt.value: - value = evaluate_ast(stmt.value, state, static_tools, custom_tools, authorized_imports) - if isinstance(stmt.target, ast.Name): - class_dict[stmt.target.id] = value - elif isinstance(stmt, ast.Assign): - value = evaluate_ast(stmt.value, state, static_tools, custom_tools, authorized_imports) - for target in stmt.targets: - if isinstance(target, ast.Name): - class_dict[target.id] = value - elif isinstance(stmt, ast.Pass): - pass - elif isinstance(stmt, ast.Expr) and stmt == class_def.body[0]: - if isinstance(stmt.value, ast.Constant) and isinstance(stmt.value.value, str): - class_dict["__doc__"] = stmt.value.value - - new_class = metaclass(class_name, tuple(bases), class_dict) # type: ignore[call-overload] - state[class_name] = new_class - return new_class # type: ignore[no-any-return] - - -def evaluate_annassign( - annassign: ast.AnnAssign, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate annotated assignments.""" - if annassign.value: - value = evaluate_ast(annassign.value, state, static_tools, custom_tools, authorized_imports) - set_value(annassign.target, value, state, static_tools, custom_tools, authorized_imports) - return value - return None - - -def evaluate_augassign( - expression: ast.AugAssign, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate augmented assignments (+=, -=, etc.).""" - - def get_current_value(target: ast.AST) -> Any: - if isinstance(target, ast.Name): - return state.get(target.id, 0) - elif isinstance(target, ast.Subscript): - obj = evaluate_ast(target.value, state, static_tools, custom_tools, authorized_imports) - key = evaluate_ast(target.slice, state, static_tools, custom_tools, authorized_imports) - return obj[key] - elif isinstance(target, ast.Attribute): - obj = evaluate_ast(target.value, state, static_tools, custom_tools, authorized_imports) - return getattr(obj, target.attr) - else: - raise InterpreterError(f"AugAssign not supported for {type(target)} targets.") - - current_value = get_current_value(expression.target) - value_to_add = evaluate_ast(expression.value, state, static_tools, custom_tools, authorized_imports) - - op_map = { - ast.Add: lambda a, b: a + b, - ast.Sub: lambda a, b: a - b, - ast.Mult: lambda a, b: a * b, - ast.Div: lambda a, b: a / b, - ast.Mod: lambda a, b: a % b, - ast.Pow: lambda a, b: a**b, - ast.FloorDiv: lambda a, b: a // b, - ast.BitAnd: lambda a, b: a & b, - ast.BitOr: lambda a, b: a | b, - ast.BitXor: lambda a, b: a ^ b, - ast.LShift: lambda a, b: a << b, - ast.RShift: lambda a, b: a >> b, - } - - op_type = type(expression.op) - if op_type in op_map: - current_value = op_map[op_type](current_value, value_to_add) - else: - raise InterpreterError(f"Operation {op_type.__name__} is not supported.") - - set_value(expression.target, current_value, state, static_tools, custom_tools, authorized_imports) - return current_value - - -def evaluate_boolop( - node: ast.BoolOp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate boolean operations (and, or).""" - is_short_circuit_value = (lambda x: not x) if isinstance(node.op, ast.And) else (lambda x: bool(x)) - - for value in node.values: - result = evaluate_ast(value, state, static_tools, custom_tools, authorized_imports) - if is_short_circuit_value(result): - return result - return result - - -def evaluate_binop( - binop: ast.BinOp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate binary operations.""" - left_val = evaluate_ast(binop.left, state, static_tools, custom_tools, authorized_imports) - right_val = evaluate_ast(binop.right, state, static_tools, custom_tools, authorized_imports) - - op_map = { - ast.Add: lambda a, b: a + b, - ast.Sub: lambda a, b: a - b, - ast.Mult: lambda a, b: a * b, - ast.Div: lambda a, b: a / b, - ast.Mod: lambda a, b: a % b, - ast.Pow: lambda a, b: a**b, - ast.FloorDiv: lambda a, b: a // b, - ast.BitAnd: lambda a, b: a & b, - ast.BitOr: lambda a, b: a | b, - ast.BitXor: lambda a, b: a ^ b, - ast.LShift: lambda a, b: a << b, - ast.RShift: lambda a, b: a >> b, - ast.MatMult: lambda a, b: a @ b, - } - - op_type = type(binop.op) - if op_type in op_map: - return op_map[op_type](left_val, right_val) - else: - raise NotImplementedError(f"Binary operation {op_type.__name__} is not implemented.") - - -def evaluate_assign( - assign: ast.Assign, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate assignment statements.""" - result = evaluate_ast(assign.value, state, static_tools, custom_tools, authorized_imports) - - if len(assign.targets) == 1: - set_value(assign.targets[0], result, state, static_tools, custom_tools, authorized_imports) - else: - for tgt in assign.targets: - set_value(tgt, result, state, static_tools, custom_tools, authorized_imports) - - return result - - -def set_value( - target: ast.AST, - value: Any, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Set a value to a target.""" - if isinstance(target, ast.Name): - if target.id in static_tools: - raise InterpreterError(f"Cannot assign to name '{target.id}': this would erase the existing tool!") - state[target.id] = value - elif isinstance(target, ast.Tuple) or isinstance(target, ast.List): - if not hasattr(value, "__iter__") or isinstance(value, (str, bytes)): - raise InterpreterError("Cannot unpack non-iterable value") - values = list(value) - if len(target.elts) != len(values): - raise InterpreterError("Cannot unpack: wrong number of values") - for i, elem in enumerate(target.elts): - set_value(elem, values[i], state, static_tools, custom_tools, authorized_imports) - elif isinstance(target, ast.Subscript): - obj = evaluate_ast(target.value, state, static_tools, custom_tools, authorized_imports) - key = evaluate_ast(target.slice, state, static_tools, custom_tools, authorized_imports) - obj[key] = value - elif isinstance(target, ast.Attribute): - obj = evaluate_ast(target.value, state, static_tools, custom_tools, authorized_imports) - setattr(obj, target.attr, value) - elif isinstance(target, ast.Starred): - # Handle starred assignment (e.g., *rest = ...) - set_value(target.value, value, state, static_tools, custom_tools, authorized_imports) - else: - raise InterpreterError(f"Cannot assign to {type(target).__name__}") - - -def evaluate_call( - call: ast.Call, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate function calls.""" - func, func_name = None, None - - if isinstance(call.func, ast.Call): - func = evaluate_ast(call.func, state, static_tools, custom_tools, authorized_imports) - elif isinstance(call.func, ast.Lambda): - func = evaluate_ast(call.func, state, static_tools, custom_tools, authorized_imports) - elif isinstance(call.func, ast.Attribute): - obj = evaluate_ast(call.func.value, state, static_tools, custom_tools, authorized_imports) - func_name = call.func.attr - if not hasattr(obj, func_name): - raise InterpreterError(f"Object {obj} has no attribute {func_name}") - func = getattr(obj, func_name) - elif isinstance(call.func, ast.Name): - func_name = call.func.id - if func_name in state: - func = state[func_name] - elif func_name in static_tools: - func = static_tools[func_name] - elif func_name in custom_tools: - func = custom_tools[func_name] - elif func_name in ERRORS: - func = ERRORS[func_name] - else: - raise InterpreterError( - f"Forbidden function evaluation: '{func_name}' is not among the allowed tools or defined in preceding code" - ) - elif isinstance(call.func, ast.Subscript): - func = evaluate_ast(call.func, state, static_tools, custom_tools, authorized_imports) - else: - raise InterpreterError(f"This is not a correct function: {call.func}") - - if not callable(func): - raise InterpreterError(f"This is not callable: {call.func}") - - # Evaluate arguments - args = [] - for arg in call.args: - if isinstance(arg, ast.Starred): - args.extend(evaluate_ast(arg.value, state, static_tools, custom_tools, authorized_imports)) - else: - args.append(evaluate_ast(arg, state, static_tools, custom_tools, authorized_imports)) - - kwargs = {} - for keyword in call.keywords: - if keyword.arg is None: - starred_dict = evaluate_ast(keyword.value, state, static_tools, custom_tools, authorized_imports) - if not isinstance(starred_dict, dict): - raise InterpreterError(f"Cannot unpack non-dict value in **kwargs: {type(starred_dict).__name__}") - kwargs.update(starred_dict) - else: - kwargs[keyword.arg] = evaluate_ast(keyword.value, state, static_tools, custom_tools, authorized_imports) - - # Handle special cases - if func_name == "super": - if not args: - if "__class__" in state and "self" in state: - return super(state["__class__"], state["self"]) - else: - raise InterpreterError("super() needs at least one argument") - cls = args[0] - if len(args) == 1: - return super(cls) - elif len(args) == 2: - return super(cls, args[1]) - else: - raise InterpreterError("super() takes at most 2 arguments") - elif func_name == "print": - state["_print_outputs"] += " ".join(map(str, args)) + "\n" - return None - else: - # Check for dangerous builtins - import inspect - - if (inspect.getmodule(func) == builtins) and inspect.isbuiltin(func) and (func not in static_tools.values()): - raise InterpreterError( - f"Invoking a builtin function that has not been explicitly added as a tool is not allowed ({func_name})." - ) - - if ( - hasattr(func, "__name__") - and func.__name__.startswith("__") - and func.__name__.endswith("__") - and (func.__name__ not in static_tools) - and (func.__name__ not in ALLOWED_DUNDER_METHODS) - ): - raise InterpreterError(f"Forbidden call to dunder function: {func.__name__}") - - return func(*args, **kwargs) - - -def evaluate_subscript( - subscript: ast.Subscript, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate subscript access.""" - index = evaluate_ast(subscript.slice, state, static_tools, custom_tools, authorized_imports) - value = evaluate_ast(subscript.value, state, static_tools, custom_tools, authorized_imports) - try: - return value[index] - except (KeyError, IndexError, TypeError) as e: - error_message = f"Could not index {value} with '{index}': {type(e).__name__}: {e}" - if isinstance(index, str) and isinstance(value, Mapping): - close_matches = difflib.get_close_matches(index, list(value.keys())) - if close_matches: - error_message += f". Maybe you meant one of these indexes instead: {close_matches}" - raise InterpreterError(error_message) from e - - -def evaluate_name( - name: ast.Name, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate name lookup.""" - if name.id in state: - return state[name.id] - elif name.id in static_tools: - return safer_func(static_tools[name.id], static_tools=static_tools, authorized_imports=authorized_imports) - elif name.id in custom_tools: - return custom_tools[name.id] - elif name.id in ERRORS: - return ERRORS[name.id] - - close_matches = difflib.get_close_matches(name.id, list(state.keys())) - if close_matches: - return state[close_matches[0]] - raise InterpreterError(f"The variable `{name.id}` is not defined.") - - -def evaluate_condition( - condition: ast.Compare, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> bool: - """Evaluate comparison operations.""" - left = evaluate_ast(condition.left, state, static_tools, custom_tools, authorized_imports) - - for op, comparator in zip(condition.ops, condition.comparators): - right = evaluate_ast(comparator, state, static_tools, custom_tools, authorized_imports) - - op_type = type(op) - if op_type == ast.Eq: - result = left == right - elif op_type == ast.NotEq: - result = left != right - elif op_type == ast.Lt: - result = left < right - elif op_type == ast.LtE: - result = left <= right - elif op_type == ast.Gt: - result = left > right - elif op_type == ast.GtE: - result = left >= right - elif op_type == ast.Is: - result = left is right - elif op_type == ast.IsNot: - result = left is not right - elif op_type == ast.In: - result = left in right - elif op_type == ast.NotIn: - result = left not in right - else: - raise InterpreterError(f"Unsupported comparison operator: {op_type}") - - if not result: - return False - left = right - - return True - - -def evaluate_if( - if_statement: ast.If, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate if statements.""" - result = None - test_result = evaluate_ast(if_statement.test, state, static_tools, custom_tools, authorized_imports) - - if test_result: - for line in if_statement.body: - result = evaluate_ast(line, state, static_tools, custom_tools, authorized_imports) - else: - for line in if_statement.orelse: - result = evaluate_ast(line, state, static_tools, custom_tools, authorized_imports) - - return result - - -def evaluate_for( - for_loop: ast.For, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Any: - """Evaluate for loops.""" - result = None - iterator = evaluate_ast(for_loop.iter, state, static_tools, custom_tools, authorized_imports) - - for counter in iterator: - set_value(for_loop.target, counter, state, static_tools, custom_tools, authorized_imports) - for node in for_loop.body: - try: - line_result = evaluate_ast(node, state, static_tools, custom_tools, authorized_imports) - if line_result is not None: - result = line_result - except BreakException: - return result - except ContinueException: - break - - return result - - -def _evaluate_comprehensions( - comprehensions: list[ast.comprehension], - evaluate_element: Callable[[dict[str, Any]], Any], - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Generator[Any, None, None]: - """Recursively evaluate nested comprehensions.""" - if not comprehensions: - yield evaluate_element(state) - return - - comprehension = comprehensions[0] - iter_value = evaluate_ast(comprehension.iter, state, static_tools, custom_tools, authorized_imports) - - for value in iter_value: - new_state = state.copy() - set_value(comprehension.target, value, new_state, static_tools, custom_tools, authorized_imports) - - if all( - evaluate_ast(if_clause, new_state, static_tools, custom_tools, authorized_imports) - for if_clause in comprehension.ifs - ): - yield from _evaluate_comprehensions( - comprehensions[1:], evaluate_element, new_state, static_tools, custom_tools, authorized_imports - ) - - -def evaluate_listcomp( - listcomp: ast.ListComp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> list[Any]: - """Evaluate list comprehensions.""" - return list( - _evaluate_comprehensions( - listcomp.generators, - lambda comp_state: evaluate_ast(listcomp.elt, comp_state, static_tools, custom_tools, authorized_imports), - state, - static_tools, - custom_tools, - authorized_imports, - ) - ) - - -def evaluate_setcomp( - setcomp: ast.SetComp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> set[Any]: - """Evaluate set comprehensions.""" - return set( - _evaluate_comprehensions( - setcomp.generators, - lambda comp_state: evaluate_ast(setcomp.elt, comp_state, static_tools, custom_tools, authorized_imports), - state, - static_tools, - custom_tools, - authorized_imports, - ) - ) - - -def evaluate_dictcomp( - dictcomp: ast.DictComp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> dict[Any, Any]: - """Evaluate dictionary comprehensions.""" - return dict( - _evaluate_comprehensions( - dictcomp.generators, - lambda comp_state: ( - evaluate_ast(dictcomp.key, comp_state, static_tools, custom_tools, authorized_imports), - evaluate_ast(dictcomp.value, comp_state, static_tools, custom_tools, authorized_imports), - ), - state, - static_tools, - custom_tools, - authorized_imports, - ) - ) - - -def evaluate_try( - try_node: ast.Try, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Evaluate try-except statements.""" - try: - for stmt in try_node.body: - evaluate_ast(stmt, state, static_tools, custom_tools, authorized_imports) - except Exception as e: - matched = False - for handler in try_node.handlers: - if handler.type is None or isinstance( - e, - evaluate_ast(handler.type, state, static_tools, custom_tools, authorized_imports), - ): - matched = True - if handler.name: - state[handler.name] = e - for stmt in handler.body: - evaluate_ast(stmt, state, static_tools, custom_tools, authorized_imports) - break - if not matched: - raise e - else: - if try_node.orelse: - for stmt in try_node.orelse: - evaluate_ast(stmt, state, static_tools, custom_tools, authorized_imports) - finally: - if try_node.finalbody: - for stmt in try_node.finalbody: - evaluate_ast(stmt, state, static_tools, custom_tools, authorized_imports) - - -def evaluate_raise( - raise_node: ast.Raise, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Evaluate raise statements.""" - if raise_node.exc is not None: - exc = evaluate_ast(raise_node.exc, state, static_tools, custom_tools, authorized_imports) - else: - exc = None - - if raise_node.cause is not None: - cause = evaluate_ast(raise_node.cause, state, static_tools, custom_tools, authorized_imports) - else: - cause = None - - if exc is not None: - if cause is not None: - raise exc from cause - else: - raise exc - else: - raise InterpreterError("Re-raise is not supported without an active exception") - - -def evaluate_assert( - assert_node: ast.Assert, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Evaluate assert statements.""" - test_result = evaluate_ast(assert_node.test, state, static_tools, custom_tools, authorized_imports) - if not test_result: - if assert_node.msg: - msg = evaluate_ast(assert_node.msg, state, static_tools, custom_tools, authorized_imports) - raise AssertionError(msg) - else: - test_code = ast.unparse(assert_node.test) - raise AssertionError(f"Assertion failed: {test_code}") - - -def evaluate_with( - with_node: ast.With, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Evaluate with statements.""" - contexts = [] - for item in with_node.items: - context_expr = evaluate_ast(item.context_expr, state, static_tools, custom_tools, authorized_imports) - if item.optional_vars: - if isinstance(item.optional_vars, ast.Name): - state[item.optional_vars.id] = context_expr.__enter__() - contexts.append(state[item.optional_vars.id]) - else: - # Handle other types of optional_vars - var_name = getattr(item.optional_vars, "id", None) - if var_name: - state[var_name] = context_expr.__enter__() - contexts.append(state[var_name]) - else: - context_var = context_expr.__enter__() - contexts.append(context_var) - - try: - for stmt in with_node.body: - evaluate_ast(stmt, state, static_tools, custom_tools, authorized_imports) - except Exception as e: - for context in reversed(contexts): - context.__exit__(type(e), e, e.__traceback__) - raise - else: - for context in reversed(contexts): - context.__exit__(None, None, None) - - -def evaluate_import(expression, state, authorized_imports): - """Evaluate import statements.""" - if isinstance(expression, ast.Import): - for alias in expression.names: - if check_import_authorized(alias.name, authorized_imports): - raw_module = import_module(alias.name) - state[alias.asname or alias.name] = get_safe_module(raw_module, authorized_imports) - else: - raise InterpreterError( - f"Import of {alias.name} is not allowed. Authorized imports are: {authorized_imports}" - ) - return None - elif isinstance(expression, ast.ImportFrom): - if check_import_authorized(expression.module, authorized_imports): - raw_module = __import__(expression.module, fromlist=[alias.name for alias in expression.names]) - module = get_safe_module(raw_module, authorized_imports) - - if expression.names[0].name == "*": - if hasattr(module, "__all__"): - for name in module.__all__: - state[name] = getattr(module, name) - else: - for name in dir(module): - if not name.startswith("_"): - state[name] = getattr(module, name) - else: - for alias in expression.names: - if hasattr(module, alias.name): - state[alias.asname or alias.name] = getattr(module, alias.name) - else: - raise InterpreterError(f"Module {expression.module} has no attribute {alias.name}") - else: - raise InterpreterError( - f"Import from {expression.module} is not allowed. Authorized imports are: {authorized_imports}" - ) - return None - - -def evaluate_generatorexp( - genexp: ast.GeneratorExp, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> Generator[Any, None, None]: - """Evaluate generator expressions.""" - - def generator(): - for gen in genexp.generators: - iter_value = evaluate_ast(gen.iter, state, static_tools, custom_tools, authorized_imports) - for value in iter_value: - new_state = state.copy() - set_value(gen.target, value, new_state, static_tools, custom_tools, authorized_imports) - if all( - evaluate_ast(if_clause, new_state, static_tools, custom_tools, authorized_imports) - for if_clause in gen.ifs - ): - yield evaluate_ast(genexp.elt, new_state, static_tools, custom_tools, authorized_imports) - - return generator() # type: ignore[no-any-return] - - -def evaluate_delete( - delete_node: ast.Delete, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """Evaluate delete statements.""" - for target in delete_node.targets: - if isinstance(target, ast.Name): - if target.id in state: - del state[target.id] - else: - raise InterpreterError(f"Cannot delete name '{target.id}': name is not defined") - elif isinstance(target, ast.Subscript): - obj = evaluate_ast(target.value, state, static_tools, custom_tools, authorized_imports) - index = evaluate_ast(target.slice, state, static_tools, custom_tools, authorized_imports) - try: - del obj[index] - except (TypeError, KeyError, IndexError) as e: - raise InterpreterError(f"Cannot delete index/key: {e}") - else: - raise InterpreterError(f"Deletion of {type(target).__name__} targets is not supported") - - -@safer_eval -def evaluate_ast( - expression: ast.AST, - state: dict[str, Any], - static_tools: dict[str, Callable], - custom_tools: dict[str, Callable], - authorized_imports: Optional[list[str]] = None, -) -> Any: - """ - Evaluate an abstract syntax tree using the content of the variables stored in a state - and only evaluating a given set of functions. - - Args: - expression: The AST node to evaluate. - state: A dictionary mapping variable names to values. - static_tools: Functions that may be called (cannot be overwritten). - custom_tools: Functions that may be called (can be overwritten). - authorized_imports: List of modules that can be imported. - - Returns: - The result of evaluating the expression. - """ - if authorized_imports is None: - authorized_imports = BASE_BUILTIN_MODULES - - # Check operation count - if state.setdefault("_operations_count", {"counter": 0})["counter"] >= MAX_OPERATIONS: - raise InterpreterError( - f"Reached the max number of operations of {MAX_OPERATIONS}. " - "Maybe there is an infinite loop somewhere in the code." - ) - state["_operations_count"]["counter"] += 1 - - common_params = (state, static_tools, custom_tools, authorized_imports) - - # Assignment nodes - if isinstance(expression, ast.Assign): - return evaluate_assign(expression, *common_params) - elif isinstance(expression, ast.AnnAssign): - return evaluate_annassign(expression, *common_params) - elif isinstance(expression, ast.AugAssign): - return evaluate_augassign(expression, *common_params) - - # Call nodes - elif isinstance(expression, ast.Call): - return evaluate_call(expression, *common_params) - - # Literal nodes - elif isinstance(expression, ast.Constant): - return expression.value - elif isinstance(expression, ast.Tuple): - return tuple(evaluate_ast(elt, *common_params) for elt in expression.elts) - elif isinstance(expression, ast.List): - return [evaluate_ast(elt, *common_params) for elt in expression.elts] - elif isinstance(expression, ast.Dict): - keys = (evaluate_ast(k, *common_params) for k in expression.keys) - values = (evaluate_ast(v, *common_params) for v in expression.values) - return dict(zip(keys, values)) - elif isinstance(expression, ast.Set): - return set(evaluate_ast(elt, *common_params) for elt in expression.elts) - - # Comprehensions - elif isinstance(expression, ast.GeneratorExp): - return evaluate_generatorexp(expression, *common_params) - elif isinstance(expression, ast.ListComp): - return evaluate_listcomp(expression, *common_params) - elif isinstance(expression, ast.DictComp): - return evaluate_dictcomp(expression, *common_params) - elif isinstance(expression, ast.SetComp): - return evaluate_setcomp(expression, *common_params) - - # Operations - elif isinstance(expression, ast.UnaryOp): - return evaluate_unaryop(expression, *common_params) - elif isinstance(expression, ast.BoolOp): - return evaluate_boolop(expression, *common_params) - elif isinstance(expression, ast.BinOp): - return evaluate_binop(expression, *common_params) - elif isinstance(expression, ast.Compare): - return evaluate_condition(expression, *common_params) - - # Control flow - elif isinstance(expression, ast.Break): - raise BreakException() - elif isinstance(expression, ast.Continue): - raise ContinueException() - elif isinstance(expression, ast.Return): - raise ReturnException(evaluate_ast(expression.value, *common_params) if expression.value else None) - elif isinstance(expression, ast.Pass): - return None - - # Functions and lambdas - elif isinstance(expression, ast.Lambda): - return evaluate_lambda(expression, *common_params) - elif isinstance(expression, ast.FunctionDef): - return evaluate_function_def(expression, *common_params) - - # Expressions and names - elif isinstance(expression, ast.Expr): - return evaluate_ast(expression.value, *common_params) - elif isinstance(expression, ast.Name): - return evaluate_name(expression, *common_params) - elif isinstance(expression, ast.Attribute): - return evaluate_attribute(expression, *common_params) - elif isinstance(expression, ast.Subscript): - return evaluate_subscript(expression, *common_params) - elif isinstance(expression, ast.Starred): - return evaluate_ast(expression.value, *common_params) - - # Control structures - elif isinstance(expression, ast.For): - return evaluate_for(expression, *common_params) - elif isinstance(expression, ast.While): - return evaluate_while(expression, *common_params) - elif isinstance(expression, ast.If): - return evaluate_if(expression, *common_params) - elif isinstance(expression, ast.IfExp): - test_val = evaluate_ast(expression.test, *common_params) - if test_val: - return evaluate_ast(expression.body, *common_params) - else: - return evaluate_ast(expression.orelse, *common_params) - - # Formatted strings - elif isinstance(expression, ast.FormattedValue): - value = evaluate_ast(expression.value, *common_params) - if not expression.format_spec: - return value - format_spec = evaluate_ast(expression.format_spec, *common_params) - return format(value, format_spec) - elif isinstance(expression, ast.JoinedStr): - return "".join(str(evaluate_ast(v, *common_params)) for v in expression.values) - - # Slices - elif isinstance(expression, ast.Slice): - return slice( - evaluate_ast(expression.lower, *common_params) if expression.lower else None, - evaluate_ast(expression.upper, *common_params) if expression.upper else None, - evaluate_ast(expression.step, *common_params) if expression.step else None, - ) - - # Imports - elif isinstance(expression, (ast.Import, ast.ImportFrom)): - return evaluate_import(expression, state, authorized_imports) - - # Classes - elif isinstance(expression, ast.ClassDef): - return evaluate_class_def(expression, *common_params) - - # Exception handling - elif isinstance(expression, ast.Try): - return evaluate_try(expression, *common_params) - elif isinstance(expression, ast.Raise): - return evaluate_raise(expression, *common_params) - elif isinstance(expression, ast.Assert): - return evaluate_assert(expression, *common_params) - - # With statements - elif isinstance(expression, ast.With): - return evaluate_with(expression, *common_params) - - # Delete - elif isinstance(expression, ast.Delete): - return evaluate_delete(expression, *common_params) - - # Python 3.8 Index node (backward compatibility) - elif hasattr(ast, "Index") and isinstance(expression, ast.Index): - value = getattr(expression, "value", None) - if value is not None: - return evaluate_ast(value, *common_params) - raise InterpreterError("Index node has no value attribute") - - else: - raise InterpreterError(f"{expression.__class__.__name__} is not supported.") - - -__all__ = ["evaluate_ast", "BASE_PYTHON_TOOLS", "PrintContainer", "MAX_OPERATIONS", "MAX_WHILE_ITERATIONS"] diff --git a/backend/app/core/agent/code_agent/interpreter/security.py b/backend/app/core/agent/code_agent/interpreter/security.py deleted file mode 100644 index 2bc640bbf..000000000 --- a/backend/app/core/agent/code_agent/interpreter/security.py +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env python -""" -Security module for CodeAgent Python interpreter. - -This module implements multi-layer security controls: -1. Module whitelist/blacklist -2. Dangerous function detection -3. Import authorization checks -4. Result safety verification -""" - -from collections.abc import Callable -from types import BuiltinFunctionType, ModuleType -from typing import Any, Optional - -from loguru import logger - - -class InterpreterError(Exception): - """Exception raised when there's an error in the interpreter.""" - - pass - - -class SecurityError(InterpreterError): - """Exception raised when a security violation is detected.""" - - pass - - -# ============================================================================ -# Dangerous Modules and Functions -# ============================================================================ - -# Modules that are never allowed -DANGEROUS_MODULES = [ - "os", - "subprocess", - "sys", - "socket", - "shutil", - "pathlib", - "io", - "multiprocessing", - "threading", - "concurrent", - "pty", - "tty", - "fcntl", - "signal", - "resource", - "gc", - "ctypes", - "cffi", - "importlib", - "builtins", - "__builtins__", - "pip", - "setuptools", - "pkg_resources", - "distutils", - "code", - "codeop", - "compile", - "ast", - "inspect", - "dis", - "runpy", - "asyncio.subprocess", - "webbrowser", - "antigravity", - "this", - "smtplib", - "smtpd", - "telnetlib", - "ftplib", -] - -# Specific functions that are dangerous -DANGEROUS_FUNCTIONS = [ - "builtins.compile", - "builtins.eval", - "builtins.exec", - "builtins.execfile", - "builtins.globals", - "builtins.locals", - "builtins.__import__", - "builtins.open", - "builtins.input", - "builtins.breakpoint", - "builtins.memoryview", - "os.system", - "os.popen", - "os.spawn", - "os.spawnl", - "os.spawnle", - "os.spawnlp", - "os.spawnlpe", - "os.spawnv", - "os.spawnve", - "os.spawnvp", - "os.spawnvpe", - "os.execl", - "os.execle", - "os.execlp", - "os.execlpe", - "os.execv", - "os.execve", - "os.execvp", - "os.execvpe", - "os.fork", - "os.forkpty", - "subprocess.run", - "subprocess.call", - "subprocess.check_call", - "subprocess.check_output", - "subprocess.Popen", - "socket.socket", -] - -# ============================================================================ -# Authorized Imports -# ============================================================================ - -# Base modules that are always safe to import -BASE_BUILTIN_MODULES = [ - "json", - "re", - "math", - "datetime", - "time", - "collections", - "itertools", - "functools", - "random", - "copy", - "typing", - "dataclasses", - "enum", - "string", - "operator", - "decimal", - "fractions", - "statistics", - "uuid", - "hashlib", - "hmac", - "base64", - "binascii", - "textwrap", - "unicodedata", - "pprint", - "heapq", - "bisect", - "array", - "struct", - "calendar", - "zoneinfo", - "contextlib", - "abc", - "warnings", - "numbers", - "cmath", - # Sub-modules - "collections.abc", - "typing.Union", - "typing.Optional", - "typing.List", - "typing.Dict", - "typing.Any", - "typing.Callable", - "datetime.datetime", - "datetime.date", - "datetime.time", - "datetime.timedelta", - "functools.wraps", - "functools.reduce", - "functools.partial", - "itertools.chain", - "itertools.product", - "itertools.permutations", - "itertools.combinations", -] - -# Data analysis modules (optional, can be enabled) -DATA_ANALYSIS_MODULES = [ - "pandas", - "pandas.DataFrame", - "pandas.Series", - "numpy", - "numpy.array", - "numpy.ndarray", - "matplotlib", - "matplotlib.pyplot", - "seaborn", - "plotly", - "plotly.express", - "plotly.graph_objects", - "scipy", - "scipy.stats", - "scipy.optimize", - "sklearn", - "sklearn.model_selection", - "sklearn.preprocessing", - "sklearn.linear_model", - "sklearn.tree", - "sklearn.ensemble", - "sklearn.cluster", - "sklearn.metrics", - "statsmodels", - "statsmodels.api", - "PIL", - "PIL.Image", - "cv2", - "openpyxl", - "xlrd", - "xlwt", - "csv", -] - -# Network modules (optional, requires explicit permission) -NETWORK_MODULES = [ - "requests", - "httpx", - "aiohttp", - "urllib.parse", - "urllib.error", - "http.client", - "http.server", -] - - -def check_import_authorized(import_to_check: str, authorized_imports: list[str]) -> bool: - """ - Check if an import is authorized. - - Args: - import_to_check: The module/submodule to import. - authorized_imports: List of authorized import paths. - - Returns: - True if the import is authorized, False otherwise. - """ - # Always block dangerous modules - for dangerous in DANGEROUS_MODULES: - if import_to_check == dangerous or import_to_check.startswith(f"{dangerous}."): - logger.warning(f"Blocked dangerous import: {import_to_check}") - return False - - # Check if module is in the whitelist - for authorized in authorized_imports: - # Exact match - if import_to_check == authorized: - return True - # Parent module match (e.g., 'json' authorizes 'json.decoder') - if import_to_check.startswith(f"{authorized}."): - return True - # Child module match (e.g., 'pandas.DataFrame' authorizes 'pandas') - if authorized.startswith(f"{import_to_check}."): - return True - - logger.warning(f"Import not authorized: {import_to_check}") - return False - - -def check_safer_result( - result: Any, - static_tools: dict[str, Callable], - authorized_imports: list[str], -) -> None: - """ - Check if the result of an operation is safe. - - Args: - result: The result to check. - static_tools: Allowed static tools. - authorized_imports: List of authorized imports. - - Raises: - SecurityError: If the result is unsafe. - """ - # Check for dangerous module access - if isinstance(result, ModuleType): - module_name = result.__name__.split(".")[0] - if module_name in DANGEROUS_MODULES: - raise SecurityError(f"Access to dangerous module '{module_name}' is forbidden") - if not check_import_authorized(result.__name__, authorized_imports): - raise SecurityError(f"Access to unauthorized module '{result.__name__}' is forbidden") - - # Check for dangerous function types - if isinstance(result, BuiltinFunctionType): - func_module = getattr(result, "__module__", "") or "" - func_name = getattr(result, "__name__", "") or "" - full_name = f"{func_module}.{func_name}" - - if full_name in DANGEROUS_FUNCTIONS: - raise SecurityError(f"Access to dangerous function '{full_name}' is forbidden") - - # Check for callable with dangerous attributes - if callable(result) and hasattr(result, "__self__"): - self_obj = result.__self__ - if isinstance(self_obj, ModuleType): - module_name = self_obj.__name__.split(".")[0] - if module_name in DANGEROUS_MODULES: - raise SecurityError(f"Access to method of dangerous module '{module_name}' is forbidden") - - -def get_allowed_imports( - base: bool = True, - data_analysis: bool = False, - network: bool = False, - custom: Optional[list[str]] = None, -) -> list[str]: - """ - Get the list of allowed imports based on configuration. - - Args: - base: Include base builtin modules. - data_analysis: Include data analysis modules (pandas, numpy, etc.). - network: Include network modules (requests, etc.). - custom: Additional custom modules to allow. - - Returns: - List of authorized import paths. - """ - allowed = [] - - if base: - allowed.extend(BASE_BUILTIN_MODULES) - - if data_analysis: - allowed.extend(DATA_ANALYSIS_MODULES) - - if network: - allowed.extend(NETWORK_MODULES) - - if custom: - allowed.extend(custom) - - return allowed - - -def is_safe_code(code: str) -> tuple[bool, str | None]: - """ - Quick static analysis to check if code might be unsafe. - - Args: - code: The Python code to check. - - Returns: - Tuple of (is_safe, error_message). - """ - import re - - # Patterns that indicate potentially dangerous code - dangerous_patterns = [ - (r"\b__\w+__\b(?!\s*\()", "Access to dunder attributes"), - (r"\bexec\s*\(", "Use of exec()"), - (r"\beval\s*\(", "Use of eval()"), - (r"\bcompile\s*\(", "Use of compile()"), - (r"\b__import__\s*\(", "Use of __import__()"), - (r"\bgetattr\s*\([^,]+,\s*['\"]__", "Dunder access via getattr()"), - (r"\bsetattr\s*\([^,]+,\s*['\"]__", "Dunder access via setattr()"), - (r"\bglobals\s*\(\)", "Access to globals()"), - (r"\blocals\s*\(\)", "Access to locals()"), - (r"\bopen\s*\([^)]*['\"]w", "File write with open()"), - (r"\bos\s*\.\s*system\s*\(", "Use of os.system()"), - (r"\bsubprocess\s*\.", "Use of subprocess module"), - (r"\bsocket\s*\.", "Use of socket module"), - ] - - for pattern, message in dangerous_patterns: - if re.search(pattern, code): - return False, message - - return True, None - - -def validate_import_statement(code: str, authorized_imports: list[str]) -> tuple[bool, str | None]: - """ - Validate import statements in code. - - Args: - code: The Python code to validate. - authorized_imports: List of authorized imports. - - Returns: - Tuple of (is_valid, error_message). - """ - import ast - - try: - tree = ast.parse(code) - except SyntaxError: - return True, None # Let the interpreter handle syntax errors - - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - if not check_import_authorized(alias.name, authorized_imports): - return False, f"Import of '{alias.name}' is not allowed" - elif isinstance(node, ast.ImportFrom): - if node.module and not check_import_authorized(node.module, authorized_imports): - return False, f"Import from '{node.module}' is not allowed" - - return True, None - - -__all__ = [ - "InterpreterError", - "SecurityError", - "DANGEROUS_MODULES", - "DANGEROUS_FUNCTIONS", - "BASE_BUILTIN_MODULES", - "DATA_ANALYSIS_MODULES", - "NETWORK_MODULES", - "check_import_authorized", - "check_safer_result", - "get_allowed_imports", - "is_safe_code", - "validate_import_statement", -] diff --git a/backend/app/core/agent/code_agent/loop.py b/backend/app/core/agent/code_agent/loop.py deleted file mode 100644 index d8490c406..000000000 --- a/backend/app/core/agent/code_agent/loop.py +++ /dev/null @@ -1,540 +0,0 @@ -#!/usr/bin/env python -""" -CodeAgent Loop - The core Thought-Code-Observation iteration engine. - -This module implements the main execution loop for CodeAgent, handling -the iterative process of thinking, generating code, executing it, -and observing results. -""" - -import asyncio -from collections.abc import AsyncGenerator, Callable -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Optional - -from loguru import logger - -from .executor import LocalPythonExecutor, PythonExecutor -from .memory import AgentMemory, PlanningStep, StepMetrics -from .parser import ( - ParsingError, - clean_code, - extract_thought_and_code, - format_observation, - validate_python_syntax, -) - -# JSON schema for structured code agent response -CODEAGENT_RESPONSE_SCHEMA = { - "type": "json_schema", - "json_schema": { - "name": "code_agent_response", - "schema": { - "type": "object", - "properties": { - "thought": { - "type": "string", - "description": "The reasoning and planning for the current step", - }, - "code": { - "type": "string", - "description": "Python code to execute", - }, - }, - "required": ["thought", "code"], - }, - }, -} - - -@dataclass -class LoopConfig: - """Configuration for the CodeAgent loop.""" - - # Maximum number of steps before stopping - max_steps: int = 20 - - # Maximum consecutive errors before stopping - max_consecutive_errors: int = 3 - - # Maximum total execution time in seconds - max_execution_time: float = 300.0 - - # Whether to enable planning - enable_planning: bool = False - - # Whether to update plan during execution - enable_plan_updates: bool = False - - # Interval (in steps) between plan updates - plan_update_interval: int = 5 - - # Maximum observation length in characters - max_observation_length: int = 10000 - - # Whether to stream step events - streaming: bool = True - - # Verbose logging - verbose: bool = False - - # Use structured outputs (JSON schema) if LLM supports it - use_structured_outputs: bool = False - - # Response format for structured outputs (passed to LLM) - response_format: dict | None = None - - -@dataclass -class StepEvent: - """Event emitted during loop execution.""" - - event_type: str # "thought", "code", "observation", "error", "final_answer", "planning" - content: Any - step_number: int - metadata: Optional[dict] = None - - def to_dict(self) -> dict: - return { - "type": self.event_type, - "content": self.content, - "step": self.step_number, - "metadata": self.metadata or {}, - } - - -class CodeAgentLoop: - """ - The core execution loop for CodeAgent. - - Implements the Thought → Code → Observation cycle: - 1. LLM generates thought and code - 2. Code is executed in the Python executor - 3. Observation (output/error) is fed back to LLM - 4. Repeat until final_answer() is called or max steps reached - """ - - def __init__( - self, - llm_call: Callable[[str], str | AsyncGenerator[str, None]], - executor: Optional[PythonExecutor] = None, - tools: Optional[dict[str, Callable]] = None, - config: Optional[LoopConfig] = None, - ): - """ - Initialize the CodeAgent loop. - - Args: - llm_call: Function to call the LLM. Takes prompt string, returns response. - Can be sync or async, can return string or async generator. - executor: Python executor to use. Defaults to LocalPythonExecutor. - tools: Tools to inject into the executor. - config: Loop configuration. - """ - self.llm_call = llm_call - self.config = config or LoopConfig() - self.memory = AgentMemory(max_steps=self.config.max_steps * 2) - - # Initialize executor - if executor is None: - self.executor = LocalPythonExecutor(enable_data_analysis=True) - else: - self.executor = executor # type: ignore[assignment] - - # Inject tools - if tools: - self.executor.send_tools(tools) - - # State - self._current_step = 0 - self._consecutive_errors = 0 - self._is_running = False - self._should_stop = False - - # Load prompts - self._system_prompt = self._load_prompt("system.md") - self._planning_prompt = self._load_prompt("planning.md") - - def _load_prompt(self, filename: str) -> str: - """Load a prompt template from file.""" - prompt_path = Path(__file__).parent / "prompts" / filename - if prompt_path.exists(): - return prompt_path.read_text(encoding="utf-8") - logger.warning(f"Prompt file not found: {prompt_path}") - return "" - - def _format_system_prompt( - self, - task: str, - tool_descriptions: str = "", - ) -> str: - """Format the system prompt with task and tools.""" - prompt = self._system_prompt - prompt = prompt.replace("{{task}}", task) - prompt = prompt.replace("{{tool_descriptions}}", tool_descriptions or "No tools available") - return prompt - - def _build_prompt(self, task: str, history: str = "") -> str: - """Build the full prompt including history.""" - tool_desc = self._get_tool_descriptions() - system_prompt = self._format_system_prompt(task, tool_desc) - - if history: - return f"{system_prompt}\n\n## Execution History\n\n{history}\n\nPlease continue executing the task." - return system_prompt - - def _get_tool_descriptions(self) -> str: - """Get descriptions of available tools.""" - tools = getattr(self.executor, "static_tools", {}) - descriptions = [] - - for name, func in tools.items(): - if name.startswith("_") or name in ("print", "final_answer"): - continue - doc = getattr(func, "__doc__", "") or "No description" - doc = doc.strip().split("\n")[0] # First line only - descriptions.append(f"- `{name}`: {doc}") - - return "\n".join(descriptions) if descriptions else "No additional tools" - - async def run(self, task: str) -> Any: - """ - Run the agent loop on a task. - - Args: - task: The task description. - - Returns: - The final answer. - - Raises: - RuntimeError: If max steps reached without final answer. - """ - result = None - async for event in self.run_stream(task): - if event.event_type == "final_answer": - result = event.content - return result - - async def run_stream(self, task: str) -> AsyncGenerator[StepEvent, None]: - """ - Run the agent loop with streaming events. - - Args: - task: The task description. - - Yields: - StepEvent objects for each significant event. - """ - import time - - self._is_running = True - self._should_stop = False - self._current_step = 0 - self._consecutive_errors = 0 - - # Initialize memory - self.memory.reset() - self.memory.task = task - self.memory.system_prompt = self._format_system_prompt(task) - - start_time = time.time() - - try: - # Optional: Initial planning - if self.config.enable_planning: - async for event in self._do_planning(task, initial=True): - yield event - - # Main execution loop - while self._current_step < self.config.max_steps and not self._should_stop: - # Check timeout - if time.time() - start_time > self.config.max_execution_time: - yield StepEvent( - event_type="error", - content="Execution timeout reached", - step_number=self._current_step, - ) - break - - # Execute one step - async for event in self._step(): - yield event - - if event.event_type == "final_answer": - self._should_stop = True - return - - if event.event_type == "error": - self._consecutive_errors += 1 - if self._consecutive_errors >= self.config.max_consecutive_errors: - yield StepEvent( - event_type="error", - content=f"Too many consecutive errors ({self._consecutive_errors})", - step_number=self._current_step, - ) - self._should_stop = True - return - else: - self._consecutive_errors = 0 - - self._current_step += 1 - - # Optional: Update planning - if ( - self.config.enable_plan_updates - and self._current_step > 0 - and self._current_step % self.config.plan_update_interval == 0 - ): - async for event in self._do_planning(task, initial=False): - yield event - - # Max steps reached without final answer - if not self._should_stop: - yield StepEvent( - event_type="error", - content=f"Max steps ({self.config.max_steps}) reached without final answer", - step_number=self._current_step, - ) - - finally: - self._is_running = False - self.memory.complete() - - async def _step(self) -> AsyncGenerator[StepEvent, None]: - """Execute a single Thought-Code-Observation step.""" - step = self.memory.create_action_step() - step.metrics = StepMetrics() - - # Build prompt with history - history = self.memory.get_history_for_prompt(include_thoughts=True) - prompt = self._build_prompt(self.memory.task, history) - - # Get LLM response - try: - llm_output = await self._call_llm(prompt) - step.llm_output = llm_output - except Exception as e: - step.error = f"LLM call failed: {str(e)}" - step.metrics.complete() - self.memory.add_step(step) - yield StepEvent("error", step.error, step.step_number) - return - - # Parse thought and code - try: - thought, code = extract_thought_and_code(llm_output) - step.thought = thought - step.code = clean_code(code) - except ParsingError as e: - step.error = f"Failed to parse LLM output: {str(e)}" - step.metrics.complete() - self.memory.add_step(step) - yield StepEvent("error", step.error, step.step_number) - return - - # Emit thought event - if step.thought: - yield StepEvent("thought", step.thought, step.step_number) - - # Emit code event - if step.code: - yield StepEvent("code", step.code, step.step_number) - else: - step.error = "No code generated" - step.metrics.complete() - self.memory.add_step(step) - yield StepEvent("error", step.error, step.step_number) - return - - # Validate syntax - is_valid, syntax_error = validate_python_syntax(step.code) - if not is_valid: - step.error = f"Syntax error: {syntax_error}" - step.observation = step.error - step.metrics.complete() - self.memory.add_step(step) - yield StepEvent("observation", step.observation, step.step_number) - return - - # Execute code - try: - result = self.executor(step.code) - - if result.is_final_answer: - step.is_final_answer = True - step.final_answer = result.output - step.observation = format_observation( - result.output, - result.logs, - max_length=self.config.max_observation_length, - ) - step.metrics.complete() - self.memory.add_step(step) - yield StepEvent("final_answer", result.output, step.step_number) - return - - if result.error: - step.error = result.error - step.observation = format_observation( - None, - result.logs, - result.error, - max_length=self.config.max_observation_length, - ) - else: - step.observation = format_observation( - result.output, - result.logs, - max_length=self.config.max_observation_length, - ) - - except Exception as e: - step.error = f"Execution error: {str(e)}" - step.observation = step.error - logger.exception(f"Error executing code: {e}") - - step.metrics.complete() - self.memory.add_step(step) - - yield StepEvent( - "observation", - step.observation, - step.step_number, - {"error": step.error} if step.error else {}, - ) - - async def _do_planning( - self, - task: str, - initial: bool = True, - ) -> AsyncGenerator[StepEvent, None]: - """Execute a planning step.""" - planning_step = PlanningStep(is_update=not initial) - planning_step.metrics = StepMetrics() - - if initial: - prompt = self._planning_prompt.split("---")[0] # Initial planning section - prompt = prompt.replace("{{task}}", task) - prompt = prompt.replace("{{tool_descriptions}}", self._get_tool_descriptions()) - else: - prompt = self._planning_prompt.split("---")[1] # Update planning section - prompt = prompt.replace("{{task}}", task) - prompt = prompt.replace("{{original_plan}}", self.memory.current_plan) - prompt = prompt.replace("{{progress}}", self._get_progress_summary()) - prompt = prompt.replace("{{issues}}", self._get_issues_summary()) - planning_step.previous_plan = self.memory.current_plan - - try: - llm_output = await self._call_llm(prompt) - planning_step.llm_output = llm_output - planning_step.plan = llm_output - self.memory.current_plan = llm_output - - planning_step.metrics.complete() - self.memory.add_step(planning_step) - - yield StepEvent( - "planning", - planning_step.plan, - self._current_step, - {"is_update": not initial}, - ) - - except Exception as e: - logger.error(f"Planning failed: {e}") - yield StepEvent( - "error", - f"Planning failed: {str(e)}", - self._current_step, - ) - - def _get_progress_summary(self) -> str: - """Get a summary of execution progress.""" - action_steps = self.memory.action_steps - completed = sum(1 for s in action_steps if s.success) - failed = sum(1 for s in action_steps if s.error) - return f"Executed {len(action_steps)} steps: {completed} succeeded, {failed} failed" - - def _get_issues_summary(self) -> str: - """Get a summary of encountered issues.""" - action_steps = self.memory.action_steps - issues = [s.error for s in action_steps if s.error] - if not issues: - return "No issues so far" - return "\n".join(f"- {issue}" for issue in issues[-3:]) # Last 3 issues - - async def _call_llm(self, prompt: str) -> str: - """Call the LLM and get response.""" - result = self.llm_call(prompt) - - # Handle async generator (streaming) - if hasattr(result, "__anext__") and not isinstance(result, str): - chunks = [] - async for chunk in result: # type: ignore[misc] - chunks.append(chunk) - return "".join(chunks) - - # Handle coroutine - if asyncio.iscoroutine(result): - return str(await result) - - # Handle sync result - return str(result) - - def stop(self) -> None: - """Request the loop to stop.""" - self._should_stop = True - - @property - def is_running(self) -> bool: - """Check if the loop is currently running.""" - return self._is_running - - @property - def current_step(self) -> int: - """Get the current step number.""" - return self._current_step - - -def create_simple_llm_call(model_name: str = "gpt-4"): - """ - Create a simple LLM call function using langchain. - - Args: - model_name: The model to use. - - Returns: - Async function that calls the LLM. - """ - try: - from langchain_openai import ChatOpenAI - - llm = ChatOpenAI(model=model_name, temperature=0) - - async def call_llm(prompt: str) -> str: - response = await llm.ainvoke(prompt) - content = response.content if hasattr(response, "content") else str(response) - if isinstance(content, list): - # Handle list of content blocks - return " ".join(str(item) for item in content) - return str(content) - - return call_llm - - except ImportError: - logger.warning("langchain_openai not installed, returning mock LLM") - - async def mock_call_llm(prompt: str) -> str: - return "Thought: This is a mock response.\n\nCode:\n```python\nfinal_answer('Mock response')\n```" - - return mock_call_llm - - -__all__ = [ - "LoopConfig", - "StepEvent", - "CodeAgentLoop", - "create_simple_llm_call", -] diff --git a/backend/app/core/agent/code_agent/memory.py b/backend/app/core/agent/code_agent/memory.py deleted file mode 100644 index c04544df3..000000000 --- a/backend/app/core/agent/code_agent/memory.py +++ /dev/null @@ -1,523 +0,0 @@ -#!/usr/bin/env python -""" -Memory management for CodeAgent. - -This module implements AgentMemory, ActionStep, and related types -for managing agent execution history and state. -""" - -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from typing import Any, Dict, Optional - - -class StepType(str, Enum): - """Types of steps in agent execution.""" - - SYSTEM_PROMPT = "system_prompt" - USER_MESSAGE = "user_message" - THOUGHT = "thought" - ACTION = "action" # Code execution - OBSERVATION = "observation" - TOOL_CALL = "tool_call" - TOOL_RESPONSE = "tool_response" - PLANNING = "planning" - FINAL_ANSWER = "final_answer" - ERROR = "error" - - -@dataclass -class StepMetrics: - """Metrics for a single step.""" - - start_time: datetime = field(default_factory=datetime.now) - end_time: datetime | None = None - duration_ms: float = 0.0 - input_tokens: int = 0 - output_tokens: int = 0 - total_tokens: int = 0 - - def complete(self) -> None: - """Mark the step as complete and calculate duration.""" - self.end_time = datetime.now() - self.duration_ms = (self.end_time - self.start_time).total_seconds() * 1000 - - -@dataclass -class ChatMessage: - """Represents a chat message for LLM interaction.""" - - role: str # "system", "user", "assistant" - content: str - - def dict(self) -> dict[str, str]: - """Convert to dictionary.""" - return {"role": self.role, "content": self.content} - - -@dataclass -class ActionStep: - """ - Represents a single code action step in the agent's execution. - - This is the core unit of the Thought-Code-Observation cycle. - """ - - # Step identification - step_number: int - - # The thought/reasoning before the action - thought: str = "" - - # The code to execute - code: str = "" - - # The observation/output from executing the code - observation: str = "" - - # Raw LLM output that produced this step - llm_output: str = "" - - # Error if any - error: str | None = None - - # Whether this step produced the final answer - is_final_answer: bool = False - - # The final answer if is_final_answer is True - final_answer: Any = None - - # Metrics - metrics: StepMetrics = field(default_factory=StepMetrics) - - # Metadata - metadata: dict = field(default_factory=dict) - - @property - def success(self) -> bool: - """Check if this step executed successfully.""" - return self.error is None - - def format_for_prompt(self, include_thought: bool = True) -> str: - """Format this step for inclusion in a prompt.""" - parts = [] - - if include_thought and self.thought: - parts.append(f"Thought: {self.thought}") - - if self.code: - parts.append(f"Code:\n```python\n{self.code}\n```") - - if self.observation: - parts.append(f"Observation: {self.observation}") - - if self.error: - parts.append(f"Error: {self.error}") - - return "\n\n".join(parts) - - def to_messages(self, summary_mode: bool = False) -> list[ChatMessage]: - """ - Convert this step to a list of chat messages. - - Args: - summary_mode: If True, include only essential information. - - Returns: - List of ChatMessage objects. - """ - messages = [] - - # Assistant message with thought and code - assistant_content = "" - if self.thought: - assistant_content += f"Thought: {self.thought}\n\n" - if self.code: - if summary_mode: - # In summary mode, just indicate code was run - assistant_content += f"Code: [executed {len(self.code.splitlines())} lines]" - else: - assistant_content += f"Code:\n```python\n{self.code}\n```" - - if assistant_content: - messages.append(ChatMessage(role="assistant", content=assistant_content.strip())) - - # User message with observation (code output is user feedback) - if self.observation or self.error: - if self.error: - obs_content = f"Error: {self.error}" - else: - if summary_mode and len(self.observation) > 500: - obs_content = f"Observation: {self.observation[:500]}... [truncated]" - else: - obs_content = f"Observation: {self.observation}" - messages.append(ChatMessage(role="user", content=obs_content)) - - return messages - - def __repr__(self) -> str: - return f"ActionStep(step={self.step_number}, success={self.success}, final={self.is_final_answer})" - - -@dataclass -class PlanningStep: - """ - Represents a planning step in the agent's execution. - - Used when the agent needs to create or update a plan. - """ - - # The plan content - plan: str = "" - - # Whether this is an update to an existing plan - is_update: bool = False - - # The previous plan (if updating) - previous_plan: str = "" - - # Raw LLM output - llm_output: str = "" - - # Metrics - metrics: StepMetrics = field(default_factory=StepMetrics) - - def format_for_prompt(self) -> str: - """Format this planning step for inclusion in a prompt.""" - if self.is_update: - return f"Updated Plan:\n{self.plan}" - return f"Plan:\n{self.plan}" - - def to_messages(self, summary_mode: bool = False) -> list[ChatMessage]: - """ - Convert this step to a list of chat messages. - - Args: - summary_mode: If True, include only essential information. - - Returns: - List of ChatMessage objects. - """ - prefix = "Updated Plan" if self.is_update else "Plan" - content = f"{prefix}:\n{self.plan}" - - if summary_mode and len(content) > 500: - content = content[:500] + "... [truncated]" - - return [ChatMessage(role="assistant", content=content)] - - -@dataclass -class ToolCallStep: - """Represents a tool call step.""" - - tool_name: str - tool_args: dict = field(default_factory=dict) - tool_result: Any = None - error: str | None = None - metrics: StepMetrics = field(default_factory=StepMetrics) - - -@dataclass -class MessageStep: - """Represents a message step (system or user).""" - - role: str # "system", "user", "assistant" - content: str - timestamp: datetime = field(default_factory=datetime.now) - - -class AgentMemory: - """ - Memory management for CodeAgent execution. - - Stores and manages the history of steps during agent execution, - including thoughts, actions, observations, and planning. - """ - - def __init__(self, max_steps: int = 100): - """ - Initialize agent memory. - - Args: - max_steps: Maximum number of steps to retain. - """ - self.max_steps = max_steps - self._steps: list[ActionStep | PlanningStep | ToolCallStep | MessageStep] = [] - self._task: str = "" - self._task_images: list[str] = [] - self._system_prompt: str = "" - self._current_plan: str = "" - self._start_time: datetime = datetime.now() - self._end_time: datetime | None = None - - # Token usage - self.total_input_tokens: int = 0 - self.total_output_tokens: int = 0 - - @property - def task(self) -> str: - """Get the current task.""" - return self._task - - @task.setter - def task(self, value: str) -> None: - """Set the current task.""" - self._task = value - - @property - def system_prompt(self) -> str: - """Get the system prompt.""" - return self._system_prompt - - @system_prompt.setter - def system_prompt(self, value: str) -> None: - """Set the system prompt.""" - self._system_prompt = value - - @property - def current_plan(self) -> str: - """Get the current plan.""" - return self._current_plan - - @current_plan.setter - def current_plan(self, value: str) -> None: - """Set the current plan.""" - self._current_plan = value - - @property - def steps(self) -> list[ActionStep | PlanningStep | ToolCallStep | MessageStep]: - """Get all steps.""" - return self._steps - - @property - def action_steps(self) -> list[ActionStep]: - """Get only action steps.""" - return [s for s in self._steps if isinstance(s, ActionStep)] - - @property - def step_count(self) -> int: - """Get the number of action steps.""" - return len(self.action_steps) - - def add_step(self, step: ActionStep | PlanningStep | ToolCallStep | MessageStep) -> None: - """ - Add a step to memory. - - Args: - step: The step to add. - """ - self._steps.append(step) - - # Trim if exceeding max - if len(self._steps) > self.max_steps: - # Keep first step (usually system prompt) and last steps - self._steps = [self._steps[0]] + self._steps[-(self.max_steps - 1) :] - - def create_action_step(self) -> ActionStep: - """Create a new action step with the next step number.""" - step_num = len(self.action_steps) + 1 - step = ActionStep(step_number=step_num) - return step - - def get_last_step(self) -> ActionStep | PlanningStep | ToolCallStep | MessageStep | None: - """Get the last step.""" - return self._steps[-1] if self._steps else None - - def get_last_action_step(self) -> ActionStep | None: - """Get the last action step.""" - action_steps = self.action_steps - return action_steps[-1] if action_steps else None - - def get_history_for_prompt( - self, - max_tokens: Optional[int] = None, - include_system: bool = True, - include_thoughts: bool = True, - ) -> str: - """ - Get formatted history for inclusion in a prompt. - - Args: - max_tokens: Maximum tokens to include (approximate). - include_system: Include system prompt. - include_thoughts: Include thought sections. - - Returns: - Formatted history string. - """ - parts = [] - - if include_system and self._system_prompt: - parts.append(f"System: {self._system_prompt}") - - if self._task: - parts.append(f"Task: {self._task}") - - if self._current_plan: - parts.append(f"Current Plan:\n{self._current_plan}") - - for step in self._steps: - if isinstance(step, ActionStep): - parts.append(step.format_for_prompt(include_thought=include_thoughts)) - elif isinstance(step, PlanningStep): - parts.append(step.format_for_prompt()) - elif isinstance(step, MessageStep): - parts.append(f"{step.role.capitalize()}: {step.content}") - - history = "\n\n---\n\n".join(parts) - - # Truncate if needed (rough approximation: 4 chars per token) - if max_tokens and len(history) > max_tokens * 4: - history = history[-(max_tokens * 4) :] - history = "...[truncated]\n\n" + history - - return history - - def get_total_duration_ms(self) -> float: - """Get total execution duration in milliseconds.""" - end = self._end_time or datetime.now() - return (end - self._start_time).total_seconds() * 1000 - - def complete(self) -> None: - """Mark execution as complete.""" - self._end_time = datetime.now() - - def reset(self) -> None: - """Reset memory to initial state.""" - self._steps = [] - self._task = "" - self._task_images = [] - self._current_plan = "" - self._start_time = datetime.now() - self._end_time = None - self.total_input_tokens = 0 - self.total_output_tokens = 0 - - def to_dict(self) -> dict: - """Convert memory to dictionary for serialization.""" - return { - "task": self._task, - "system_prompt": self._system_prompt, - "current_plan": self._current_plan, - "step_count": self.step_count, - "total_duration_ms": self.get_total_duration_ms(), - "total_input_tokens": self.total_input_tokens, - "total_output_tokens": self.total_output_tokens, - "steps": [ - { - "type": type(s).__name__, - "data": s.__dict__ if hasattr(s, "__dict__") else str(s), - } - for s in self._steps - ], - } - - def to_messages(self, summary_mode: bool = False) -> list[ChatMessage]: - """ - Convert the entire memory to a list of chat messages. - - This is useful for sending the conversation history to an LLM - in a standardized message format. - - Args: - summary_mode: If True, use condensed representations. - - Returns: - List of ChatMessage objects representing the conversation. - """ - messages = [] - - # System prompt - if self._system_prompt: - messages.append(ChatMessage(role="system", content=self._system_prompt)) - - # Task as user message - if self._task: - task_content = f"Task: {self._task}" - if self._current_plan: - task_content += f"\n\nCurrent Plan:\n{self._current_plan}" - messages.append(ChatMessage(role="user", content=task_content)) - - # All steps - for step in self._steps: - if hasattr(step, "to_messages"): - step_messages = step.to_messages(summary_mode=summary_mode) - messages.extend(step_messages) - elif isinstance(step, MessageStep): - messages.append(ChatMessage(role=step.role, content=step.content)) - - return messages - - def get_full_steps(self) -> list[Dict[str, Any]]: - """ - Get all steps as dictionaries with full information. - - Returns: - List of step dictionaries. - """ - result: list[Dict[str, Any]] = [] - for step in self._steps: - step_dict: Dict[str, Any] = { - "type": type(step).__name__, - } - if isinstance(step, ActionStep): - step_dict.update( - { - "step_number": step.step_number, - "thought": step.thought, - "code": step.code, - "observation": step.observation, - "error": step.error, - "is_final_answer": step.is_final_answer, - "success": step.success, - } - ) - elif isinstance(step, PlanningStep): - step_dict.update( - { - "plan": step.plan, - "is_update": step.is_update, - } - ) - elif isinstance(step, MessageStep): - step_dict.update( - { - "role": step.role, - "content": step.content, - } - ) - result.append(step_dict) - return result - - def return_full_code(self) -> str: - """ - Return all executed code as a single concatenated string. - - Useful for debugging or reviewing what code was run. - - Returns: - All executed code blocks joined with newlines. - """ - code_blocks = [] - for step in self.action_steps: - if step.code: - code_blocks.append(f"# Step {step.step_number}") - code_blocks.append(step.code) - return "\n\n".join(code_blocks) - - def __repr__(self) -> str: - return f"AgentMemory(steps={len(self._steps)}, action_steps={self.step_count})" - - -__all__ = [ - "StepType", - "StepMetrics", - "ChatMessage", - "ActionStep", - "PlanningStep", - "ToolCallStep", - "MessageStep", - "AgentMemory", -] diff --git a/backend/app/core/agent/code_agent/monitoring.py b/backend/app/core/agent/code_agent/monitoring.py deleted file mode 100644 index 1294f4441..000000000 --- a/backend/app/core/agent/code_agent/monitoring.py +++ /dev/null @@ -1,595 +0,0 @@ -#!/usr/bin/env python -""" -Monitoring and logging for CodeAgent. - -This module provides monitoring, token tracking, timing, and logging -utilities for agent execution. -""" - -from __future__ import annotations - -import time -from dataclasses import dataclass, field -from enum import IntEnum -from typing import TYPE_CHECKING, Any, Optional - -try: - from rich import box - from rich.console import Console, Group - from rich.panel import Panel - from rich.rule import Rule - from rich.syntax import Syntax - from rich.table import Table - from rich.tree import Tree - - HAS_RICH = True -except ImportError: - HAS_RICH = False - Console = None # type: ignore[assignment, misc] - -if TYPE_CHECKING: - from .memory import ActionStep, PlanningStep - - -__all__ = ["AgentLogger", "LogLevel", "Monitor", "TokenUsage", "Timing"] - - -# Color constants for rich output -YELLOW_HEX = "#d4b702" -BLUE_HEX = "#1E90FF" - - -@dataclass -class TokenUsage: - """ - Token usage information for a step or entire run. - - Attributes: - input_tokens: Number of tokens in the input/prompt. - output_tokens: Number of tokens in the output/response. - total_tokens: Total tokens used (auto-calculated). - """ - - input_tokens: int - output_tokens: int - total_tokens: int = field(init=False) - - def __post_init__(self): - self.total_tokens = self.input_tokens + self.output_tokens - - def __add__(self, other: "TokenUsage") -> "TokenUsage": - """Add two TokenUsage instances together.""" - return TokenUsage( - input_tokens=self.input_tokens + other.input_tokens, - output_tokens=self.output_tokens + other.output_tokens, - ) - - def dict(self) -> dict[str, int]: - """Convert to dictionary.""" - return { - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "total_tokens": self.total_tokens, - } - - def __repr__(self) -> str: - return f"TokenUsage(input={self.input_tokens}, output={self.output_tokens}, total={self.total_tokens})" - - -@dataclass -class Timing: - """ - Timing information for a step or run. - - Attributes: - start_time: Unix timestamp when the operation started. - end_time: Unix timestamp when the operation ended (None if still running). - """ - - start_time: float - end_time: float | None = None - - @classmethod - def start_now(cls) -> "Timing": - """Create a new Timing starting now.""" - return cls(start_time=time.time()) - - def stop(self) -> None: - """Mark the timing as complete.""" - self.end_time = time.time() - - @property - def duration(self) -> float | None: - """Get the duration in seconds (None if not complete).""" - if self.end_time is None: - return None - return self.end_time - self.start_time - - @property - def duration_ms(self) -> float | None: - """Get the duration in milliseconds (None if not complete).""" - duration = self.duration - return duration * 1000 if duration is not None else None - - def dict(self) -> dict[str, Any]: - """Convert to dictionary.""" - return { - "start_time": self.start_time, - "end_time": self.end_time, - "duration": self.duration, - } - - def __repr__(self) -> str: - return f"Timing(duration={self.duration:.3f}s)" if self.duration else "Timing(running)" - - -class Monitor: - """ - Monitors agent execution, tracking step durations and token usage. - - The monitor collects metrics during agent execution and can provide - summary statistics at the end. - - Attributes: - step_durations: List of durations for each step. - total_input_token_count: Total input tokens across all steps. - total_output_token_count: Total output tokens across all steps. - """ - - def __init__(self, logger: "AgentLogger | None" = None): - """ - Initialize the monitor. - - Args: - logger: Optional logger for output. If None, metrics are tracked silently. - """ - self.logger = logger - self.step_durations: list[float] = [] - self.total_input_token_count: int = 0 - self.total_output_token_count: int = 0 - self._start_time: float | None = None - self._end_time: float | None = None - - def start(self) -> None: - """Start monitoring a new run.""" - self._start_time = time.time() - self._end_time = None - - def stop(self) -> None: - """Stop monitoring the current run.""" - self._end_time = time.time() - - def get_total_token_counts(self) -> TokenUsage: - """Get the total token usage across all steps.""" - return TokenUsage( - input_tokens=self.total_input_token_count, - output_tokens=self.total_output_token_count, - ) - - def get_total_duration(self) -> float | None: - """Get the total run duration in seconds.""" - if self._start_time is None: - return None - end = self._end_time or time.time() - return end - self._start_time - - def reset(self) -> None: - """Reset all metrics.""" - self.step_durations = [] - self.total_input_token_count = 0 - self.total_output_token_count = 0 - self._start_time = None - self._end_time = None - - def update_metrics( - self, - step_log: "ActionStep | PlanningStep | None" = None, - duration: float | None = None, - token_usage: TokenUsage | None = None, - ) -> None: - """ - Update metrics with a new step's data. - - Args: - step_log: A memory step with timing and token_usage attributes. - duration: Alternatively, provide duration directly. - token_usage: Alternatively, provide token usage directly. - """ - # Get duration from step_log if available - if step_log is not None: - if hasattr(step_log, "metrics") and step_log.metrics is not None: - step_duration = step_log.metrics.duration_ms / 1000 if step_log.metrics.duration_ms else None - else: - step_duration = duration - else: - step_duration = duration - - if step_duration is not None: - self.step_durations.append(step_duration) - - # Update token counts - if token_usage is not None: - self.total_input_token_count += token_usage.input_tokens - self.total_output_token_count += token_usage.output_tokens - elif step_log is not None: - if hasattr(step_log, "metrics") and step_log.metrics is not None: - self.total_input_token_count += step_log.metrics.input_tokens - self.total_output_token_count += step_log.metrics.output_tokens - - # Log if logger is available - if self.logger is not None: - step_num = len(self.step_durations) - console_outputs = f"[Step {step_num}" - if step_duration is not None: - console_outputs += f": Duration {step_duration:.2f}s" - if self.total_input_token_count > 0 or self.total_output_token_count > 0: - console_outputs += ( - f" | Tokens: {self.total_input_token_count:,} in, {self.total_output_token_count:,} out" - ) - console_outputs += "]" - self.logger.log(console_outputs, level=LogLevel.DEBUG) - - def get_summary(self) -> dict[str, Any]: - """Get a summary of all metrics.""" - return { - "step_count": len(self.step_durations), - "total_duration": self.get_total_duration(), - "step_durations": self.step_durations, - "average_step_duration": ( - sum(self.step_durations) / len(self.step_durations) if self.step_durations else None - ), - "token_usage": self.get_total_token_counts().dict(), - } - - def __repr__(self) -> str: - return f"Monitor(steps={len(self.step_durations)}, tokens={self.get_total_token_counts()})" - - -class LogLevel(IntEnum): - """Log levels for agent output.""" - - OFF = -1 # No output - ERROR = 0 # Only errors - INFO = 1 # Normal output (default) - DEBUG = 2 # Detailed output - - -class AgentLogger: - """ - Logger for agent execution with rich formatting support. - - Provides methods for logging various types of content including - code, markdown, tasks, and structured data with optional rich formatting. - """ - - console: Optional[Console] - - def __init__( - self, - level: LogLevel = LogLevel.INFO, - console: Console | None = None, - use_rich: bool = True, - ): - """ - Initialize the logger. - - Args: - level: Minimum log level to display. - console: Optional rich Console instance. - use_rich: Whether to use rich formatting (requires rich package). - """ - self.level = level - self._use_rich = use_rich and HAS_RICH - - if self._use_rich: - if console is not None: - self.console = console - else: - self.console = Console(highlight=False) - else: - self.console = None - - def log( - self, - *args, - level: int | str | LogLevel = LogLevel.INFO, - **kwargs, - ) -> None: - """ - Log a message. - - Args: - *args: Arguments to print. - level: Log level for this message. - **kwargs: Additional arguments for print/console.print. - """ - # Convert string level to LogLevel - if isinstance(level, str): - level = LogLevel[level.upper()] - - # Check if we should log - if level > self.level: - return - - if self._use_rich and self.console: - self.console.print(*args, **kwargs) - else: - # Strip rich markup for plain output - print(*args) - - def log_error(self, error_message: str) -> None: - """Log an error message.""" - if self._use_rich and self.console: - self.console.print( - _escape_brackets(error_message), - style="bold red", - ) - else: - print(f"ERROR: {error_message}") - - def log_code( - self, - title: str, - content: str, - level: LogLevel = LogLevel.INFO, - ) -> None: - """ - Log a code block with syntax highlighting. - - Args: - title: Title for the code block. - content: The code content. - level: Log level. - """ - if level > self.level: - return - - if self._use_rich and self.console: - self.console.print( - Panel( - Syntax( - content, - lexer="python", - theme="monokai", - word_wrap=True, - ), - title=f"[bold]{title}", - title_align="left", - box=box.HORIZONTALS, - ) - ) - else: - print(f"\n=== {title} ===") - print(content) - print("=" * (len(title) + 8)) - - def log_markdown( - self, - content: str, - title: str | None = None, - level: LogLevel = LogLevel.INFO, - style: str = YELLOW_HEX, - ) -> None: - """ - Log markdown content. - - Args: - content: Markdown content to log. - title: Optional title. - level: Log level. - style: Color style for the title. - """ - if level > self.level: - return - - if self._use_rich and self.console: - markdown_content = Syntax( - content, - lexer="markdown", - theme="github-dark", - word_wrap=True, - ) - if title: - self.console.print( - Group( - Rule( - f"[bold italic]{title}", - align="left", - style=style, - ), - markdown_content, - ) - ) - else: - self.console.print(markdown_content) - else: - if title: - print(f"\n--- {title} ---") - print(content) - - def log_task( - self, - content: str, - subtitle: str = "", - title: str | None = None, - level: LogLevel = LogLevel.INFO, - ) -> None: - """ - Log a new task. - - Args: - content: Task description. - subtitle: Subtitle text. - title: Optional title override. - level: Log level. - """ - if level > self.level: - return - - if self._use_rich and self.console: - panel_title = "[bold]New run" - if title: - panel_title += f" - {title}" - - self.console.print( - Panel( - f"\n[bold]{_escape_brackets(content)}\n", - title=panel_title, - subtitle=subtitle, - border_style=YELLOW_HEX, - subtitle_align="left", - ) - ) - else: - print(f"\n{'=' * 50}") - if title: - print(f"New run - {title}") - else: - print("New run") - print(f"{'=' * 50}") - print(content) - if subtitle: - print(f"({subtitle})") - print() - - def log_rule(self, title: str, level: LogLevel = LogLevel.INFO) -> None: - """ - Log a horizontal rule with title. - - Args: - title: Text to display in the rule. - level: Log level. - """ - if level > self.level: - return - - if self._use_rich and self.console: - self.console.print( - Rule( - f"[bold white]{title}", - characters="━", - style=YELLOW_HEX, - ) - ) - else: - print(f"\n{'━' * 20} {title} {'━' * 20}") - - def log_step( - self, - step_number: int, - thought: str = "", - code: str = "", - observation: str = "", - level: LogLevel = LogLevel.INFO, - ) -> None: - """ - Log a complete agent step. - - Args: - step_number: The step number. - thought: The agent's reasoning. - code: The code generated. - observation: The execution result. - level: Log level. - """ - if level > self.level: - return - - self.log_rule(f"Step {step_number}", level=level) - - if thought: - self.log_markdown(thought, title="Thought", level=level) - - if code: - self.log_code("Code", code, level=level) - - if observation: - self.log_markdown(observation, title="Observation", level=level) - - def log_final_answer( - self, - answer: Any, - level: LogLevel = LogLevel.INFO, - ) -> None: - """ - Log the final answer. - - Args: - answer: The final answer. - level: Log level. - """ - if level > self.level: - return - - if self._use_rich and self.console: - self.console.print( - Panel( - f"[bold green]{_escape_brackets(str(answer))}", - title="[bold]Final Answer", - border_style="green", - ) - ) - else: - print(f"\n{'=' * 50}") - print("FINAL ANSWER:") - print(answer) - print(f"{'=' * 50}\n") - - def visualize_agent_tree(self, agent) -> None: - """ - Visualize the agent hierarchy as a tree. - - Args: - agent: The root agent to visualize. - """ - if not self._use_rich or not self.console: - # Simple text representation - print(f"Agent: {agent.name}") - if hasattr(agent, "tools") and agent.tools: - print(" Tools:") - for name in agent.tools: - print(f" - {name}") - return - - def create_tools_section(tools_dict): - table = Table(show_header=True, header_style="bold") - table.add_column("Name", style=BLUE_HEX) - table.add_column("Description") - - for name, tool in tools_dict.items(): - description = getattr(tool, "description", str(tool)) - if len(description) > 80: - description = description[:77] + "..." - table.add_row(name, description) - - return Group(f"🛠️ [italic {BLUE_HEX}]Tools:", table) - - def get_agent_headline(agent, name: str | None = None): - name_part = f"{name} | " if name else "" - class_name = agent.__class__.__name__ - return f"[bold {YELLOW_HEX}]{name_part}{class_name}" - - def build_agent_tree(parent_tree, agent_obj): - if hasattr(agent_obj, "tools") and agent_obj.tools: - parent_tree.add(create_tools_section(agent_obj.tools)) - - if hasattr(agent_obj, "managed_agents") and agent_obj.managed_agents: - agents_branch = parent_tree.add(f"🤖 [italic {BLUE_HEX}]Managed agents:") - for name, managed_agent in agent_obj.managed_agents.items(): - agent_tree = agents_branch.add(get_agent_headline(managed_agent, name)) - if hasattr(managed_agent, "description"): - agent_tree.add(f"📝 Description: {managed_agent.description}") - build_agent_tree(agent_tree, managed_agent) - - main_tree = Tree(get_agent_headline(agent)) - if hasattr(agent, "description") and agent.description: - main_tree.add(f"📝 Description: {agent.description}") - build_agent_tree(main_tree, agent) - self.console.print(main_tree) - - -def _escape_brackets(text: str) -> str: - """Escape brackets for rich output.""" - if not isinstance(text, str): - text = str(text) - return text.replace("[", "\\[").replace("]", "\\]") diff --git a/backend/app/core/agent/code_agent/parser.py b/backend/app/core/agent/code_agent/parser.py deleted file mode 100644 index 60c5b0cac..000000000 --- a/backend/app/core/agent/code_agent/parser.py +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env python -""" -Code Parser for CodeAgent. - -This module provides utilities for parsing code blocks from LLM outputs -and fixing common issues with generated code. -""" - -import re -from typing import Any, Optional - - -class ParsingError(Exception): - """Exception raised when code parsing fails.""" - - pass - - -# Regex patterns for code block extraction -CODE_BLOCK_PATTERN = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.DOTALL | re.IGNORECASE) - -# Pattern for extracting thought and code sections -THOUGHT_PATTERN = re.compile( - r"(?:Thought|思考|分析|想法)[:：]\s*(.*?)(?=(?:Code|代码|```)|$)", re.DOTALL | re.IGNORECASE -) - -# Pattern for final_answer calls -FINAL_ANSWER_PATTERN = re.compile(r"final_answer\s*\((.*)\)\s*$", re.DOTALL) - - -def parse_code_blobs(llm_output: str) -> str: - """ - Extract Python code from LLM output. - - Handles various formats: - - Code wrapped in ```python ... ``` blocks - - Code wrapped in ``` ... ``` blocks - - Plain code without markers - - Args: - llm_output: Raw LLM output that may contain code. - - Returns: - Extracted Python code. - - Raises: - ParsingError: If no valid code can be extracted. - """ - if not llm_output or not llm_output.strip(): - raise ParsingError("Empty LLM output - no code to parse") - - # Try to find code blocks with python marker - matches = CODE_BLOCK_PATTERN.findall(llm_output) - if matches: - # Return the last code block (usually the most relevant) - code: str = str(matches[-1]).strip() - if code: - return code - - # Try to find any code block (without language marker) - generic_pattern = re.compile(r"```\s*\n(.*?)```", re.DOTALL) - matches = generic_pattern.findall(llm_output) - if matches: - code = str(matches[-1]).strip() - if code: - return code - - # Check if the entire output looks like code (heuristic) - lines = llm_output.strip().split("\n") - code_indicators = [ - "import ", - "from ", - "def ", - "class ", - "if ", - "for ", - "while ", - "return ", - "print(", - "=", - "#", - "try:", - "except:", - "with ", - ] - - code_like_lines = sum(1 for line in lines if any(line.strip().startswith(ind) for ind in code_indicators)) - - # If more than 50% of lines look like code, treat whole output as code - if code_like_lines > len(lines) * 0.5: - return llm_output.strip() - - raise ParsingError( - "Could not find code block in LLM output. Expected code wrapped in ```python ... ``` or ``` ... ```" - ) - - -def extract_thought_and_code(llm_output: str) -> tuple[str, str]: - """ - Extract both thought and code from LLM output. - - Args: - llm_output: Raw LLM output containing thought and code. - - Returns: - Tuple of (thought, code). - """ - thought = "" - code = "" - - # Try to extract thought section - thought_match = THOUGHT_PATTERN.search(llm_output) - if thought_match: - thought = thought_match.group(1).strip() - - # Extract code - try: - code = parse_code_blobs(llm_output) - except ParsingError: - # If no code block found, check if there's code after "Code:" marker - code_marker_pattern = re.compile(r"(?:Code|代码)[:：]\s*(.*)$", re.DOTALL | re.IGNORECASE) - code_match = code_marker_pattern.search(llm_output) - if code_match: - code = code_match.group(1).strip() - - return thought, code - - -def fix_final_answer_code(code: str) -> str: - """ - Fix common issues with final_answer code. - - Issues fixed: - - Missing quotes around string arguments - - Incorrect function call syntax - - Adding final_answer wrapper if missing - - Args: - code: The code that may have issues with final_answer. - - Returns: - Fixed code. - """ - # Check if code already has final_answer - if "final_answer" in code: - return code - - # If code ends with a simple expression, wrap it in final_answer - lines = code.strip().split("\n") - if lines: - last_line = lines[-1].strip() - - # Check if last line is a simple return or expression - if last_line.startswith("return "): - # Convert "return x" to "final_answer(x)" - value = last_line[7:].strip() - lines[-1] = f"final_answer({value})" - elif ( - not last_line.endswith(":") - and "=" not in last_line - and not last_line.startswith("#") - and not last_line.startswith("print(") - and last_line - ): - # Wrap simple expressions - lines[-1] = f"final_answer({last_line})" - - return "\n".join(lines) - - -def clean_code(code: str) -> str: - """ - Clean code by removing unnecessary elements. - - - Removes markdown artifacts - - Removes leading/trailing whitespace - - Normalizes newlines - - Args: - code: The code to clean. - - Returns: - Cleaned code. - """ - # Remove markdown artifacts - code = code.strip() - - # Remove ```python or ``` at start - if code.startswith("```python"): - code = code[9:] - elif code.startswith("```py"): - code = code[5:] - elif code.startswith("```"): - code = code[3:] - - # Remove ``` at end - if code.endswith("```"): - code = code[:-3] - - # Remove leading/trailing whitespace again - code = code.strip() - - # Normalize newlines - code = code.replace("\r\n", "\n").replace("\r", "\n") - - return code - - -def validate_python_syntax(code: str) -> tuple[bool, str | None]: - """ - Validate Python syntax. - - Args: - code: The code to validate. - - Returns: - Tuple of (is_valid, error_message). - """ - import ast - - try: - ast.parse(code) - return True, None - except SyntaxError as e: - return False, f"SyntaxError at line {e.lineno}: {e.msg}" - - -def extract_imports(code: str) -> list[str]: - """ - Extract import statements from code. - - Args: - code: The code to analyze. - - Returns: - List of imported module names. - """ - import ast - - imports = [] - try: - tree = ast.parse(code) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - imports.append(alias.name.split(".")[0]) - elif isinstance(node, ast.ImportFrom): - if node.module: - imports.append(node.module.split(".")[0]) - except SyntaxError: - # Fallback to regex if parsing fails - import_pattern = re.compile(r"^\s*(?:from\s+(\S+)|import\s+(\S+))", re.MULTILINE) - for match in import_pattern.finditer(code): - module = match.group(1) or match.group(2) - if module: - imports.append(module.split(".")[0]) - - return list(set(imports)) - - -def format_observation( - output: Any, - logs: str = "", - error: Optional[str] = None, - max_length: int = 10000, -) -> str: - """ - Format execution output as an observation string. - - Args: - output: The output value from execution. - logs: Captured print output. - error: Error message if any. - max_length: Maximum length before truncation. - - Returns: - Formatted observation string. - """ - parts = [] - - if error: - parts.append(f"Error: {error}") - else: - if logs: - parts.append(f"Print output:\n{logs.strip()}") - - if output is not None: - output_str = str(output) - if len(output_str) > max_length: - output_str = output_str[:max_length] + "... [truncated]" - parts.append(f"Result: {output_str}") - - observation = "\n".join(parts) if parts else "Execution completed with no output." - - # Truncate if too long - if len(observation) > max_length: - observation = observation[:max_length] + "\n... [output truncated]" - - return observation - - -def split_code_into_steps(code: str) -> list[str]: - """ - Split code into logical steps for gradual execution. - - This is useful for debugging and step-by-step execution. - - Args: - code: The code to split. - - Returns: - List of code snippets representing logical steps. - """ - import ast - - steps = [] - try: - tree = ast.parse(code) - for node in tree.body: - step_code = ast.unparse(node) - steps.append(step_code) - except SyntaxError: - # If parsing fails, return the whole code as one step - steps = [code] - - return steps - - -__all__ = [ - "ParsingError", - "parse_code_blobs", - "extract_thought_and_code", - "fix_final_answer_code", - "clean_code", - "validate_python_syntax", - "extract_imports", - "format_observation", - "split_code_into_steps", -] diff --git a/backend/app/core/agent/code_agent/planning.py b/backend/app/core/agent/code_agent/planning.py deleted file mode 100644 index ab9e8991a..000000000 --- a/backend/app/core/agent/code_agent/planning.py +++ /dev/null @@ -1,532 +0,0 @@ -#!/usr/bin/env python -""" -Planning Engine for CodeAgent. - -This module provides planning capabilities for complex multi-step tasks, -including initial plan generation and dynamic plan updates. -""" - -from collections.abc import Callable -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from pathlib import Path -from typing import Optional - -from loguru import logger - - -class PlanStatus(str, Enum): - """Status of a plan step.""" - - PENDING = "pending" - IN_PROGRESS = "in_progress" - COMPLETED = "completed" - SKIPPED = "skipped" - FAILED = "failed" - BLOCKED = "blocked" - - -@dataclass -class PlanStep: - """A single step in a plan.""" - - step_id: int - name: str - description: str - method: str = "" - expected_output: str = "" - status: PlanStatus = PlanStatus.PENDING - dependencies: list[int] = field(default_factory=list) - actual_output: str = "" - error: str | None = None - started_at: datetime | None = None - completed_at: datetime | None = None - - def to_dict(self) -> dict: - return { - "step_id": self.step_id, - "name": self.name, - "description": self.description, - "method": self.method, - "expected_output": self.expected_output, - "status": self.status.value, - "dependencies": self.dependencies, - "actual_output": self.actual_output, - "error": self.error, - } - - def format_for_prompt(self, include_status: bool = True) -> str: - """Format step for inclusion in prompt.""" - status_symbol = { - PlanStatus.PENDING: "○", - PlanStatus.IN_PROGRESS: "●", - PlanStatus.COMPLETED: "✓", - PlanStatus.SKIPPED: "⊘", - PlanStatus.FAILED: "✗", - PlanStatus.BLOCKED: "⊗", - } - - status_str = f" {status_symbol.get(self.status, '?')}" if include_status else "" - - lines = [f"### Step {self.step_id}: {self.name}{status_str}"] - lines.append(f"- Purpose: {self.description}") - if self.method: - lines.append(f"- Method: {self.method}") - if self.expected_output: - lines.append(f"- Expected output: {self.expected_output}") - if self.dependencies: - lines.append(f"- Dependencies: step {', '.join(map(str, self.dependencies))}") - if self.actual_output and self.status == PlanStatus.COMPLETED: - lines.append(f"- Actual output: {self.actual_output[:200]}...") - if self.error: - lines.append(f"- Error: {self.error}") - - return "\n".join(lines) - - -@dataclass -class Plan: - """A complete execution plan.""" - - task: str - goal: str = "" - steps: list[PlanStep] = field(default_factory=list) - notes: list[str] = field(default_factory=list) - created_at: datetime = field(default_factory=datetime.now) - updated_at: datetime = field(default_factory=datetime.now) - version: int = 1 - - @property - def current_step(self) -> PlanStep | None: - """Get the current step to execute.""" - for step in self.steps: - if step.status in (PlanStatus.PENDING, PlanStatus.IN_PROGRESS): - # Check dependencies - deps_completed = all( - (dep_step := self.get_step(dep_id)) is not None and dep_step.status == PlanStatus.COMPLETED - for dep_id in step.dependencies - ) - if deps_completed: - return step - return None - - def get_step(self, step_id: int) -> PlanStep | None: - """Get a step by ID.""" - for step in self.steps: - if step.step_id == step_id: - return step - return None - - @property - def completed_steps(self) -> list[PlanStep]: - """Get all completed steps.""" - return [s for s in self.steps if s.status == PlanStatus.COMPLETED] - - @property - def pending_steps(self) -> list[PlanStep]: - """Get all pending steps.""" - return [s for s in self.steps if s.status == PlanStatus.PENDING] - - @property - def progress_pct(self) -> float: - """Get completion percentage.""" - if not self.steps: - return 0.0 - completed = len(self.completed_steps) - return completed / len(self.steps) * 100 - - def mark_step_started(self, step_id: int) -> None: - """Mark a step as in progress.""" - step = self.get_step(step_id) - if step: - step.status = PlanStatus.IN_PROGRESS - step.started_at = datetime.now() - self.updated_at = datetime.now() - - def mark_step_completed(self, step_id: int, output: str = "") -> None: - """Mark a step as completed.""" - step = self.get_step(step_id) - if step: - step.status = PlanStatus.COMPLETED - step.completed_at = datetime.now() - step.actual_output = output - self.updated_at = datetime.now() - - def mark_step_failed(self, step_id: int, error: str) -> None: - """Mark a step as failed.""" - step = self.get_step(step_id) - if step: - step.status = PlanStatus.FAILED - step.error = error - self.updated_at = datetime.now() - - def format_for_prompt(self, include_completed: bool = True) -> str: - """Format plan for inclusion in prompt.""" - lines = [ - "## Task Goal", - f"{self.goal or self.task}", - "", - f"## Execution Steps (progress: {self.progress_pct:.0f}%)", - ] - - for step in self.steps: - if step.status == PlanStatus.COMPLETED and not include_completed: - lines.append(f"### Step {step.step_id}: {step.name} ✓ (completed)") - else: - lines.append("") - lines.append(step.format_for_prompt()) - - if self.notes: - lines.append("") - lines.append("## Notes") - for note in self.notes: - lines.append(f"- {note}") - - return "\n".join(lines) - - def to_dict(self) -> dict: - return { - "task": self.task, - "goal": self.goal, - "steps": [s.to_dict() for s in self.steps], - "notes": self.notes, - "progress_pct": self.progress_pct, - "version": self.version, - } - - -class PlanningEngine: - """ - Engine for generating and managing execution plans. - - Features: - - Initial plan generation from task description - - Dynamic plan updates based on execution progress - - Dependency tracking between steps - - Plan versioning - - Example: - >>> engine = PlanningEngine(llm_call=my_llm) - >>> plan = await engine.create_plan("Build a web scraper") - >>> print(plan.format_for_prompt()) - """ - - def __init__( - self, - llm_call: Optional[Callable[[str], str]] = None, - max_steps: int = 10, - auto_update_interval: int = 3, - ): - """ - Initialize the planning engine. - - Args: - llm_call: Function to call LLM for plan generation. - max_steps: Maximum number of steps in a plan. - auto_update_interval: Steps between automatic plan updates. - """ - self.llm_call = llm_call - self.max_steps = max_steps - self.auto_update_interval = auto_update_interval - - self._current_plan: Plan | None = None - self._plan_history: list[Plan] = [] - - # Load prompts - self._planning_prompt = self._load_prompt() - - def _load_prompt(self) -> str: - """Load planning prompt template.""" - prompt_path = Path(__file__).parent / "prompts" / "planning.md" - if prompt_path.exists(): - return prompt_path.read_text(encoding="utf-8") - return "" - - async def create_plan( - self, - task: str, - tools: Optional[list[str]] = None, - context: str = "", - ) -> Plan: - """ - Create an initial plan for a task. - - Args: - task: Task description. - tools: Available tools. - context: Additional context. - - Returns: - Generated Plan object. - """ - prompt = f"""Please create an execution plan for the following task. - -## Task -{task} - -## Available Tools -{", ".join(tools) if tools else "No specific tool restrictions"} - -## Context -{context or "None"} - -## Requirements -1. Break the task into at most {self.max_steps} steps -2. Each step should be specific and actionable -3. Indicate dependencies between steps -4. Consider potential risks and mitigation strategies - -## Output Format -Please output in the following JSON format: -```json -{{{{ - "goal": "Overall task goal", - "steps": [ - {{{{ - "step_id": 1, - "name": "Step name", - "description": "Step description", - "method": "Implementation method", - "expected_output": "Expected output", - "dependencies": [] - }}}} - ], - "notes": ["Note 1", "Note 2"] -}}}} -```""" - - if self.llm_call: - try: - import json - - response = await self._call_llm(prompt) - - # Extract JSON from response - json_match = response.find("```json") - if json_match != -1: - json_end = response.find("```", json_match + 7) - json_str = response[json_match + 7 : json_end] - else: - json_str = response - - plan_data = json.loads(json_str) - - plan = Plan( - task=task, - goal=plan_data.get("goal", task), - notes=plan_data.get("notes", []), - ) - - for step_data in plan_data.get("steps", []): - step = PlanStep( - step_id=step_data.get("step_id", len(plan.steps) + 1), - name=step_data.get("name", f"Step {len(plan.steps) + 1}"), - description=step_data.get("description", ""), - method=step_data.get("method", ""), - expected_output=step_data.get("expected_output", ""), - dependencies=step_data.get("dependencies", []), - ) - plan.steps.append(step) - - self._current_plan = plan - logger.info(f"Created plan with {len(plan.steps)} steps") - return plan - - except Exception as e: - logger.error(f"Failed to parse plan: {e}") - - # Fallback: simple single-step plan - plan = Plan( - task=task, - goal=task, - steps=[ - PlanStep( - step_id=1, - name="Execute task", - description=task, - ) - ], - ) - self._current_plan = plan - return plan - - async def update_plan( - self, - progress: str, - issues: Optional[list[str]] = None, - ) -> Plan: - """ - Update the current plan based on progress. - - Args: - progress: Description of current progress. - issues: List of issues encountered. - - Returns: - Updated Plan object. - """ - if not self._current_plan: - raise ValueError("No current plan to update") - - prompt = f"""Please update the execution plan based on current progress. - -## Original Plan -{self._current_plan.format_for_prompt()} - -## Current Progress -{progress} - -## Issues Encountered -{chr(10).join(f"- {issue}" for issue in (issues or [])) or "None so far"} - -## Requirements -1. Keep completed steps -2. Adjust subsequent steps based on actual progress -3. Add necessary new steps -4. Remove steps that are no longer needed - -## Output Format -Please output the updated plan in the same JSON format as the original plan.""" - - if self.llm_call: - try: - import json - - response = await self._call_llm(prompt) - - # Extract JSON - json_match = response.find("```json") - if json_match != -1: - json_end = response.find("```", json_match + 7) - json_str = response[json_match + 7 : json_end] - else: - json_str = response - - plan_data = json.loads(json_str) - - # Archive current plan - self._plan_history.append(self._current_plan) - - # Create updated plan - new_plan = Plan( - task=self._current_plan.task, - goal=plan_data.get("goal", self._current_plan.goal), - notes=plan_data.get("notes", []), - version=self._current_plan.version + 1, - ) - - for step_data in plan_data.get("steps", []): - # Check if step exists in old plan - old_step = self._current_plan.get_step(step_data.get("step_id", -1)) - - step = PlanStep( - step_id=step_data.get("step_id", len(new_plan.steps) + 1), - name=step_data.get("name", f"Step {len(new_plan.steps) + 1}"), - description=step_data.get("description", ""), - method=step_data.get("method", ""), - expected_output=step_data.get("expected_output", ""), - dependencies=step_data.get("dependencies", []), - status=old_step.status if old_step else PlanStatus.PENDING, - actual_output=old_step.actual_output if old_step else "", - ) - new_plan.steps.append(step) - - self._current_plan = new_plan - logger.info(f"Updated plan to version {new_plan.version}") - return new_plan - - except Exception as e: - logger.error(f"Failed to update plan: {e}") - - return self._current_plan - - async def _call_llm(self, prompt: str) -> str: - """Call LLM and handle async.""" - import asyncio - - if self.llm_call is None: - raise ValueError("llm_call is not set") - - result = self.llm_call(prompt) - - if asyncio.iscoroutine(result): - return str(await result) - - if hasattr(result, "__anext__") and not isinstance(result, str): - chunks = [] - async for chunk in result: # type: ignore[misc] - chunks.append(str(chunk)) - return "".join(chunks) - - return str(result) - - @property - def current_plan(self) -> Plan | None: - """Get the current plan.""" - return self._current_plan - - @property - def current_step(self) -> PlanStep | None: - """Get the current step to execute.""" - if self._current_plan: - return self._current_plan.current_step - return None - - def advance_step(self, output: str = "") -> PlanStep | None: - """ - Mark current step as completed and move to next. - - Args: - output: Output from the completed step. - - Returns: - The next step to execute, or None if done. - """ - if not self._current_plan: - return None - - current = self._current_plan.current_step - if current: - self._current_plan.mark_step_completed(current.step_id, output) - - return self._current_plan.current_step - - def fail_step(self, error: str) -> None: - """Mark current step as failed.""" - if self._current_plan: - current = self._current_plan.current_step - if current: - self._current_plan.mark_step_failed(current.step_id, error) - - def reset(self) -> None: - """Reset the planning engine.""" - if self._current_plan: - self._plan_history.append(self._current_plan) - self._current_plan = None - - -def create_planning_engine( - llm: Optional[Callable] = None, - **kwargs, -) -> PlanningEngine: - """ - Factory function to create a PlanningEngine. - - Args: - llm: LLM function for plan generation. - **kwargs: Additional arguments for PlanningEngine. - - Returns: - Configured PlanningEngine instance. - """ - return PlanningEngine(llm_call=llm, **kwargs) - - -__all__ = [ - "PlanStatus", - "PlanStep", - "Plan", - "PlanningEngine", - "create_planning_engine", -] diff --git a/backend/app/core/agent/code_agent/prompts/planning.md b/backend/app/core/agent/code_agent/prompts/planning.md deleted file mode 100644 index 1c5ee9a07..000000000 --- a/backend/app/core/agent/code_agent/prompts/planning.md +++ /dev/null @@ -1,141 +0,0 @@ -# CodeAgent 规划提示词 - -## 初始规划模板 - -你正在执行一个需要多步骤的任务。在开始之前，请先制定一个计划。 - -### 任务 -{{task}} - -### 可用工具 -{{tool_descriptions}} - -### 规划要求 - -请制定一个清晰的执行计划，包含： -1. **目标理解**: 明确任务的最终目标 -2. **分解步骤**: 将任务分解为可执行的步骤 -3. **依赖分析**: 标明步骤之间的依赖关系 -4. **风险评估**: 可能遇到的问题及应对策略 - -### 输出格式 - -请按以下格式输出你的计划： - -``` -## 任务目标 -[简明描述任务目标] - -## 执行步骤 - -### 步骤 1: [步骤名称] -- 目的: [这一步要做什么] -- 方法: [具体如何实现] -- 预期输出: [期望得到什么结果] - -### 步骤 2: [步骤名称] -- 目的: ... -- 方法: ... -- 预期输出: ... - -[继续添加步骤...] - -## 注意事项 -[可能的风险和应对策略] -``` - ---- - -## 规划更新模板 - -任务执行过程中，你可能需要根据观察结果更新计划。 - -### 当前任务 -{{task}} - -### 原计划 -{{original_plan}} - -### 执行进度 -{{progress}} - -### 遇到的问题 -{{issues}} - -### 更新要求 - -请根据当前状态更新计划： -1. 保留已完成或仍然适用的步骤 -2. 调整受影响的步骤 -3. 添加新的必要步骤 -4. 解释变更原因 - -### 输出格式 - -``` -## 计划更新说明 -[解释为什么需要更新计划] - -## 更新后的执行步骤 - -### 步骤 X: [步骤名称] ✓ 已完成 -[简要说明完成情况] - -### 步骤 Y: [步骤名称] ← 当前 -- 目的: ... -- 方法: ... -- 预期输出: ... - -### 步骤 Z: [新增/修改的步骤] -- 目的: ... -- 方法: ... -- 预期输出: ... - -## 调整原因 -[详细解释为什么做这些调整] -``` - ---- - -## 规划总结模板 - -任务完成后，请总结执行情况。 - -### 原计划 -{{original_plan}} - -### 实际执行 -{{execution_history}} - -### 总结要求 - -请总结任务执行情况： -1. 目标是否达成 -2. 执行过程中的关键发现 -3. 与原计划的偏差及原因 -4. 可复用的经验 - -### 输出格式 - -``` -## 任务总结 - -### 完成状态 -[成功/部分成功/失败] - -### 最终结果 -[任务的最终输出] - -### 执行回顾 -- 总步骤数: X -- 成功步骤: Y -- 调整次数: Z - -### 关键发现 -1. [发现 1] -2. [发现 2] -... - -### 经验总结 -[可复用的经验和教训] -``` diff --git a/backend/app/core/agent/code_agent/prompts/system.md b/backend/app/core/agent/code_agent/prompts/system.md deleted file mode 100644 index 6a5152596..000000000 --- a/backend/app/core/agent/code_agent/prompts/system.md +++ /dev/null @@ -1,179 +0,0 @@ -# CodeAgent 系统提示词 - -你是一个专业的代码执行 Agent。你将通过编写和执行 Python 代码来解决任务。 - -## 执行流程 - -你必须遵循以下 **Thought → Code → Observation** 迭代循环： - -1. **Thought（思考）**: 分析当前状态，思考下一步该做什么 -2. **Code（代码）**: 编写 Python 代码来执行操作 -3. **Observation（观察）**: 查看代码执行结果 -4. 重复以上步骤直到任务完成 - -## 代码规则 - -### 规则 1: 始终使用 print() 输出观察结果 -你**必须**使用 `print()` 来输出需要观察的结果。代码的返回值不会自动显示。 - -```python -# 正确 ✓ -result = calculate_something() -print(f"计算结果: {result}") - -# 错误 ✗ -result = calculate_something() # 这不会显示任何内容 -``` - -### 规则 2: 使用 final_answer() 返回最终结果 -当你确定得到了最终答案时，必须调用 `final_answer(answer)` 函数： - -```python -# 返回文本答案 -final_answer("答案是 42") - -# 返回计算结果 -result = complex_calculation() -final_answer(result) - -# 返回数据结构 -final_answer({"key": "value", "data": [1, 2, 3]}) -``` - -### 规则 3: 可使用的工具 -你可以直接调用以下已注入的工具： -{{tool_descriptions}} - -工具调用示例： -```python -# 调用工具 -result = tool_name(arg1, arg2, key=value) -print(result) -``` - -### 规则 4: 变量持久化 -变量会在多次代码执行之间保持。你可以在后续步骤中使用之前定义的变量。 - -```python -# 第一次执行 -data = load_data() -processed = preprocess(data) - -# 第二次执行 - 可以使用之前的变量 -result = analyze(processed) -print(result) -``` - -### 规则 5: 使用授权的导入 -你可以导入以下模块： -- 基础模块：json, re, math, datetime, time, collections, itertools, functools, random, copy, typing -- 数据分析：pandas, numpy, matplotlib, seaborn, sklearn, scipy (如果已启用) - -```python -import json -import pandas as pd -import numpy as np - -data = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) -print(data.describe()) -``` - -### 规则 6: 禁止危险操作 -以下操作被禁止： -- 文件系统操作（除非通过工具） -- 网络请求（除非通过工具） -- 系统命令执行 -- 使用 eval/exec/compile -- 访问 dunder 属性（__xxx__） - -### 规则 7: 代码格式 -始终将代码放在 markdown 代码块中： -```python -# 你的代码 -``` - -### 规则 8: 错误处理 -如果代码执行出错，你会在 Observation 中看到错误信息。请分析错误并修正代码。 - -```python -# 如果之前的代码出错，分析错误原因 -# 然后编写修正后的代码 -try: - result = risky_operation() - print(result) -except Exception as e: - print(f"操作失败: {e}") - # 尝试替代方案 - result = alternative_approach() - print(result) -``` - -### 规则 9: 逐步解决复杂问题 -对于复杂任务，分解为多个步骤： - -```python -# 步骤 1: 数据加载 -data = load_data("file.csv") -print(f"已加载 {len(data)} 条记录") -``` - -观察结果后继续： - -```python -# 步骤 2: 数据处理 -cleaned = clean_data(data) -print(f"清理后剩余 {len(cleaned)} 条记录") -``` - -### 规则 10: 输出最终答案 -当你确信已经完成任务时，调用 final_answer()： - -```python -# 完成所有处理后 -summary = generate_summary(results) -final_answer(summary) -``` - -## 输出格式 - -每次响应应遵循以下格式： - -``` -Thought: [你的思考过程] - -Code: -```python -[你的 Python 代码] -``` -``` - -## 示例 - -**任务**: 计算 1 到 100 的质数之和 - -**响应**: - -Thought: 我需要找出 1 到 100 之间的所有质数，然后求和。我将编写一个函数来判断质数，然后遍历并求和。 - -Code: -```python -def is_prime(n): - if n < 2: - return False - for i in range(2, int(n**0.5) + 1): - if n % i == 0: - return False - return True - -primes = [n for n in range(2, 101) if is_prime(n)] -total = sum(primes) -print(f"质数列表: {primes}") -print(f"质数之和: {total}") -final_answer(total) -``` - ---- - -现在，请解决以下任务： - -{{task}} diff --git a/backend/app/core/agent/code_agent/tools.py b/backend/app/core/agent/code_agent/tools.py deleted file mode 100644 index 2f012d366..000000000 --- a/backend/app/core/agent/code_agent/tools.py +++ /dev/null @@ -1,567 +0,0 @@ -#!/usr/bin/env python -""" -Tool system for CodeAgent. - -This module implements the Tool base class and @tool decorator for creating -production-ready tools with parameter validation and schema generation. -""" - -from __future__ import annotations - -import inspect -import json -import textwrap -from abc import ABC, abstractmethod -from collections.abc import Callable -from functools import wraps -from typing import Any, Union, get_args, get_origin, get_type_hints - -# Authorized types for tool inputs/outputs -AUTHORIZED_TYPES = [ - "string", - "boolean", - "integer", - "number", - "image", - "audio", - "array", - "object", - "any", - "null", -] - -# Python type to JSON schema type conversion -TYPE_CONVERSION = { - str: "string", - int: "integer", - float: "number", - bool: "boolean", - list: "array", - dict: "object", - type(None): "null", -} - - -def python_type_to_json_type(python_type: type) -> str: - """Convert a Python type to JSON schema type.""" - # Handle None type - if python_type is type(None): - return "null" - - # Handle basic types - if python_type in TYPE_CONVERSION: - return TYPE_CONVERSION[python_type] - - # Handle Optional types (Union with None) - origin = get_origin(python_type) - if origin is Union: - args = get_args(python_type) - # Filter out NoneType - non_none_args = [a for a in args if a is not type(None)] - if len(non_none_args) == 1: - return python_type_to_json_type(non_none_args[0]) - return "any" - - # Handle List, Dict etc - if origin is list: - return "array" - if origin is dict: - return "object" - - # Default to any for complex types - return "any" - - -def get_json_type(value: Any) -> str: - """Get the JSON schema type of a value.""" - if value is None: - return "null" - return python_type_to_json_type(type(value)) - - -def is_valid_name(name: str) -> bool: - """Check if a name is a valid Python identifier and not a reserved keyword.""" - import keyword - - return name.isidentifier() and not keyword.iskeyword(name) - - -class BaseTool(ABC): - """Abstract base class for all tools.""" - - name: str - - @abstractmethod - def __call__(self, *args, **kwargs) -> Any: - pass - - -class Tool(BaseTool): - """ - A base class for tools used by the agent. - - Subclass this and implement the `forward` method along with the following - class attributes: - - - **name** (`str`) -- A unique name for the tool. - - **description** (`str`) -- A description of what the tool does. - - **inputs** (`dict`) -- A dictionary describing each input parameter. - - **output_type** (`str`) -- The type of the output. - - **output_schema** (`dict`, optional) -- JSON schema for structured output. - - Example: - >>> class MyTool(Tool): - ... name = "my_tool" - ... description = "Does something useful" - ... inputs = { - ... "query": {"type": "string", "description": "The query to process"} - ... } - ... output_type = "string" - ... - ... def forward(self, query: str) -> str: - ... return f"Processed: {query}" - """ - - name: str - description: str - inputs: dict[str, dict[str, str | type | bool]] - output_type: str - output_schema: dict[str, Any] | None = None - - def __init__(self, *args, **kwargs): - self.is_initialized = False - - def __init_subclass__(cls, **kwargs): - """Validate subclass attributes after class definition.""" - super().__init_subclass__(**kwargs) - # Validation will be done on first instantiation - - def validate_arguments(self) -> None: - """Validate that the tool has all required attributes.""" - required_attributes = { - "description": str, - "name": str, - "inputs": dict, - "output_type": str, - } - - # Check required attributes exist and have correct types - for attr, expected_type in required_attributes.items(): - attr_value = getattr(self, attr, None) - if attr_value is None: - raise TypeError(f"Tool must have attribute '{attr}'.") - if not isinstance(attr_value, expected_type): - raise TypeError( - f"Attribute '{attr}' should be {expected_type.__name__}, got {type(attr_value).__name__}." - ) - - # Validate name is a valid identifier - if not is_valid_name(self.name): - raise ValueError(f"Invalid tool name '{self.name}': must be a valid Python identifier") - - # Validate inputs schema - for input_name, input_content in self.inputs.items(): - if not isinstance(input_content, dict): - raise TypeError(f"Input '{input_name}' should be a dictionary.") - - if "type" not in input_content or "description" not in input_content: - raise ValueError(f"Input '{input_name}' must have 'type' and 'description' keys.") - - # Validate type is authorized - input_type = input_content["type"] - if isinstance(input_type, str): - if input_type not in AUTHORIZED_TYPES: - raise ValueError( - f"Input '{input_name}' has invalid type '{input_type}'. Must be one of {AUTHORIZED_TYPES}" - ) - elif isinstance(input_type, list): - for t in input_type: - if t not in AUTHORIZED_TYPES: - raise ValueError( - f"Input '{input_name}' has invalid type '{t}'. Must be one of {AUTHORIZED_TYPES}" - ) - - # Validate output type - if self.output_type not in AUTHORIZED_TYPES: - raise ValueError(f"output_type '{self.output_type}' must be one of {AUTHORIZED_TYPES}") - - # Validate forward method signature matches inputs - if hasattr(self, "forward") and callable(self.forward): - sig = inspect.signature(self.forward) - params = [k for k in sig.parameters.keys() if k != "self"] - expected_params = set(self.inputs.keys()) - actual_params = set(params) - - if actual_params != expected_params: - raise ValueError( - f"Tool '{self.name}' forward() parameters {actual_params} don't match inputs {expected_params}" - ) - - def forward(self, *args, **kwargs) -> Any: - """ - Execute the tool's main logic. Override this in subclasses. - - Args: - *args: Positional arguments. - **kwargs: Keyword arguments matching the inputs schema. - - Returns: - The tool's output. - """ - raise NotImplementedError("Implement forward() in your Tool subclass.") - - def setup(self) -> None: - """ - Optional setup method for expensive operations. - - Override this for operations like loading models that should only - happen once, on first use. - """ - self.is_initialized = True - - def __call__(self, *args, **kwargs) -> Any: - """ - Execute the tool. - - Handles lazy initialization and argument conversion. - """ - if not self.is_initialized: - self.setup() - - # Handle case where arguments are passed as a single dict - if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], dict): - potential_kwargs = args[0] - if all(key in self.inputs for key in potential_kwargs): - args = () - kwargs = potential_kwargs - - return self.forward(*args, **kwargs) - - def to_code_prompt(self) -> str: - """Generate a code-style prompt for the LLM to understand this tool.""" - # Build signature - args_parts = [] - for arg_name, arg_schema in self.inputs.items(): - arg_type = arg_schema["type"] - nullable = arg_schema.get("nullable", False) - if nullable: - args_parts.append(f"{arg_name}: {arg_type} | None = None") - else: - args_parts.append(f"{arg_name}: {arg_type}") - - args_signature = ", ".join(args_parts) - - # Determine return type - has_schema = self.output_schema is not None - output_type = "dict" if has_schema else self.output_type - - tool_signature = f"({args_signature}) -> {output_type}" - - # Build docstring - tool_doc = self.description - - if has_schema: - tool_doc += "\n\nNote: This tool returns structured output as a dictionary." - - # Add arguments documentation - if self.inputs: - args_descriptions = "\n".join( - f"{arg_name}: {arg_schema['description']}" for arg_name, arg_schema in self.inputs.items() - ) - tool_doc += f"\n\nArgs:\n{textwrap.indent(args_descriptions, ' ')}" - - # Add return type documentation - if has_schema: - formatted_schema = json.dumps(self.output_schema, indent=4) - indented_schema = textwrap.indent(formatted_schema, " ") - tool_doc += f"\n\nReturns:\n dict: Structured output following this schema:\n{indented_schema}" - - tool_doc = f'"""{tool_doc}\n"""' - return f"def {self.name}{tool_signature}:\n{textwrap.indent(tool_doc, ' ')}" - - def to_dict(self) -> dict: - """Convert the tool to a dictionary representation.""" - result = { - "name": self.name, - "description": self.description, - "inputs": self.inputs, - "output_type": self.output_type, - } - if self.output_schema: - result["output_schema"] = self.output_schema - return result - - def __repr__(self) -> str: - return f"Tool(name='{self.name}')" - - -def tool(func: Callable) -> Tool: - """ - Decorator to convert a function into a Tool instance. - - The function should have: - - Type hints for all parameters - - A return type hint - - A docstring with description and Args section - - Example: - >>> @tool - ... def web_search(query: str) -> str: - ... '''Search the web for information. - ... - ... Args: - ... query: The search query - ... ''' - ... return search_engine.search(query) - - >>> web_search.name - 'web_search' - >>> web_search.inputs - {'query': {'type': 'string', 'description': 'The search query'}} - - Args: - func: The function to convert. - - Returns: - A Tool instance wrapping the function. - """ - # Get function name - func_name = func.__name__ - - # Get docstring - docstring = inspect.getdoc(func) or "" - - # Parse docstring to get description and argument descriptions - description, arg_descriptions = _parse_docstring(docstring) - - # Get type hints - try: - type_hints = get_type_hints(func) - except Exception: - type_hints = {} - - # Get signature - sig = inspect.signature(func) - - # Build inputs schema - inputs: dict[str, dict[str, str | bool]] = {} - for param_name, param in sig.parameters.items(): - if param_name == "self": - continue - - # Get type - if param_name in type_hints: - param_type = python_type_to_json_type(type_hints[param_name]) - else: - param_type = "any" - - # Get description - param_desc = arg_descriptions.get(param_name, f"The {param_name} parameter") - - # Check if nullable (has default of None) - nullable = param.default is None and param.default is not inspect.Parameter.empty - - input_schema: dict[str, str | bool] = { - "type": param_type, - "description": param_desc, - } - if nullable or param.default is not inspect.Parameter.empty: - input_schema["nullable"] = True - - inputs[param_name] = input_schema - - # Get output type - if "return" in type_hints: - output_type = python_type_to_json_type(type_hints["return"]) - else: - output_type = "any" - - # Create dynamic Tool subclass - class SimpleTool(Tool): - def __init__(self): - self.is_initialized = True - - # Set class attributes - SimpleTool.name = func_name - SimpleTool.description = description - SimpleTool.inputs = inputs # type: ignore[assignment] - SimpleTool.output_type = output_type - - # Bind the function to forward - @wraps(func) - def forward_method(self, *args, **kwargs): - return func(*args, **kwargs) - - SimpleTool.forward = forward_method # type: ignore[method-assign] - - # Create instance - tool_instance = SimpleTool() - - # Copy function attributes - tool_instance.__doc__ = func.__doc__ - tool_instance.__module__ = func.__module__ - - return tool_instance - - -def _parse_docstring(docstring: str) -> tuple[str, dict[str, str]]: - """ - Parse a docstring to extract description and argument descriptions. - - Args: - docstring: The docstring to parse. - - Returns: - Tuple of (description, {arg_name: arg_description}) - """ - if not docstring: - return "", {} - - lines = docstring.strip().split("\n") - - description_lines = [] - arg_descriptions = {} - - current_section = "description" - current_arg = None - current_arg_desc = [] - - for line in lines: - stripped = line.strip() - - # Check for Args section - if stripped.lower() in ("args:", "arguments:", "parameters:"): - current_section = "args" - continue - - # Check for Returns section (end of args) - if stripped.lower() in ("returns:", "return:", "yields:", "raises:", "examples:"): - # Save last arg if any - if current_arg and current_arg_desc: - arg_descriptions[current_arg] = " ".join(current_arg_desc).strip() - current_section = "other" - continue - - if current_section == "description": - if stripped: - description_lines.append(stripped) - - elif current_section == "args": - # Check if this is a new argument line (name: description) - if ":" in stripped and not stripped.startswith(" "): - # Save previous arg if any - if current_arg and current_arg_desc: - arg_descriptions[current_arg] = " ".join(current_arg_desc).strip() - - # Parse new arg - parts = stripped.split(":", 1) - arg_name = parts[0].strip() - # Handle type annotations in docstring like "query (str): description" - if "(" in arg_name: - arg_name = arg_name.split("(")[0].strip() - current_arg = arg_name - current_arg_desc = [parts[1].strip()] if len(parts) > 1 else [] - - elif current_arg and stripped: - # Continuation of previous arg description - current_arg_desc.append(stripped) - - # Save last arg if any - if current_arg and current_arg_desc: - arg_descriptions[current_arg] = " ".join(current_arg_desc).strip() - - description = " ".join(description_lines) - - return description, arg_descriptions - - -def validate_tool_arguments(tool: Tool, arguments: Any) -> None: - """ - Validate tool arguments against the tool's input schema. - - Args: - tool: The tool to validate against. - arguments: The arguments to validate (dict or single value). - - Raises: - ValueError: If an argument is missing or invalid. - TypeError: If an argument has the wrong type. - """ - if isinstance(arguments, dict): - # Check for unknown arguments - for key in arguments: - if key not in tool.inputs: - raise ValueError(f"Unknown argument '{key}' for tool '{tool.name}'") - - # Check each argument - for key, value in arguments.items(): - expected_type = tool.inputs[key]["type"] - actual_type = get_json_type(value) - nullable = tool.inputs[key].get("nullable", False) - - # Allow null for nullable parameters - if actual_type == "null" and nullable: - continue - - # Allow integer for number type - if actual_type == "integer" and expected_type == "number": - continue - - # Allow any type if expected is "any" - if expected_type == "any": - continue - - # Handle list of types - if isinstance(expected_type, list): - if actual_type not in expected_type: - raise TypeError(f"Argument '{key}' has type '{actual_type}' but expected one of {expected_type}") - elif actual_type != expected_type: - raise TypeError(f"Argument '{key}' has type '{actual_type}' but expected '{expected_type}'") - - # Check required arguments are present - for key, schema in tool.inputs.items(): - if key not in arguments and not schema.get("nullable", False): - raise ValueError(f"Missing required argument '{key}' for tool '{tool.name}'") - - else: - # Single value - check against first input - if len(tool.inputs) != 1: - raise ValueError(f"Tool '{tool.name}' expects {len(tool.inputs)} arguments, but received a single value") - - expected_type = list(tool.inputs.values())[0]["type"] - actual_type = get_json_type(arguments) - - if expected_type != "any" and actual_type != expected_type: - # Allow integer for number - if not (actual_type == "integer" and expected_type == "number"): - raise TypeError(f"Argument has type '{actual_type}' but expected '{expected_type}'") - - -class FinalAnswerTool(Tool): - """Built-in tool for returning the final answer.""" - - name = "final_answer" - description = "Return the final answer to the user's task." - inputs = {"answer": {"type": "any", "description": "The final answer to return."}} - output_type = "any" - - def forward(self, answer: Any) -> Any: - """Return the final answer.""" - return answer - - -def create_final_answer_tool() -> FinalAnswerTool: - """Create a final_answer tool instance.""" - return FinalAnswerTool() - - -__all__ = [ - "AUTHORIZED_TYPES", - "Tool", - "tool", - "validate_tool_arguments", - "FinalAnswerTool", - "create_final_answer_tool", - "python_type_to_json_type", - "get_json_type", -] diff --git a/backend/app/core/agent/code_agent/utils.py b/backend/app/core/agent/code_agent/utils.py deleted file mode 100644 index e60b3d800..000000000 --- a/backend/app/core/agent/code_agent/utils.py +++ /dev/null @@ -1,446 +0,0 @@ -#!/usr/bin/env python -""" -Utility functions and classes for CodeAgent. - -This module provides rate limiting, retry mechanisms, and other utilities -for reliable agent execution. -""" - -from __future__ import annotations - -import asyncio -import random -import time -from collections.abc import Callable -from dataclasses import dataclass -from functools import wraps -from typing import TypeVar - -from loguru import logger - -__all__ = [ - "RateLimiter", - "Retrying", - "retry", - "is_rate_limit_error", - "is_transient_error", -] - - -T = TypeVar("T") - - -def is_rate_limit_error(error: Exception) -> bool: - """ - Check if an error is a rate limit error. - - Args: - error: The exception to check. - - Returns: - True if this appears to be a rate limit error. - """ - error_str = str(error).lower() - error_type = type(error).__name__.lower() - - rate_limit_indicators = [ - "rate limit", - "ratelimit", - "rate_limit", - "too many requests", - "429", - "quota exceeded", - "quota_exceeded", - "throttle", - "throttling", - ] - - return any(indicator in error_str or indicator in error_type for indicator in rate_limit_indicators) - - -def is_transient_error(error: Exception) -> bool: - """ - Check if an error is transient and worth retrying. - - Args: - error: The exception to check. - - Returns: - True if this is a transient error that might succeed on retry. - """ - error_str = str(error).lower() - error_type = type(error).__name__.lower() - - transient_indicators = [ - "timeout", - "timed out", - "connection", - "network", - "temporary", - "service unavailable", - "503", - "502", - "504", - "internal server error", - "500", - "overloaded", - "capacity", - ] - - # Also consider rate limits as transient - if is_rate_limit_error(error): - return True - - return any(indicator in error_str or indicator in error_type for indicator in transient_indicators) - - -class RateLimiter: - """ - Rate limiter for API calls. - - Implements a simple rate limiting strategy based on requests per minute. - Can be used to prevent hitting API rate limits. - - Example: - >>> limiter = RateLimiter(requests_per_minute=60) - >>> for request in requests: - ... await limiter.throttle() # or limiter.throttle_sync() - ... response = await api.call(request) - """ - - def __init__( - self, - requests_per_minute: float | None = None, - requests_per_second: float | None = None, - ): - """ - Initialize the rate limiter. - - Args: - requests_per_minute: Maximum requests per minute (mutually exclusive with requests_per_second). - requests_per_second: Maximum requests per second (takes precedence). - """ - if requests_per_second is not None: - self._min_interval = 1.0 / requests_per_second - elif requests_per_minute is not None: - self._min_interval = 60.0 / requests_per_minute - else: - self._min_interval = 0.0 - - self._last_request_time: float = 0.0 - self._lock = asyncio.Lock() - self._sync_lock_time: float = 0.0 - - @property - def is_enabled(self) -> bool: - """Check if rate limiting is enabled.""" - return self._min_interval > 0 - - async def throttle(self) -> float: - """ - Wait if necessary to respect rate limits. - - Returns: - The time waited in seconds. - """ - if not self.is_enabled: - return 0.0 - - async with self._lock: - now = time.time() - time_since_last = now - self._last_request_time - - if time_since_last < self._min_interval: - wait_time = self._min_interval - time_since_last - await asyncio.sleep(wait_time) - self._last_request_time = time.time() - return wait_time - - self._last_request_time = now - return 0.0 - - def throttle_sync(self) -> float: - """ - Synchronous version of throttle. - - Returns: - The time waited in seconds. - """ - if not self.is_enabled: - return 0.0 - - now = time.time() - time_since_last = now - self._sync_lock_time - - if time_since_last < self._min_interval: - wait_time = self._min_interval - time_since_last - time.sleep(wait_time) - self._sync_lock_time = time.time() - return wait_time - - self._sync_lock_time = now - return 0.0 - - def reset(self) -> None: - """Reset the rate limiter state.""" - self._last_request_time = 0.0 - self._sync_lock_time = 0.0 - - -@dataclass -class RetryConfig: - """Configuration for retry behavior.""" - - max_attempts: int = 3 - wait_seconds: float = 1.0 - exponential_base: float = 2.0 - max_wait_seconds: float = 60.0 - jitter: bool = True - jitter_factor: float = 0.1 - - def get_wait_time(self, attempt: int) -> float: - """ - Calculate wait time for a given attempt. - - Args: - attempt: The attempt number (0-indexed). - - Returns: - Wait time in seconds. - """ - # Exponential backoff - wait = self.wait_seconds * (self.exponential_base**attempt) - - # Cap at max wait - wait = min(wait, self.max_wait_seconds) - - # Add jitter if enabled - if self.jitter: - jitter_range = wait * self.jitter_factor - wait += random.uniform(-jitter_range, jitter_range) - - return max(0, wait) - - -class Retrying: - """ - Retry mechanism with exponential backoff. - - Provides configurable retry logic for operations that may fail transiently. - - Example: - >>> retrying = Retrying(max_attempts=3, wait_seconds=1.0) - >>> result = await retrying(api_call, query="test") - - # Or as decorator: - >>> @Retrying(max_attempts=3) - ... async def my_api_call(): - ... return await api.call() - """ - - def __init__( - self, - max_attempts: int = 3, - wait_seconds: float = 1.0, - exponential_base: float = 2.0, - max_wait_seconds: float = 60.0, - jitter: bool = True, - retry_predicate: Callable[[Exception], bool] | None = None, - on_retry: Callable[[Exception, int], None] | None = None, - ): - """ - Initialize the retry mechanism. - - Args: - max_attempts: Maximum number of attempts (including first try). - wait_seconds: Initial wait time between retries. - exponential_base: Base for exponential backoff. - max_wait_seconds: Maximum wait time between retries. - jitter: Whether to add random jitter to wait times. - retry_predicate: Function to determine if an error should trigger retry. - Defaults to is_transient_error. - on_retry: Optional callback called before each retry. - """ - self.config = RetryConfig( - max_attempts=max_attempts, - wait_seconds=wait_seconds, - exponential_base=exponential_base, - max_wait_seconds=max_wait_seconds, - jitter=jitter, - ) - self.retry_predicate = retry_predicate or is_transient_error - self.on_retry = on_retry - - async def __call__( - self, - func: Callable[..., T], - *args, - **kwargs, - ) -> T: - """ - Execute a function with retry logic. - - Args: - func: The function to execute (can be sync or async). - *args: Positional arguments for the function. - **kwargs: Keyword arguments for the function. - - Returns: - The function's return value. - - Raises: - The last exception if all retries fail. - """ - last_exception: Exception | None = None - - for attempt in range(self.config.max_attempts): - try: - result = func(*args, **kwargs) - if asyncio.iscoroutine(result): - return await result # type: ignore[no-any-return] - return result # type: ignore[no-any-return] - - except Exception as e: - last_exception = e - - # Check if we should retry - if not self.retry_predicate(e): - raise - - # Check if we have retries left - if attempt + 1 >= self.config.max_attempts: - raise - - # Calculate wait time - wait_time = self.config.get_wait_time(attempt) - - # Call retry callback - if self.on_retry: - self.on_retry(e, attempt + 1) - - logger.warning( - f"Retry {attempt + 1}/{self.config.max_attempts - 1}: " - f"{type(e).__name__}: {e}. Waiting {wait_time:.2f}s..." - ) - - await asyncio.sleep(wait_time) - - # Should not reach here, but just in case - if last_exception: - raise last_exception - raise RuntimeError("Unexpected state in retry logic") - - def call_sync( - self, - func: Callable[..., T], - *args, - **kwargs, - ) -> T: - """ - Synchronous version of __call__. - - Args: - func: The function to execute (must be sync). - *args: Positional arguments for the function. - **kwargs: Keyword arguments for the function. - - Returns: - The function's return value. - """ - last_exception: Exception | None = None - - for attempt in range(self.config.max_attempts): - try: - return func(*args, **kwargs) - - except Exception as e: - last_exception = e - - if not self.retry_predicate(e): - raise - - if attempt + 1 >= self.config.max_attempts: - raise - - wait_time = self.config.get_wait_time(attempt) - - if self.on_retry: - self.on_retry(e, attempt + 1) - - logger.warning( - f"Retry {attempt + 1}/{self.config.max_attempts - 1}: " - f"{type(e).__name__}: {e}. Waiting {wait_time:.2f}s..." - ) - - time.sleep(wait_time) - - if last_exception: - raise last_exception - raise RuntimeError("Unexpected state in retry logic") - - def wrap(self, func: Callable[..., T]) -> Callable[..., T]: - """ - Wrap a function with retry logic. - - Args: - func: The function to wrap. - - Returns: - A wrapped function with retry logic. - """ - if asyncio.iscoroutinefunction(func): - - @wraps(func) - async def async_wrapper(*args, **kwargs): - return await self(func, *args, **kwargs) - - return async_wrapper - else: - - @wraps(func) - def sync_wrapper(*args, **kwargs): - return self.call_sync(func, *args, **kwargs) - - return sync_wrapper - - -def retry( - max_attempts: int = 3, - wait_seconds: float = 1.0, - exponential_base: float = 2.0, - max_wait_seconds: float = 60.0, - jitter: bool = True, - retry_predicate: Callable[[Exception], bool] | None = None, -) -> Callable: - """ - Decorator for adding retry logic to a function. - - Example: - >>> @retry(max_attempts=3, wait_seconds=1.0) - ... async def my_api_call(): - ... return await api.call() - - Args: - max_attempts: Maximum number of attempts. - wait_seconds: Initial wait time between retries. - exponential_base: Base for exponential backoff. - max_wait_seconds: Maximum wait time. - jitter: Whether to add jitter. - retry_predicate: Function to determine if error is retryable. - - Returns: - Decorator function. - """ - retrying = Retrying( - max_attempts=max_attempts, - wait_seconds=wait_seconds, - exponential_base=exponential_base, - max_wait_seconds=max_wait_seconds, - jitter=jitter, - retry_predicate=retry_predicate, - ) - - def decorator(func: Callable[..., T]) -> Callable[..., T]: - return retrying.wrap(func) - - return decorator diff --git a/backend/app/core/agent/langfuse_callback.py b/backend/app/core/agent/langfuse_callback.py deleted file mode 100644 index 08a5c14ee..000000000 --- a/backend/app/core/agent/langfuse_callback.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Langfuse callback handler for LLM observability. - -Integrates Langfuse tracing with LangChain/LangGraph agents to track: -- LLM calls (prompts, responses, tokens, costs) -- Tool calls and results -- Agent execution traces -- User interactions -""" - -import os -from typing import Any - -from loguru import logger - -try: - from langfuse.langchain import CallbackHandler as LangfuseCallbackHandler - - LANGFUSE_AVAILABLE = True -except ImportError: - LANGFUSE_AVAILABLE = False - logger.warning("langfuse not installed. Langfuse tracing will be disabled.") - - -def get_langfuse_callbacks(enabled: bool = True, **kwargs: Any) -> list[Any]: - """ - Get list of Langfuse callbacks for use with LangChain/LangGraph. - - Environment variables are automatically read from .env: - - LANGFUSE_PUBLIC_KEY - - LANGFUSE_SECRET_KEY - - LANGFUSE_HOST (optional, defaults to https://cloud.langfuse.com) - - Returns a list that can be used in two ways: - 1. Via with_config: runnable.with_config({"callbacks": [...]}) - 2. Via invoke: agent.invoke(..., config={"callbacks": [...]}) - - Example: - # Simple usage - environment variables from .env - langfuse_handler = CallbackHandler() - config = { - "callbacks": [langfuse_handler], - "configurable": {...} - } - result = graph.astream(input=initial_state, config=config) - - Args: - enabled: Whether to enable Langfuse tracing - **kwargs: Additional arguments (for backward compatibility, but not used) - - Returns: - List of callback handlers (empty list if disabled or unavailable) - """ - if not enabled: - logger.debug("[langfuse] Langfuse tracing is disabled") - return [] - - if not LANGFUSE_AVAILABLE: - logger.warning("[langfuse] Langfuse package not installed, skipping callback creation") - return [] - - # Check if environment variables are set - public_key = os.getenv("LANGFUSE_PUBLIC_KEY") - secret_key = os.getenv("LANGFUSE_SECRET_KEY") - host = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com") - - # Print configuration (mask sensitive keys) - def _mask_key(k): - return f"{k[:8]}...{k[-4:]}" if k and len(k) > 12 else "***" if k else None - - logger.info( - f"[langfuse] Configuration: enabled={enabled}, " - f"public_key={_mask_key(public_key)}, " - f"secret_key={'***' if secret_key else None}, " - f"host={host}" - ) - - if not public_key or not secret_key: - logger.warning( - "[langfuse] Langfuse keys not found in environment variables. " - "Please set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY in .env file" - ) - return [] - - try: - # Create handler with trace_id from context for cross-system correlation - from app.core.trace_context import get_trace_id - - trace_id = get_trace_id() - handler = LangfuseCallbackHandler( - trace_context={"trace_id": trace_id} if trace_id else None, - ) - logger.info(f"[langfuse] Langfuse callback handler created successfully (host: {host})") - return [handler] - except Exception as e: - logger.error(f"[langfuse] Failed to create Langfuse callback handler: {e}") - return [] diff --git a/backend/app/core/agent/memory/__init__.py b/backend/app/core/agent/memory/__init__.py deleted file mode 100644 index 04f5fe449..000000000 --- a/backend/app/core/agent/memory/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Memory subsystem for the core agent. - -Note: Use local/core import paths (app.core.agent...) instead of the legacy -app.agent... package layout. -""" - -from app.schemas.memory import UserMemory - -from .manager import MemoryManager -from .strategies import ( - MemoryOptimizationStrategy, - MemoryOptimizationStrategyFactory, - MemoryOptimizationStrategyType, - SummarizeStrategy, -) - -__all__ = [ - "MemoryManager", - "UserMemory", - "MemoryOptimizationStrategy", - "MemoryOptimizationStrategyType", - "MemoryOptimizationStrategyFactory", - "SummarizeStrategy", -] diff --git a/backend/app/core/agent/memory/manager.py b/backend/app/core/agent/memory/manager.py deleted file mode 100644 index b76872031..000000000 --- a/backend/app/core/agent/memory/manager.py +++ /dev/null @@ -1,2064 +0,0 @@ -from dataclasses import dataclass -from textwrap import dedent -from typing import Any, Callable, Dict, List, Literal, Optional, Type, Union - -from langchain_core.language_models import BaseChatModel -from langchain_core.messages.chat import ChatMessage as Message -from loguru import logger -from pydantic import BaseModel, Field - -from app.core.agent.memory.strategies import ( - MemoryOptimizationStrategy, - MemoryOptimizationStrategyFactory, - MemoryOptimizationStrategyType, -) - -# Import DEFAULT_USER_ID for consistent user_id handling -from app.core.constants import DEFAULT_USER_ID -from app.core.tools.tool import EnhancedTool -from app.schemas.memory import UserMemory -from app.services.memory_service import MemoryService -from app.utils.datetime import utc_now -from app.utils.prompts import get_json_output_prompt -from app.utils.string import parse_response_model_str - - -class MemorySearchResponse(BaseModel): - """Model for Memory Search Response.""" - - memory_ids: List[str] = Field( - ..., - description="The IDs of the memories that are most semantically similar to the query.", - ) - - -@dataclass -class MemoryManager: - """Memory Manager""" - - # Model used for memory management - model: Optional[BaseChatModel] = None - - # Provide the system message for the manager as a string. If not provided, the default system message will be used. - system_message: Optional[str] = None - # Provide the memory capture instructions for the manager as a string. If not provided, the default memory capture instructions will be used. - memory_capture_instructions: Optional[str] = None - # Additional instructions for the manager. These instructions are appended to the default system message. - additional_instructions: Optional[str] = None - - # Whether memories were created in the last run - memories_updated: bool = False - - # ----- db tools --------- - # Whether to delete memories - delete_memories: bool = True - # Whether to clear memories - clear_memories: bool = True - # Whether to update memories - update_memories: bool = True - # whether to add memories - add_memories: bool = True - - # The database to store memories - db: Optional[Union[MemoryService]] = None - - debug_mode: bool = False - - def __init__( - self, - model: Optional[Union[BaseChatModel, str]] = None, - system_message: Optional[str] = None, - memory_capture_instructions: Optional[str] = None, - additional_instructions: Optional[str] = None, - db: Optional[Union[MemoryService]] = None, - delete_memories: bool = False, - update_memories: bool = True, - add_memories: bool = True, - clear_memories: bool = False, - debug_mode: bool = False, - ): - self.model = model # type: ignore[assignment] - self.system_message = system_message - self.memory_capture_instructions = memory_capture_instructions - self.additional_instructions = additional_instructions - self.db = db - self.delete_memories = delete_memories - self.update_memories = update_memories - self.add_memories = add_memories - self.clear_memories = clear_memories - self.debug_mode = debug_mode - - # model configuration is injected by the external caller; no secondary conversion here - - def get_model(self) -> BaseChatModel: - if self.model is None: - raise ValueError("A model parameter is required when creating MemoryManager") - return self.model - - def _get_message_content_string(self, msg: Message) -> str: - """Extract content string from message, supporting both get_content_string() and content attribute. - - This method handles different message types: - - Messages with get_content_string() method (e.g., app.models.message.Message) - - Messages with content attribute (e.g., langchain_core.messages.chat.ChatMessage) - """ - if hasattr(msg, "get_content_string"): - result = msg.get_content_string() - return str(result) if result is not None else "" - elif hasattr(msg, "content"): - content = msg.content - if isinstance(content, str): - return content - elif isinstance(content, list): - # Handle content_blocks format - text_parts = [] - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text_parts.append(block.get("text", "")) - elif isinstance(block, str): - text_parts.append(block) - return " ".join(text_parts) if text_parts else str(content) - else: - return str(content) if content else "" - return "" - - def read_from_db(self, user_id: Optional[str] = None): - if self.db: - # If no user_id is provided, read all memories - if user_id is None: - all_memories: List[UserMemory] = self.db.get_user_memories() # type: ignore - else: - all_memories = self.db.get_user_memories(user_id=user_id) # type: ignore - - memories: Dict[str, List[UserMemory]] = {} - for memory in all_memories: - if memory.user_id is not None and memory.memory_id is not None: - memories.setdefault(memory.user_id, []).append(memory) - - return memories - return None - - async def aread_from_db(self, user_id: Optional[str] = None): - if self.db: - if isinstance(self.db, MemoryService): - # If no user_id is provided, read all memories - if user_id is None: - all_memories: List[UserMemory] = await self.db.get_user_memories() # type: ignore - else: - all_memories = await self.db.get_user_memories(user_id=user_id) # type: ignore - else: - if user_id is None: - all_memories = self.db.get_user_memories() # type: ignore - else: - all_memories = self.db.get_user_memories(user_id=user_id) # type: ignore - - memories: Dict[str, List[UserMemory]] = {} - for memory in all_memories: - if memory.user_id is not None and memory.memory_id is not None: - memories.setdefault(memory.user_id, []).append(memory) - - return memories - return None - - def set_log_level(self): - """Log level is configured externally; this is a reserved interface.""" - pass - - def initialize(self, user_id: Optional[str] = None): - self.set_log_level() - - def _build_tool_map(self, tools: List[Callable]) -> Dict[str, Callable]: - """Build a mapping from tool name to tool callable. - - Supports both: - - EnhancedTool/BaseTool objects (with .name attribute) - - Regular functions (with __name__ attribute) - """ - tool_map = {} - for tool in tools: - if hasattr(tool, "name"): # EnhancedTool or BaseTool - tool_map[tool.name] = tool - elif hasattr(tool, "__name__"): # Regular function - tool_map[tool.__name__] = tool - return tool_map - - def _execute_tool_calls(self, tool_calls: List[Dict], tool_map: Dict[str, Callable]) -> bool: - """Execute tool calls synchronously. - - Args: - tool_calls: List of tool call dicts with 'name' and 'args' keys - tool_map: Mapping from tool name to tool callable - - Returns: - True if any tools were executed - """ - if not tool_calls: - return False - - logger.info(f"Executing {len(tool_calls)} tool calls...") - executed = False - - for tool_call in tool_calls: - tool_name = tool_call.get("name") - tool_args = tool_call.get("args", {}) - - if tool_name not in tool_map: - logger.warning(f"Unknown tool: {tool_name}") - continue - - try: - logger.info(f"Executing tool: {tool_name} with args: {tool_args}") - tool = tool_map[tool_name] - - # Handle different tool types - if hasattr(tool, "invoke"): - result = tool.invoke(tool_args) - elif hasattr(tool, "run"): - result = tool.run(**tool_args) - elif callable(tool): - result = tool(**tool_args) - else: - result = f"Tool {tool_name} is not callable" - - logger.info(f"Tool {tool_name} result: {result}") - executed = True - except Exception as e: - logger.error(f"Error executing tool {tool_name}: {e}") - - return executed - - async def _aexecute_tool_calls(self, tool_calls: List[Dict], tool_map: Dict[str, Callable]) -> bool: - """Execute tool calls asynchronously. - - Args: - tool_calls: List of tool call dicts with 'name' and 'args' keys - tool_map: Mapping from tool name to tool callable - - Returns: - True if any tools were executed - """ - if not tool_calls: - return False - - logger.info(f"Executing {len(tool_calls)} tool calls (async)...") - executed = False - - for tool_call in tool_calls: - tool_name = tool_call.get("name") - tool_args = tool_call.get("args", {}) - - if tool_name not in tool_map: - logger.warning(f"Unknown tool: {tool_name}") - continue - - try: - logger.info(f"Executing tool: {tool_name} with args: {tool_args}") - tool = tool_map[tool_name] - - # Handle different tool types - prefer async methods - if hasattr(tool, "ainvoke"): - result = await tool.ainvoke(tool_args) - elif hasattr(tool, "invoke"): - result = tool.invoke(tool_args) - elif hasattr(tool, "run"): - result = tool.run(**tool_args) - elif callable(tool): - result = tool(**tool_args) - else: - result = f"Tool {tool_name} is not callable" - - logger.info(f"Tool {tool_name} result: {result}") - executed = True - except Exception as e: - logger.error(f"Error executing tool {tool_name}: {e}") - - return executed - - # -*- Public Functions - def get_user_memories(self, user_id: Optional[str] = None) -> Optional[List[UserMemory]]: - """Get the user memories for a given user id""" - if self.db: - if user_id is None: - user_id = DEFAULT_USER_ID - # Refresh from the Db - memories = self.read_from_db(user_id=user_id) - if memories is None: - return [] - result = memories.get(user_id, []) - return result if isinstance(result, list) else [] - else: - logger.warning("Memory Db not provided.") - return [] - - async def aget_user_memories(self, user_id: Optional[str] = None) -> Optional[List[UserMemory]]: - """Get the user memories for a given user id""" - if self.db: - if user_id is None: - user_id = DEFAULT_USER_ID - # Refresh from the Db - memories = await self.aread_from_db(user_id=user_id) - if memories is None: - return [] - result = memories.get(user_id, []) - return result if isinstance(result, list) else [] - else: - logger.warning("Memory Db not provided.") - return [] - - def get_user_memory(self, memory_id: str, user_id: Optional[str] = None) -> Optional[UserMemory]: - """Get the user memory for a given user id""" - if self.db: - if user_id is None: - user_id = DEFAULT_USER_ID - # Refresh from the DB - memories = self.read_from_db(user_id=user_id) - if memories is None: - return None - memories_for_user = memories.get(user_id, []) - if not isinstance(memories_for_user, list): - return None - for memory in memories_for_user: - if memory.memory_id == memory_id: - return memory # type: ignore[no-any-return] - return None - else: - logger.warning("Memory Db not provided.") - return None - - def add_user_memory( - self, - memory: UserMemory, - user_id: Optional[str] = None, - ) -> Optional[str]: - """Add a user memory for a given user id - Args: - memory (UserMemory): The memory to add - user_id (Optional[str]): The user id to add the memory to. If not provided, the memory is added to the "default" user. - Returns: - str: The id of the memory - """ - if self.db: - if memory.memory_id is None: - from uuid import uuid4 - - memory_id = memory.memory_id or str(uuid4()) - memory.memory_id = memory_id - - if user_id is None: - user_id = DEFAULT_USER_ID - memory.user_id = user_id - - if not memory.updated_at: - memory.updated_at = int(utc_now().timestamp()) - - self._upsert_db_memory(memory=memory) - return memory.memory_id - - else: - logger.warning("Memory Db not provided.") - return None - - def replace_user_memory( - self, - memory_id: str, - memory: UserMemory, - user_id: Optional[str] = None, - ) -> Optional[str]: - """Replace a user memory for a given user id - Args: - memory_id (str): The id of the memory to replace - memory (UserMemory): The memory to add - user_id (Optional[str]): The user id to add the memory to. If not provided, the memory is added to the "default" user. - Returns: - str: The id of the memory - """ - if self.db: - if user_id is None: - user_id = DEFAULT_USER_ID - - if not memory.updated_at: - memory.updated_at = int(utc_now().timestamp()) - - memory.memory_id = memory_id - memory.user_id = user_id - - self._upsert_db_memory(memory=memory) - - return memory.memory_id - else: - logger.warning("Memory Db not provided.") - return None - - def clear(self) -> None: - """Clears the memory.""" - if self.db: - result = self.db.clear_memories() - if hasattr(result, "__await__"): - import asyncio - - asyncio.create_task(result) # type: ignore[unused-coroutine] - - def delete_user_memory( - self, - memory_id: str, - user_id: Optional[str] = None, - ) -> None: - """Delete a user memory for a given user id - Args: - memory_id (str): The id of the memory to delete - user_id (Optional[str]): The user id to delete the memory from. If not provided, the memory is deleted from the "default" user. - """ - if user_id is None: - user_id = "default" - - if self.db: - self._delete_db_memory(memory_id=memory_id, user_id=user_id) - else: - logger.warning("Memory DB not provided.") - return None - - def clear_user_memories(self, user_id: Optional[str] = None) -> None: - """Clear all memories for a specific user. - - Args: - user_id (Optional[str]): The user id to clear memories for. If not provided, clears memories for the "default" user. - """ - if user_id is None: - logger.warning("Using default user id.") - user_id = "default" - - if not self.db: - logger.warning("Memory DB not provided.") - return - - if isinstance(self.db, MemoryService): - raise ValueError( - "clear_user_memories() is not supported with an async DB. Please use aclear_user_memories() instead." - ) - - # current implementation: fetch all then batch-delete by ID; can be optimized to delete by user_id directly - memories = self.get_user_memories(user_id=user_id) - if not memories: - logger.debug(f"No memories found for user {user_id}") - return - - # Extract memory IDs - memory_ids = [mem.memory_id for mem in memories if mem.memory_id] - - if memory_ids: - # Delete all memories in a single batch operation - self.db.delete_user_memories(memory_ids=memory_ids, user_id=user_id) - logger.debug(f"Cleared {len(memory_ids)} memories for user {user_id}") - - async def aclear_user_memories(self, user_id: Optional[str] = None) -> None: - """Clear all memories for a specific user (async). - - Args: - user_id (Optional[str]): The user id to clear memories for. If not provided, clears memories for the "default" user. - """ - if user_id is None: - user_id = "default" - - if not self.db: - logger.warning("Memory DB not provided.") - return - - if isinstance(self.db, MemoryService): - memories = await self.aget_user_memories(user_id=user_id) - else: - memories = self.get_user_memories(user_id=user_id) - - if not memories: - logger.debug(f"No memories found for user {user_id}") - return - - # Extract memory IDs - memory_ids = [mem.memory_id for mem in memories if mem.memory_id] - - if memory_ids: - # Delete all memories in a single batch operation - if isinstance(self.db, MemoryService): - await self.db.delete_user_memories(memory_ids=memory_ids, user_id=user_id) - else: - self.db.delete_user_memories(memory_ids=memory_ids, user_id=user_id) - logger.debug(f"Cleared {len(memory_ids)} memories for user {user_id}") - - # -*- Agent Functions - def create_user_memories( - self, - message: Optional[str] = None, - messages: Optional[List[Message]] = None, - agent_id: Optional[str] = None, - team_id: Optional[str] = None, - user_id: Optional[str] = None, - ) -> str: - """Creates memories from multiple messages and adds them to the memory db.""" - self.set_log_level() - - if self.db is None: - logger.warning("MemoryDb not provided.") - return "Please provide a db to store memories" - - if isinstance(self.db, MemoryService): - raise ValueError( - "create_user_memories() is not supported with an async DB. Please use acreate_user_memories() instead." - ) - - if not messages and not message: - raise ValueError("You must provide either a message or a list of messages") - - if message: - messages = [Message(role="user", content=message)] - - if not messages or not isinstance(messages, list): - raise ValueError("Invalid messages list") - - if user_id is None: - user_id = "default" - - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - existing_memories = memories.get(user_id, []) # type: ignore - existing_memories = [{"memory_id": memory.memory_id, "memory": memory.memory} for memory in existing_memories] - response = self.create_or_update_memories( # type: ignore - messages=messages, - existing_memories=existing_memories, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - db=self.db, - update_memories=self.update_memories, - add_memories=self.add_memories, - ) - - # We refresh from the DB - self.read_from_db(user_id=user_id) - return response - - async def acreate_user_memories( - self, - message: Optional[str] = None, - messages: Optional[List[Message]] = None, - agent_id: Optional[str] = None, - team_id: Optional[str] = None, - user_id: Optional[str] = None, - ) -> str: - logger.info( - f"Creating memories for user {user_id}, message: {message}, messages: {messages}, agent_id: {agent_id}, team_id: {team_id}" - ) - """Creates memories from multiple messages and adds them to the memory db.""" - self.set_log_level() - - if self.db is None: - logger.warning("MemoryDb not provided.") - return "Please provide a db to store memories" - - if not messages and not message: - raise ValueError("You must provide either a message or a list of messages") - - if message: - messages = [Message(role="user", content=message)] - - if not messages or not isinstance(messages, list): - raise ValueError("Invalid messages list") - - if user_id is None: - user_id = "default" - - if isinstance(self.db, MemoryService): - memories = await self.aread_from_db(user_id=user_id) - else: - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - existing_memories = memories.get(user_id, []) # type: ignore - existing_memories = [{"memory_id": memory.memory_id, "memory": memory.memory} for memory in existing_memories] - - response = await self.acreate_or_update_memories( # type: ignore - messages=messages, - existing_memories=existing_memories, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - db=self.db, - update_memories=self.update_memories, - add_memories=self.add_memories, - ) - - # We refresh from the DB - if isinstance(self.db, MemoryService): - memories = await self.aread_from_db(user_id=user_id) - else: - memories = self.read_from_db(user_id=user_id) - - return response - - def update_memory_task(self, task: str, user_id: Optional[str] = None) -> str: - """Updates the memory with a task""" - - if not self.db: - logger.warning("MemoryDb not provided.") - return "Please provide a db to store memories" - - if not isinstance(self.db, MemoryService): - raise ValueError( - "update_memory_task() is not supported with an async DB. Please use aupdate_memory_task() instead." - ) - - if user_id is None: - user_id = "default" - - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - existing_memories = memories.get(user_id, []) # type: ignore - existing_memories = [{"memory_id": memory.memory_id, "memory": memory.memory} for memory in existing_memories] - # The memory manager updates the DB directly - response = self.run_memory_task( # type: ignore - task=task, - existing_memories=existing_memories, - user_id=user_id, - db=self.db, - delete_memories=self.delete_memories, - update_memories=self.update_memories, - add_memories=self.add_memories, - clear_memories=self.clear_memories, - ) - - # We refresh from the DB - self.read_from_db(user_id=user_id) - - return response - - async def aupdate_memory_task(self, task: str, user_id: Optional[str] = None) -> str: - """Updates the memory with a task""" - self.set_log_level() - - if not self.db: - logger.warning("MemoryDb not provided.") - return "Please provide a db to store memories" - - if user_id is None: - user_id = "default" - - if isinstance(self.db, MemoryService): - memories = await self.aread_from_db(user_id=user_id) - else: - memories = self.read_from_db(user_id=user_id) - - if memories is None: - memories = {} - - existing_memories = memories.get(user_id, []) # type: ignore - existing_memories = [{"memory_id": memory.memory_id, "memory": memory.memory} for memory in existing_memories] - # The memory manager updates the DB directly - response = await self.arun_memory_task( # type: ignore - task=task, - existing_memories=existing_memories, - user_id=user_id, - db=self.db, - delete_memories=self.delete_memories, - update_memories=self.update_memories, - add_memories=self.add_memories, - clear_memories=self.clear_memories, - ) - - # We refresh from the DB - if isinstance(self.db, MemoryService): - await self.aread_from_db(user_id=user_id) - else: - self.read_from_db(user_id=user_id) - - return response - - # -*- Memory Db Functions - def _upsert_db_memory(self, memory: UserMemory) -> str: - """Use this function to add a memory to the database.""" - try: - if not self.db: - raise ValueError("Memory db not initialized") - result = self.db.upsert_user_memory(memory=memory) - if hasattr(result, "__await__"): - import asyncio - - asyncio.create_task(result) # type: ignore[unused-coroutine] - return "Memory added successfully" - except Exception as e: - logger.warning(f"Error storing memory in db: {e}") - return f"Error adding memory: {e}" - - def _delete_db_memory(self, memory_id: str, user_id: Optional[str] = None) -> str: - """Use this function to delete a memory from the database.""" - try: - if not self.db: - raise ValueError("Memory db not initialized") - - if user_id is None: - user_id = DEFAULT_USER_ID - - result = self.db.delete_user_memory(memory_id=memory_id, user_id=user_id) - if hasattr(result, "__await__"): - import asyncio - - asyncio.create_task(result) # type: ignore[unused-coroutine] - return "Memory deleted successfully" - except Exception as e: - logger.warning(f"Error deleting memory in db: {e}") - return f"Error deleting memory: {e}" - - # -*- Utility Functions - def search_user_memories( - self, - query: Optional[str] = None, - limit: Optional[int] = None, - retrieval_method: Optional[Literal["last_n", "first_n", "agentic"]] = None, - user_id: Optional[str] = None, - ) -> List[UserMemory]: - """Search through user memories using the specified retrieval method. - - Args: - query: The search query for agentic search. Required if retrieval_method is "agentic". - limit: Maximum number of memories to return. Defaults to self.retrieval_limit if not specified. Optional. - retrieval_method: The method to use for retrieving memories. Defaults to self.retrieval if not specified. - - "last_n": Return the most recent memories - - "first_n": Return the oldest memories - - "agentic": Return memories most similar to the query, but using an agentic approach - user_id: The user to search for. Optional. - - Returns: - A list of UserMemory objects matching the search criteria. - """ - - if user_id is None: - user_id = "default" - - self.set_log_level() - - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - if not memories: - return [] - - # Use default retrieval method if not specified - retrieval_method = retrieval_method - # Use default limit if not specified - limit = limit - - # Handle different retrieval methods - if retrieval_method == "agentic": - if not query: - raise ValueError("Query is required for agentic search") - - return self._search_user_memories_agentic(user_id=user_id, query=query, limit=limit) - - elif retrieval_method == "first_n": - return self._get_first_n_memories(user_id=user_id, limit=limit) - - else: # Default to last_n - return self._get_last_n_memories(user_id=user_id, limit=limit) - - def _get_response_format(self) -> Union[Dict[str, Any], Type[BaseModel]]: - """Get response format for structured output.""" - return MemorySearchResponse - - def _search_user_memories_agentic(self, user_id: str, query: str, limit: Optional[int] = None) -> List[UserMemory]: - """Search through user memories using agentic search.""" - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - if not memories: - return [] - - model = self.get_model() - - response_format = self._get_response_format() - - logger.debug("Searching for memories", center=True) - - # Get all memories as a list - user_memories: List[UserMemory] = memories[user_id] - system_message_str = "Your task is to search through user memories and return the IDs of the memories that are related to the query.\n" - system_message_str += "\n\n" - for memory in user_memories: - system_message_str += f"ID: {memory.memory_id}\n" - system_message_str += f"Memory: {memory.memory}\n" - if memory.topics: - system_message_str += f"Topics: {','.join(memory.topics)}\n" - system_message_str += "\n" - system_message_str = system_message_str.strip() - system_message_str += "\n\n\n" - system_message_str += "REMEMBER: Only return the IDs of the memories that are related to the query." - - if response_format == {"type": "json_object"}: - # MemorySearchResponse is a class, not a type, so pass it directly - system_message_str += "\n" + get_json_output_prompt(MemorySearchResponse) # type: ignore[arg-type] # type: ignore - - messages_for_model = [ - Message(role="system", content=system_message_str), - Message( - role="user", - content=f"Return the IDs of the memories related to the following query: {query}", - ), - ] - - # Generate a response from the Model using LangChain API - # Use with_structured_output for structured responses - memory_search: Optional[MemorySearchResponse] = None - try: - model_with_structure = model.with_structured_output(MemorySearchResponse) - memory_search_raw = model_with_structure.invoke(messages_for_model) - if isinstance(memory_search_raw, MemorySearchResponse): - memory_search = memory_search_raw - elif isinstance(memory_search_raw, BaseModel): - memory_search = memory_search_raw # type: ignore[assignment] - else: - memory_search = None - except Exception: - # Fallback to regular invoke and parse response - try: - response = model.invoke(messages_for_model) - if isinstance(response.content, str): - memory_search = parse_response_model_str(response.content, MemorySearchResponse) # type: ignore - else: - memory_search = None - except Exception as e: - logger.warning(f"Failed to search memories: {e}") - return [] - - if memory_search is None: - logger.warning("Failed to convert memory_search response to MemorySearchResponse") - return [] - - memories_to_return = [] - if memory_search: - for memory_id in memory_search.memory_ids: - for memory in user_memories: - if memory.memory_id == memory_id: - memories_to_return.append(memory) - return memories_to_return[:limit] - - def _get_last_n_memories(self, user_id: str, limit: Optional[int] = None) -> List[UserMemory]: - """Get the most recent user memories. - - Args: - limit: Maximum number of memories to return. - - Returns: - A list of the most recent UserMemory objects. - """ - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - memories_list = memories.get(user_id, []) - - # Sort memories by updated_at timestamp if available - if memories_list: - # Sort memories by updated_at timestamp (newest first) - # If updated_at is None, place at the beginning of the list - sorted_memories_list = sorted( - memories_list, - key=lambda m: m.updated_at if m.updated_at is not None else 0, - ) - else: - sorted_memories_list = [] - - if limit is not None and limit > 0: - sorted_memories_list = sorted_memories_list[-limit:] - - return sorted_memories_list - - def _get_first_n_memories(self, user_id: str, limit: Optional[int] = None) -> List[UserMemory]: - """Get the oldest user memories. - - Args: - limit: Maximum number of memories to return. - - Returns: - A list of the oldest UserMemory objects. - """ - memories = self.read_from_db(user_id=user_id) - if memories is None: - memories = {} - - MAX_UNIX_TS = 2**63 - 1 - memories_list = memories.get(user_id, []) - # Sort memories by updated_at timestamp if available - if memories_list: - # Sort memories by updated_at timestamp (oldest first) - # If updated_at is None, place at the end of the list - sorted_memories_list = sorted( - memories_list, - key=lambda m: m.updated_at if m.updated_at is not None else MAX_UNIX_TS, - ) - - else: - sorted_memories_list = [] - - if limit is not None and limit > 0: - sorted_memories_list = sorted_memories_list[:limit] - - return sorted_memories_list - - # -*- Async Utility Functions for search_user_memories - async def asearch_user_memories( - self, - query: Optional[str] = None, - limit: Optional[int] = None, - retrieval_method: Optional[Literal["last_n", "first_n", "agentic"]] = None, - user_id: Optional[str] = None, - ) -> List[UserMemory]: - """Async version: Search through user memories using the specified retrieval method. - - Args: - query: The search query for agentic search. Required if retrieval_method is "agentic". - limit: Maximum number of memories to return. Defaults to self.retrieval_limit if not specified. Optional. - retrieval_method: The method to use for retrieving memories. Defaults to self.retrieval if not specified. - - "last_n": Return the most recent memories - - "first_n": Return the oldest memories - - "agentic": Return memories most similar to the query, but using an agentic approach - user_id: The user to search for. Optional. - - Returns: - A list of UserMemory objects matching the search criteria. - """ - if user_id is None: - user_id = "default" - - self.set_log_level() - - memories = await self.aread_from_db(user_id=user_id) - if memories is None: - memories = {} - - if not memories: - return [] - - # Handle different retrieval methods - if retrieval_method == "agentic": - if not query: - raise ValueError("Query is required for agentic search") - - return await self._asearch_user_memories_agentic(user_id=user_id, query=query, limit=limit) - - elif retrieval_method == "first_n": - return await self._aget_first_n_memories(user_id=user_id, limit=limit) - - else: # Default to last_n - return await self._aget_last_n_memories(user_id=user_id, limit=limit) - - async def _aget_last_n_memories(self, user_id: str, limit: Optional[int] = None) -> List[UserMemory]: - """Async version: Get the most recent user memories.""" - memories = await self.aread_from_db(user_id=user_id) - if memories is None: - memories = {} - - memories_list = memories.get(user_id, []) - - if memories_list: - sorted_memories_list = sorted( - memories_list, - key=lambda m: m.updated_at if m.updated_at is not None else 0, - ) - else: - sorted_memories_list = [] - - if limit is not None and limit > 0: - sorted_memories_list = sorted_memories_list[-limit:] - - return sorted_memories_list - - async def _aget_first_n_memories(self, user_id: str, limit: Optional[int] = None) -> List[UserMemory]: - """Async version: Get the oldest user memories.""" - memories = await self.aread_from_db(user_id=user_id) - if memories is None: - memories = {} - - MAX_UNIX_TS = 2**63 - 1 - memories_list = memories.get(user_id, []) - - if memories_list: - sorted_memories_list = sorted( - memories_list, - key=lambda m: m.updated_at if m.updated_at is not None else MAX_UNIX_TS, - ) - else: - sorted_memories_list = [] - - if limit is not None and limit > 0: - sorted_memories_list = sorted_memories_list[:limit] - - return sorted_memories_list - - async def _asearch_user_memories_agentic( - self, user_id: str, query: str, limit: Optional[int] = None - ) -> List[UserMemory]: - """Async version: Search through user memories using agentic search.""" - memories = await self.aread_from_db(user_id=user_id) - if memories is None: - memories = {} - - if not memories: - return [] - - model = self.get_model() - response_format = self._get_response_format() - - logger.debug("Searching for memories (async)", center=True) - - user_memories: List[UserMemory] = memories.get(user_id, []) - if not user_memories: - return [] - - system_message_str = "Your task is to search through user memories and return the IDs of the memories that are related to the query.\n" - system_message_str += "\n\n" - for memory in user_memories: - system_message_str += f"ID: {memory.memory_id}\n" - system_message_str += f"Memory: {memory.memory}\n" - if memory.topics: - system_message_str += f"Topics: {','.join(memory.topics)}\n" - system_message_str += "\n" - system_message_str = system_message_str.strip() - system_message_str += "\n\n\n" - system_message_str += "REMEMBER: Only return the IDs of the memories that are related to the query." - - if response_format == {"type": "json_object"}: - # MemorySearchResponse is a class, not a type, so pass it directly - system_message_str += "\n" + get_json_output_prompt(MemorySearchResponse) # type: ignore[arg-type] - - messages_for_model = [ - Message(role="system", content=system_message_str), - Message( - role="user", - content=f"Return the IDs of the memories related to the following query: {query}", - ), - ] - - # Generate a response from the Model using LangChain API - # Use with_structured_output for structured responses - memory_search: Optional[MemorySearchResponse] = None - try: - model_with_structure = model.with_structured_output(MemorySearchResponse) - memory_search_raw = await model_with_structure.ainvoke(messages_for_model) - if isinstance(memory_search_raw, MemorySearchResponse): - memory_search = memory_search_raw - elif isinstance(memory_search_raw, BaseModel): - memory_search = memory_search_raw # type: ignore[assignment] - else: - memory_search = None - except Exception: - # Fallback to regular ainvoke and parse response - try: - response = await model.ainvoke(messages_for_model) - if isinstance(response.content, str): - memory_search_parsed = parse_response_model_str(response.content, MemorySearchResponse) - if isinstance(memory_search_parsed, MemorySearchResponse): - memory_search = memory_search_parsed - else: - memory_search = None # type: ignore[assignment] - else: - memory_search = None - except Exception as e: - logger.warning(f"Failed to search memories (async): {e}") - return [] - - if memory_search is None: - logger.warning("Failed to convert memory_search response to MemorySearchResponse") - return [] - - memories_to_return = [] - if memory_search: - for memory_id in memory_search.memory_ids: - for memory in user_memories: - if memory.memory_id == memory_id: - memories_to_return.append(memory) - return memories_to_return[:limit] - - def optimize_memories( - self, - user_id: Optional[str] = None, - strategy: Union[ - MemoryOptimizationStrategyType, MemoryOptimizationStrategy - ] = MemoryOptimizationStrategyType.SUMMARIZE, - apply: bool = True, - ) -> List[UserMemory]: - """Optimize user memories using the specified strategy. - - Args: - user_id: User ID to optimize memories for. Defaults to "default". - strategy: Optimization strategy. Can be: - - Enum: MemoryOptimizationStrategyType.SUMMARIZE - - Instance: Custom MemoryOptimizationStrategy instance - apply: If True, automatically replace memories in database. - - Returns: - List of optimized UserMemory objects. - """ - if user_id is None: - user_id = "default" - - if isinstance(self.db, MemoryService): - raise ValueError( - "optimize_memories() is not supported with an async DB. Please use aoptimize_memories() instead." - ) - - # Get user memories - memories = self.get_user_memories(user_id=user_id) - if not memories: - logger.debug("No memories to optimize") - return [] - - # Get strategy instance - if isinstance(strategy, MemoryOptimizationStrategyType): - strategy_instance = MemoryOptimizationStrategyFactory.create_strategy(strategy) - else: - # Already a strategy instance - strategy_instance = strategy - - # Optimize memories using strategy - optimization_model = self.get_model() - optimized_memories = strategy_instance.optimize(memories=memories, model=optimization_model) # type: ignore[arg-type] - - # Apply to database if requested - if apply: - logger.debug(f"Applying optimized memories to database for user {user_id}") - - if not self.db: - logger.warning("Memory DB not provided. Cannot apply optimized memories.") - return optimized_memories - - # Clear all existing memories for the user - self.clear_user_memories(user_id=user_id) - - # Add all optimized memories - for opt_mem in optimized_memories: - # Ensure memory has an ID (generate if needed for new memories) - if not opt_mem.memory_id: - from uuid import uuid4 - - opt_mem.memory_id = str(uuid4()) - - self.db.upsert_user_memory(memory=opt_mem) - - optimized_tokens = strategy_instance.count_tokens(optimized_memories) - logger.debug(f"Optimization complete. New token count: {optimized_tokens}") - - return optimized_memories - - async def aoptimize_memories( - self, - user_id: Optional[str] = None, - strategy: Union[ - MemoryOptimizationStrategyType, MemoryOptimizationStrategy - ] = MemoryOptimizationStrategyType.SUMMARIZE, - apply: bool = True, - ) -> List[UserMemory]: - """Async version of optimize_memories. - - Args: - user_id: User ID to optimize memories for. Defaults to "default". - strategy: Optimization strategy. Can be: - - Enum: MemoryOptimizationStrategyType.SUMMARIZE - - Instance: Custom MemoryOptimizationStrategy instance - apply: If True, automatically replace memories in database. - - Returns: - List of optimized UserMemory objects. - """ - if user_id is None: - user_id = "default" - - # Get user memories - handle both sync and async DBs - if isinstance(self.db, MemoryService): - memories = await self.aget_user_memories(user_id=user_id) - else: - memories = self.get_user_memories(user_id=user_id) - - if not memories: - logger.debug("No memories to optimize") - return [] - - # Get strategy instance - if isinstance(strategy, MemoryOptimizationStrategyType): - strategy_instance = MemoryOptimizationStrategyFactory.create_strategy(strategy) - else: - # Already a strategy instance - strategy_instance = strategy - - # Optimize memories using strategy (async) - optimization_model = self.get_model() - optimized_memories = await strategy_instance.aoptimize(memories=memories, model=optimization_model) # type: ignore[arg-type] - - # Apply to database if requested - if apply: - logger.debug(f"Optimizing memories for user {user_id}") - - if not self.db: - logger.warning("Memory DB not provided. Cannot apply optimized memories.") - return optimized_memories - - # Clear all existing memories for the user - await self.aclear_user_memories(user_id=user_id) - - # Add all optimized memories - for opt_mem in optimized_memories: - # Ensure memory has an ID (generate if needed for new memories) - if not opt_mem.memory_id: - from uuid import uuid4 - - opt_mem.memory_id = str(uuid4()) - - if isinstance(self.db, MemoryService): - await self.db.upsert_user_memory(memory=opt_mem) - elif isinstance(self.db, MemoryService): - self.db.upsert_user_memory(memory=opt_mem) - - optimized_tokens = strategy_instance.count_tokens(optimized_memories) - logger.debug(f"Memory optimization complete. New token count: {optimized_tokens}") - - return optimized_memories - - # --Memory Manager Functions-- - def determine_tools_for_model(self, tools: List[Callable]) -> List[Union[EnhancedTool, dict]]: - # Have to reset each time, because of different user IDs - _function_names = [] - _functions: List[Union[EnhancedTool, dict]] = [] - - for tool in tools: - try: - function_name = tool.__name__ - if function_name in _function_names: - continue - _function_names.append(function_name) - func = EnhancedTool.from_callable( - tool, - name=function_name, - description=tool.__doc__, - ) - _functions.append(func) - logger.debug(f"Added function {func.name}") - except Exception as e: - logger.warning(f"Could not add function {tool}: {e}") - return _functions - - def get_system_message( - self, - existing_memories: Optional[List[Dict[str, Any]]] = None, - enable_delete_memory: bool = True, - enable_clear_memory: bool = True, - enable_update_memory: bool = True, - enable_add_memory: bool = True, - ) -> Message: - if self.system_message is not None: - return Message(role="system", content=self.system_message) - - memory_capture_instructions = self.memory_capture_instructions or dedent( - """\ - Memories should capture personal information about the user that is relevant to the current conversation, such as: - - Personal facts: name, age, occupation, location, interests, and preferences - - Opinions and preferences: what the user likes, dislikes, enjoys, or finds frustrating - - Significant life events or experiences shared by the user - - Important context about the user's current situation, challenges, or goals - - Any other details that offer meaningful insight into the user's personality, perspective, or needs - """ - ) - - # -*- Return a system message for the memory manager - system_prompt_lines = [ - "You are a Memory Manager that is responsible for managing information and preferences about the user. " - "You will be provided with a criteria for memories to capture in the section and a list of existing memories in the section.", - "", - "## When to add or update memories", - "- Your first task is to decide if a memory needs to be added, updated, or deleted based on the user's message OR if no changes are needed.", - "- If the user's message meets the criteria in the section and that information is not already captured in the section, you should capture it as a memory.", - "- If the users messages does not meet the criteria in the section, no memory updates are needed.", - "- If the existing memories in the section capture all relevant information, no memory updates are needed.", - "", - "## How to add or update memories", - "- If you decide to add a new memory, create memories that captures key information, as if you were storing it for future reference.", - "- Memories should be a brief, third-person statements that encapsulate the most important aspect of the user's input, without adding any extraneous information.", - " - Example: If the user's message is 'I'm going to the gym', a memory could be `John Doe goes to the gym regularly`.", - " - Example: If the user's message is 'My name is John Doe', a memory could be `User's name is John Doe`.", - "- Don't make a single memory too long or complex, create multiple memories if needed to capture all the information.", - "- Don't repeat the same information in multiple memories. Rather update existing memories if needed.", - "- If a user asks for a memory to be updated or forgotten, remove all reference to the information that should be forgotten. Don't say 'The user used to like ...`", - "- When updating a memory, append the existing memory with new information rather than completely overwriting it.", - "- When a user's preferences change, update the relevant memories to reflect the new preferences but also capture what the user's preferences used to be and what has changed.", - "", - "## Criteria for creating memories", - "Use the following criteria to determine if a user's message should be captured as a memory.", - "", - "", - memory_capture_instructions, - "", - "", - "## Updating memories", - "You will also be provided with a list of existing memories in the section. You can:", - " - Decide to make no changes.", - ] - if enable_add_memory: - system_prompt_lines.append(" - Decide to add a new memory, using the `add_memory` tool.") - if enable_update_memory: - system_prompt_lines.append(" - Decide to update an existing memory, using the `update_memory` tool.") - if enable_delete_memory: - system_prompt_lines.append(" - Decide to delete an existing memory, using the `delete_memory` tool.") - if enable_clear_memory: - system_prompt_lines.append(" - Decide to clear all memories, using the `clear_memory` tool.") - - system_prompt_lines += [ - "You can call multiple tools in a single response if needed. ", - "Only add or update memories if it is necessary to capture key information provided by the user.", - ] - - if existing_memories and len(existing_memories) > 0: - system_prompt_lines.append("\n") - for existing_memory in existing_memories: - system_prompt_lines.append(f"ID: {existing_memory['memory_id']}") - system_prompt_lines.append(f"Memory: {existing_memory['memory']}") - system_prompt_lines.append("") - system_prompt_lines.append("") - - if self.additional_instructions: - system_prompt_lines.append(self.additional_instructions) - - return Message(role="system", content="\n".join(system_prompt_lines)) - - def create_or_update_memories( - self, - messages: List[Message], - existing_memories: List[Dict[str, Any]], - user_id: str, - db: MemoryService, - agent_id: Optional[str] = None, - team_id: Optional[str] = None, - update_memories: bool = True, - add_memories: bool = True, - ) -> str: - if self.model is None: - logger.error("No model provided for memory manager") - return "No model provided for memory manager" - - logger.debug("MemoryManager Start", center=True) - - if len(messages) == 1: - input_string = self._get_message_content_string(messages[0]) - else: - input_string = f"{', '.join([self._get_message_content_string(m) for m in messages if m.role == 'user' and m.content])}" - - # Use original model directly - response() method doesn't modify model state - # and LangChain models are thread-safe - model_copy = self.model - # Update the Model (set defaults, add logit etc.) - _tools = self.determine_tools_for_model( - self._get_db_tools( - user_id, - db, - input_string, - agent_id=agent_id, - team_id=team_id, - enable_add_memory=add_memories, - enable_update_memory=update_memories, - enable_delete_memory=True, - enable_clear_memory=False, - ), - ) - - # Prepare the List of messages to send to the Model - messages_for_model: List[Message] = [ - self.get_system_message( - existing_memories=existing_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - enable_delete_memory=True, - enable_clear_memory=False, - ), - *messages, - ] - - # Generate a response from the Model (includes running function calls) - # Use LangChain API: bind_tools() + invoke() - model_with_tools = model_copy.bind_tools(_tools) if _tools else model_copy - response = model_with_tools.invoke(messages_for_model) - - # Execute tool calls if present - if response.tool_calls is not None and len(response.tool_calls) > 0: - logger.info(f"Model returned {len(response.tool_calls)} tool calls, executing...") - # Build a map of tool name -> tool (supports both functions and EnhancedTool objects) - tool_map = {} - for tool in _tools: - if hasattr(tool, "name"): # EnhancedTool or BaseTool - tool_map[tool.name] = tool - elif hasattr(tool, "__name__"): # Regular function - tool_map[tool.__name__] = tool - - for tool_call in response.tool_calls: - tool_name = tool_call.get("name") - tool_args = tool_call.get("args", {}) - - if tool_name in tool_map: - try: - logger.info(f"Executing tool: {tool_name} with args: {tool_args}") - tool = tool_map[tool_name] - # Handle both callable functions and tool objects with invoke/run - if hasattr(tool, "invoke"): - result = tool.invoke(tool_args) - elif hasattr(tool, "run"): - result = tool.run(**tool_args) - elif callable(tool): - result = tool(**tool_args) - else: - result = f"Tool {tool_name} is not callable" - logger.info(f"Tool {tool_name} result: {result}") - except Exception as e: - logger.error(f"Error executing tool {tool_name}: {e}") - else: - logger.warning(f"Unknown tool: {tool_name}") - - self.memories_updated = True - else: - logger.debug("Model did not return any tool calls") - - logger.debug("MemoryManager End", center=True) - - content = response.content if hasattr(response, "content") else "No response from model" - if isinstance(content, list): - return " ".join(str(item) for item in content) - return str(content) if content is not None else "No response from model" - - async def acreate_or_update_memories( - self, - messages: List[Message], - existing_memories: List[Dict[str, Any]], - user_id: str, - db: Union[MemoryService], - agent_id: Optional[str] = None, - team_id: Optional[str] = None, - update_memories: bool = True, - add_memories: bool = True, - ) -> str: - if self.model is None: - logger.error("No model provided for memory manager") - return "No model provided for memory manager" - - logger.debug("MemoryManager Start", center=True) - - if len(messages) == 1: - input_string = self._get_message_content_string(messages[0]) - else: - input_string = f"{', '.join([self._get_message_content_string(m) for m in messages if m.role == 'user' and m.content])}" - - # Use original model directly - response() method doesn't modify model state - # and LangChain models are thread-safe - model_copy = self.model - # Update the Model (set defaults, add logit etc.) - if isinstance(db, MemoryService): - _tools = self.determine_tools_for_model( - await self._aget_db_tools( - user_id, - db, - input_string, - agent_id=agent_id, - team_id=team_id, - enable_add_memory=add_memories, - enable_update_memory=update_memories, - enable_delete_memory=True, - enable_clear_memory=False, - ), - ) - else: - _tools = self.determine_tools_for_model( - self._get_db_tools( - user_id, - db, - input_string, - agent_id=agent_id, - team_id=team_id, - enable_add_memory=add_memories, - enable_update_memory=update_memories, - enable_delete_memory=True, - enable_clear_memory=False, - ), - ) - - # Prepare the List of messages to send to the Model - messages_for_model: List[Message] = [ - self.get_system_message( - existing_memories=existing_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - enable_delete_memory=True, - enable_clear_memory=False, - ), - *messages, - ] - - # Generate a response from the Model (includes running function calls) - # Use LangChain API: bind_tools() + ainvoke() - model_with_tools = model_copy.bind_tools(_tools) if _tools else model_copy - response = await model_with_tools.ainvoke(messages_for_model) - - # Execute tool calls if present - if response.tool_calls is not None and len(response.tool_calls) > 0: - logger.info(f"Model returned {len(response.tool_calls)} tool calls, executing...") - # Build a map of tool name -> tool (supports both functions and EnhancedTool objects) - tool_map = {} - for tool in _tools: - if hasattr(tool, "name"): # EnhancedTool or BaseTool - tool_map[tool.name] = tool - elif hasattr(tool, "__name__"): # Regular function - tool_map[tool.__name__] = tool - - for tool_call in response.tool_calls: - tool_name = tool_call.get("name") - tool_args = tool_call.get("args", {}) - - if tool_name in tool_map: - try: - logger.info(f"Executing tool: {tool_name} with args: {tool_args}") - tool = tool_map[tool_name] - # Handle both callable functions and tool objects with invoke/ainvoke - if hasattr(tool, "ainvoke"): - result = await tool.ainvoke(tool_args) - elif hasattr(tool, "invoke"): - result = tool.invoke(tool_args) - elif hasattr(tool, "run"): - result = tool.run(**tool_args) - elif callable(tool): - result = tool(**tool_args) - else: - result = f"Tool {tool_name} is not callable" - logger.info(f"Tool {tool_name} result: {result}") - except Exception as e: - logger.error(f"Error executing tool {tool_name}: {e}") - else: - logger.warning(f"Unknown tool: {tool_name}") - - self.memories_updated = True - else: - logger.debug("Model did not return any tool calls") - - logger.debug("MemoryManager End", center=True) - - content = response.content if hasattr(response, "content") else "No response from model" - if isinstance(content, list): - return " ".join(str(item) for item in content) - return str(content) if content is not None else "No response from model" - - def run_memory_task( - self, - task: str, - existing_memories: List[Dict[str, Any]], - user_id: str, - db: MemoryService, - delete_memories: bool = True, - update_memories: bool = True, - add_memories: bool = True, - clear_memories: bool = True, - ) -> str: - if self.model is None: - logger.error("No model provided for memory manager") - return "No model provided for memory manager" - - logger.debug("MemoryManager Start", center=True) - - # Use original model directly - response() method doesn't modify model state - # and LangChain models are thread-safe - model_copy = self.model - # Update the Model (set defaults, add logit etc.) - _tools = self.determine_tools_for_model( - self._get_db_tools( - user_id, - db, - task, - enable_delete_memory=delete_memories, - enable_clear_memory=clear_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - ), - ) - - # Prepare the List of messages to send to the Model - messages_for_model: List[Message] = [ - self.get_system_message( - existing_memories, - enable_delete_memory=delete_memories, - enable_clear_memory=clear_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - ), - # For models that require a non-system message - Message(role="user", content=task), - ] - - # Generate a response from the Model (includes running function calls) - # Use LangChain API: bind_tools() + invoke() - model_with_tools = model_copy.bind_tools(_tools) if _tools else model_copy - response = model_with_tools.invoke(messages_for_model) - - # Execute tool calls if present - if response.tool_calls is not None and len(response.tool_calls) > 0: - logger.info(f"Model returned {len(response.tool_calls)} tool calls, executing...") - # Build a map of tool name -> tool (supports both functions and EnhancedTool objects) - tool_map = {} - for tool in _tools: - if hasattr(tool, "name"): # EnhancedTool or BaseTool - tool_map[tool.name] = tool - elif hasattr(tool, "__name__"): # Regular function - tool_map[tool.__name__] = tool - - for tool_call in response.tool_calls: - tool_name = tool_call.get("name") - tool_args = tool_call.get("args", {}) - - if tool_name in tool_map: - try: - logger.info(f"Executing tool: {tool_name} with args: {tool_args}") - tool = tool_map[tool_name] - # Handle both callable functions and tool objects with invoke/run - if hasattr(tool, "invoke"): - result = tool.invoke(tool_args) - elif hasattr(tool, "run"): - result = tool.run(**tool_args) - elif callable(tool): - result = tool(**tool_args) - else: - result = f"Tool {tool_name} is not callable" - logger.info(f"Tool {tool_name} result: {result}") - except Exception as e: - logger.error(f"Error executing tool {tool_name}: {e}") - else: - logger.warning(f"Unknown tool: {tool_name}") - - self.memories_updated = True - else: - logger.debug("Model did not return any tool calls") - - logger.debug("MemoryManager End", center=True) - - content = response.content if hasattr(response, "content") else "No response from model" - if isinstance(content, list): - return " ".join(str(item) for item in content) - return str(content) if content is not None else "No response from model" - - async def arun_memory_task( - self, - task: str, - existing_memories: List[Dict[str, Any]], - user_id: str, - db: Union[MemoryService], - delete_memories: bool = True, - clear_memories: bool = True, - update_memories: bool = True, - add_memories: bool = True, - ) -> str: - if self.model is None: - logger.error("No model provided for memory manager") - return "No model provided for memory manager" - - logger.debug("MemoryManager Start", center=True) - - # Use original model directly - response() method doesn't modify model state - # and LangChain models are thread-safe - model_copy = self.model - # Update the Model (set defaults, add logit etc.) - if isinstance(db, MemoryService): - _tools = self.determine_tools_for_model( - await self._aget_db_tools( - user_id, - db, - task, - enable_delete_memory=delete_memories, - enable_clear_memory=clear_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - ), - ) - else: - _tools = self.determine_tools_for_model( - self._get_db_tools( - user_id, - db, - task, - enable_delete_memory=delete_memories, - enable_clear_memory=clear_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - ), - ) - - # Prepare the List of messages to send to the Model - messages_for_model: List[Message] = [ - self.get_system_message( - existing_memories, - enable_delete_memory=delete_memories, - enable_clear_memory=clear_memories, - enable_update_memory=update_memories, - enable_add_memory=add_memories, - ), - # For models that require a non-system message - Message(role="user", content=task), - ] - - # Generate a response from the Model (includes running function calls) - # Use LangChain API: bind_tools() + ainvoke() - model_with_tools = model_copy.bind_tools(_tools) if _tools else model_copy - response = await model_with_tools.ainvoke(messages_for_model) - - # Execute tool calls if present - if response.tool_calls is not None and len(response.tool_calls) > 0: - logger.info(f"Model returned {len(response.tool_calls)} tool calls, executing...") - # Build a map of tool name -> tool (supports both functions and EnhancedTool objects) - tool_map = {} - for tool in _tools: - if hasattr(tool, "name"): # EnhancedTool or BaseTool - tool_map[tool.name] = tool - elif hasattr(tool, "__name__"): # Regular function - tool_map[tool.__name__] = tool - - for tool_call in response.tool_calls: - tool_name = tool_call.get("name") - tool_args = tool_call.get("args", {}) - - if tool_name in tool_map: - try: - logger.info(f"Executing tool: {tool_name} with args: {tool_args}") - tool = tool_map[tool_name] - # Handle both callable functions and tool objects with invoke/ainvoke - if hasattr(tool, "ainvoke"): - result = await tool.ainvoke(tool_args) - elif hasattr(tool, "invoke"): - result = tool.invoke(tool_args) - elif hasattr(tool, "run"): - result = tool.run(**tool_args) - elif callable(tool): - result = tool(**tool_args) - else: - result = f"Tool {tool_name} is not callable" - logger.info(f"Tool {tool_name} result: {result}") - except Exception as e: - logger.error(f"Error executing tool {tool_name}: {e}") - else: - logger.warning(f"Unknown tool: {tool_name}") - - self.memories_updated = True - else: - logger.debug("Model did not return any tool calls") - - logger.debug("MemoryManager End", center=True) - - content = response.content if hasattr(response, "content") else "No response from model" - if isinstance(content, list): - return " ".join(str(item) for item in content) - return str(content) if content is not None else "No response from model" - - # -*- DB Functions - def _get_db_tools( - self, - user_id: str, - db: MemoryService, - input_string: str, - enable_add_memory: bool = True, - enable_update_memory: bool = True, - enable_delete_memory: bool = True, - enable_clear_memory: bool = True, - agent_id: Optional[str] = None, - team_id: Optional[str] = None, - ) -> List[Callable]: - def _run_async(coro): - """Helper to run async code in sync context""" - import asyncio - - try: - # Event loop is running, need to use a different approach - import threading - - result = None - exception = None - - def run_in_thread(): - nonlocal result, exception - try: - new_loop = asyncio.new_event_loop() - asyncio.set_event_loop(new_loop) - result = new_loop.run_until_complete(coro) - new_loop.close() - except Exception as e: - exception = e - - thread = threading.Thread(target=run_in_thread) - thread.start() - thread.join() - - if exception: - raise exception - return result - except RuntimeError: - # No event loop running, safe to use asyncio.run - return asyncio.run(coro) - - def add_memory(memory: str, topics: Optional[List[str]] = None) -> str: - """Use this function to add a memory to the database. - Args: - memory (str): The memory to be added. - topics (Optional[List[str]]): The topics of the memory (e.g. ["name", "hobbies", "location"]). - Returns: - str: A message indicating if the memory was added successfully or not. - """ - import asyncio - from uuid import uuid4 - - from app.schemas.memory import UserMemory - - try: - memory_id = str(uuid4()) - # Run async method in sync context - try: - # Event loop is running, schedule coroutine - import nest_asyncio - - nest_asyncio.apply() - asyncio.run( - db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - memory=memory, - topics=topics, - input=input_string, - ) - ) - ) - except RuntimeError: - # No event loop running, safe to use asyncio.run - asyncio.run( - db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - memory=memory, - topics=topics, - input=input_string, - ) - ) - ) - logger.debug(f"Memory added: {memory_id}") - return "Memory added successfully" - except Exception as e: - logger.warning(f"Error storing memory in db: {e}") - return f"Error adding memory: {e}" - - def update_memory(memory_id: str, memory: str, topics: Optional[List[str]] = None) -> str: - """Use this function to update an existing memory in the database. - Args: - memory_id (str): The id of the memory to be updated. - memory (str): The updated memory. - topics (Optional[List[str]]): The topics of the memory (e.g. ["name", "hobbies", "location"]). - Returns: - str: A message indicating if the memory was updated successfully or not. - """ - from app.schemas.memory import UserMemory - - if memory == "": - return "Can't update memory with empty string. Use the delete memory function if available." - - try: - _run_async( - db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - memory=memory, - topics=topics, - user_id=user_id, - input=input_string, - ) - ) - ) - logger.debug("Memory updated") - return "Memory updated successfully" - except Exception as e: - logger.warning(f"Error storing memory in db: {e}") - return f"Error adding memory: {e}" - - def delete_memory(memory_id: str) -> str: - """Use this function to delete a single memory from the database. - Args: - memory_id (str): The id of the memory to be deleted. - Returns: - str: A message indicating if the memory was deleted successfully or not. - """ - try: - _run_async(db.delete_user_memory(memory_id=memory_id, user_id=user_id)) - logger.debug("Memory deleted") - return "Memory deleted successfully" - except Exception as e: - logger.warning(f"Error deleting memory in db: {e}") - return f"Error deleting memory: {e}" - - def clear_memory() -> str: - """Use this function to remove all (or clear all) memories from the database. - - Returns: - str: A message indicating if the memory was cleared successfully or not. - """ - _run_async(db.clear_memories()) - logger.debug("Memory cleared") - return "Memory cleared successfully" - - functions: List[Callable] = [] - if enable_add_memory: - functions.append(add_memory) - if enable_update_memory: - functions.append(update_memory) - if enable_delete_memory: - functions.append(delete_memory) - if enable_clear_memory: - functions.append(clear_memory) - return functions - - async def _aget_db_tools( - self, - user_id: str, - db: Union[MemoryService], - input_string: str, - enable_add_memory: bool = True, - enable_update_memory: bool = True, - enable_delete_memory: bool = True, - enable_clear_memory: bool = True, - agent_id: Optional[str] = None, - team_id: Optional[str] = None, - ) -> List[Callable]: - async def add_memory(memory: str, topics: Optional[List[str]] = None) -> str: - """Use this function to add a memory to the database. - Args: - memory (str): The memory to be added. - topics (Optional[List[str]]): The topics of the memory (e.g. ["name", "hobbies", "location"]). - Returns: - str: A message indicating if the memory was added successfully or not. - """ - from uuid import uuid4 - - from app.schemas.memory import UserMemory - - try: - memory_id = str(uuid4()) - if isinstance(db, MemoryService): - await db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - memory=memory, - topics=topics, - input=input_string, - ) - ) - else: - db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - user_id=user_id, - agent_id=agent_id, - team_id=team_id, - memory=memory, - topics=topics, - input=input_string, - ) - ) - logger.debug(f"Memory added: {memory_id}") - return "Memory added successfully" - except Exception as e: - logger.warning(f"Error storing memory in db: {e}") - return f"Error adding memory: {e}" - - async def update_memory(memory_id: str, memory: str, topics: Optional[List[str]] = None) -> str: - """Use this function to update an existing memory in the database. - Args: - memory_id (str): The id of the memory to be updated. - memory (str): The updated memory. - topics (Optional[List[str]]): The topics of the memory (e.g. ["name", "hobbies", "location"]). - Returns: - str: A message indicating if the memory was updated successfully or not. - """ - from app.schemas.memory import UserMemory - - if memory == "": - return "Can't update memory with empty string. Use the delete memory function if available." - - try: - if isinstance(db, MemoryService): - await db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - memory=memory, - topics=topics, - input=input_string, - ) - ) - else: - db.upsert_user_memory( - UserMemory( - memory_id=memory_id, - memory=memory, - topics=topics, - input=input_string, - ) - ) - logger.debug("Memory updated") - return "Memory updated successfully" - except Exception as e: - logger.warning(f"Error storing memory in db: {e}") - return f"Error adding memory: {e}" - - async def delete_memory(memory_id: str) -> str: - """Use this function to delete a single memory from the database. - Args: - memory_id (str): The id of the memory to be deleted. - Returns: - str: A message indicating if the memory was deleted successfully or not. - """ - try: - if isinstance(db, MemoryService): - await db.delete_user_memory(memory_id=memory_id, user_id=user_id) - else: - db.delete_user_memory(memory_id=memory_id, user_id=user_id) - logger.debug("Memory deleted") - return "Memory deleted successfully" - except Exception as e: - logger.warning(f"Error deleting memory in db: {e}") - return f"Error deleting memory: {e}" - - async def clear_memory() -> str: - """Use this function to remove all (or clear all) memories from the database. - - Returns: - str: A message indicating if the memory was cleared successfully or not. - """ - if isinstance(db, MemoryService): - await db.clear_memories() - else: - db.clear_memories() - logger.debug("Memory cleared") - return "Memory cleared successfully" - - functions: List[Callable] = [] - if enable_add_memory: - functions.append(add_memory) - if enable_update_memory: - functions.append(update_memory) - if enable_delete_memory: - functions.append(delete_memory) - if enable_clear_memory: - functions.append(clear_memory) - return functions diff --git a/backend/app/core/agent/memory/strategies/__init__.py b/backend/app/core/agent/memory/strategies/__init__.py deleted file mode 100644 index 285682875..000000000 --- a/backend/app/core/agent/memory/strategies/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Memory optimization strategy implementations.""" - -from app.core.agent.memory.strategies.base import MemoryOptimizationStrategy -from app.core.agent.memory.strategies.summarize import SummarizeStrategy -from app.core.agent.memory.strategies.types import ( - MemoryOptimizationStrategyFactory, - MemoryOptimizationStrategyType, -) - -__all__ = [ - "MemoryOptimizationStrategy", - "MemoryOptimizationStrategyFactory", - "MemoryOptimizationStrategyType", - "SummarizeStrategy", -] diff --git a/backend/app/core/agent/memory/strategies/base.py b/backend/app/core/agent/memory/strategies/base.py deleted file mode 100644 index 737f1f745..000000000 --- a/backend/app/core/agent/memory/strategies/base.py +++ /dev/null @@ -1,68 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List - -from langchain_core.language_models.llms import LLM - -from app.schemas.memory import UserMemory -from app.utils.tokens import count_tokens as count_text_tokens - - -class MemoryOptimizationStrategy(ABC): - """Abstract base class for memory optimization strategies. - - Subclasses must implement optimize() and aoptimize(). - get_system_prompt() is optional and only needed for LLM-based strategies. - """ - - def get_system_prompt(self) -> str: - """Get system prompt for this optimization strategy. - - Returns: - System prompt string for LLM-based strategies. - """ - raise NotImplementedError - - @abstractmethod - def optimize( - self, - memories: List[UserMemory], - model: LLM, - ) -> List[UserMemory]: - """Optimize memories synchronously. - - Args: - memories: List of UserMemory objects to optimize - model: Model to use for optimization (if needed) - - Returns: - List of optimized UserMemory objects - """ - raise NotImplementedError - - @abstractmethod - async def aoptimize( - self, - memories: List[UserMemory], - model: LLM, - ) -> List[UserMemory]: - """Optimize memories asynchronously. - - Args: - memories: List of UserMemory objects to optimize - model: Model to use for optimization (if needed) - - Returns: - List of optimized UserMemory objects - """ - raise NotImplementedError - - def count_tokens(self, memories: List[UserMemory]) -> int: - """Count total tokens across all memories. - - Args: - memories: List of UserMemory objects - - Returns: - Total token count using tiktoken (or fallback estimation) - """ - return sum(count_text_tokens(mem.memory or "") for mem in memories) diff --git a/backend/app/core/agent/memory/strategies/summarize.py b/backend/app/core/agent/memory/strategies/summarize.py deleted file mode 100644 index f5f3eba7b..000000000 --- a/backend/app/core/agent/memory/strategies/summarize.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Summarize strategy: Combine all memories into single comprehensive summary.""" - -from textwrap import dedent -from typing import List -from uuid import uuid4 - -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages.chat import ChatMessage as Message -from loguru import logger - -from app.core.agent.memory.strategies import MemoryOptimizationStrategy -from app.schemas.memory import UserMemory -from app.utils.datetime import utc_now - - -class SummarizeStrategy(MemoryOptimizationStrategy): - """Combine all memories into single comprehensive summary. - - This strategy summarizes all memories into one coherent narrative, - achieving maximum compression by eliminating redundancy. All - metadata (topics, user_id) is preserved in the summarized memory. - """ - - def _get_system_prompt(self) -> str: - """Get system prompt for memory summarization. - - Returns: - System prompt string for LLM - """ - return dedent("""\ - You are a memory compression assistant. Your task is to summarize multiple memories about a user - into a single comprehensive summary while preserving all key facts. - - Requirements: - - Combine related information from all memories - - Preserve all factual information - - Remove redundancy and consolidate repeated facts - - Create a coherent narrative about the user - - Maintain third-person perspective - - Do not add information not present in the original memories - - Return only the summarized memory text, nothing else.\ - """) - - def optimize( - self, - memories: List[UserMemory], - model: BaseChatModel, # type: ignore[override] - ) -> List[UserMemory]: - """Summarize multiple memories into single comprehensive summary. - - Args: - memories: List of UserMemory objects to summarize - model: Model to use for summarization - - Returns: - List containing single summarized UserMemory object - - Raises: - ValueError: If memories list is empty or if user_id cannot be determined - """ - # Validate memories list - if not memories: - raise ValueError("No Memories found") - - # Extract user_id from first memory - user_id = memories[0].user_id - if user_id is None: - raise ValueError("Cannot determine user_id: first memory does not have a valid user_id or is None") - - # Collect all memory contents - memory_contents = [mem.memory for mem in memories if mem.memory] - - # Combine topics - get unique topics from all memories - all_topics: List[str] = [] - for mem in memories: - if mem.topics: - all_topics.extend(mem.topics) - summarized_topics = list(set(all_topics)) if all_topics else None - - # Check if agent_id and team_id are consistent - agent_ids = {mem.agent_id for mem in memories if mem.agent_id} - summarized_agent_id = list(agent_ids)[0] if len(agent_ids) == 1 else None - - team_ids = {mem.team_id for mem in memories if mem.team_id} - summarized_team_id = list(team_ids)[0] if len(team_ids) == 1 else None - - # Create comprehensive prompt for summarization - combined_content = "\n\n".join([f"Memory {i + 1}: {content}" for i, content in enumerate(memory_contents)]) - - system_prompt = self._get_system_prompt() - - messages_for_model = [ - Message(role="system", content=system_prompt), - Message(role="user", content=f"Summarize these memories into a single summary:\n\n{combined_content}"), - ] - - # Generate summarized content - response = model.invoke(messages_for_model) - summarized_content = response.content if hasattr(response, "content") else " ".join(memory_contents) - content_str = str(summarized_content) if not isinstance(summarized_content, str) else summarized_content - if isinstance(content_str, list): - content_str = " ".join(str(item) for item in content_str) - if not content_str: - content_str = " ".join(memory_contents) - - # Generate new memory_id - new_memory_id = str(uuid4()) - - # Create summarized memory - summarized_memory = UserMemory( - memory_id=new_memory_id, - memory=content_str.strip() if isinstance(content_str, str) else "", - topics=summarized_topics, - user_id=user_id, - agent_id=summarized_agent_id, - team_id=summarized_team_id, - updated_at=int(utc_now().timestamp()), - ) - - logger.debug( - f"Summarized {len(memories)} memories into 1: {self.count_tokens(memories)} -> {self.count_tokens([summarized_memory])} tokens" - ) - - return [summarized_memory] - - async def aoptimize( - self, - memories: List[UserMemory], - model: BaseChatModel, # type: ignore[override] - ) -> List[UserMemory]: - """Async version: Summarize multiple memories into single comprehensive summary. - - Args: - memories: List of UserMemory objects to summarize - model: Model to use for summarization - - Returns: - List containing single summarized UserMemory object - - Raises: - ValueError: If memories list is empty or if user_id cannot be determined - """ - # Validate memories list - if not memories: - raise ValueError("No Memories found") - - # Extract user_id from first memory - user_id = memories[0].user_id - if user_id is None: - raise ValueError("Cannot determine user_id: first memory does not have a valid user_id or is None") - - # Collect all memory contents - memory_contents = [mem.memory for mem in memories if mem.memory] - - # Combine topics - get unique topics from all memories - all_topics: List[str] = [] - for mem in memories: - if mem.topics: - all_topics.extend(mem.topics) - summarized_topics = list(set(all_topics)) if all_topics else None - - # Check if agent_id and team_id are consistent - agent_ids = {mem.agent_id for mem in memories if mem.agent_id} - summarized_agent_id = list(agent_ids)[0] if len(agent_ids) == 1 else None - - team_ids = {mem.team_id for mem in memories if mem.team_id} - summarized_team_id = list(team_ids)[0] if len(team_ids) == 1 else None - - # Create comprehensive prompt for summarization - combined_content = "\n\n".join([f"Memory {i + 1}: {content}" for i, content in enumerate(memory_contents)]) - - system_prompt = self._get_system_prompt() - - messages_for_model = [ - Message(role="system", content=system_prompt), - Message(role="user", content=f"Summarize these memories into a single summary:\n\n{combined_content}"), - ] - - # Generate summarized content (async) - response = await model.ainvoke(messages_for_model) - summarized_content = response.content if hasattr(response, "content") else " ".join(memory_contents) - content_str = str(summarized_content) if not isinstance(summarized_content, str) else summarized_content - if isinstance(content_str, list): - content_str = " ".join(str(item) for item in content_str) - if not content_str: - content_str = " ".join(memory_contents) - - # Generate new memory_id - new_memory_id = str(uuid4()) - - # Create summarized memory - summarized_memory = UserMemory( - memory_id=new_memory_id, - memory=content_str.strip() if isinstance(content_str, str) else "", - topics=summarized_topics, - user_id=user_id, - agent_id=summarized_agent_id, - team_id=summarized_team_id, - updated_at=int(utc_now().timestamp()), - ) - - logger.debug( - f"Summarized {len(memories)} memories into 1: {self.count_tokens(memories)} -> {self.count_tokens([summarized_memory])} tokens" - ) - - return [summarized_memory] diff --git a/backend/app/core/agent/memory/strategies/types.py b/backend/app/core/agent/memory/strategies/types.py deleted file mode 100644 index 3e226ab8d..000000000 --- a/backend/app/core/agent/memory/strategies/types.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Memory optimization strategy types and factory.""" - -from enum import Enum - -from app.core.agent.memory.strategies import MemoryOptimizationStrategy - - -class MemoryOptimizationStrategyType(str, Enum): - """Enumeration of available memory optimization strategies.""" - - SUMMARIZE = "summarize" - - -class MemoryOptimizationStrategyFactory: - """Factory for creating memory optimization strategy instances.""" - - @classmethod - def create_strategy(cls, strategy_type: MemoryOptimizationStrategyType, **kwargs) -> MemoryOptimizationStrategy: - """Create an instance of the optimization strategy with given parameters. - - Args: - strategy_type: Type of strategy to create - **kwargs: Additional parameters for strategy initialization - - Returns: - MemoryOptimizationStrategy instance - """ - strategy_map = { - MemoryOptimizationStrategyType.SUMMARIZE: cls._create_summarize_strategy, - } - return strategy_map[strategy_type](**kwargs) - - @classmethod - def _create_summarize_strategy(cls, **kwargs) -> MemoryOptimizationStrategy: - from app.core.agent.memory.strategies.summarize import SummarizeStrategy - - return SummarizeStrategy(**kwargs) diff --git a/backend/app/core/agent/middleware/SKILL_MIDDLEWARE.md b/backend/app/core/agent/middleware/SKILL_MIDDLEWARE.md deleted file mode 100644 index e64262647..000000000 --- a/backend/app/core/agent/middleware/SKILL_MIDDLEWARE.md +++ /dev/null @@ -1,566 +0,0 @@ -# Skill Middleware 使用说明 - -## 概述 - -Skill Middleware 实现了**渐进式技能披露（Progressive Disclosure）**模式，允许 Agent 按需加载技能内容，而不是一次性加载所有技能信息。这种方式可以： - -- **减少上下文使用**：只加载当前任务需要的 2-3 个技能，而不是所有可用技能 -- **提高可扩展性**：可以添加数十或数百个技能而不会使上下文窗口过载 -- **简化对话历史**：单一 Agent 使用一个对话线程 -- **支持团队自治**：不同团队可以独立开发和维护专业技能 - -## 架构设计 - -### 分层架构 - -``` -Middleware Layer (SkillsMiddleware) - ↓ 读取预加载文件 -Sandbox Backend (/workspace/skills/) - ↓ 预加载 -Service Layer (SkillService) - ↓ -Repository Layer (SkillRepository) - ↓ -Database (PostgreSQL) -``` - -**注意**：详细架构设计请参考 [Skill 架构文档](../../skill/ARCHITECTURE.md) - -### 工作流程 - -```mermaid -graph TD - A[构建 Agent] --> B[SkillSandboxLoader 预加载技能文件] - B --> C[文件写入 /workspace/skills/] - C --> D[Agent with SkillsMiddleware] - D --> E[系统提示注入技能描述] - E --> F[用户请求] - F --> G[Agent 决定需要技能] - G --> H[Agent 直接读取沙箱文件] - H --> I[/workspace/skills/skill-name/SKILL.md] - I --> J[Agent 使用技能完成任务] -``` - -## 核心组件 - -### 1. SkillsMiddleware (deepagents) - -技能中间件使用系统库 `deepagents.middleware.skills.SkillsMiddleware`，负责： -- 在 `before_agent` 钩子中从 BackendProtocol 加载可用技能列表 -- 在 `wrap_model_call` 中将技能描述注入系统提示 -- 支持多个 sources，后面的源覆盖前面的同名技能 -- 技能必须预加载到 `/workspace/skills/` 目录（通过 `SkillSandboxLoader`） - -### 2. SkillSandboxLoader - -**位置**：`core/skill/sandbox_loader.py` - -负责在构建时预加载技能文件到沙箱： -- 从数据库加载技能及其文件 -- 将文件写入 `/workspace/skills/{skill_name}/` 目录 -- 支持增量加载（避免重复加载已存在的技能） -- 在 DeepAgents 构建时自动调用 - -**预加载时机**： -- DeepAgents 构建时：通过 `_preload_skills_to_backend` 自动预加载 -- 常规 Agent：如果配置了技能，也会在构建时预加载 - -### 3. Agent 直接读取文件 - -**重要**：Agent 应该直接读取沙箱中的技能文件，而不是通过工具加载。 - -技能文件结构： -``` -/workspace/skills/ -├── skill-name-1/ -│ ├── SKILL.md # 技能说明（包含 Instructions） -│ ├── file1.py -│ └── subdir/ -│ └── file2.py -└── skill-name-2/ - └── SKILL.md -``` - -Agent 读取方式： -1. **读取 SKILL.md 获取 Instructions**： - ```python - # Agent 可以通过 FilesystemMiddleware 读取 - content = read_file("/workspace/skills/pdf-skill/SKILL.md") - ``` - -2. **读取其他文件**： - ```python - # Agent 可以根据需要读取技能的其他文件 - code = read_file("/workspace/skills/pdf-skill/utils.py") - ``` - -**为什么直接读取文件？** -- ✅ 文件已经预加载到沙箱，无需重复加载 -- ✅ 架构更清晰：预加载阶段和运行时阶段分离 -- ✅ 性能更好：避免数据库查询和格式化开销 -- ✅ Agent 可以按需读取，只加载需要的文件 - -### 4. SkillFormatter - -**位置**：`core/skill/formatter.py` - -技能内容格式化器，纯函数实现： -- `format_skill_content()`: 格式化单个技能内容 -- `format_skill_list()`: 格式化技能列表为 Markdown -- 无副作用，可独立测试 - -### 5. SkillService - -服务层提供以下方法： -- `list_skills()`: 获取用户可访问的技能列表 -- `get_skill_by_name()`: 根据名称查找技能（不区分大小写） -- `format_skill_content()`: 格式化技能内容为字符串（使用 `SkillFormatter`，主要用于 API 响应） - -## 使用方法 - -### 1. 在常规 Agent 中启用技能 - -```python -from app.core.agent.sample_agent import get_agent -from app.core.database import async_session_factory - -# 创建带技能支持的 Agent -agent = await get_agent( - user_id="user-123", - enable_skills=True, # 启用技能中间件（默认 True） - skill_user_id="user-123", # 技能过滤的用户ID（默认使用 user_id） - # ... 其他参数 -) -``` - -### 2. 在 Deep Agents 中启用技能 - -Deep Agents 会自动为所有节点（Manager 和 Worker）启用技能支持： - -```python -from app.core.graph.deep_agents_builder import DeepAgentsGraphBuilder - -builder = DeepAgentsGraphBuilder( - graph=graph, - nodes=nodes, - edges=edges, - user_id="user-123", - # ... 其他参数 -) - -# 构建时自动集成技能中间件 -agent_graph = await builder.build() -``` - -### 3. 创建技能 - -通过 API 创建技能： - -```python -POST /api/v1/skills -{ - "name": "pdf-skill", - "description": "处理 PDF 文件的专业技能", - "content": "# PDF 处理技能\n\n## 功能\n- 解析 PDF 内容\n- 提取文本\n- 处理表格数据", - "tags": ["pdf", "document"], - "is_public": false -} -``` - -### 4. Agent 使用技能 - -Agent 会自动看到可用技能的描述，并直接读取沙箱中的文件： - -``` -用户: 帮我处理这个 PDF 文件 - -Agent: 我看到有一个 pdf-skill 技能可以处理 PDF 文件。 - 让我读取这个技能的详细说明... - -[Agent 读取 /workspace/skills/pdf-skill/SKILL.md] - -Agent: 已读取 PDF 处理技能说明。现在我可以： - - 解析 PDF 内容 - - 提取文本 - - 处理表格数据 - - 请提供 PDF 文件路径... -``` - -**Agent 读取技能文件的步骤**： -1. SkillsMiddleware 已在系统提示中注入技能描述 -2. Agent 根据描述决定使用哪个技能 -3. Agent 通过 FilesystemMiddleware 读取 `/workspace/skills/{skill_name}/SKILL.md` -4. Agent 根据需要读取其他文件（如 Python 代码、配置文件等） -5. Agent 使用技能完成任务 - -## 配置选项 - -### SkillMiddleware 参数 - -```python -from deepagents.middleware.skills import SkillsMiddleware - -# SkillsMiddleware 需要 backend 和 sources -middleware = SkillsMiddleware( - backend=backend, # BackendProtocol 实例 - sources=["/workspace/skills/"], # 技能源路径列表 -) -``` - -如果没有 backend，可以使用适配器： - -```python -from app.core.agent.midware.skill_adapter import DatabaseSkillAdapter - -middleware = DatabaseSkillAdapter( - user_id="user-123", # 用户ID，用于过滤技能 - skill_ids=[uuid1, uuid2], # 可选：特定技能ID列表 - db_session_factory=async_session_factory, # 数据库会话工厂 -) -``` - -**SkillsMiddleware 参数说明：** - -- `backend` (BackendProtocol): 必需，后端协议实例，用于读取技能文件 -- `sources` (List[str]): 必需，技能源路径列表，例如 `["/workspace/skills/"]` - - 技能必须通过 `SkillSandboxLoader` 预加载到这些路径 - - 支持多个源，后面的源覆盖前面的同名技能 - -**DatabaseSkillAdapter 参数说明：** - -- `user_id` (Optional[str]): 用户ID，用于过滤用户可访问的技能 - - 如果为 `None`，只加载公开技能 - - 如果提供，加载用户自己的技能和公开技能 - -- `skill_ids` (Optional[List[UUID]]): 可选，特定技能ID列表 - - 如果提供，只加载指定的技能 - - 如果为 `None`，加载所有可访问的技能 - -- `db_session_factory` (Optional[Callable]): 异步数据库会话工厂 - - 默认使用 `async_session_factory` - - 用于创建数据库会话以查询技能 - -- `backend_factory` (Optional[Callable]): 后端工厂函数 - - 默认使用 `StateBackend` - - 用于创建临时后端存储技能文件 - -### 默认系统提示模板 - -```python -""" -## Available Skills - -{skills_prompt} - -When you need detailed information about a skill, read the skill files -directly from /workspace/skills/{skill_name}/. Start with SKILL.md for -instructions, then read other files as needed. -""" -``` - -## 技能数据结构 - -### Skill 模型 - -```python -{ - "id": "uuid", - "name": "skill-name", # 技能名称（唯一标识） - "description": "简短描述", # 显示在系统提示中 - "content": "完整内容...", # SKILL.md 的 body 部分（Instructions） - "tags": ["tag1", "tag2"], - "is_public": false, - "owner_id": "user-id", - "files": [...] # 关联的文件 -} -``` - -### 技能文件结构 - -技能文件预加载到沙箱后的结构： - -``` -/workspace/skills/pdf-skill/ -├── SKILL.md # 技能说明文件（包含 Instructions） -├── utils.py # 工具函数 -└── examples/ - └── example.pdf # 示例文件 -``` - -**SKILL.md 格式**： -```markdown ---- -name: pdf-skill -description: 处理 PDF 文件的专业技能 -tags: [pdf, document] ---- - -# PDF 处理技能 - -## 功能 -- 解析 PDF 内容 -- 提取文本 -- 处理表格数据 - -## 使用方法 -... -``` - -Agent 应该： -1. 读取 `/workspace/skills/{skill_name}/SKILL.md` 获取完整说明 -2. 根据需要读取其他文件（如 Python 代码、配置文件等） -3. 使用技能完成任务 - -## 权限控制 - -技能系统遵循以下权限规则： - -1. **私有技能**：只有拥有者可以访问 -2. **公开技能**：所有用户都可以访问 -3. **系统技能**：`owner_id` 为 `None` 的技能，所有用户可访问 - -权限检查在 `SkillService` 层进行，确保 Agent 只能访问有权限的技能。 - -## Agent 读取技能文件 - -### 文件读取方式 - -Agent 应该通过 FilesystemMiddleware 直接读取沙箱中的技能文件： - -```python -# 读取技能说明 -skill_instructions = read_file("/workspace/skills/pdf-skill/SKILL.md") - -# 读取其他文件 -utility_code = read_file("/workspace/skills/pdf-skill/utils.py") -``` - -### 文件路径规范 - -技能文件预加载后的路径结构： -- 技能根目录：`/workspace/skills/{skill_name}/` -- 技能说明：`/workspace/skills/{skill_name}/SKILL.md` -- 其他文件：`/workspace/skills/{skill_name}/{relative_path}` - -### 预加载保证 - -**重要**：所有技能都会在 Agent 构建时通过 `SkillSandboxLoader` 预加载到沙箱： -- DeepAgents：在 `_preload_skills_to_backend` 中自动预加载 -- 常规 Agent：如果配置了技能，也会在构建时预加载 -- 支持增量加载：已加载的技能不会重复加载 - -## 故障排除 - -### 1. 技能未加载 - -**问题**：Agent 看不到可用技能 - -**解决方案**： -- 检查 `enable_skills` 参数是否为 `True` -- 确认 `user_id` 正确设置 -- 检查数据库中是否有可访问的技能（用户自己的或公开的） - -### 2. 技能文件未找到 - -**问题**：Agent 无法读取 `/workspace/skills/{skill_name}/SKILL.md` - -**可能原因**： -- 技能未预加载到沙箱 -- 技能名称拼写错误 -- 用户没有访问该技能的权限 - -**解决方案**： -- 检查技能是否在节点配置中正确配置（`config.skills`） -- 确认 `SkillSandboxLoader` 已成功预加载技能 -- 检查日志中是否有预加载错误信息 -- 确认技能是公开的或属于当前用户 - -### 3. 技能文件读取失败 - -**问题**：Agent 无法读取技能文件 - -**可能原因**： -- FilesystemMiddleware 未启用 -- 文件路径错误 -- 权限问题 - -**解决方案**： -- 确保 FilesystemMiddleware 已添加到 Agent 中间件列表 -- 检查文件路径是否正确（使用 `/workspace/skills/{skill_name}/SKILL.md`） -- 确认 backend 已正确创建并启动 - -### 4. 技能内容过长 - -**问题**：技能内容超过上下文限制 - -**解决方案**： -- 将大技能拆分为多个小技能 -- 使用技能文件存储详细内容 -- 考虑使用分页加载（未来功能） - -## 最佳实践 - -### 1. 技能命名 - -- 使用清晰、描述性的名称 -- 使用小写字母和连字符：`pdf-skill`, `sql-query`, `data-analysis` -- 避免使用特殊字符和空格 - -### 2. 技能描述 - -- 保持简短（1-2 句话） -- 明确说明技能的用途 -- 帮助 Agent 判断是否需要加载该技能 - -### 3. 技能内容 - -- 结构化组织内容（使用 Markdown） -- 提供清晰的指令和示例 -- 包含相关的业务逻辑和规则 - -### 4. 技能文件 - -- 将大型内容存储在技能文件中 -- 使用适当的文件类型标识 -- 保持文件内容简洁明了 - -## 示例 - -### 完整示例：SQL 查询助手 - -```python -# 1. 创建技能 -skill_data = { - "name": "sales-analytics", - "description": "销售数据分析的数据库模式和业务逻辑", - "content": """# Sales Analytics Schema - -## Tables - -### customers -- customer_id (PRIMARY KEY) -- name -- email -- signup_date -- status (active/inactive) -- customer_tier (bronze/silver/gold/platinum) - -### orders -- order_id (PRIMARY KEY) -- customer_id (FOREIGN KEY -> customers) -- order_date -- status (pending/completed/cancelled/refunded) -- total_amount -- sales_region (north/south/east/west) - -## Business Logic - -**Active customers**: status = 'active' AND signup_date <= CURRENT_DATE - INTERVAL '90 days' - -**Revenue calculation**: Only count orders with status = 'completed' -""", - "tags": ["sql", "sales", "analytics"], - "is_public": True -} - -# 2. 创建 Agent -agent = await get_agent( - user_id="user-123", - enable_skills=True, - system_prompt="You are a SQL query assistant." -) - -# 3. Agent 自动使用技能 -result = await agent.ainvoke({ - "messages": [{ - "role": "user", - "content": "Write a SQL query to find all active customers who made orders over $1000" - }] -}) - -# Agent 会： -# 1. 看到 "sales-analytics" 技能描述（通过 SkillsMiddleware） -# 2. 读取 /workspace/skills/sales-analytics/SKILL.md 获取完整说明 -# 3. 获取完整的数据库模式和业务逻辑 -# 4. 编写正确的 SQL 查询 -``` - -## API 参考 - -### SkillSandboxLoader 类 - -**位置**：`app.core.skill.sandbox_loader` - -#### `SkillSandboxLoader(skill_service: SkillService, user_id: Optional[str] = None)` - -负责将技能文件从数据库加载到沙箱文件系统。 - -**参数**： -- `skill_service`: SkillService 实例，用于数据库操作 -- `user_id`: 用户ID，用于权限检查（默认 None） - -**方法**： - -- `async load_skill_to_sandbox(skill_id: uuid.UUID, backend: BackendProtocol, user_id: Optional[str] = None) -> bool` - - 加载单个技能到沙箱 - - 返回 True 如果成功，False 如果失败 - -- `async load_skills_to_sandbox(skill_ids: list[uuid.UUID], backend: BackendProtocol, user_id: Optional[str] = None) -> dict[uuid.UUID, bool]` - - 批量加载技能到沙箱 - - `backend`: BackendProtocol 实例 - - `user_id`: 用户ID，用于权限检查（可选） - - 返回字典，映射 skill_id 到加载状态（True/False） - - 注意：每次加载前会清理现有目录，确保文件是最新的 - -**文件组织**： -- 技能文件写入 `/workspace/skills/{skill_name}/` -- 保持原有的文件路径结构 - -### SkillService 方法 - -#### `get_skill_by_name(skill_name: str, current_user_id: Optional[str] = None) -> Optional[Skill]` - -根据名称查找技能（不区分大小写）。 - -**参数：** -- `skill_name`: 技能名称 -- `current_user_id`: 当前用户ID，用于权限检查 - -**返回：** -- `Skill` 对象，如果未找到或无权访问则返回 `None` - -#### `format_skill_content(skill: Skill) -> str` - -格式化技能内容为字符串。 - -**参数：** -- `skill`: Skill 对象（应包含 files 关系） - -**返回：** -- 格式化后的技能内容字符串 - -## 未来增强 - -计划中的功能： - -- [ ] 技能版本控制 -- [ ] 技能使用分析 -- [ ] 技能搜索和过滤 -- [ ] 技能文件缓存优化 -- [ ] 支持技能文件增量更新 - -## 相关文档 - -- [Skill 架构设计文档](../../skill/ARCHITECTURE.md) - 完整的架构设计和分层说明 -- [LangChain Skills Pattern](https://docs.langchain.com/oss/python/langchain/multi-agent/skills-sql-assistant) -- [Progressive Disclosure](https://www.anthropic.com/engineering/equipping-agents-for-the-real-world-with-agent-skills) -- [Skill API 文档](../../../api/v1/skills.py) - -## 支持 - -如有问题或建议，请： -1. 查看日志文件获取详细错误信息 -2. 检查数据库连接和权限设置 -3. 联系开发团队 diff --git a/backend/app/core/agent/middleware/__init__.py b/backend/app/core/agent/middleware/__init__.py deleted file mode 100644 index b5d7fd619..000000000 --- a/backend/app/core/agent/middleware/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Middleware package initialization. -""" - -from .logging import LoggingMiddleware -from .memory_iteration_with_db import AgentMemoryIterationMiddleware - -__all__ = [ - "AgentMemoryMiddleware", - "AgentMemoryIterationMiddleware", - "LoggingMiddleware", -] diff --git a/backend/app/core/agent/middleware/logging.py b/backend/app/core/agent/middleware/logging.py deleted file mode 100644 index 753d7cb1e..000000000 --- a/backend/app/core/agent/middleware/logging.py +++ /dev/null @@ -1,647 +0,0 @@ -"""Logging middleware - Provide comprehensive operation logs and audit trails.""" - -import json -import logging -import time -import traceback -from collections.abc import Awaitable, Callable -from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional - -if TYPE_CHECKING: - pass - -from deepagents.backends.protocol import BackendProtocol -from langchain.agents.middleware.types import AgentMiddleware, AgentState, ModelRequest, ModelResponse -from loguru import logger -from typing_extensions import NotRequired - - -class LoggingState(AgentState): - """State for the logging middleware.""" - - session_id: NotRequired[str] # type: ignore[valid-type] - """Session ID.""" - - log_config: NotRequired[Dict[str, Any]] # type: ignore[valid-type] - """Log configuration.""" - - interaction_count: NotRequired[int] # type: ignore[valid-type] - """Interaction count.""" - - session_start_time: NotRequired[float] # type: ignore[valid-type] - """Session start time.""" - - last_activity: NotRequired[float] # type: ignore[valid-type] - """Last activity time.""" - - -class LoggingMiddleware(AgentMiddleware): - """Logging middleware. - - Provide comprehensive operation logging: - - Conversation history - - Tool call logs - - Performance metrics - - Error tracking - - User behavior analysis - - Session statistics - """ - - state_schema = LoggingState - - def __init__( - self, - *, - backend: BackendProtocol, - log_path: str = "/logs/", - session_id: Optional[str] = None, - enable_conversation_logging: bool = True, - enable_tool_logging: bool = True, - enable_performance_logging: bool = True, - enable_error_logging: bool = True, - log_level: str = "INFO", - max_log_files: int = 10, - max_file_size: int = 10 * 1024 * 1024, # 10MB - rotate_interval: int = 24, # hours - ) -> None: - """Initialize the logging middleware.""" - self.backend = backend - self.log_path = log_path.rstrip("/") + "/" - self.session_id = session_id or self._generate_session_id() - self.enable_conversation_logging = enable_conversation_logging - self.enable_tool_logging = enable_tool_logging - self.enable_performance_logging = enable_performance_logging - self.enable_error_logging = enable_error_logging - self.log_level = getattr(logging, log_level.upper()) - self.max_log_files = max_log_files - self.max_file_size = max_file_size - self.rotate_interval = rotate_interval - - # log file paths - self.conversation_log_path = f"{self.log_path}conversations/{self.session_id}.jsonl" - self.tool_log_path = f"{self.log_path}tools/{self.session_id}.jsonl" - self.performance_log_path = f"{self.log_path}performance/{self.session_id}.jsonl" - self.error_log_path = f"{self.log_path}errors/{self.session_id}.jsonl" - - # initialize log directories - self._init_log_directories() - - def _generate_session_id(self) -> str: - """Generate a session ID.""" - import uuid - - return str(uuid.uuid4())[:12] - - def _init_log_directories(self) -> None: - """Initialize log directories.""" - directories = [ - f"{self.log_path}conversations", - f"{self.log_path}tools", - f"{self.log_path}performance", - f"{self.log_path}errors", - f"{self.log_path}sessions", - ] - - for directory in directories: - try: - self.backend.write(f"{directory}/.gitkeep", "") - except Exception as e: - logger.warning(f"Failed to initialize log directory {directory}: {e}") - - def _write_log_entry(self, log_path: str, entry: Dict[str, Any]) -> None: - """Write a log entry.""" - try: - # add timestamp - entry["timestamp"] = datetime.now().isoformat() - entry["session_id"] = self.session_id - - # write log - log_line = json.dumps(entry, ensure_ascii=False, separators=(",", ":")) - try: - # check if file exists and read existing content (BackendProtocol extension may have exists) - existing_content = "" - exists_fn = getattr(self.backend, "exists", None) - if exists_fn is not None and callable(exists_fn) and exists_fn(log_path): - existing_content = self.backend.read(log_path) or "" - - # add new line - new_content = existing_content + log_line + "\n" - self.backend.write(log_path, new_content) - except Exception: - # if append fails, try writing directly - self.backend.write(log_path, log_line + "\n") - except Exception as e: - logger.warning(f"Failed to write log entry to {log_path}: {e}") - - def _log_conversation_entry(self, entry_type: str, content: str, metadata: Optional[Dict[str, Any]] = None) -> None: - """Log a conversation entry.""" - if not self.enable_conversation_logging: - return - - entry = { - "type": entry_type, - "content": content, - "metadata": metadata or {}, - "length": len(content), - } - - self._write_log_entry(self.conversation_log_path, entry) - - def _log_tool_call( - self, - tool_name: str, - tool_args: Dict[str, Any], - result: Any = None, - execution_time: Optional[float] = None, - error: Optional[str] = None, - ) -> None: - """Log a tool call.""" - if not self.enable_tool_logging: - return - - entry = { - "tool_name": tool_name, - "tool_args": tool_args, - "result_type": type(result).__name__ if result else "None", - "execution_time_seconds": execution_time, - "error": error, - "success": error is None, - } - - # if result is large, log only a summary - if result and len(str(result)) > 1000: - entry["result_summary"] = f"{type(result).__name__} object, size: {len(str(result))} chars" - else: - entry["result"] = result - - self._write_log_entry(self.tool_log_path, entry) - - def _log_performance_metrics(self, operation: str, metrics: Dict[str, Any]) -> None: - """Log performance metrics.""" - if not self.enable_performance_logging: - return - - entry = {"operation": operation, "metrics": metrics} - - self._write_log_entry(self.performance_log_path, entry) - - def _log_error(self, error_type: str, error_message: str, context: Optional[Dict[str, Any]] = None) -> None: - """Log an error.""" - if not self.enable_error_logging: - return - - entry = { - "error_type": error_type, - "error_message": error_message, - "context": context or {}, - "traceback": (traceback.format_exc() if traceback.format_exc().strip() != "NoneType: None" else None), - } - - self._write_log_entry(self.error_log_path, entry) - - def _update_session_stats(self, state: Dict[str, Any]) -> None: # type: ignore[assignment] - """Update session statistics.""" - try: - session_stats = { - "session_id": self.session_id, - "interaction_count": state.get("interaction_count", 0), - "session_start_time": state.get("session_start_time", time.time()), - "last_activity": time.time(), - "total_duration": time.time() - state.get("session_start_time", time.time()), - "log_config": { - "conversation_logging": self.enable_conversation_logging, - "tool_logging": self.enable_tool_logging, - "performance_logging": self.enable_performance_logging, - "error_logging": self.enable_error_logging, - }, - } - - # save session statistics - session_path = f"{self.log_path}sessions/{self.session_id}.json" - self.backend.write(session_path, json.dumps(session_stats, indent=2)) - except Exception as e: - logger.warning(f"Failed to update session stats: {e}") - - def _extract_conversation_content(self, request: ModelRequest) -> str: - """Extract conversation content from a request.""" - if hasattr(request, "content") and request.content: - return str(request.content) - elif hasattr(request, "messages") and request.messages: - # get the last user message - for msg in reversed(request.messages): - # compatible with LangChain Message objects and dict format - role = None - if hasattr(msg, "type"): - role = "user" if msg.type == "human" else "assistant" if msg.type == "ai" else None - elif isinstance(msg, dict): - role = msg.get("role") - - if role == "user": - if hasattr(msg, "content"): - content = msg.content - elif isinstance(msg, dict): - content = msg.get("content", "") - else: - content = "" - return str(content) if content is not None else "" - return "" - - def _extract_response_content(self, response: ModelResponse) -> str: - """Extract response content.""" - if hasattr(response, "content") and response.content: - return str(response.content) - elif hasattr(response, "messages") and response.messages: - # get the last assistant message - for msg in reversed(response.messages): - # compatible with LangChain Message objects and dict format - role = None - if hasattr(msg, "type"): - role = "user" if msg.type == "human" else "assistant" if msg.type == "ai" else None - elif isinstance(msg, dict): - role = msg.get("role") - - if role == "assistant": - if hasattr(msg, "content"): - content = msg.content - elif isinstance(msg, dict): - content = msg.get("content", "") - else: - content = "" - return str(content) if content is not None else "" - return "" - - def _extract_tool_calls(self, response: ModelResponse) -> List[Dict[str, Any]]: - """Extract tool calls from a response.""" - tool_calls = [] - - if hasattr(response, "tool_calls") and response.tool_calls: - tool_calls.extend(response.tool_calls) - elif hasattr(response, "messages") and response.messages: - for msg in response.messages: - if hasattr(msg, "tool_calls") and msg.tool_calls: - tool_calls.extend(msg.tool_calls) - - return tool_calls - - def before_agent( - self, - state: AgentState[Any], # type: ignore[assignment] - runtime, - ) -> dict[str, Any]: # type: ignore[override] - """Initialize logging before agent execution.""" - session_id = self.session_id - - log_config = { - "conversation_logging": self.enable_conversation_logging, - "tool_logging": self.enable_tool_logging, - "performance_logging": self.enable_performance_logging, - "error_logging": self.enable_error_logging, - "log_level": self.log_level, - "max_log_files": self.max_log_files, - "max_file_size": self.max_file_size, - } - - return { - "session_id": session_id, - "log_config": log_config, - "interaction_count": 0, - "session_start_time": time.time(), - "last_activity": time.time(), - "messages": [], # Add required messages key - } - - async def abefore_agent( - self, - state: AgentState[Any], # type: ignore[assignment] - runtime, - ) -> dict[str, Any]: # type: ignore[override] - """Async: initialize logging before agent execution.""" - return self.before_agent(state, runtime) - - def wrap_model_call( - self, - request: ModelRequest, - handler: Callable[[ModelRequest], ModelResponse], - ) -> ModelResponse: - """Wrap model call and log details.""" - start_time = time.time() - - # extract request content - request_content = self._extract_conversation_content(request) - - # log user input - if request_content: - self._log_conversation_entry( - "user_input", - request_content, - {"source": "model_request", "timestamp": datetime.now().isoformat()}, - ) - - try: - # execute model call - response = handler(request) - - # compute execution time - execution_time = time.time() - start_time - - # extract response content - response_content = self._extract_response_content(response) - - # log AI response - if response_content: - self._log_conversation_entry( - "assistant_response", - response_content, - { - "source": "model_response", - "execution_time": execution_time, - "response_length": len(response_content), - }, - ) - - # log tool calls - tool_calls = self._extract_tool_calls(response) - if tool_calls: - for tool_call in tool_calls: - time.time() - tool_name = tool_call.get("name", "unknown") - tool_args = tool_call.get("args", {}) - - try: - # only log the call here; actual execution is handled by other middleware - self._log_tool_call( - tool_name, - tool_args, - execution_time=0.0, # actual execution time is recorded elsewhere - result=None, - ) - except Exception as e: - self._log_error( - "tool_logging_error", - f"Failed to log tool call: {str(e)}", - {"tool_name": tool_name, "tool_args": tool_args}, - ) - - # log performance metrics - self._log_performance_metrics( - "model_call", - { - "execution_time": execution_time, - "request_length": len(request_content) if request_content else 0, - "response_length": len(response_content) if response_content else 0, - "tool_calls_count": len(tool_calls), - }, - ) - - # update interaction count - state_dict = dict(request.state) # type: ignore[arg-type] - current_count = state_dict.get("interaction_count", 0) - if not isinstance(current_count, int): - current_count = 0 - state_dict["interaction_count"] = current_count + 1 # type: ignore[assignment] - state_dict["last_activity"] = time.time() # type: ignore[assignment] - - # update session statistics - self._update_session_stats(state_dict) # type: ignore[arg-type] - - return response - - except Exception as e: - # log error - error_msg = str(e) - execution_time = time.time() - start_time - - # provide more detailed context for "No generations found in stream" errors - context = { - "request_content_preview": ( - request_content[:100] + "..." if len(request_content) > 100 else request_content - ), - "execution_time": execution_time, - } - - # if the error is "No generations found in stream", add extra diagnostics - if "No generations found in stream" in error_msg: - context.update( - { - "error_type": "stream_timeout_or_empty", - "diagnosis": ( - "This error typically occurs when: " - "1) The model stream timed out (default 60s), " - "2) The API returned an empty stream, or " - "3) Network connectivity issues. " - f"Execution time was {execution_time:.2f}s. " - "Consider increasing the model timeout or checking network connectivity." - ), - } - ) - - self._log_error( - "model_call_error", - error_msg, - context, - ) - raise - - async def awrap_model_call( - self, - request: ModelRequest, - handler: Callable[[ModelRequest], Awaitable[ModelResponse]], - ) -> ModelResponse: - """Async: wrap model call and log details.""" - start_time = time.time() - - # extract request content - request_content = self._extract_conversation_content(request) - - # log user input - if request_content: - self._log_conversation_entry( - "user_input", - request_content, - { - "source": "model_request_async", - "timestamp": datetime.now().isoformat(), - }, - ) - - try: - # execute model call - response = await handler(request) - - # compute execution time - execution_time = time.time() - start_time - - # extract response content - response_content = self._extract_response_content(response) - - # log AI response - if response_content: - self._log_conversation_entry( - "assistant_response", - response_content, - { - "source": "model_response_async", - "execution_time": execution_time, - "response_length": len(response_content), - }, - ) - - # log tool calls - tool_calls = self._extract_tool_calls(response) - if tool_calls: - for tool_call in tool_calls: - tool_name = tool_call.get("name", "unknown") - tool_args = tool_call.get("args", {}) - - try: - self._log_tool_call(tool_name, tool_args, execution_time=0.0, result=None) - except Exception as e: - self._log_error( - "tool_logging_error", - f"Failed to log tool call: {str(e)}", - {"tool_name": tool_name, "tool_args": tool_args}, - ) - - # log performance metrics - self._log_performance_metrics( - "model_call_async", - { - "execution_time": execution_time, - "request_length": len(request_content) if request_content else 0, - "response_length": len(response_content) if response_content else 0, - "tool_calls_count": len(tool_calls), - }, - ) - - # update interaction count - state_dict = dict(request.state) # type: ignore[arg-type] - current_count = state_dict.get("interaction_count", 0) - if not isinstance(current_count, int): - current_count = 0 - state_dict["interaction_count"] = current_count + 1 # type: ignore[assignment] - state_dict["last_activity"] = time.time() # type: ignore[assignment] - - # update session statistics - self._update_session_stats(state_dict) # type: ignore[arg-type] - - return response - - except Exception as e: - # log error - error_msg = str(e) - execution_time = time.time() - start_time - - # provide more detailed context for "No generations found in stream" errors - context = { - "request_content_preview": ( - request_content[:100] + "..." if len(request_content) > 100 else request_content - ), - "execution_time": execution_time, - } - - # if the error is "No generations found in stream", add extra diagnostics - if "No generations found in stream" in error_msg: - context.update( - { - "error_type": "stream_timeout_or_empty", - "diagnosis": ( - "This error typically occurs when: " - "1) The model stream timed out (default 60s), " - "2) The API returned an empty stream, or " - "3) Network connectivity issues. " - f"Execution time was {execution_time:.2f}s. " - "Consider increasing the model timeout or checking network connectivity." - ), - } - ) - - self._log_error( - "model_call_error", - error_msg, - context, - ) - raise - - def get_session_statistics(self) -> Dict[str, Any]: - """Get session statistics.""" - try: - session_path = f"{self.log_path}sessions/{self.session_id}.json" - session_data = self.backend.read(session_path) - if session_data: - result = json.loads(session_data) - return result if isinstance(result, dict) else {"error": "Invalid session data format"} - except Exception: - logger.debug("Failed to read session statistics", exc_info=True) - - return {"error": "Session statistics not available"} - - def get_recent_conversations(self, limit: int = 10) -> List[Dict[str, Any]]: - """Get recent conversation records.""" - try: - conversation_data = self.backend.read(self.conversation_log_path) - if conversation_data: - lines = conversation_data.strip().split("\n") - recent_lines = lines[-limit:] if len(lines) > limit else lines - - return [json.loads(line) for line in recent_lines if line.strip()] - except Exception: - logger.debug("Failed to read recent conversations", exc_info=True) - - return [] - - def get_error_summary(self) -> Dict[str, Any]: - """Get error summary.""" - try: - error_data = self.backend.read(self.error_log_path) - if not error_data: - return {"total_errors": 0} - - lines = error_data.strip().split("\n") - error_entries = [] - for line in lines: - if line.strip(): - try: - entry = json.loads(line) - if isinstance(entry, dict): - error_entries.append(entry) - except json.JSONDecodeError: - continue - - error_types: Dict[str, int] = {} - recent_errors: List[Dict[str, Any]] = [] - - for entry in error_entries[-20:]: # last 20 errors - error_type = entry.get("error_type", "unknown") - error_types[error_type] = error_types.get(error_type, 0) + 1 - - if len(recent_errors) < 10: - recent_errors.append( - { - "timestamp": entry.get("timestamp"), - "error_type": error_type, - "error_message": entry.get("error_message", "")[:100], - } - ) - - return { - "total_errors": len(error_entries), - "error_types": error_types, - "recent_errors": recent_errors, - } - except Exception: - return {"error": "Failed to generate error summary"} - - def cleanup_old_logs(self, days_to_keep: int = 30) -> None: - """Clean up old log files.""" - time.time() - (days_to_keep * 24 * 60 * 60) - - try: - # concrete cleanup logic needs to be implemented here; - # limited by BackendProtocol, this is a placeholder - pass - except Exception as e: - logger.warning(f"Failed to cleanup old logs: {e}") - - -# (no additional imports needed) diff --git a/backend/app/core/agent/middleware/memory_iteration_with_db.py b/backend/app/core/agent/middleware/memory_iteration_with_db.py deleted file mode 100644 index b245c76c1..000000000 --- a/backend/app/core/agent/middleware/memory_iteration_with_db.py +++ /dev/null @@ -1,432 +0,0 @@ -"""MemoryManager-driven memory middleware. - -Before model call: -- Retrieve relevant long-term memories for the current user input (supports last_n / first_n / agentic) -- Inject retrieved memories as structured fragments into the system prompt to enrich context - -After model call: -- Submit the current user input to MemoryManager, which decides whether to add/update/delete memories based on capture rules -""" - -import asyncio -from collections.abc import Awaitable, Callable -from typing import TYPE_CHECKING, Any, List, Literal, Optional - -from langchain.agents.middleware.types import AgentMiddleware, AgentState, ModelRequest, ModelResponse -from langchain_core.messages import AIMessage, HumanMessage -from loguru import logger -from typing_extensions import NotRequired - -from app.core.agent.memory.manager import MemoryManager -from app.schemas.memory import UserMemory - -if TYPE_CHECKING: - pass - - -class AgenticMemoryState(AgentState): - """Extended state for the Agentic Memory middleware.""" - - user_id: NotRequired[str | None] # type: ignore[valid-type] - agent_memory_context: NotRequired[str | None] # type: ignore[valid-type] - - -class AgentMemoryIterationMiddleware(AgentMiddleware): - """Agent memory middleware using MemoryManager. - - - before: retrieve user-related memories from MemoryManager and inject into system prompt - - after: pass user input to MemoryManager for memory write/update - - Args: - memory_manager: Pre-configured MemoryManager instance (must have model/db). - retrieval_method: Retrieval method — "last_n" | "first_n" | "agentic". - retrieval_limit: Maximum number of memories to retrieve. - context_header: Header for the memory fragment injected into the system prompt. - enable_writeback: Whether to write memories after the model call. - capture_source: Source for memory capture — "user" or "assistant" (default "user"). - """ - - priority = 50 # medium priority, runs alongside skill middleware - state_schema = AgenticMemoryState - - def __init__( - self, - *, - memory_manager: MemoryManager, - retrieval_method: str = "last_n", - retrieval_limit: int = 5, - context_header: str = "## Relevant User Memories", - enable_writeback: bool = True, - capture_source: str = "user", - user_id: Optional[str] = None, - ) -> None: - self.memory_manager = memory_manager - self.retrieval_method = retrieval_method - self.retrieval_limit = retrieval_limit - self.context_header = context_header - self.enable_writeback = enable_writeback - self.capture_source = capture_source - self.user_id = user_id - - if self.memory_manager is None: - raise ValueError("AgentMemoryManagerMiddleware requires a MemoryManager instance") - - logger.info( - f"AgentMemoryIterationMiddleware initialized: " - f"retrieval_method={retrieval_method}, retrieval_limit={retrieval_limit}, " - f"enable_writeback={enable_writeback}, capture_source={capture_source}, " - f"user_id={user_id}" - ) - - # --------------------------- - # Helpers - # --------------------------- - def _get_user_id(self) -> Optional[str]: - """Get the current user's user_id. - - Returns: - user_id string, or None if not available. - """ - if not self.user_id: - logger.warning("No user_id configured in middleware instance") - return self.user_id - - def _extract_user_input(self, request: ModelRequest) -> Optional[str]: - """Extract user input text from the request (LangGraph ModelRequest format).""" - # get the last HumanMessage from the message list - if hasattr(request, "messages") and request.messages: - try: - # iterate messages in reverse to find the last HumanMessage - for msg in reversed(request.messages): - # check if it is a HumanMessage type - if isinstance(msg, HumanMessage): - content = getattr(msg, "content", None) - if content: - # handle different content types - if isinstance(content, str): - extracted = content - elif isinstance(content, list): - # handle content_blocks format - text_parts = [] - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text_parts.append(block.get("text", "")) - elif isinstance(block, str): - text_parts.append(block) - extracted = " ".join(text_parts) if text_parts else str(content) - else: - extracted = str(content) - - if extracted and extracted.strip(): - logger.debug(f"Extracted user input from HumanMessage: {extracted[:100]}...") - return extracted - - # compatibility: check the message's type attribute (LangChain message types) - elif hasattr(msg, "type"): - if msg.type == "human": - content = getattr(msg, "content", None) - if content: - extracted = content if isinstance(content, str) else str(content) - if extracted and extracted.strip(): - logger.debug( - f"Extracted user input from message type 'human': {extracted[:100]}..." - ) - return extracted - - # compatibility: check dict-format messages - elif isinstance(msg, dict): - msg_type = msg.get("type") or msg.get("role") - if msg_type in ("human", "user"): - content = msg.get("content") - if content: - extracted = content if isinstance(content, str) else str(content) - if extracted and extracted.strip(): - logger.debug(f"Extracted user input from dict message: {extracted[:100]}...") - return extracted - except Exception as e: - logger.warning(f"Failed to extract user input from messages: {e}", exc_info=True) - - logger.debug("No user input found in request messages") - return None - - def _extract_assistant_response(self, response: ModelResponse) -> Optional[str]: - """Extract assistant response text from ModelResponse (LangGraph ModelResponse format).""" - # prefer response.content if it is a string - if hasattr(response, "content"): - content = response.content - if isinstance(content, str): - if content.strip(): - logger.debug(f"Extracted assistant response from response.content: {content[:100]}...") - return content - - # get the last AIMessage from the message list - if hasattr(response, "messages") and response.messages: - try: - # iterate messages in reverse to find the last AIMessage - for msg in reversed(response.messages): - # check if it is an AIMessage type - if isinstance(msg, AIMessage): - content = getattr(msg, "content", None) - if content: - # handle different content types - if isinstance(content, str): - extracted = content - elif isinstance(content, list): - # handle content_blocks format - text_parts = [] - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text_parts.append(block.get("text", "")) - elif isinstance(block, str): - text_parts.append(block) - extracted = " ".join(text_parts) if text_parts else str(content) - else: - extracted = str(content) - - if extracted and extracted.strip(): - logger.debug(f"Extracted assistant response from AIMessage: {extracted[:100]}...") - return extracted - - # compatibility: check the message's type attribute (LangChain message types) - elif hasattr(msg, "type"): - if msg.type == "ai": - content = getattr(msg, "content", None) - if content: - extracted = content if isinstance(content, str) else str(content) - if extracted and extracted.strip(): - logger.debug( - f"Extracted assistant response from message type 'ai': {extracted[:100]}..." - ) - return extracted - - # compatibility: check dict-format messages - elif isinstance(msg, dict): - msg_type = msg.get("type") or msg.get("role") - if msg_type in ("ai", "assistant"): - content = msg.get("content") - if content: - extracted = content if isinstance(content, str) else str(content) - if extracted and extracted.strip(): - logger.debug( - f"Extracted assistant response from dict message: {extracted[:100]}..." - ) - return extracted - except Exception as e: - logger.warning(f"Failed to extract assistant response from messages: {e}", exc_info=True) - - logger.debug("No assistant response found in response") - return None - - def _format_memories(self, memories: List[UserMemory]) -> str: - """Format a list of UserMemory objects into a system prompt fragment.""" - if not memories: - return "" - - lines: List[str] = [self.context_header] - for i, mem in enumerate(memories, 1): - bullet = f"- {mem.memory}" if isinstance(mem.memory, str) else f"- {mem.memory!r}" - meta_parts: List[str] = [] - if mem.topics: - meta_parts.append(f"topics={','.join(mem.topics)}") - if mem.memory_id: - meta_parts.append(f"id={mem.memory_id}") - meta_str = f" ({'; '.join(meta_parts)})" if meta_parts else "" - lines.append(f"{i}. {bullet}{meta_str}") - - return "\n".join(lines) - - async def _build_memory_context(self, request: ModelRequest, user_id: str) -> str: - """Retrieve memories from MemoryManager per config and build context (async).""" - query: Optional[str] = None - if self.retrieval_method == "agentic": - query = self._extract_user_input(request) - logger.info( - f"Retrieving memories with agentic method for user_id={user_id}, " - f"query={query[:100] if query else None}..." - ) - else: - logger.info( - f"Retrieving memories with {self.retrieval_method} method for user_id={user_id}, " - f"limit={self.retrieval_limit}" - ) - - try: - retrieval_method_literal: Literal["last_n", "first_n", "agentic"] | None = None - if self.retrieval_method in ("last_n", "first_n", "agentic"): - retrieval_method_literal = self.retrieval_method # type: ignore[assignment] - memories = await self.memory_manager.asearch_user_memories( - query=query, - limit=self.retrieval_limit, - retrieval_method=retrieval_method_literal, - user_id=user_id, - ) - memory_count = len(memories) if memories else 0 - logger.info(f"Memory retrieval completed for user_id={user_id}: found {memory_count} memories") - except Exception as e: - logger.warning(f"Memory retrieval failed for user_id={user_id}: {e}") - memories = [] - - formatted_context = self._format_memories(memories or []) - if formatted_context: - logger.debug(f"Formatted memory context length: {len(formatted_context)} characters") - else: - logger.debug("No memory context generated") - return formatted_context - - # --------------------------- - # Lifecycle Hooks - # --------------------------- - def before_agent( - self, - state: AgentState[Any], # type: ignore[override] - runtime, # type: ignore[no-untyped-def] - ) -> dict[str, Any]: # type: ignore[override] - """Perform necessary initialization here.""" - user_id = self._get_user_id() - if user_id: - logger.debug(f"Initializing MemoryManager for user_id={user_id}") - try: - self.memory_manager.initialize(user_id=user_id) - logger.debug(f"MemoryManager initialized successfully for user_id={user_id}") - except Exception as e: - logger.warning(f"MemoryManager initialize failed for user_id={user_id}: {e}") - else: - logger.warning("Skipping MemoryManager initialization: no user_id available") - - return {"messages": []} - - async def abefore_agent( - self, - state: AgentState[Any], # type: ignore[override] - runtime, # type: ignore[no-untyped-def] - ) -> dict[str, Any]: # type: ignore[override] - return self.before_agent(state, runtime) - - def wrap_model_call( - self, - request: ModelRequest, - handler: Callable[[ModelRequest], ModelResponse], - ) -> ModelResponse: - """Inject memories before model call; write back memories after call (sync version, uses asyncio.run internally).""" - user_id = self._get_user_id() - - if not user_id: - logger.warning("Skipping memory operations: no user_id available") - return handler(request) - - # build and inject memory context - # simplified: use asyncio.run() directly for the async method; - # if already inside an event loop, LangGraph calls awrap_model_call instead - memory_context = asyncio.run(self._build_memory_context(request, user_id)) - - if memory_context: - # store in state for downstream use or debugging - try: - request.state["agent_memory_context"] = memory_context # type: ignore[typeddict-unknown-key] - except Exception: - logger.debug("Failed to store memory context in request state", exc_info=True) - - logger.info(f"Injecting memory context into system prompt for user_id={user_id}") - if request.system_prompt: - request.system_prompt = f"{memory_context}\n\n{request.system_prompt}" # type: ignore[misc, assignment] - else: - request.system_prompt = memory_context # type: ignore[misc, assignment] - else: - logger.debug(f"No memory context to inject for user_id={user_id}") - - # call model - logger.debug(f"Calling model handler for user_id={user_id}") - response = handler(request) - - # write back memories: use user input as the basis for memory capture - if self.enable_writeback: - logger.info(f"Attempting to write back memory for user_id={user_id}, capture_source={self.capture_source}") - try: - message_text: Optional[str] = None - if self.capture_source == "assistant": - # extract assistant response content from ModelResponse - message_text = self._extract_assistant_response(response) - else: - # default: capture from user input - message_text = self._extract_user_input(request) - - if message_text and message_text.strip(): - logger.info( - f"Writing memory for user_id={user_id}, " - f"message_length={len(message_text)}, capture_source={self.capture_source}" - ) - # simplified: use asyncio.run() directly for the async method; - # if already inside an event loop, LangGraph calls awrap_model_call instead - asyncio.run(self.memory_manager.acreate_user_memories(message=message_text, user_id=user_id)) - logger.info(f"Memory writeback completed successfully for user_id={user_id}") - else: - logger.debug(f"No message text to write back for user_id={user_id}") - except Exception as e: - logger.error(f"Failed to write back memory for user_id={user_id}: {e}", exc_info=True) - else: - logger.debug(f"Memory writeback disabled for user_id={user_id}") - - return response - - async def awrap_model_call( - self, - request: ModelRequest, - handler: Callable[[ModelRequest], Awaitable[ModelResponse]], - ) -> ModelResponse: - """Async version: inject memories before model call; write back memories after call.""" - user_id = self._get_user_id() - - if not user_id: - logger.warning("Skipping memory operations: no user_id available") - return await handler(request) - - # build and inject memory context - memory_context = await self._build_memory_context(request, user_id) - if memory_context: - try: - request.state["agent_memory_context"] = memory_context # type: ignore[typeddict-unknown-key] - except Exception: - logger.debug("Failed to store memory context in request state", exc_info=True) - - logger.info(f"Injecting memory context into system prompt for user_id={user_id}") - if request.system_prompt: - request.system_prompt = f"{memory_context}\n\n{request.system_prompt}" # type: ignore[misc, assignment] - else: - request.system_prompt = memory_context # type: ignore[misc, assignment] - else: - logger.debug(f"No memory context to inject for user_id={user_id}") - - # call model - logger.debug(f"Calling model handler for user_id={user_id}") - response = await handler(request) - - # write back memories - if self.enable_writeback: - logger.info(f"Attempting to write back memory for user_id={user_id}, capture_source={self.capture_source}") - try: - message_text: Optional[str] = None - if self.capture_source == "assistant": - # extract assistant response content from ModelResponse - message_text = self._extract_assistant_response(response) - else: - # default: capture from user input - message_text = self._extract_user_input(request) - - if message_text and message_text.strip(): - logger.info( - f"Writing memory for user_id={user_id}, " - f"message_length={len(message_text)}, capture_source={self.capture_source}" - ) - await self.memory_manager.acreate_user_memories( - message=message_text, - user_id=user_id, - ) - logger.info(f"Memory writeback completed successfully for user_id={user_id}") - else: - logger.debug(f"No message text to write back for user_id={user_id}") - except Exception as e: - logger.error(f"Failed to write back memory for user_id={user_id}: {e}", exc_info=True) - else: - logger.debug(f"Memory writeback disabled for user_id={user_id}") - - return response diff --git a/backend/app/core/agent/node_tools.py b/backend/app/core/agent/node_tools.py deleted file mode 100644 index 563988904..000000000 --- a/backend/app/core/agent/node_tools.py +++ /dev/null @@ -1,468 +0,0 @@ -""" -Node tools resolution. - -Parses `GraphNode` tool configuration (persisted in DB) and resolves it into a -LangChain-compatible tools list for `create_agent(..., tools=[...])`. - -Frontend stores tools under: -- node.data.config.tools = { builtin: string[], mcp: string[] } - where mcp entries are in format `${server_name}::${toolName}`. -Backend also has a dedicated `GraphNode.tools` JSONB field; we support both. -""" - -from __future__ import annotations - -import re -import uuid -from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Set - -from loguru import logger - -# MCP tools are now loaded from ToolRegistry instead of direct connections -# Import default user ID constant -from app.core.constants import DEFAULT_USER_ID -from app.core.tools.tool import EnhancedTool, ToolMetadata, ToolSourceType -from app.models.graph import GraphNode -from app.utils.sandbox_paths import get_user_sandbox_host_dir - - -def _first_dict(*candidates: Any) -> Optional[dict]: - for c in candidates: - if isinstance(c, dict): - return c - return None - - -def extract_tools_config(node: GraphNode) -> Optional[dict]: - """ - Extract tools config dict from a GraphNode. - - Preference order: - 1) node.data.config.tools (canonical) - 2) node.data.tools (legacy) - """ - data = node.data or {} - config = data.get("config", {}) if isinstance(data, dict) else {} - tools_from_config = config.get("tools") if isinstance(config, dict) else None - - tools_dict = _first_dict(tools_from_config, data.get("tools") if isinstance(data, dict) else None) - if not tools_dict: - return None - return tools_dict - - -def _parse_mcp_ids(mcp_ids: Iterable[str]) -> Dict[str, Set[str]]: - """Parse MCP tool IDs (format: server_name::tool_name)""" - result: Dict[str, Set[str]] = {} - - for raw in mcp_ids: - if not raw: - continue - - # Split by "::" separator - if "::" not in raw: - logger.warning( - f"[_parse_mcp_ids] Invalid format (missing '::'): '{raw}'. Expected format: 'server_name::tool_name'" - ) - continue - - server_name, tool_name = raw.split("::", 1) - server_name = (server_name or "").strip() - tool_name = (tool_name or "").strip() - - if not server_name: - logger.warning(f"[_parse_mcp_ids] Missing server name in: '{raw}'") - continue - - if not tool_name: - logger.warning(f"[_parse_mcp_ids] Missing tool name in: '{raw}'") - continue - - result.setdefault(server_name, set()).add(tool_name) - - return result - - -def _alias_tool(*, name: str, description: str, callable_func: Any) -> EnhancedTool: - """Create an EnhancedTool with a stable user-facing `name`.""" - return EnhancedTool.from_callable( # type: ignore - callable_func=callable_func, - name=name, - description=description, - tool_metadata=ToolMetadata(source_type=ToolSourceType.BUILTIN, tags={"builtin"}, category="node"), - ) - - -def _resolve_builtin_tools(*, builtin_ids: List[str], root_dir: Path, user_id: str, backend: Any = None) -> List[Any]: - """ - Resolve builtin tool IDs into LangChain tools. - - Resolve builtin tool IDs (e.g. ``tavily_search``, ``preview_skill``) - into concrete LangChain tool implementations. - - Args: - backend: Optional sandbox backend adapter (PydanticSandboxAdapter). - When provided, preview_skill reads files from inside the container. - """ - # Try to get tools from registry first - from app.core.tools.tool_registry import get_global_registry - - registry = get_global_registry() - - # Lazy imports to avoid import-time failures when optional dependencies - # (e.g. `tavily-python`) are not installed. - from app.core.tools.builtin.preview_skill import preview_skill_in_sandbox - - # Bind preview_skill with the user's sandbox root path - # NOTE: We use a real wrapper function instead of functools.partial because - # langchain's create_schema_from_function cannot introspect partial objects. - _sandbox_root = str(get_user_sandbox_host_dir(str(user_id))) - - def bound_preview_skill(skill_name: str, skills_subdir: str = "skills") -> str: - """Preview a skill's files and validate its metadata.""" - # The Agent sees paths inside the container (e.g. /workspace/skills/...) - # Normalize the subdir by stripping /workspace prefix. - normalized_subdir = skills_subdir.strip("/") - if normalized_subdir.startswith("workspace/"): - normalized_subdir = normalized_subdir[len("workspace/") :] - if not normalized_subdir: - normalized_subdir = "skills" - - # If a backend is available, read files directly from inside the container. - # This avoids relying on Docker volume mount sync (unreliable on macOS). - if backend is not None: - return _preview_skill_from_backend(backend, skill_name, normalized_subdir) - - # Fallback: read from host filesystem - return preview_skill_in_sandbox( - skill_name=skill_name, - sandbox_root=_sandbox_root, - skills_subdir=normalized_subdir, - ) - - # Research tools - get from registry only - research_tools = {} - for tool_id in ["tavily_search", "think_tool"]: - registry_tool = registry.get_tool(tool_id) - if registry_tool: - research_tools[tool_id] = registry_tool - logger.debug(f"[node_tools] Found {tool_id} in registry") - else: - logger.warning(f"[node_tools] Tool '{tool_id}' not found in registry, skipping") - - # Canonical mapping for UI-friendly IDs -> tool implementations. - aliases: Dict[str, Any] = { - "preview_skill": _alias_tool( - name="preview_skill", - description="Preview a skill generated in the sandbox. Reads all files and returns JSON with contents and validation.", - callable_func=bound_preview_skill, - ), - **research_tools, # Add research tools to aliases - } - - resolved: List[Any] = [] - for tool_id in builtin_ids: - if not tool_id: - continue - t = aliases.get(tool_id) - if t is None: - logger.warning(f"[node_tools] Unknown builtin tool id '{tool_id}', skipping") - continue - resolved.append(t) - return resolved - - -def _preview_skill_from_backend(backend: Any, skill_name: str, skills_subdir: str = "skills") -> str: - """Preview a skill by reading files from inside the Docker container via backend. - - This uses the backend adapter's ls_info() and read() methods to access - files directly inside the container, avoiding host filesystem sync issues. - """ - import json as _json - - from app.core.skill.validators import validate_skill_description, validate_skill_name - from app.core.skill.yaml_parser import parse_skill_md - - skill_dir = f"/workspace/{skills_subdir}/{skill_name}" - errors: List[str] = [] - warnings: List[str] = [] - - def _safe_ls(path: str) -> List[Dict[str, Any]]: - try: - return list(backend.ls_info(path)) - except Exception as e: - logger.warning(f"[preview_skill] Failed to list {path}: {e}") - return [] - - dir_listing = _safe_ls(skill_dir) - resolved_subdir = skills_subdir - - if not dir_listing and skills_subdir == "skills": - for entry in _safe_ls("/workspace"): - path = str(entry.get("path", "")) - if not path or not entry.get("is_dir"): - continue - thread_name = Path(path.rstrip("/")).name - candidate_subdir = f"{thread_name}/skills" - candidate_dir = f"/workspace/{candidate_subdir}/{skill_name}" - candidate_listing = _safe_ls(candidate_dir) - if candidate_listing: - skill_dir = candidate_dir - dir_listing = candidate_listing - resolved_subdir = candidate_subdir - break - - if not dir_listing: - return _json.dumps( - { - "skill_name": skill_name, - "files": [], - "validation": { - "valid": False, - "errors": [f"Skill directory not found: {resolved_subdir}/{skill_name}"], - "warnings": [], - }, - } - ) - - # Recursively collect all files from the skill directory - files: List[Dict[str, Any]] = [] - _collect_files_from_backend(backend, skill_dir, skill_dir, files) - - # Locate and validate SKILL.md - skill_md_entry = next((f for f in files if f["path"] == "SKILL.md"), None) - if skill_md_entry is None: - errors.append("SKILL.md not found in skill directory") - else: - frontmatter, body = parse_skill_md(skill_md_entry["content"]) - fm_name = frontmatter.get("name", "") - if not fm_name: - errors.append("Missing required field 'name' in SKILL.md frontmatter") - else: - name_valid, name_err = validate_skill_name(fm_name) - if not name_valid: - errors.append(f"Invalid skill name: {name_err}") - - fm_desc = frontmatter.get("description", "") - if not fm_desc: - errors.append("Missing required field 'description' in SKILL.md frontmatter") - else: - desc_valid, desc_err = validate_skill_description(str(fm_desc)) - if not desc_valid: - errors.append(f"Invalid skill description: {desc_err}") - - if not body or not body.strip(): - warnings.append("SKILL.md body is empty; consider adding usage documentation") - - valid = len(errors) == 0 - return _json.dumps( - { - "skill_name": skill_name, - "files": files, - "validation": { - "valid": valid, - "errors": errors, - "warnings": warnings, - }, - } - ) - - -def _collect_files_from_backend(backend: Any, dir_path: str, base_dir: str, files: List[Dict[str, Any]]) -> None: - """Recursively collect files from inside the container via backend.""" - import os - - _EXCLUDED_DIRS = {"__pycache__", ".git", "node_modules", ".mypy_cache", ".pytest_cache"} - - try: - entries = backend.ls_info(dir_path) - except Exception: - return - - for entry in entries: - path = entry.get("path", "").rstrip("/") - is_dir = entry.get("is_dir", False) - - if not path: - continue - - basename = os.path.basename(path) - - if is_dir: - if basename in _EXCLUDED_DIRS: - continue - _collect_files_from_backend(backend, path, base_dir, files) - else: - # Get relative path - rel_path = path - if path.startswith(base_dir): - rel_path = path[len(base_dir) :].lstrip("/") - - if not rel_path: - continue - - # Read file content - try: - raw_content = backend.read(path, offset=0, limit=100000) - content = raw_content if isinstance(raw_content, str) else str(raw_content) - # Strip line numbers added by format_read_response - # format_content_with_line_numbers outputs: "{line_num}\t{content}" - # e.g. " 1\t---" or " 2\tname: foo" - # Also handle " | " separator format. - stripped_lines = [] - for line in content.splitlines(): - # Try tab-separated format first: " N\tcontent" - m = re.match(r"^\s*\d+(?:\.\d+)?\t", line) - if m: - stripped_lines.append(line[m.end() :]) - # Fallback: pipe-separated format " N | content" - elif " | " in line[:12]: - stripped_lines.append(line.split(" | ", 1)[1]) - else: - stripped_lines.append(line) - content = "\n".join(stripped_lines) - except Exception: - logger.debug("Skipping unreadable skill file: %s", rel_path, exc_info=True) - continue - - # Detect file type - ext = os.path.splitext(rel_path)[1].lower() - _EXT_MAP = { - ".py": "python", - ".md": "markdown", - ".json": "json", - ".yaml": "yaml", - ".yml": "yaml", - ".txt": "text", - ".sh": "shell", - ".js": "javascript", - ".ts": "typescript", - ".html": "html", - ".css": "css", - } - file_type = _EXT_MAP.get(ext, "other") - - files.append( - { - "path": rel_path, - "content": content, - "file_type": file_type, - "size": entry.get("size", len(content)), - } - ) - - -def _normalize_user_id(user_id: Any | None) -> str: - """ - Normalize user_id to a string format. - - Converts UUID objects to strings, handles None by returning DEFAULT_USER_ID. - Ensures all user_id values are strings (UUID format). - - Args: - user_id: User ID (can be UUID object, string, or None) - - Returns: - Normalized user_id as string (UUID format) - """ - if user_id is None: - return DEFAULT_USER_ID - - # Convert UUID object to string if needed - if isinstance(user_id, uuid.UUID): - return str(user_id) - - # Already a string - if isinstance(user_id, str): - return user_id - - # Fallback: convert to string - return str(user_id) - - -async def resolve_tools_for_node( - node: GraphNode, *, user_id: str | None = None, backend: Any = None -) -> Optional[List[Any]]: - """ - Resolve tools list for a node. - - Process flow: - 1. Extract tools config from node - 2. Parse builtin tools → resolve to tool objects - 3. Parse MCP tools → resolve server names → get tools - 4. Return combined tool list - - MCP server identification: server name (unique per user) - - Args: - node: GraphNode to resolve tools for - user_id: User ID (normalized to string UUID format) - - Returns: - - None: means "no explicit tools config" (caller may use defaults) - - [] / [..]: explicit tool list - """ - # Normalize user_id - normalized_user_id = _normalize_user_id(user_id) - - logger.debug(f"[resolve_tools_for_node] Starting resolution for node_id={node.id}, user_id={normalized_user_id}") - - # Step 1: Extract tools config - cfg = extract_tools_config(node) - if cfg is None: - logger.debug(f"[resolve_tools_for_node] No tools config found for node_id={node.id}") - return None - - logger.debug(f"[resolve_tools_for_node] Tools config: {cfg}") - - builtin_ids = cfg.get("builtin") if isinstance(cfg, dict) else None - mcp_ids = cfg.get("mcp") if isinstance(cfg, dict) else None - - builtin_ids_list = list(builtin_ids) if isinstance(builtin_ids, list) else [] - mcp_ids_list = list(mcp_ids) if isinstance(mcp_ids, list) else [] - - logger.debug(f"[resolve_tools_for_node] Parsed config: builtin_ids={builtin_ids_list}, mcp_ids={mcp_ids_list}") - - root_dir = Path(f"/tmp/{normalized_user_id}") - tools: List[Any] = [] - - # Step 2: Resolve builtin tools - if builtin_ids_list: - logger.debug(f"[resolve_tools_for_node] Resolving {len(builtin_ids_list)} builtin tools") - builtin_tools = _resolve_builtin_tools( - builtin_ids=builtin_ids_list, root_dir=root_dir, user_id=normalized_user_id, backend=backend - ) - logger.debug(f"[resolve_tools_for_node] Resolved {len(builtin_tools)} builtin tools") - tools.extend(builtin_tools) - - # Step 3: Resolve MCP tools from Registry with instance validation - if mcp_ids_list: - logger.debug(f"[resolve_tools_for_node] Resolving {len(mcp_ids_list)} MCP tools") - - from app.core.database import async_session_factory - from app.core.tools.mcp_tool_utils import resolve_mcp_tools_from_list - - async with async_session_factory() as db: - mcp_tools = await resolve_mcp_tools_from_list(mcp_ids_list, normalized_user_id, db) - - logger.debug(f"[resolve_tools_for_node] Retrieved {len(mcp_tools)} MCP tools") - tools.extend(mcp_tools) - - logger.debug( - f"[resolve_tools_for_node] Final resolution for node_id={node.id} | " - f"builtin_selected={len(builtin_ids_list)} | mcp_selected={len(mcp_ids_list)} | " - f"tools_resolved={len(tools)}" - ) - - # Final check: ensure no ToolMetadata objects in the list - from app.core.tools.tool import ToolMetadata - - for i, tool in enumerate(tools): - if isinstance(tool, ToolMetadata): - logger.error( - f"[resolve_tools_for_node] ERROR: ToolMetadata object found at index {i} " - f"in tools list! This should not happen. metadata: {tool}" - ) - - return tools diff --git a/backend/app/core/agent/sample_agent.py b/backend/app/core/agent/sample_agent.py deleted file mode 100644 index 31412ff3e..000000000 --- a/backend/app/core/agent/sample_agent.py +++ /dev/null @@ -1,304 +0,0 @@ -""" -Example Agent implementation. - -Implements a sample chatbot node using LangChain v1 create_agent API. -""" - -from typing import Any, cast - -from deepagents.middleware import FilesystemMiddleware -from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware -from dotenv import load_dotenv -from langchain.agents import create_agent -from langchain.agents.middleware import TodoListMiddleware -from langchain.agents.middleware.summarization import SummarizationMiddleware -from langchain_core.runnables import Runnable -from langchain_openai import ChatOpenAI -from pydantic import SecretStr - -from app.common.exceptions import ModelConfigError -from app.core.agent.backends.filesystem_sandbox import FilesystemSandboxBackend -from app.core.agent.middleware import LoggingMiddleware -from app.services.model_service import MODEL_NAME_REQUIRED, MODEL_NO_CREDENTIALS - -load_dotenv() - - -def get_default_model( - llm_model: str | None = None, - api_key: str | None = None, - base_url: str | None = None, - max_tokens: int = 4096, - timeout: int | None = None, -) -> ChatOpenAI: - """ - Create and return a ChatOpenAI model instance. - - Args: - llm_model: LLM model name (required). - api_key: API key for authentication (required). - base_url: Optional base URL for the API endpoint. - max_tokens: Maximum completion tokens. Defaults to 4096. - timeout: Optional timeout in seconds. Defaults to 120 seconds. - - Returns: - ChatOpenAI: Configured ChatOpenAI model instance with streaming enabled. - - Raises: - ModelConfigError: If llm_model or api_key is not provided. - """ - api_key_value = api_key - base_url_value = base_url - - if not llm_model: - raise ModelConfigError( - MODEL_NAME_REQUIRED, - "Model name is required but was not specified.", - ) - - model_name = llm_model - - if not api_key_value: - raise ModelConfigError( - MODEL_NO_CREDENTIALS, - f'No valid API key provided for model "{model_name}".', - params={"model": model_name}, - ) - - secret_api_key = SecretStr(api_key_value) - - # Set default timeout to 120 seconds if not provided - # This helps prevent "No generations found in stream" errors - # that occur when the default 60s timeout is exceeded - timeout_value = timeout if timeout is not None else 120 - - # Create and return ChatOpenAI instance - model = ChatOpenAI( - model=model_name, - api_key=secret_api_key, - base_url=base_url_value, - max_completion_tokens=max_tokens, - streaming=True, # Enable streaming output - timeout=timeout_value, # Set timeout to prevent premature stream termination - ) - - return model - - -async def get_agent( - checkpointer: Any | None = None, - llm_model: str | None = None, - api_key: str | None = None, - base_url: str | None = None, - max_tokens: int = 4096, - user_id: str | None = None, - system_prompt: str | None = None, - tools: list[Any] | None = None, - enable_todo_list: bool = True, - enable_skills: bool = True, - skill_user_id: str | None = None, - agent_name: str | None = None, - model: Any | None = None, - node_middleware: list[Any] | None = None, -) -> Runnable: - """ - Create and return the Agent graph. - - Args: - checkpointer: Optional checkpointer for state persistence. - llm_model: LLM model name. - api_key: API key. - base_url: API base URL. - max_tokens: Maximum completion tokens. - user_id: User ID (UUID), used to create an isolated workspace directory. - system_prompt: System prompt for the agent. - tools: List of tools for the agent. - enable_todo_list: Whether to enable TodoListMiddleware. - Set to False for DeepAgents subagents to avoid state conflicts. - enable_skills: Whether to enable SkillMiddleware for progressive skill disclosure. - skill_user_id: User ID for skill filtering (defaults to user_id). - agent_name: Name of the agent (for tagging). - model: Optional pre-created model instance. - node_middleware: List of middleware instances from node configuration (e.g., from resolve_middleware_for_node). - - Returns: - Runnable: The compiled Agent graph. - """ - # if a model object is provided, use it directly; otherwise create one via get_default_model - if model is None: - model = get_default_model( - llm_model=llm_model, - api_key=api_key, - base_url=base_url, - max_tokens=max_tokens, - ) - else: - from loguru import logger - - # Tools resolution: - # - If caller provides `tools`, use exactly that list (can be empty to disable tools). - # Try to get tools from ToolRegistry first, then use provided tools as fallback. - # - Otherwise, return empty list (tools should be explicitly provided via resolve_tools_for_node). - from loguru import logger - - from app.core.tools.tool import EnhancedTool, ToolMetadata - from app.core.tools.tool_registry import get_global_registry - - if tools is None: - tools = [] - else: - # check if tools is a ToolMetadata object (should not be passed directly as tools param) - if isinstance(tools, ToolMetadata): - logger.error(f"[get_agent] ERROR: tools parameter is a ToolMetadata object, not a list! metadata: {tools}") - logger.warning("[get_agent] Converting ToolMetadata to empty list") - tools = [] - # check if tools is a list or iterable - elif not isinstance(tools, (list, tuple)): - logger.warning(f"[get_agent] tools is not a list/tuple, type: {type(tools)}, converting to list") - try: - tools = list(tools) if hasattr(tools, "__iter__") else [tools] - except Exception as e: - logger.error(f"[get_agent] Failed to convert tools to list: {e}") - tools = [] - - # get tools from ToolRegistry if registered - registry = get_global_registry() - resolved_tools = [] - for tool in tools: - if isinstance(tool, ToolMetadata): - continue - - if isinstance(tool, EnhancedTool): - resolved_tools.append(tool) - continue - - if isinstance(tool, str): - # try direct lookup first (builtin tools) - registry_tool = registry.get_tool(tool) - if registry_tool: - resolved_tools.append(registry_tool) - continue - - # try parsing as MCP tool (format: server_name::tool_name) - if "::" in tool: - from app.core.tools.mcp_tool_utils import parse_mcp_tool_name - - server_name, tool_name = parse_mcp_tool_name(tool) - if server_name and tool_name: - mcp_tool = registry.get_mcp_tool(server_name, tool_name) - if mcp_tool: - resolved_tools.append(mcp_tool) - continue - - # unable to resolve; log warning and skip (do not add string to tools list) - logger.warning(f"[get_agent] Unable to resolve tool '{tool}', skipping") - continue - - resolved_tools.append(tool) - - tools = resolved_tools - - # Create per-user isolated workspace directory - # Normalize user_id to ensure it's a string (UUID format) - from app.core.agent.node_tools import _normalize_user_id - - normalized_user_id = _normalize_user_id(user_id) - root_dir = f"./logs/{normalized_user_id}" - backend = FilesystemSandboxBackend( - root_dir=root_dir, - virtual_mode=True, # Use virtual filesystem (in-memory) - ) - - # Build middleware list - middleware = [ - FilesystemMiddleware(backend=backend), - PatchToolCallsMiddleware(), - SummarizationMiddleware(model=model, max_tokens_before_summary=170000, messages_to_keep=10), - LoggingMiddleware(backend=backend), - ] - - # Only add TodoListMiddleware if enabled (disabled for DeepAgents subagents) - if enable_todo_list: - middleware.insert(0, cast(Any, TodoListMiddleware())) - - # Add SkillsMiddleware if enabled - if enable_skills: - # Use deepagents SkillsMiddleware if backend is available - # This injects skill descriptions into system prompt from /workspace/skills/ - # Skills must be preloaded to /workspace/skills/ via SkillSandboxLoader - if backend: - try: - from deepagents.middleware.skills import SkillsMiddleware - - skills_middleware = SkillsMiddleware( - backend=backend, - sources=["/workspace/skills/"], # Path where skills should be preloaded - ) - # Insert after TodoListMiddleware if it exists, otherwise at the beginning - if enable_todo_list: - middleware.insert(1, skills_middleware) - else: - middleware.insert(0, skills_middleware) - logger.debug( - "[get_agent] Added deepagents SkillsMiddleware (backend available, " - "reading from /workspace/skills/). " - "Agents can read skill files directly from the sandbox." - ) - except ImportError: - logger.warning( - "[get_agent] deepagents SkillsMiddleware not available. Skills descriptions will not be injected." - ) - except Exception as e: - logger.warning( - f"[get_agent] Failed to create deepagents SkillsMiddleware: {e}. " - "Skills descriptions will not be injected." - ) - else: - logger.debug( - "[get_agent] No backend available for SkillsMiddleware. " - "Skills descriptions will not be injected. " - "Ensure skills are preloaded via SkillSandboxLoader if backend is needed." - ) - - # Add node-specific middleware (from resolve_middleware_for_node) - # These are middleware instances created from node configuration (e.g., MemoryMiddleware) - if node_middleware: - # Sort middleware by priority (lower number = higher priority = executed first) - node_middleware.sort(key=lambda mw: getattr(mw, "priority", 100)) - - # Insert node middleware after default middleware but before the end - # This ensures they have access to the full middleware chain - for mw in node_middleware: - middleware.append(mw) - logger.debug( - f"[get_agent] Added {len(node_middleware)} node middleware instance(s) " - f"for agent '{agent_name or 'unknown'}' (sorted by priority)" - ) - - # Create agent (callbacks will be configured at invoke time) - from langchain_core.runnables import RunnableConfig - - agent_config: RunnableConfig = {"recursion_limit": 1000} # type: ignore[assignment] - if agent_name: - agent_config["tags"] = [f"Agent:{agent_name}"] # type: ignore[assignment] - - agent: Runnable = create_agent( - model, - tools=tools, - system_prompt=system_prompt, - checkpointer=checkpointer, - middleware=middleware, - ).with_config(agent_config) - - return agent - - -if __name__ == "__main__": - import asyncio - - async def main(): - agent = await get_agent() - result = await agent.ainvoke({"messages": [{"role": "user", "content": "What is 1231972 / 8723?"}]}) - print(result) - - asyncio.run(main()) diff --git a/backend/app/core/ai_adapter.py b/backend/app/core/ai_adapter.py deleted file mode 100644 index fc997a160..000000000 --- a/backend/app/core/ai_adapter.py +++ /dev/null @@ -1,432 +0,0 @@ -"""AI model adapter for web use (decoupled from CLI runtime and dynamic imports). - -This refactor removes: -- sys.path manipulations to locate backend CLI code -- _import_cli_modules dynamic loader and the global cli_modules -- Tight coupling to backend runtime construction inside this layer - -Design goals: -- Keep a stable, minimal interface for the web layer (stream_response + memory helpers) -- Allow optional injection of an engine that implements a simple streaming protocol -- Provide a safe fallback behavior when no engine is available -""" - -from pathlib import Path -from typing import Any, AsyncGenerator, Dict, List, Optional, Protocol - -from .settings import settings - - -class AgentEngine(Protocol): - """Protocol for pluggable agent engines used by AgentBridge. - - Implementations should provide a `stream` method that yields tuples compatible - with downstream processing in AgentBridge._process_stream_chunk. - - Expected yield format per chunk: - (namespace: str, stream_mode: str, data: Any) - where stream_mode is typically one of: "messages", "updates". - """ - - def stream( - self, - input: Dict[str, Any], - stream_mode: List[str], - subgraphs: bool, - config: Dict[str, Any], - durability: str, - ): ... - - -class AgentBridge: - """Adapter class to bridge AI logic with the web interface. - - Independent from backend CLI code. An engine implementing AgentEngine may be - injected optionally to enable full functionality. Without an engine, the adapter - provides a graceful mock response suitable for UI integration tests. - """ - - def __init__( - self, - session_id: str, - workspace_path: str, - engine: Optional[AgentEngine] = None, - ): - """Initialize AI adapter for a specific session. - - Args: - session_id: Unique identifier for the web session - workspace_path: Path to the user's workspace directory - engine: Optional agent engine implementation to power the adapter - """ - self.session_id = session_id - self.workspace_path = Path(workspace_path) - self.engine = engine - - # Create per-session directories using configured workspace root - workspace_root = Path(settings.WORKSPACE_ROOT) - self.session_dir = workspace_root / "sessions" / session_id - self.session_dir.mkdir(parents=True, exist_ok=True) - - # Memory directory - self.memory_dir = workspace_root / "memories" / session_id - self.memory_dir.mkdir(parents=True, exist_ok=True) - - # Streaming state - self.pending_text = "" # Accumulated text buffer - self.tool_call_buffers: Dict[str, Dict[str, Any]] = {} # Tool call buffers - self.last_chunk_time: float = 0 # Last time we received a text chunk - self.chunk_timeout: float = 2.0 # Timeout in seconds before flushing buffer - self.is_thinking = False - self.sent_thinking = False - self.has_sent_thinking_for_current_request = False - - def set_engine(self, engine: AgentEngine): - """Attach or replace the underlying engine at runtime.""" - self.engine = engine - - async def stream_response( - self, message: str, file_references: Optional[List[str]] = None - ) -> AsyncGenerator[Dict[str, Any], None]: - """Stream AI response for web interface. - - Args: - message: User message - file_references: List of file paths to include in context - - Yields: - Dict containing streaming response chunks - """ - # Reset thinking state for each request - self.sent_thinking = False - self.has_sent_thinking_for_current_request = False - - # If no engine is provided, return a mock response - if not self.engine: - yield { - "type": "message", - "content": f"I received your message: '{message}'. This is a mock AI response. The full version will be provided by an injected engine.", - "session_id": self.session_id, - } - return - - # Immediately send thinking status - yield { - "type": "status", - "content": "AI is thinking...", - "session_id": self.session_id, - "metadata": {"state": "thinking"}, - } - - # Build full input (include referenced files content if any) - full_input = self._build_input_with_files(message, file_references or []) - - # Engine config - config = { - "configurable": {"thread_id": self.session_id}, - "metadata": {"session_id": self.session_id, "source": "web"}, - } - - try: - # Stream from the engine - for chunk in self.engine.stream( - {"messages": [{"role": "user", "content": full_input}]}, - stream_mode=["messages", "updates"], - subgraphs=True, - config=config, - durability="exit", - ): - processed_chunk = self._process_stream_chunk(chunk) - if processed_chunk: - yield processed_chunk - - # On stream end, flush any remaining buffered text - final_chunks = self.flush_pending_text(final=True) - for final_chunk in final_chunks: - yield final_chunk - - except Exception as e: - # Attempt to flush buffered text even on error - error_chunks = self.flush_pending_text(final=True) - for chunk in error_chunks: - yield chunk - - yield { - "type": "error", - "content": f"AI response error: {str(e)}", - "session_id": self.session_id, - } - - def _build_input_with_files(self, message: str, file_paths: List[str]) -> str: - """Build input message with file contents included.""" - if not file_paths: - return message - - context_parts = [message, "\n\n## Referenced Files\n"] - - for file_path in file_paths: - try: - full_path = self.workspace_path / file_path - if full_path.exists(): - content = full_path.read_text(encoding="utf-8") - # Limit file size to avoid overwhelming context - if len(content) > 50000: - content = content[:50000] + "\n... (file truncated)" - - context_parts.append(f"\n### {full_path.name}\nPath: `{file_path}`\n```\n{content}\n```") - else: - context_parts.append(f"\n### {Path(file_path).name}\n[Error: File not found - {file_path}]") - except Exception as e: - context_parts.append(f"\n### {Path(file_path).name}\n[Error reading file: {e}]") - - return "\n".join(context_parts) - - def _process_stream_chunk(self, chunk) -> Optional[Dict[str, Any]]: - """Process streaming chunk from AI engine with CLI-style buffering.""" - import time - - if not isinstance(chunk, tuple) or len(chunk) != 3: - return None - - namespace, stream_mode, data = chunk - current_time = time.time() - results: List[Dict[str, Any]] = [] - - if stream_mode == "messages": - if isinstance(data, tuple) and len(data) == 2: - message, metadata = data - - # Handle AI message content - if hasattr(message, "content_blocks"): - # First handle tool call chunks - for block in message.content_blocks: - if block.get("type") == "tool_call_chunk": - tool_name = block.get("name") - tool_args = block.get("args", {}) - tool_call_id = block.get("id", "default") - - # Buffer tool call data - if tool_call_id not in self.tool_call_buffers: - self.tool_call_buffers[tool_call_id] = { - "name": tool_name, - "args": "", - "complete": False, - } - - buffer = self.tool_call_buffers[tool_call_id] - if tool_args: - buffer["args"] += tool_args - - # Check completion - if block.get("complete", False): - buffer["complete"] = True - results.append( - { - "type": "tool_call", - "tool": buffer["name"], - "args": buffer["args"], - "session_id": self.session_id, - "tool_call_id": tool_call_id, - "complete": True, - } - ) - del self.tool_call_buffers[tool_call_id] - - # Then handle text chunks - for block in message.content_blocks: - if block.get("type") == "text": - text_content = block.get("text", "") - if text_content: - # Detect structured tool output and format nicely - if self._is_tool_output(text_content): - formatted_output = self._format_tool_output(text_content) - if formatted_output: - results.append( - { - "type": "tool_result", - "content": formatted_output, - "session_id": self.session_id, - } - ) - else: - # Accumulate text to buffer - self.pending_text += text_content - self.last_chunk_time = current_time - - elif stream_mode == "updates": - # Handle updates (including HITL interrupts) - if isinstance(data, dict): - if "__interrupt__" in data: - interrupt_data = data["__interrupt__"] - if interrupt_data and interrupt_data.get("action_requests"): - results.append( - { - "type": "approval_request", - "approval_data": interrupt_data, - "session_id": self.session_id, - } - ) - - elif "todos" in data: - results.append( - { - "type": "todos", - "todos": data["todos"], - "session_id": self.session_id, - } - ) - - # Decide whether to flush buffered text (only if no active tool calls) - should_flush_text = False - if self.pending_text and not self.tool_call_buffers: - time_elapsed = current_time - self.last_chunk_time - - # Condition 1: timeout exceeded - if time_elapsed > self.chunk_timeout: - should_flush_text = True - # Condition 2: likely complete sentence and not too short - elif self._has_complete_sentence(self.pending_text) and len(self.pending_text) > 30: - should_flush_text = True - # Condition 3: very long buffer - elif len(self.pending_text) > 200: - should_flush_text = True - - if should_flush_text: - text_to_send = self.pending_text.rstrip() - if text_to_send: - results.append( - { - "type": "message", - "content": text_to_send, - "session_id": self.session_id, - "is_stream": True, - } - ) - self.pending_text = "" - self.last_chunk_time = current_time - - # Priority: tool_call > tool_result > status > other > text - if results: - tool_call_messages = [r for r in results if r.get("type") == "tool_call"] - tool_result_messages = [r for r in results if r.get("type") == "tool_result"] - status_messages = [r for r in results if r.get("type") == "status"] - other_messages = [ - r for r in results if r.get("type") not in ["tool_call", "tool_result", "status", "message"] - ] - text_messages = [r for r in results if r.get("type") == "message"] - - if tool_call_messages: - return tool_call_messages[0] - elif tool_result_messages: - return tool_result_messages[0] - elif status_messages: - return status_messages[0] - elif other_messages: - return other_messages[0] - elif text_messages: - return text_messages[0] - - return None - - def _is_tool_output(self, text: str) -> bool: - """Check if text is a JSON array likely representing tool output.""" - import json - - text_stripped = text.strip() - if text_stripped.startswith("[") and text_stripped.endswith("]"): - try: - parsed = json.loads(text_stripped) - return isinstance(parsed, list) and len(parsed) > 0 - except json.JSONDecodeError: - return False - return False - - def _format_tool_output(self, text: str) -> Optional[str]: - """Format tool output into a friendly text.""" - import json - - try: - items = json.loads(text.strip()) - if not isinstance(items, list): - return None - - formatted_lines: List[str] = [] - for item in items: - if isinstance(item, list) and len(item) > 0: - main_item = item[0] if isinstance(item[0], str) else str(item[0]) - - if main_item.startswith("/") or ("/" in main_item and "." in main_item): - formatted_lines.append(f"📁 {main_item}") - for sub_item in item[1:]: - if isinstance(sub_item, str) and sub_item.strip(): - if sub_item.startswith("/"): - formatted_lines.append(f" 📄 {sub_item}") - else: - formatted_lines.append(f" • {sub_item}") - else: - formatted_lines.append(f"• {main_item}") - for sub_item in item[1:]: - if isinstance(sub_item, str) and sub_item.strip(): - formatted_lines.append(f" • {sub_item}") - elif isinstance(item, str): - if item.startswith("/"): - formatted_lines.append(f"📁 {item}") - else: - formatted_lines.append(f"• {item}") - - return "\n".join(formatted_lines) if formatted_lines else None - - except (json.JSONDecodeError, Exception): - return None - - def _has_complete_sentence(self, text: str) -> bool: - """Heuristic detection for complete sentences to decide flush timing.""" - import re - - text_stripped = text.strip() - if len(text_stripped) < 20: - return False - - end_chars = [".", "!", "?", "。", "！", "？", "\n"] - ends_with_sentence = any(text_stripped.endswith(char) for char in end_chars) - - sentence_patterns = [ - r".*[。！？]\s*$", - r"[.!?]\s*$", - r"：\s*.*[。！？.!?]", - r"\s*\n\s*$", - ] - has_sentence_structure = any(re.match(pattern, text_stripped) for pattern in sentence_patterns) - - avoid_split_patterns = [ - r".*```$", - r".*`[^`]*$", - r".*\d+\.$", - r".*[-*+]\s*$", - ] - should_avoid_split = any(re.match(pattern, text_stripped) for pattern in avoid_split_patterns) - - return ends_with_sentence and has_sentence_structure and not should_avoid_split - - def flush_pending_text(self, final: bool = False) -> List[Dict[str, Any]]: - """Force flush accumulated text buffer.""" - results: List[Dict[str, Any]] = [] - if self.pending_text and (final or self.pending_text.strip()): - text_to_send = self.pending_text.rstrip() - if text_to_send: - results.append( - { - "type": "message", - "content": text_to_send, - "session_id": self.session_id, - "is_stream": not final, - } - ) - self.pending_text = "" - - if final: - self.sent_thinking = False - self.is_thinking = False - self.has_sent_thinking_for_current_request = False - - return results diff --git a/backend/app/core/code_executor.py b/backend/app/core/code_executor.py deleted file mode 100644 index 11d07d07f..000000000 --- a/backend/app/core/code_executor.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Code Executor — execute user LangGraph code in a sandboxed environment. - -Security model: -- Builtins blacklist: open, eval, exec, compile, globals, locals, vars, dir removed -- Import guard: blocklist + allowlist -- Exec timeout: 10 seconds via signal.alarm (Unix) -- TypedDict/Annotated compatibility: uses synthetic module namespace -""" - -from __future__ import annotations - -import builtins -import signal -import sys -import types -from typing import Any - -from langgraph.graph import StateGraph -from loguru import logger - -# --------------------------------------------------------------------------- -# Import guard -# --------------------------------------------------------------------------- - -ALLOWED_MODULES = frozenset( - { - "langgraph", - "langchain", - "langchain_core", - "langchain_community", - "langchain_openai", - "langchain_anthropic", - "langchain_google_genai", - "langchain_deepseek", - "typing", - "typing_extensions", - "operator", - "functools", - "itertools", - "json", - "re", - "math", - "datetime", - "collections", - "enum", - "dataclasses", - "abc", - "copy", - "textwrap", - "string", - "hashlib", - "base64", - "uuid", - "pydantic", - } -) - -BLOCKED_MODULES = frozenset( - { - "os", - "sys", - "subprocess", - "shutil", - "pathlib", - "socket", - "http", - "urllib", - "requests", - "httpx", - "importlib", - "ctypes", - "signal", - "multiprocessing", - "threading", - "asyncio", - "pickle", - "shelve", - "marshal", - "code", - "codeop", - "compileall", - "io", - "tempfile", - "glob", - } -) - -_real_import = builtins.__import__ - - -def _safe_import(name: str, *args: Any, **kwargs: Any) -> Any: - top_level = name.split(".")[0] - if top_level in BLOCKED_MODULES: - raise ImportError(f"Import of '{name}' is blocked for security reasons.") - if top_level not in ALLOWED_MODULES: - raise ImportError(f"Import of '{name}' is not allowed. Allowed: {', '.join(sorted(ALLOWED_MODULES))}") - return _real_import(name, *args, **kwargs) - - -# --------------------------------------------------------------------------- -# Safe builtins (full builtins minus dangerous functions) -# --------------------------------------------------------------------------- - -_BLOCKED_BUILTINS = frozenset( - { - "open", - "eval", - "exec", - "compile", - "globals", - "locals", - "vars", - "dir", - "breakpoint", - "exit", - "quit", - "__import__", - "input", # blocks stdin in server context - } -) - -_safe_builtins = {k: v for k, v in builtins.__dict__.items() if k not in _BLOCKED_BUILTINS} -_safe_builtins["__import__"] = _safe_import - -# Exec timeout (seconds) -EXEC_TIMEOUT = 10 - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def _timeout_handler(signum: int, frame: Any) -> None: - raise TimeoutError(f"Code execution timed out ({EXEC_TIMEOUT}s limit)") - - -def execute_code(code: str) -> StateGraph: - """Execute user code and return the StateGraph instance. - - Security: - - Dangerous builtins removed (open, eval, exec, compile, etc.) - - Import whitelist enforced - - 10 second execution timeout via SIGALRM - """ - logger.info(f"[CodeExecutor] Executing user code ({len(code)} chars)") - - # Create synthetic module for TypedDict compatibility - module_name = "__langgraph_user_code__" - module = types.ModuleType(module_name) - module.__dict__["__builtins__"] = _safe_builtins # restricted builtins - module.__dict__["__name__"] = module_name - - old_module = sys.modules.get(module_name) - sys.modules[module_name] = module - - try: - # Patch builtins.__import__ for duration of exec - original_import = builtins.__import__ - builtins.__import__ = _safe_import - - # Set exec timeout (Unix only) - old_handler = signal.signal(signal.SIGALRM, _timeout_handler) - signal.alarm(EXEC_TIMEOUT) - - try: - exec(code, module.__dict__) - finally: - signal.alarm(0) # cancel timeout - signal.signal(signal.SIGALRM, old_handler) - builtins.__import__ = original_import - - # Find StateGraph instances - graphs = [v for v in module.__dict__.values() if isinstance(v, StateGraph)] - - if not graphs: - raise ValueError( - "No StateGraph instance found in your code. " - "Make sure you create a StateGraph variable, e.g.:\n" - " graph = StateGraph(MyState)" - ) - - if len(graphs) > 1: - raise ValueError( - f"Found {len(graphs)} StateGraph instances. Only one StateGraph per code file is supported." - ) - - logger.info("[CodeExecutor] StateGraph extracted successfully") - return graphs[0] - - finally: - if old_module is not None: - sys.modules[module_name] = old_module - else: - sys.modules.pop(module_name, None) diff --git a/backend/app/core/constants.py b/backend/app/core/constants.py deleted file mode 100644 index 8bc50f020..000000000 --- a/backend/app/core/constants.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Core constants for the application. - -Centralized location for shared constants to ensure consistency across modules. -""" - -# Default user ID: nil UUID (00000000-0000-0000-0000-000000000000) -# Using nil UUID ensures type consistency and avoids confusion with real UUIDs. -# This is used when user_id is None or not provided, ensuring all user_id values -# are strings in UUID format. -DEFAULT_USER_ID = "00000000-0000-0000-0000-000000000000" diff --git a/backend/app/core/copilot/__init__.py b/backend/app/core/copilot/__init__.py deleted file mode 100644 index 6fd23ad52..000000000 --- a/backend/app/core/copilot/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Copilot Module - AI-powered graph building assistant. - -This module provides the core functionality for the Copilot feature, -which helps users build agent workflows through natural language instructions. - -Submodules: -- action_types: Pydantic models for requests/responses -- graph_analyzer: Graph topology analysis utilities -- prompt_builder: System prompt construction -- response_parser: Response parsing utilities -- tools: LangChain tools for graph manipulation -- agent: Copilot agent creation -""" - -from app.core.copilot.action_types import ( - ConnectNodesPayload, - CopilotContentEvent, - CopilotDoneEvent, - CopilotErrorEvent, - CopilotHistoryResponse, - CopilotMessage, - CopilotRequest, - CopilotResponse, - CopilotResultEvent, - CopilotStatusEvent, - CopilotStreamEvent, - CopilotThoughtStep, - CopilotThoughtStepEvent, - CopilotToolCall, - CopilotToolCallEvent, - CopilotToolResultEvent, - CreateNodePayload, - DeleteNodePayload, - GraphAction, - GraphActionType, - UpdateConfigPayload, -) -from app.core.copilot.action_validator import ( - extract_existing_node_ids, - filter_invalid_actions, - validate_actions, -) -from app.core.copilot.agent import get_copilot_agent -from app.core.copilot.message_builder import build_langchain_messages -from app.core.copilot.response_parser import ( - expand_action_payload, - extract_actions_from_agent_result, - parse_thought_to_steps, - try_extract_thought_field, -) -from app.core.copilot.tool_output_parser import parse_tool_output -from app.core.copilot.tools import reset_node_registry - -__all__ = [ - # Action types - "GraphActionType", - "GraphAction", - "CopilotRequest", - "CopilotResponse", - "CreateNodePayload", - "ConnectNodesPayload", - "DeleteNodePayload", - "UpdateConfigPayload", - # Stream event types (contract for WebSocket/SSE) - "CopilotStatusEvent", - "CopilotContentEvent", - "CopilotThoughtStepEvent", - "CopilotToolCallEvent", - "CopilotToolResultEvent", - "CopilotResultEvent", - "CopilotDoneEvent", - "CopilotErrorEvent", - "CopilotStreamEvent", - # Message persistence types - "CopilotMessage", - "CopilotThoughtStep", - "CopilotToolCall", - "CopilotHistoryResponse", - # Agent - "get_copilot_agent", - # Response parser - "try_extract_thought_field", - "parse_thought_to_steps", - "extract_actions_from_agent_result", - "expand_action_payload", - # Tool output parser - "parse_tool_output", - # Message builder - "build_langchain_messages", - # Tools - "reset_node_registry", - # Validator - "validate_actions", - "extract_existing_node_ids", - "filter_invalid_actions", -] diff --git a/backend/app/core/copilot/action_applier.py b/backend/app/core/copilot/action_applier.py deleted file mode 100644 index a2d46718b..000000000 --- a/backend/app/core/copilot/action_applier.py +++ /dev/null @@ -1,245 +0,0 @@ -""" -Action Applier - Apply Copilot actions to graph state. - -Converts Copilot actions (CREATE_NODE, CONNECT_NODES, etc.) to complete -graph state (nodes and edges) that can be saved to the database. - -This module replicates the logic from frontend ActionProcessor to ensure -consistency between frontend and backend graph state. - -Node defaults: NODE_DEFAULT_CONFIGS and NODE_LABELS must be kept in sync -with frontend app/workspace/.../services/nodeRegistry.tsx (defaultConfig and -label per type). See docs/schemas/README.md. -""" - -from typing import Any, Dict, List, Tuple - -from loguru import logger - -# Node type default configurations (must match frontend nodeRegistry) -NODE_DEFAULT_CONFIGS: Dict[str, Dict[str, Any]] = { - "agent": { - "provider_name": "", - "model_name": "", - "temp": 0.7, - "systemPrompt": "", - "enableMemory": False, - "memory_provider_name": "", - "memory_model_name": "", - "memoryPrompt": "Summarize the interaction highlights and key facts learned about the user.", - "useDeepAgents": False, - "description": "", - }, - "condition": { - "expression": "", - "trueLabel": "Yes", - "falseLabel": "No", - }, - "condition_agent": { - "instruction": "Analyze and route", - "options": ["Option A", "Option B"], - }, - "http": { - "method": "GET", - "url": "https://api.example.com", - }, - "custom_function": { - "name": "my_tool", - "description": "", - "parameters": [], - }, - "direct_reply": { - "template": "Hello user", - }, - "human_input": { - "prompt": "Please approve", - }, - "router_node": { - "routes": [], - "defaultRoute": "default", - }, - "loop_condition_node": { - "conditionType": "while", - "listVariable": "items", - "condition": "loop_count < 3", - "maxIterations": 5, - }, -} - -# Node type labels (matching frontend nodeRegistry) -NODE_LABELS: Dict[str, str] = { - "agent": "Agent", - "condition": "Condition", - "condition_agent": "Condition Agent", - "http": "HTTP Request", - "custom_function": "Custom Tool", - "direct_reply": "Direct Reply", - "human_input": "Human Input", - "router_node": "Router", - "loop_condition_node": "Loop Condition", -} - - -def get_node_default_config(node_type: str) -> Dict[str, Any]: - """Get default configuration for a node type.""" - return NODE_DEFAULT_CONFIGS.get(node_type, {}).copy() - - -def get_node_label(node_type: str) -> str: - """Get default label for a node type.""" - return NODE_LABELS.get(node_type, "Node") - - -def apply_actions_to_graph_state( - current_nodes: List[Dict[str, Any]], - current_edges: List[Dict[str, Any]], - actions: List[Dict[str, Any]], -) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: - """ - Apply Copilot actions to current graph state and return updated nodes and edges. - - This function replicates the logic from frontend ActionProcessor.processActions - to ensure consistency between frontend and backend. - - Args: - current_nodes: Current nodes in the graph (from graph_context) - current_edges: Current edges in the graph (from graph_context) - actions: List of actions to apply (CREATE_NODE, CONNECT_NODES, etc.) - - Returns: - Tuple of (updated_nodes, updated_edges) in format ready for GraphService.save_graph_state - """ - # Clone current state to apply diffs - processed_nodes: List[Dict[str, Any]] = [node.copy() for node in current_nodes] - processed_edges: List[Dict[str, Any]] = [edge.copy() for edge in current_edges] - - # Track existing node IDs for idempotent CREATE_NODE - existing_node_ids: set = {n.get("id") for n in processed_nodes if n.get("id")} - - for action in actions: - action_type = action.get("type") - payload = action.get("payload", {}) - - try: - if action_type == "CREATE_NODE": - node_id = payload.get("id") or f"ai_{hash(str(action)) % 1000000}" - node_type = payload.get("type", "agent") - label = payload.get("label") - position = payload.get("position", {"x": 0, "y": 0}) - config = payload.get("config", {}) - - if node_id in existing_node_ids: - logger.warning(f"[ActionApplier] Skipping duplicate node: {node_id}") - continue - - # Get default config and merge with action config - base_config = get_node_default_config(node_type) - merged_config = {**base_config, **config} - - # Use default label if not provided - node_label = label or get_node_label(node_type) - - new_node: Dict[str, Any] = { - "id": node_id, - "type": "custom", - "position": position, - "data": { - "label": node_label, - "type": node_type, - "config": merged_config, - }, - } - - processed_nodes.append(new_node) - existing_node_ids.add(node_id) - logger.debug(f"[ActionApplier] Created node: {node_id}, type: {node_type}, label: {node_label}") - - elif action_type == "CONNECT_NODES": - source = payload.get("source") - target = payload.get("target") - - if source and target: - # Check if edge already exists - edge_exists = any(e.get("source") == source and e.get("target") == target for e in processed_edges) - - if not edge_exists: - # Create edge in format expected by GraphService.save_graph_state - # Format matches frontend: { id, source, target, data: {} } - new_edge: Dict[str, Any] = { - "id": f"e-{source}-{target}", - "source": source, - "target": target, - "animated": True, - "style": {"stroke": "#cbd5e1", "strokeWidth": 1.5}, - "data": {}, # Must exist, even if empty (required by backend) - } - - processed_edges.append(new_edge) - logger.debug(f"[ActionApplier] Created edge: {source} -> {target}") - - elif action_type == "DELETE_NODE": - node_id = payload.get("id") - - if node_id: - # Remove node - processed_nodes = [n for n in processed_nodes if n.get("id") != node_id] - - # Remove edges connected to this node - processed_edges = [ - e for e in processed_edges if e.get("source") != node_id and e.get("target") != node_id - ] - - logger.debug(f"[ActionApplier] Deleted node: {node_id}") - - elif action_type == "UPDATE_CONFIG": - node_id = payload.get("id") - config_updates = payload.get("config", {}) - - if node_id and config_updates: - for node in processed_nodes: - if node.get("id") == node_id: - node_data = node.get("data", {}) - if isinstance(node_data, dict): - existing_config = node_data.get("config", {}) - if isinstance(existing_config, dict): - # Merge config updates - node_data["config"] = {**existing_config, **config_updates} - logger.debug(f"[ActionApplier] Updated config for node: {node_id}") - break - - elif action_type == "UPDATE_POSITION": - node_id = payload.get("id") - position = payload.get("position") - - if node_id and position: - for node in processed_nodes: - if node.get("id") == node_id: - node["position"] = position - logger.debug(f"[ActionApplier] Updated position for node: {node_id}") - break - - except Exception as e: - logger.error(f"[ActionApplier] Error processing action {action_type}: {e}", exc_info=True) - # Continue processing other actions even if one fails - - # Deduplicate edges (based on source-target combination) - seen_edges: Dict[Tuple[str, str], bool] = {} - deduplicated_edges: List[Dict[str, Any]] = [] - - for edge in processed_edges: - source = edge.get("source") - target = edge.get("target") - - if source and target: - edge_key = (source, target) - if edge_key not in seen_edges: - seen_edges[edge_key] = True - deduplicated_edges.append(edge) - - logger.info( - f"[ActionApplier] Applied {len(actions)} actions: " - f"nodes {len(current_nodes)} -> {len(processed_nodes)}, " - f"edges {len(current_edges)} -> {len(deduplicated_edges)}" - ) - - return processed_nodes, deduplicated_edges diff --git a/backend/app/core/copilot/action_types.py b/backend/app/core/copilot/action_types.py deleted file mode 100644 index 77cb510fd..000000000 --- a/backend/app/core/copilot/action_types.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Copilot Action Types - Pydantic models for Copilot API. - -Defines the data models used in Copilot requests and responses, -including graph actions (CREATE_NODE, CONNECT_NODES, etc.). -""" - -import uuid -from datetime import datetime -from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Union - -from pydantic import BaseModel, Field - - -class GraphActionType(str, Enum): - """Graph action types that can be executed by Copilot.""" - - CREATE_NODE = "CREATE_NODE" - CONNECT_NODES = "CONNECT_NODES" - DELETE_NODE = "DELETE_NODE" - UPDATE_CONFIG = "UPDATE_CONFIG" - UPDATE_POSITION = "UPDATE_POSITION" - - -class GraphAction(BaseModel): - """Single graph action to be executed.""" - - type: GraphActionType = Field(..., description="Action type") - payload: Dict[str, Any] = Field(..., description="Action payload") - reasoning: str = Field(..., description="Reasoning for the action") - - -# ==================== Message Persistence Types ==================== - - -class CopilotThoughtStep(BaseModel): - """Single thought step in AI reasoning process.""" - - index: int = Field(..., description="Step index (1-based)") - content: str = Field(..., description="Thought content") - - -class CopilotToolCall(BaseModel): - """Record of a tool call made by the AI.""" - - tool: str = Field(..., description="Tool name") - input: Dict[str, Any] = Field(default_factory=dict, description="Tool input parameters") - output: Optional[Dict[str, Any]] = Field(default=None, description="Tool output (optional)") - - -class CopilotMessage(BaseModel): - """Single message in Copilot conversation history.""" - - id: str = Field(default_factory=lambda: f"msg_{uuid.uuid4().hex[:12]}", description="Unique message ID") - role: str = Field(..., description="Message role: 'user' or 'assistant'") - content: str = Field(..., description="Message text content") - created_at: datetime = Field(default_factory=datetime.utcnow, description="Message creation time") - - # Fields only for assistant messages - actions: Optional[List[Dict[str, Any]]] = Field(default=None, description="Graph actions executed") - thought_steps: Optional[List[CopilotThoughtStep]] = Field(default=None, description="AI thinking process") - tool_calls: Optional[List[CopilotToolCall]] = Field(default=None, description="Tool calls made") - - -class CopilotHistoryResponse(BaseModel): - """Response for Copilot history API.""" - - graph_id: str = Field(..., description="Associated graph ID") - messages: List[CopilotMessage] = Field(default_factory=list, description="Conversation messages") - created_at: Optional[datetime] = Field(default=None, description="Session creation time") - updated_at: Optional[datetime] = Field(default=None, description="Session last update time") - - -class CopilotRequest(BaseModel): - """Copilot request for generating graph actions.""" - - prompt: str = Field(..., description="User prompt") - graph_context: Dict[str, Any] = Field(default_factory=dict, description="Current graph state (nodes, edges)") - graph_id: Optional[str] = Field(default=None, description="Graph ID for history persistence") - conversation_history: Optional[List[Dict[str, Any]]] = Field( - default=None, - description="Previous conversation messages. Only 'role' and 'content' are used for context. " - "Format: [{'role': 'user'|'assistant', 'content': '...', 'actions'?: ...}, ...]", - ) - mode: str = Field(default="deepagents", description="Copilot engine mode: 'standard' or 'deepagents'") - model: Optional[str] = Field(default=None, description="Model identifier, format: provider:model_name") - - -class CopilotResponse(BaseModel): - """Copilot response with message and actions.""" - - message: str = Field(..., description="Chat message to the user") - actions: List[GraphAction] = Field(default_factory=list, description="Array of actions to execute") - - -# ==================== Tool Call Result Types ==================== - - -class CreateNodePayload(BaseModel): - """Payload for CREATE_NODE action.""" - - id: str = Field(..., description="Unique node ID") - type: str = Field(..., description="Node type (agent, condition, etc.)") - label: str = Field(..., description="Human-readable label") - position: Dict[str, float] = Field(..., description="Node position {x, y}") - config: Dict[str, Any] = Field(default_factory=dict, description="Node configuration") - - -class ConnectNodesPayload(BaseModel): - """Payload for CONNECT_NODES action.""" - - source: str = Field(..., description="Source node ID") - target: str = Field(..., description="Target node ID") - - -class DeleteNodePayload(BaseModel): - """Payload for DELETE_NODE action.""" - - id: str = Field(..., description="Node ID to delete") - - -class UpdateConfigPayload(BaseModel): - """Payload for UPDATE_CONFIG action.""" - - id: str = Field(..., description="Node ID to update") - config: Dict[str, Any] = Field(..., description="Configuration to merge") - - -# ==================== Stream Event Types (WebSocket / SSE contract) ==================== - - -class CopilotStatusEvent(BaseModel): - """Stream event: progress status.""" - - type: Literal["status"] = "status" - stage: str = Field(..., description="Stage identifier (e.g. thinking, processing)") - message: str = Field(..., description="Human-readable status message") - - -class CopilotContentEvent(BaseModel): - """Stream event: streaming AI response content.""" - - type: Literal["content"] = "content" - content: str = Field(..., description="Content chunk") - - -class CopilotThoughtStepEvent(BaseModel): - """Stream event: single thought step in AI reasoning.""" - - type: Literal["thought_step"] = "thought_step" - step: Dict[str, Any] = Field(..., description="Step with index and content (e.g. {index, content})") - - -class CopilotToolCallEvent(BaseModel): - """Stream event: tool invocation started.""" - - type: Literal["tool_call"] = "tool_call" - tool: str = Field(..., description="Tool name") - input: Dict[str, Any] = Field(default_factory=dict, description="Tool input parameters") - - -class CopilotToolResultEvent(BaseModel): - """Stream event: tool execution result (action payload).""" - - type: Literal["tool_result"] = "tool_result" - action: Dict[str, Any] = Field(..., description="Action dict: type, payload, reasoning (GraphAction-compatible)") - - -class CopilotResultEvent(BaseModel): - """Stream event: final result with message and actions.""" - - type: Literal["result"] = "result" - message: str = Field(..., description="Final assistant message") - actions: List[Dict[str, Any]] = Field(default_factory=list, description="List of GraphAction-compatible dicts") - batch: Optional[bool] = Field(default=None, description="Optional batch flag for frontend") - - -class CopilotDoneEvent(BaseModel): - """Stream event: stream finished.""" - - type: Literal["done"] = "done" - - -class CopilotErrorEvent(BaseModel): - """Stream event: error occurred.""" - - type: Literal["error"] = "error" - message: str = Field(..., description="Error message") - code: str = Field(..., description="Error code for frontend mapping (e.g. CREDENTIAL_ERROR, UNKNOWN_ERROR)") - - -# Union of all stream events (for schema export and optional validation) -CopilotStreamEvent = Union[ - CopilotStatusEvent, - CopilotContentEvent, - CopilotThoughtStepEvent, - CopilotToolCallEvent, - CopilotToolResultEvent, - CopilotResultEvent, - CopilotDoneEvent, - CopilotErrorEvent, -] diff --git a/backend/app/core/copilot/action_validator.py b/backend/app/core/copilot/action_validator.py deleted file mode 100644 index eb823d66a..000000000 --- a/backend/app/core/copilot/action_validator.py +++ /dev/null @@ -1,233 +0,0 @@ -""" -Action Validator - Validates Copilot-generated actions before execution. - -Provides validation for: -- Node ID references (existing + newly created) -- Graph topology (orphan nodes, connectivity) -- Action consistency -""" - -from typing import Any, Dict, List, Optional, Set, Tuple - -from loguru import logger - - -class ActionValidationResult: - """Result of action validation.""" - - def __init__(self): - self.is_valid: bool = True - self.errors: List[str] = [] - self.warnings: List[str] = [] - self.fixed_actions: Optional[List[Dict[str, Any]]] = None - - def add_error(self, message: str): - """Add an error (validation failed).""" - self.is_valid = False - self.errors.append(message) - - def add_warning(self, message: str): - """Add a warning (validation passed with issues).""" - self.warnings.append(message) - - -def validate_actions( - actions: List[Dict[str, Any]], - existing_node_ids: Set[str], -) -> ActionValidationResult: - """ - Validate a list of actions for consistency and correctness. - - Checks: - 1. All node references in CONNECT_NODES, DELETE_NODE, UPDATE_CONFIG exist - 2. No duplicate node IDs are created - 3. Graph connectivity (warns about orphan nodes) - - Args: - actions: List of action dicts from Copilot - existing_node_ids: Set of node IDs already in the graph - - Returns: - ActionValidationResult with validation status and details - """ - result = ActionValidationResult() - - # Track nodes created in this batch - created_node_ids: Set[str] = set() - created_node_labels: Dict[str, str] = {} # id -> label - - # Track connections - connections: List[Tuple[str, str]] = [] - - # All valid IDs (existing + newly created) - all_valid_ids = existing_node_ids.copy() - - for i, action in enumerate(actions): - action_type = action.get("type", "") - payload = action.get("payload", {}) - - if action_type == "CREATE_NODE": - node_id = payload.get("id", "") - label = payload.get("label", "") - - if not node_id: - result.add_error(f"Action {i}: CREATE_NODE missing node ID") - continue - - # Check for duplicate ID - if node_id in all_valid_ids: - result.add_warning(f"Action {i}: Duplicate node ID '{node_id}' - may overwrite existing node") - - created_node_ids.add(node_id) - created_node_labels[node_id] = label - all_valid_ids.add(node_id) - - elif action_type == "CONNECT_NODES": - source = payload.get("source", "") - target = payload.get("target", "") - - if not source: - result.add_error(f"Action {i}: CONNECT_NODES missing source ID") - elif source not in all_valid_ids: - result.add_error(f"Action {i}: CONNECT_NODES source '{source}' not found") - - if not target: - result.add_error(f"Action {i}: CONNECT_NODES missing target ID") - elif target not in all_valid_ids: - result.add_error(f"Action {i}: CONNECT_NODES target '{target}' not found") - - if source and target: - connections.append((source, target)) - - elif action_type == "DELETE_NODE": - node_id = payload.get("id", "") - - if not node_id: - result.add_error(f"Action {i}: DELETE_NODE missing node ID") - elif node_id not in all_valid_ids: - result.add_error(f"Action {i}: DELETE_NODE node '{node_id}' not found") - else: - # Remove from valid IDs (node will be deleted) - all_valid_ids.discard(node_id) - - elif action_type == "UPDATE_CONFIG": - node_id = payload.get("id", "") - - if not node_id: - result.add_error(f"Action {i}: UPDATE_CONFIG missing node ID") - elif node_id not in all_valid_ids: - result.add_error(f"Action {i}: UPDATE_CONFIG node '{node_id}' not found") - - # Check for orphan nodes (newly created nodes not connected to anything) - if created_node_ids: - connected_nodes: Set[str] = set() - for src, tgt in connections: - connected_nodes.add(src) - connected_nodes.add(tgt) - - # Include existing nodes as potentially connected - connected_nodes.update(existing_node_ids) - - orphan_nodes = created_node_ids - connected_nodes - - # Only warn if multiple nodes were created (single node is fine without connections) - if orphan_nodes and len(created_node_ids) > 1: - for node_id in orphan_nodes: - label = created_node_labels.get(node_id, node_id) - result.add_warning(f"Orphan node: '{label}' ({node_id}) is not connected to the workflow") - - logger.info( - f"[ActionValidator] Validation complete: valid={result.is_valid}, errors={len(result.errors)}, warnings={len(result.warnings)}" - ) - - return result - - -def extract_existing_node_ids(graph_context: Dict[str, Any]) -> Set[str]: - """ - Extract existing node IDs from graph context. - - Args: - graph_context: Graph context dict with nodes - - Returns: - Set of existing node IDs - """ - node_ids: Set[str] = set() - - nodes = graph_context.get("nodes", []) - for node in nodes: - # Handle ReactFlow format - node_id = node.get("id") - if node_id: - node_ids.add(node_id) - - return node_ids - - -def filter_invalid_actions( - actions: List[Dict[str, Any]], - existing_node_ids: Set[str], -) -> Tuple[List[Dict[str, Any]], List[str]]: - """ - Filter out invalid actions and return valid ones with removal reasons. - - Args: - actions: List of action dicts - existing_node_ids: Set of existing node IDs - - Returns: - Tuple of (valid_actions, removed_reasons) - """ - valid_actions: List[Dict[str, Any]] = [] - removed_reasons: List[str] = [] - - # Track nodes as we process - all_valid_ids = existing_node_ids.copy() - - for i, action in enumerate(actions): - action_type = action.get("type", "") - payload = action.get("payload", {}) - keep = True - reason = "" - - if action_type == "CREATE_NODE": - node_id = payload.get("id", "") - if node_id: - all_valid_ids.add(node_id) - else: - keep = False - reason = f"Action {i}: CREATE_NODE has no ID" - - elif action_type == "CONNECT_NODES": - source = payload.get("source", "") - target = payload.get("target", "") - - if source not in all_valid_ids: - keep = False - reason = f"Action {i}: CONNECT_NODES source '{source}' not found" - elif target not in all_valid_ids: - keep = False - reason = f"Action {i}: CONNECT_NODES target '{target}' not found" - - elif action_type == "DELETE_NODE": - node_id = payload.get("id", "") - if node_id not in all_valid_ids: - keep = False - reason = f"Action {i}: DELETE_NODE node '{node_id}' not found" - else: - all_valid_ids.discard(node_id) - - elif action_type == "UPDATE_CONFIG": - node_id = payload.get("id", "") - if node_id not in all_valid_ids: - keep = False - reason = f"Action {i}: UPDATE_CONFIG node '{node_id}' not found" - - if keep: - valid_actions.append(action) - else: - removed_reasons.append(reason) - logger.warning(f"[ActionValidator] Removing invalid action: {reason}") - - return valid_actions, removed_reasons diff --git a/backend/app/core/copilot/agent.py b/backend/app/core/copilot/agent.py deleted file mode 100644 index f1487b83c..000000000 --- a/backend/app/core/copilot/agent.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Copilot Agent - Create and manage the Copilot Agent instance. - -Uses the same infrastructure as sample_agent to create an Agent -specialized for graph manipulation tasks. -""" - -from typing import Any, Dict, List, Optional - -from langchain_core.runnables import Runnable -from loguru import logger -from sqlalchemy.ext.asyncio import AsyncSession - -from app.core.agent.sample_agent import get_agent -from app.core.copilot.prompt_builder import build_copilot_system_prompt -from app.core.copilot.tools import get_copilot_tools, set_current_graph_context, set_preloaded_models - - -async def _preload_available_models(db: AsyncSession) -> List[Dict[str, Any]]: - """ - Preload available chat models from database. - - Args: - db: Database session - - Returns: - List of available model info dicts - """ - try: - from app.core.model import ModelType - from app.services.model_service import ModelService - - service = ModelService(db) - models = await service.get_available_models(model_type=ModelType.CHAT) - logger.debug(f"[_preload_available_models] Loaded {len(models)} chat models") - return models - except Exception as e: - logger.warning(f"[_preload_available_models] Failed to load models: {e}") - return [] - - -async def get_copilot_agent( - graph_context: Dict[str, Any], - user_id: Optional[str] = None, - max_tokens: int = 4096, - db: Optional[AsyncSession] = None, - model: Any = None, -) -> Runnable: - """ - Create a Copilot Agent with graph manipulation tools. - - The agent is configured with: - - System prompt built from graph context - - Tools for creating/connecting/deleting nodes - - Available models list for intelligent model selection - - TodoList and Skills middleware disabled (not needed for Copilot) - - Args: - graph_context: Current graph state with nodes and edges - user_id: Optional user ID for workspace isolation - max_tokens: Maximum tokens for completion - db: Optional database session for loading available models - model: Pre-created LangChain model instance (from ModelResolver) - - Returns: - Configured Runnable agent - """ - logger.debug(f"[get_copilot_agent] Creating Copilot agent for user_id={user_id}") - - # Preload available models if db session provided - available_models: List[Dict[str, Any]] = [] - if db: - available_models = await _preload_available_models(db) - # Set models for list_models tool - set_preloaded_models(available_models) - - # Set current graph context for analysis tools (auto_layout, analyze_workflow) - set_current_graph_context(graph_context) - - # Build system prompt from graph context (with available models) - system_prompt = build_copilot_system_prompt(graph_context, available_models) - - # Get Copilot tools - tools = get_copilot_tools() - - logger.debug(f"[get_copilot_agent] System prompt length: {len(system_prompt)}") - logger.debug(f"[get_copilot_agent] Tools count: {len(tools)}") - - # Create agent using the same infrastructure as sample_agent - agent = await get_agent( - model=model, - max_tokens=max_tokens, - user_id=user_id, - system_prompt=system_prompt, - tools=tools, - enable_todo_list=False, - enable_skills=False, - agent_name="Copilot", - ) - - logger.debug("[get_copilot_agent] Agent created successfully") - return agent diff --git a/backend/app/core/copilot/exceptions.py b/backend/app/core/copilot/exceptions.py deleted file mode 100644 index 31025ed24..000000000 --- a/backend/app/core/copilot/exceptions.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Copilot-specific exceptions for better error handling. - -Provides specialized exception classes for Copilot operations, -enabling better error categorization and user-friendly error messages. -""" - -from typing import Any - -from fastapi import status - -from app.common.exceptions import AppException, BadRequestException - - -class CopilotException(AppException): - """Base exception for Copilot operations.""" - - def __init__( - self, - message: str = "Copilot operation failed", - *, - code: int | None = 5001, - data: Any = None, - status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR, - ): - super().__init__(status_code=status_code, message=message, code=code, data=data) - - -class CopilotLLMError(CopilotException): - """LLM-related errors (API failures, rate limits, etc.).""" - - def __init__( - self, - message: str = "LLM service error", - *, - code: int | None = 5101, - data: Any = None, - original_error: Exception | None = None, - ): - if original_error: - message = f"{message}: {str(original_error)}" - if data is None: - data = {"error_type": type(original_error).__name__} - super().__init__( - status_code=status.HTTP_502_BAD_GATEWAY, - message=message, - code=code, - data=data, - ) - - -class CopilotValidationError(BadRequestException): - """Action validation errors.""" - - def __init__( - self, - message: str = "Action validation failed", - *, - code: int | None = 5102, - data: Any = None, - ): - super().__init__(message=message, code=code, data=data) - - -class CopilotSessionError(CopilotException): - """Session management errors (Redis unavailable, session not found, etc.).""" - - def __init__( - self, - message: str = "Session management error", - *, - code: int | None = 5103, - data: Any = None, - status_code: int = status.HTTP_503_SERVICE_UNAVAILABLE, - ): - super().__init__( - status_code=status_code, - message=message, - code=code, - data=data, - ) - - -class CopilotCredentialError(CopilotException): - """Credential-related errors (missing API key, invalid credentials, etc.).""" - - def __init__( - self, - message: str = "Credential error", - *, - code: int | None = 5105, - data: Any = None, - ): - super().__init__( - status_code=status.HTTP_401_UNAUTHORIZED, - message=message, - code=code, - data=data, - ) - - -class CopilotAgentError(CopilotException): - """Agent execution errors (tool failures, recursion limits, etc.).""" - - def __init__( - self, - message: str = "Agent execution error", - *, - code: int | None = 5106, - data: Any = None, - original_error: Exception | None = None, - ): - if original_error: - message = f"{message}: {str(original_error)}" - if data is None: - data = {"error_type": type(original_error).__name__} - super().__init__( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - message=message, - code=code, - data=data, - ) diff --git a/backend/app/core/copilot/graph_analyzer.py b/backend/app/core/copilot/graph_analyzer.py deleted file mode 100644 index 3974c3966..000000000 --- a/backend/app/core/copilot/graph_analyzer.py +++ /dev/null @@ -1,515 +0,0 @@ -""" -Graph Analyzer - Utility functions for analyzing graph topology. - -Provides functions to normalize nodes, analyze topology, and extract -node configurations for the Copilot system prompt. -""" - -from typing import Any, Dict, List, Optional - - -def normalize_node(node: Dict[str, Any]) -> Dict[str, Any]: - """ - Normalize node format from ReactFlow format. - - Frontend sends complete ReactFlow format: - { - "id": "...", - "type": "custom", - "position": {...}, - "data": { - "type": "agent", - "label": "...", - "config": {...}, - "tools": {...} // optional - } - } - - Consistent with base_builder._get_node_type logic: - - Priority: data.type -> node.type -> default "agent" - """ - data = node.get("data", {}) - if isinstance(data, dict): - node_type = data.get("type") or node.get("type") or "agent" - return { - "id": node.get("id"), - "type": node_type, - "label": data.get("label", ""), - "position": node.get("position", {}), - "config": data.get("config", {}), - "tools": data.get("tools"), - "prompt": node.get("prompt"), - } - else: - # Fallback: if data doesn't exist or is malformed - return { - "id": node.get("id"), - "type": node.get("type", "agent"), - "label": node.get("label", ""), - "position": node.get("position", {}), - "config": node.get("config", {}), - "tools": node.get("tools"), - "prompt": node.get("prompt"), - } - - -def extract_system_prompt(normalized_node: Dict[str, Any]) -> Optional[str]: - """ - Extract system prompt from node configuration. - - Extraction order (consistent with base_builder._get_system_prompt_from_node): - 1. node.prompt (GraphNode.prompt, if exists) - 2. data.config.systemPrompt - 3. data.config.prompt - 4. None - """ - prompt = normalized_node.get("prompt") - if prompt: - return str(prompt) if not isinstance(prompt, str) else prompt - - config = normalized_node.get("config", {}) - if isinstance(config, dict): - return config.get("systemPrompt", "") or config.get("prompt", "") or None - return None - - -def extract_tools_config(normalized_node: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """ - Extract tools configuration from node. - - Tools config format: - { - "builtin": ["tool_id1", "tool_id2", ...], - "mcp": ["uuid_or_url1", "uuid_or_url2", ...] - } - - Tools may be stored in: - 1. data.config.tools (primary location) - 2. data.tools (some nodes may define here) - - Returns tools config dict or None if not found. - """ - config = normalized_node.get("config", {}) - if isinstance(config, dict): - config_tools = config.get("tools") - if isinstance(config_tools, dict): - return config_tools - - node_tools = normalized_node.get("tools") - if isinstance(node_tools, dict): - return node_tools - - return None - - -def format_tools_summary(tools_config: Optional[Dict[str, Any]]) -> str: - """ - Format tools configuration as a readable summary string. - - Used in system prompt to display tool information. - """ - if not tools_config: - return "None" - - parts = [] - builtin = tools_config.get("builtin", []) - mcp = tools_config.get("mcp", []) - - if builtin: - parts.append(f"builtin: {', '.join(builtin)}") - if mcp: - parts.append(f"mcp: {len(mcp)} tool(s)") - - return "; ".join(parts) if parts else "None" - - -def analyze_graph_topology(normalized_nodes: List[Dict], edges: List[Dict]) -> Dict[str, Any]: - """ - Analyze graph topology structure. - - Topology analysis includes: - 1. Build connection relationships (incoming/outgoing edges) - 2. Identify root nodes (in-degree 0) and leaf nodes (out-degree 0) - 3. Analyze DeepAgents hierarchy (Manager/Worker roles) - - DeepAgents role logic (consistent with _build_recursive): - - Worker: no children (leaf node) - - Manager: has children (non-leaf node) - """ - node_ids = {n["id"] for n in normalized_nodes} - - # Build connection relationships - incoming: Dict[str, List[str]] = {node_id: [] for node_id in node_ids} - outgoing: Dict[str, List[str]] = {node_id: [] for node_id in node_ids} - - for edge in edges: - source = edge.get("source") - target = edge.get("target") - if ( - source is not None - and target is not None - and isinstance(source, str) - and isinstance(target, str) - and source in node_ids - and target in node_ids - ): - outgoing[source].append(target) - incoming[target].append(source) - - # Find root nodes (in-degree 0) and leaf nodes (out-degree 0) - root_nodes = [n["id"] for n in normalized_nodes if len(incoming.get(n["id"], [])) == 0] - leaf_nodes = [n["id"] for n in normalized_nodes if len(outgoing.get(n["id"], [])) == 0] - - # Analyze DeepAgents hierarchy - deep_agents_hierarchy: Dict[str, Dict[str, Any]] = {} - for node in normalized_nodes: - node_id = node["id"] - config = node.get("config", {}) - - if config.get("useDeepAgents", False) is True: - children = outgoing.get(node_id, []) - - # Role determination: - # - No children = Worker (Leaf Node) - # - Has children = Manager (DeepAgent) - if not children: - role = "Worker" - else: - role = "Manager" - - deep_agents_hierarchy[node_id] = { - "role": role, - "children": children, - "description": config.get("description", ""), - } - - return { - "rootNodes": root_nodes, - "leafNodes": leaf_nodes, - "deepAgentsHierarchy": deep_agents_hierarchy, - "incoming": incoming, - "outgoing": outgoing, - } - - -def generate_topology_description( - normalized_nodes: List[Dict], topology: Dict[str, Any], node_map: Dict[str, Dict] -) -> str: - """ - Generate structured topology description text showing graph flow structure. - - Example output: - "Current Flow: [Start: Support Agent] -> [Condition: Sentiment Check] -> [Branch A: Thank You] / [Branch B: Human Escalation]" - - For DeepAgents: - "DeepAgents Hierarchy: [Manager: Research Coordinator] -> [Worker: Research SubAgent]" - """ - if not normalized_nodes: - return "Current Flow: (Empty graph - no nodes yet)" - - root_nodes = topology.get("rootNodes", []) - outgoing = topology.get("outgoing", {}) - deep_agents_hierarchy = topology.get("deepAgentsHierarchy", {}) - - # Build node ID to node info mapping - node_id_to_info: Dict[str, Dict[str, Any]] = {} - for node in normalized_nodes: - node_id = node["id"] - node_type = node.get("type", "agent") - label = node.get("label", "") or node_id[:8] - config = node.get("config", {}) - is_deep_agent = config.get("useDeepAgents", False) is True - - # Generate node display name - type_abbr = { - "agent": "Agent", - "condition": "Condition", - "condition_agent": "AI Decision", - "http": "HTTP", - "custom_function": "Custom", - "direct_reply": "Reply", - "human_input": "Human", - "execute_flow": "SubFlow", - "iteration": "Loop", - }.get(node_type, node_type) - - display_name = f"{type_abbr}: {label}" - if is_deep_agent: - role = deep_agents_hierarchy.get(node_id, {}).get("role", "") - if role: - display_name += f" [{role}]" - - node_id_to_info[node_id] = { - "display": display_name, - "type": node_type, - "label": label, - "is_deep_agent": is_deep_agent, - "role": deep_agents_hierarchy.get(node_id, {}).get("role"), - } - - # Generate main flow description - flow_parts: List[str] = [] - - # If there are DeepAgents, describe hierarchy first - if deep_agents_hierarchy: - flow_parts.append("DeepAgents Hierarchy:") - for node_id, info in deep_agents_hierarchy.items(): - node_info = node_id_to_info.get(node_id, {}) - role = info.get("role", "") - children = info.get("children", []) - - if role == "Manager": - manager_name = node_info.get("display", node_id[:8]) - child_displays = [] - for cid in children: - child_info = node_id_to_info.get(cid, {}) - child_display = child_info.get("display", cid[:8]) - child_displays.append(f"[{child_display}]") - - if child_displays: - flow_parts.append(f" [{role}: {manager_name}] -> {', '.join(child_displays)}") - else: - flow_parts.append(f" [{role}: {manager_name}] (no subagents)") - - # Generate main flow path - if root_nodes: - flow_parts.append("Current Flow:") - - def traverse_path(node_id: str, visited: set, depth: int = 0) -> List[str]: - """Recursively traverse path, handle branches""" - if node_id in visited or depth > 20: - return ["[...]"] - - visited.add(node_id) - node_info = node_id_to_info.get(node_id, {}) - display = node_info.get("display", node_id[:8]) - - children = outgoing.get(node_id, []) - - if not children: - return [f"[{display}]"] - - if len(children) == 1: - child_path = traverse_path(children[0], visited.copy(), depth + 1) - return [f"[{display}]"] + child_path - else: - branch_paths = [] - for i, child_id in enumerate(children): - child_path = traverse_path(child_id, visited.copy(), depth + 1) - branch_label = chr(65 + i) # A, B, C... - if len(child_path) == 1: - branch_paths.append(f"[Branch {branch_label}: {child_path[0]}]") - else: - branch_paths.append(f"[Branch {branch_label}: {child_path[0]} -> ...]") - - return [f"[{display}]"] + [f" -> {' / '.join(branch_paths)}"] - - # Traverse from each root node - all_paths = [] - for root_id in root_nodes[:3]: # Show max 3 root nodes - path = traverse_path(root_id, set()) - all_paths.append(" -> ".join(path)) - - if all_paths: - if len(all_paths) == 1: - flow_parts.append(f" {all_paths[0]}") - else: - flow_parts.append(f" {' | '.join(all_paths)}") - else: - flow_parts.append("Current Flow: (No clear entry point - disconnected nodes)") - - return "\n".join(flow_parts) - - -def build_enhanced_node_data(normalized_nodes: List[Dict], topology: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Build enhanced context data for each node. - - Returns a list of node data with extracted configurations - for the system prompt. - """ - existing_nodes = [] - - for normalized_node in normalized_nodes: - node_id = normalized_node["id"] - config = normalized_node.get("config", {}) - - # Extract system prompt - system_prompt = extract_system_prompt(normalized_node) - - provider_name = config.get("provider_name") - model_name = config.get("model_name") - - # Extract tools configuration - tools_config = extract_tools_config(normalized_node) - tools_summary = format_tools_summary(tools_config) - - # Check if DeepAgents is enabled - is_deep_agent = config.get("useDeepAgents", False) is True - - # Get DeepAgents role and children from topology analysis - deep_agent_info = topology["deepAgentsHierarchy"].get(node_id, {}) - role = deep_agent_info.get("role") if is_deep_agent else None - children = topology["outgoing"].get(node_id, []) - - node_data = { - "id": node_id, - "type": normalized_node.get("type", "agent"), - "label": normalized_node.get("label", ""), - "position": normalized_node.get("position", {}), - "systemPrompt": system_prompt, - "provider_name": provider_name, - "model_name": model_name, - "tools": tools_summary, - "config": config, - "isDeepAgent": is_deep_agent, - "role": role, - "children": children, - } - - existing_nodes.append(node_data) - - return existing_nodes - - -def calculate_positions_for_nodes( - base_x: float, - base_y: float, - node_count: int, - layout_type: str = "deepagents", - x_spacing: float = 250, - y_spacing: float = 150, -) -> List[Dict[str, float]]: - """ - Unified function to calculate positions for multiple nodes. - - Supports different layout patterns: - - "deepagents": Manager on left, SubAgents on right (vertical stack) - - "horizontal": All nodes in a horizontal row - - "vertical": All nodes in a vertical column - - Args: - base_x: Base X coordinate - base_y: Base Y coordinate - node_count: Total number of nodes to calculate positions for - layout_type: Layout pattern ("deepagents", "horizontal", "vertical") - x_spacing: Horizontal spacing between nodes - y_spacing: Vertical spacing between nodes - - Returns: - List of position dicts, each with {"x": float, "y": float} - """ - positions: List[Dict[str, float]] = [] - - if layout_type == "deepagents": - # DeepAgents layout: First node (Manager) on left, rest (SubAgents) on right - if node_count == 0: - return positions - - # First node (Manager) at base position - positions.append({"x": base_x, "y": base_y}) - - # Remaining nodes (SubAgents) on the right, stacked vertically - for i in range(1, node_count): - positions.append({"x": base_x + x_spacing, "y": base_y + (i - 1) * y_spacing}) - - elif layout_type == "horizontal": - # Horizontal layout: All nodes in a row - for i in range(node_count): - positions.append({"x": base_x + i * x_spacing, "y": base_y}) - - elif layout_type == "vertical": - # Vertical layout: All nodes in a column - for i in range(node_count): - positions.append({"x": base_x, "y": base_y + i * y_spacing}) - - else: - # Default to horizontal if unknown layout type - for i in range(node_count): - positions.append({"x": base_x + i * x_spacing, "y": base_y}) - - return positions - - -def calculate_positions_for_deepagents( - base_x: float, - base_y: float, - manager_count: int = 1, - subagent_count: int = 5, - x_spacing: float = 250, - y_spacing: float = 150, -) -> Dict[str, List[Dict[str, float]]]: - """ - Calculate positions for DeepAgents workflow (Manager + SubAgents). - - This is a convenience wrapper around calculate_positions_for_nodes() - that separates Manager and SubAgent positions. - - Args: - base_x: Base X coordinate - base_y: Base Y coordinate - manager_count: Number of Manager nodes (typically 1) - subagent_count: Number of SubAgent nodes - x_spacing: Horizontal spacing from Manager to SubAgents - y_spacing: Vertical spacing between SubAgents - - Returns: - { - "manager": [{"x": float, "y": float}, ...], - "subagents": [{"x": float, "y": float}, ...] - } - """ - result: Dict[str, List[Dict[str, float]]] = {"manager": [], "subagents": []} - - # Calculate Manager positions (on the left) - if manager_count > 0: - manager_positions = calculate_positions_for_nodes( - base_x=base_x, - base_y=base_y, - node_count=manager_count, - layout_type="vertical", - x_spacing=0, - y_spacing=y_spacing, - ) - result["manager"] = manager_positions - - # Calculate SubAgent positions (on the right, vertical stack) - if subagent_count > 0: - subagent_positions = calculate_positions_for_nodes( - base_x=base_x + x_spacing, - base_y=base_y, - node_count=subagent_count, - layout_type="vertical", - x_spacing=0, - y_spacing=y_spacing, - ) - result["subagents"] = subagent_positions - - return result - - -def calculate_next_position(normalized_nodes: List[Dict]) -> Dict[str, float]: - """ - Calculate the next available position for a single new node. - - Primary flow: 250px to the right, same Y - Empty graph: default starting position (100, 100) - - This function uses calculate_positions_for_nodes() internally for consistency. - """ - if not normalized_nodes: - return {"x": 100, "y": 100} - - last_node = normalized_nodes[-1] - last_position = last_node.get("position", {"x": 0, "y": 0}) - - # Use unified function for consistency - positions = calculate_positions_for_nodes( - base_x=last_position.get("x", 0), - base_y=last_position.get("y", 100), - node_count=1, - layout_type="horizontal", - x_spacing=250, - y_spacing=150, - ) - return positions[0] if positions else {"x": 100, "y": 100} diff --git a/backend/app/core/copilot/message_builder.py b/backend/app/core/copilot/message_builder.py deleted file mode 100644 index 955ed6c67..000000000 --- a/backend/app/core/copilot/message_builder.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Message Builder - Unified builder for LangChain messages from conversation history. - -Converts conversation history (List[Dict]) to LangChain message objects -(HumanMessage, AIMessage) for agent invocation. -""" - -from typing import Dict, List, Optional - -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage - - -def build_langchain_messages( - prompt: str, - conversation_history: Optional[List[Dict[str, str]]] = None, -) -> List[BaseMessage]: - """ - Build LangChain messages list from prompt and conversation history. - - Converts conversation history dicts to LangChain message objects: - - {"role": "user", "content": "..."} -> HumanMessage - - {"role": "assistant", "content": "..."} -> AIMessage - - Args: - prompt: Current user prompt - conversation_history: Optional list of previous messages in format - [{"role": "user"|"assistant", "content": "..."}, ...] - - Returns: - List of LangChain message objects (HumanMessage, AIMessage) - """ - messages: List[BaseMessage] = [] - - # Add conversation history if provided - if conversation_history: - for msg in conversation_history: - if isinstance(msg, dict) and "role" in msg and "content" in msg: - role = msg["role"] - content = msg.get("content", "") - if role == "user" and content: - messages.append(HumanMessage(content=content)) - elif role == "assistant" and content: - messages.append(AIMessage(content=content)) - - # Add current user message - messages.append(HumanMessage(content=prompt)) - - return messages diff --git a/backend/app/core/copilot/prompt_builder.py b/backend/app/core/copilot/prompt_builder.py deleted file mode 100644 index 262401e8a..000000000 --- a/backend/app/core/copilot/prompt_builder.py +++ /dev/null @@ -1,374 +0,0 @@ -""" -Prompt Builder - System prompt construction for Copilot. - -Builds the comprehensive system prompt that guides the AI in generating -graph actions based on user requests and current graph context. - -OPTIMIZED VERSION: Modular prompt construction with direct value injection. -""" - -import json -from typing import Any, Dict, List, Optional - -from app.core.copilot.graph_analyzer import ( - analyze_graph_topology, - calculate_next_position, - calculate_positions_for_deepagents, - generate_topology_description, - normalize_node, -) -from app.utils.datetime import utc_now - - -def build_copilot_system_prompt( - graph_context: Dict[str, Any], - available_models: Optional[List[Dict[str, Any]]] = None, -) -> str: - """ - Build the comprehensive system prompt for Copilot. - - This prompt guides the AI in: - - Understanding current graph structure - - Making decisions about node creation/modification - - Following best practices for workflow design - - Selecting appropriate models for agent nodes - - Args: - graph_context: Current graph state with nodes and edges - available_models: Optional list of available models for model selection - - Returns: - Complete system prompt string - """ - # Extract nodes and edges from graph context - nodes = graph_context.get("nodes", []) - edges = graph_context.get("edges", []) - - # Normalize all nodes to extract data structure - normalized_nodes = [normalize_node(node) for node in nodes] - - # Analyze graph topology - topology = analyze_graph_topology(normalized_nodes, edges) - - # Build enhanced context data for each node (simplified) - existing_nodes = _build_simplified_node_data(normalized_nodes, topology) - - # Pre-calculate next available position - DIRECT VALUES - next_pos = calculate_next_position(normalized_nodes) - - # Build node map for topology description - node_map = {node["id"]: node for node in normalized_nodes} - - # Generate structured topology description - topology_description = generate_topology_description(normalized_nodes, topology, node_map) - - # Build available models summary for context - models_summary = _build_models_summary(available_models or []) - - # Get current time for temporal context in search operations - current_time = utc_now().isoformat() - - # Detect if graph has DeepAgents (to conditionally load those instructions) - has_deep_agents = any(node.get("config", {}).get("useDeepAgents", False) for node in normalized_nodes) - - # Build optimized prompt with direct values - return _get_optimized_system_prompt( - topology_description=topology_description, - existing_nodes=existing_nodes, - edges=edges, - topology=topology, - next_position_x=next_pos["x"], - next_position_y=next_pos["y"], - models_summary=models_summary, - current_time=current_time, - has_deep_agents=has_deep_agents, - ) - - -def _build_simplified_node_data(normalized_nodes: List[Dict], topology: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Build simplified context data for each node. - Only includes essential information for decision making. - """ - existing_nodes = [] - - for node in normalized_nodes: - node_id = node["id"] - config = node.get("config", {}) - - # Check if DeepAgents is enabled - is_deep_agent = config.get("useDeepAgents", False) is True - - # Get DeepAgents role from topology analysis - deep_agent_info = topology["deepAgentsHierarchy"].get(node_id, {}) - role = deep_agent_info.get("role") if is_deep_agent else None - - # Simplified node data - only essential fields - node_data = { - "id": node_id, - "type": node.get("type", "agent"), - "label": node.get("label", ""), - } - - # Only add optional fields if they exist and are meaningful - if is_deep_agent: - node_data["isDeepAgent"] = True - node_data["role"] = role - - if config.get("description"): - node_data["description"] = config["description"] - - existing_nodes.append(node_data) - - return existing_nodes - - -def _build_models_summary(models: List[Dict[str, Any]]) -> Dict[str, Any]: - """ - Build a summary of available models for the context. - - Args: - models: List of model info dicts from ModelService - - Returns: - Summarized model info for prompt context - """ - if not models: - return {"count": 0, "models": []} - - # Filter to only available models - available = [m for m in models if m.get("is_available", False)] - - # Build simplified model list - model_list = [ - { - "name": m.get("name"), - "displayName": m.get("display_name"), - "provider": m.get("provider_name"), - } - for m in available - ] - - return { - "count": len(model_list), - "models": model_list, - } - - -def _get_optimized_system_prompt( - topology_description: str, - existing_nodes: List[Dict[str, Any]], - edges: List[Dict[str, Any]], - topology: Dict[str, Any], - next_position_x: float, - next_position_y: float, - models_summary: Dict[str, Any], - current_time: str, - has_deep_agents: bool, -) -> str: - """ - Build an optimized system prompt with direct value injection. - - Key optimizations: - 1. Direct position values (no JSON parsing needed) - 2. Modular sections based on context - 3. Condensed instructions - 4. Semantic ID support for node references - """ - # Build node list string (simplified) - nodes_str = json.dumps(existing_nodes, indent=2, ensure_ascii=False) if existing_nodes else "[]" - edges_str = ( - json.dumps([{"source": e.get("source"), "target": e.get("target")} for e in edges], indent=2) if edges else "[]" - ) - - # Build model list (simplified) - model_names = [m["name"] for m in models_summary.get("models", [])][:5] - - # Base prompt (always included) - Senior AI Solutions Architect persona - prompt = f"""You are a Senior AI Solutions Architect using a visual builder tool. - -YOUR ROLE: You are NOT a chatbot. You are a co-pilot for building PRODUCTION-GRADE multi-agent systems. - -YOUR GOAL: Extract concrete requirements from the user to build a robust graph architecture. - -BEHAVIORAL RULES: -1. DO NOT simply agree with the user. If a request is vague (e.g., "Build a support bot"), YOU MUST ASK for specific architectural details: - - What is the specific Trigger? (Webhook, Schedule, Manual) - - What Tools/APIs are required? - - What is the specific Output format? (JSON, Markdown) - - Are there specific logic constraints? - - -Decision Rule: -- DeepAgents (DEFAULT): Multi-step tasks, research, analysis, complex workflows (2+ steps) -- Single Agent: ONLY for trivial single-step tasks (translation, formatting, simple classification) - - - -- create_node: Create nodes, returns ID. Parameters: use_deep_agents (True for Manager, False for SubAgents), description (REQUIRED for DeepAgents). See for rules. -- connect_nodes: Connect nodes by ID or label (@label:Name). For DeepAgents: Star topology (Manager → ALL SubAgents). -- delete_node / update_config: Modify existing nodes. -- think: Self-reflection at planning (before creating) and validation (after completion) stages (use in workflow). -- tavily_search: Research unfamiliar domains before creating agents. -- auto_layout: Rearrange nodes (horizontal/vertical/tree/grid layouts). -- analyze_workflow: Analyze graph structure and suggest optimizations. -- list_models: Query available LLM models for agent configuration. - - - -Next Position: x={next_position_x}, y={next_position_y} -Available Models: {", ".join(model_names[:3]) if model_names else "none configured"} -Note: For DeepAgents workflows, use pre-calculated positions from section below. - - - -{topology_description} -Nodes: {nodes_str} -Edges: {edges_str} - -""" - - # DeepAgents guidance - Optimized and structured - prompt += """ - -Structure: 1 Manager (use_deep_agents=True) + 3-8 SubAgents (use_deep_agents=False) -Topology: Star pattern - Manager connects to ALL SubAgents directly (NOT chain) -Note: Parameter name in create_node tool is "use_deep_agents" (maps to useDeepAgents in config) - - -1. Decompose task into phases: Information Gathering → Processing → Synthesis → Quality Control -2. Design roles by expertise: Researcher, Analyst, Synthesizer, Validator, Specialist -3. Single responsibility: Each SubAgent does ONE thing. If role has "and", SPLIT IT. - - - -- use_deep_agents=True (REQUIRED in create_node tool) -- description: "[Team Name]-team: [One-sentence goal]" (REQUIRED) -- systemPrompt MUST list ALL SubAgents with their capabilities -- Use task() tool to delegate to subagents -- DO NOT perform specialist tasks yourself -- DO NOT add tools_builtin or tools_mcp (Manager uses internal task() only) - - - -- use_deep_agents=False (REQUIRED in create_node tool) -- description: "[Action verb] [what] [output format]" (REQUIRED) -- systemPrompt: Role definition, single task, output format, quality standards -- Specialized tools allowed (tools_builtin, tools_mcp) - NOT for Manager - - -""" - - # Pre-calculate DeepAgents positions using unified function (1 Manager + 3 SubAgents example) - deepagents_positions = calculate_positions_for_deepagents( - base_x=next_position_x, base_y=next_position_y, manager_count=1, subagent_count=3, x_spacing=250, y_spacing=150 - ) - - # Extract calculated positions (use unified function results, no hardcoded fallbacks) - manager_pos = ( - deepagents_positions["manager"][0] - if deepagents_positions["manager"] - else {"x": next_position_x, "y": next_position_y} - ) - subagent1_pos = ( - deepagents_positions["subagents"][0] - if len(deepagents_positions["subagents"]) > 0 - else {"x": next_position_x + 250, "y": next_position_y} - ) - subagent2_pos = ( - deepagents_positions["subagents"][1] - if len(deepagents_positions["subagents"]) > 1 - else {"x": next_position_x + 250, "y": next_position_y + 150} - ) - subagent3_pos = ( - deepagents_positions["subagents"][2] - if len(deepagents_positions["subagents"]) > 2 - else {"x": next_position_x + 250, "y": next_position_y + 300} - ) - - # Task-first: treat input as task by default (vague-requirements overrides for goal-only requests) - prompt += """ - -Treat user input as a graph-building or graph-modification TASK by default. -- Prefer interpreting the message as: add node(s), connect nodes, delete node, update config, or arrange layout. -- Do not treat the Copilot as general chat; it is for producing and updating the workflow graph. -- When the user already gave a CONCRETE task (e.g. "add a weather agent", "connect node A to B"), proceed with tools. - -""" - - # Vague requirements: do NOT generate; MUST ask for Trigger, Tools/APIs, Output format, logic constraints - prompt += """ - -When the user describes a GOAL or SCENARIO without concrete specs (e.g. "Build a support bot", "build me a customer service bot", "build a Q&A system"): -- DO NOT call create_node, connect_nodes, or any graph-modification tools yet. -- YOU MUST ASK for specific architectural details before generating: - - What is the specific Trigger? (Webhook, Schedule, Manual) - - What Tools/APIs are required? - - What is the specific Output format? (JSON, Markdown) - - Are there specific logic constraints? -- Reply in one short paragraph with these questions; do not generate the graph until the user provides sufficient detail (or explicitly asks you to decide). -If the user already gave a CONCRETE task (e.g. "add a weather agent", "connect node A to B", "delete node X"), treat as normal and use tools. - -""" - - # Execution workflow - Optimized with clear algorithms - prompt += f""" - - -Apply decision rule from : -- Simple single-step? → Single agent -- Multi-step/complex? → DeepAgents (DEFAULT) - - - -1. think(stage="planning", nodes=[...]) - validate roles and count -2. create_node(Manager) - Coordinator first, use_deep_agents=True -3. create_node(SubAgent1, SubAgent2, ...) - Specialists, use_deep_agents=False -4. connect_nodes(Manager → each SubAgent) - Star topology -5. think(stage="validation", nodes=[...], connections=[...]) - validate connections and topology - - - -Pre-calculated positions for DeepAgents workflow (1 Manager + 3 SubAgents example): -Manager: x={manager_pos["x"]}, y={manager_pos["y"]} -SubAgent1: x={subagent1_pos["x"]}, y={subagent1_pos["y"]} -SubAgent2: x={subagent2_pos["x"]}, y={subagent2_pos["y"]} -SubAgent3: x={subagent3_pos["x"]}, y={subagent3_pos["y"]} -Pattern: Additional SubAgents continue vertically: y + 150 for each next SubAgent (x remains {subagent1_pos["x"]}) -For single nodes (non-DeepAgents): Use nextPosition from section above. - - - -For EVERY agent node: -- [ ] Clear ROLE definition -- [ ] Specific TASK description -- [ ] OUTPUT FORMAT specification -- [ ] DO NOT / constraints section -- [ ] Manager lists ALL SubAgents -- [ ] SubAgents have single responsibility - - - -❌ Generic prompts ("You are a helpful assistant") -❌ Missing output format -❌ Chain topology (SubAgent1 → SubAgent2 → SubAgent3) -❌ Manager without SubAgent list -❌ SubAgent with multiple responsibilities - - - - -Single agent mode: ONLY for translation, formatting, simple classification (trivial single-step tasks). -All other tasks → Use DeepAgents (see ). - - - -Only when the user message is clearly JUST a greeting or off-topic chitchat (e.g. "hi", "hello", "how are you", unrelated question): -- Reply in ONE short sentence that redirects them to give a concrete graph-building task. -- Example redirect: "Please describe what you want to do on the canvas, e.g. add an agent, connect two nodes, or arrange the layout." -- Do NOT call any graph-modification tools for such messages. -In all other cases, treat the input as a task and use tools to modify the graph. - -""" - - return prompt diff --git a/backend/app/core/copilot/response_parser.py b/backend/app/core/copilot/response_parser.py deleted file mode 100644 index b9a3c3c07..000000000 --- a/backend/app/core/copilot/response_parser.py +++ /dev/null @@ -1,248 +0,0 @@ -""" -Response Parser - Parse and extract information from Copilot responses. - -Provides functions for parsing streaming JSON responses and extracting -thought steps for real-time UI updates. -""" - -import json -import re -from typing import Any, Dict, List, Optional - -from app.core.copilot.action_types import GraphAction, GraphActionType - - -def try_extract_thought_field(json_content: str) -> Optional[str]: - """ - Try to extract the 'thought' field from partial JSON content during streaming. - - Uses multiple strategies: - 1. Try to parse as complete JSON first - 2. Fall back to regex extraction if JSON parsing fails - - Args: - json_content: Partial or complete JSON string - - Returns: - The thought content if found, None otherwise. - """ - # Strategy 1: Try parsing as complete JSON - try: - data = json.loads(json_content) - if isinstance(data, dict) and "thought" in data: - thought = data.get("thought", "") - if thought: - return str(thought) - except (json.JSONDecodeError, ValueError): - # JSON not complete yet, try regex extraction - pass - - # Strategy 2: Use regex to extract thought field (handles partial JSON) - # Match "thought": "..." with proper handling of escaped quotes and newlines - pattern = r'"thought"\s*:\s*"((?:[^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*)"' - - match = re.search(pattern, json_content, re.DOTALL) - if match: - thought_content = match.group(1) - # Unescape JSON string (handle \n, \", etc.) - try: - thought_content = json.loads(f'"{thought_content}"') - return str(thought_content) if thought_content is not None else None - except (json.JSONDecodeError, ValueError): - # If unescaping fails, might be incomplete - return None to wait for more data - return None - - return None - - -def parse_thought_to_steps(thought: str) -> List[Dict[str, Any]]: - """ - Parse the thought field into structured steps. - - Expected format examples: - 1. Numbered list: "1. Step one\n2. Step two\n3. Step three" - 2. Bullet points: "- Step one\n- Step two" - 3. Plain text with line breaks - - Args: - thought: Raw thought string from AI response - - Returns: - List of step objects with index and content. - """ - if not thought or not thought.strip(): - return [] - - NUMBERED_PATTERNS = [r"^\d+\.", r"^\d+\)", r"^Step \d+:", r"^Step\d+:"] - BULLET_PATTERN = r"^[-*•]\s+" - - steps: List[Dict[str, Any]] = [] - lines = thought.strip().split("\n") - current_step_index = 1 - current_content: List[str] = [] - - def finish_current_step() -> None: - """Helper to finish current step and reset state.""" - nonlocal current_step_index, current_content - if current_content: - steps.append({"index": current_step_index, "content": " ".join(current_content).strip()}) - current_step_index += 1 - current_content = [] - - for line in lines: - line = line.strip() - if not line: - finish_current_step() - continue - - # Check for numbered list pattern (1. 2. 3. etc.) - numbered_match = False - for pattern in NUMBERED_PATTERNS: - if re.match(pattern, line, re.IGNORECASE): - finish_current_step() - # Extract content after number - content = re.sub(pattern, "", line, flags=re.IGNORECASE).strip() - if content: - current_content.append(content) - numbered_match = True - break - - # Check for bullet points (-, *, •) - if not numbered_match and re.match(BULLET_PATTERN, line): - finish_current_step() - # Extract content after bullet - content = re.sub(BULLET_PATTERN, "", line).strip() - if content: - current_content.append(content) - elif not numbered_match: - # Regular line - add to current content - current_content.append(line) - - # Add final step if there's content - finish_current_step() - - # If no structured format detected, treat entire thought as one step - if not steps: - steps.append({"index": 1, "content": thought.strip()}) - - return steps - - -def extract_actions_from_agent_result( - result: Dict[str, Any], - filter_non_actions: bool = False, -) -> List[GraphAction]: - """ - Extract GraphAction objects from agent result messages. - - Looks for tool messages in the result and extracts actions from them. - Handles various message formats and content types. - - Args: - result: Agent result dict with 'messages' key - filter_non_actions: Whether to filter out NON_ACTION_TYPES (default: False) - - Returns: - List of GraphAction objects - """ - import json - import re - - from loguru import logger - - actions = [] - output_messages = result.get("messages", []) - - logger.info(f"[ResponseParser] Extracting actions from {len(output_messages)} messages") - - for idx, msg in enumerate(output_messages): - logger.debug( - f"[ResponseParser] Message {idx}: type={getattr(msg, 'type', 'unknown')}, class={msg.__class__.__name__}" - ) - - # Check for tool messages (contain actual results) - if hasattr(msg, "type") and msg.type == "tool": - logger.info(f"[ResponseParser] Found tool message: {msg}") - try: - content = msg.content - logger.debug(f"[ResponseParser] Tool message content type: {type(content)}, content: {content}") - - if isinstance(content, str): - try: - action_data = json.loads(content) - except json.JSONDecodeError: - # Try to extract JSON from string - json_match = re.search(r'\{[^{}]*"type"[^{}]*\}', content) - if json_match: - action_data = json.loads(json_match.group()) - else: - logger.warning(f"[ResponseParser] Could not parse tool message content as JSON: {content}") - continue - else: - action_data = content - - # Use the unified expand_action_payload method - expanded = expand_action_payload(action_data, filter_non_actions=filter_non_actions) - if not expanded: - logger.warning(f"[ResponseParser] Tool message is not an action payload: {action_data}") - continue - - for a in expanded: - logger.info(f"[ResponseParser] Extracted action: {a.get('type')}") - action_type = GraphActionType(a["type"]) - actions.append( - GraphAction( - type=action_type, - payload=a.get("payload", {}), - reasoning=a.get("reasoning", ""), - ) - ) - except (json.JSONDecodeError, ValueError, KeyError) as e: - logger.error(f"[ResponseParser] Error extracting action: {e}") - pass - - logger.info(f"[ResponseParser] Extracted {len(actions)} actions from result") - return actions - - -def expand_action_payload(payload: Any, filter_non_actions: bool = True) -> List[Dict[str, Any]]: - """ - Normalize tool outputs into a list of action dicts. - - Supported shapes: - - {"type": "...", "payload": {...}, "reasoning": "..."} -> [action] - - {"actions": [action, action, ...], ...} -> actions - - Args: - payload: Tool output payload (dict, string, or None) - filter_non_actions: Whether to filter out NON_ACTION_TYPES (default: True) - - Returns: - List of action dicts - """ - from loguru import logger - - NON_ACTION_TYPES = {"THINK", "think"} # Self-reflection tool output - - if payload is None: - return [] - if isinstance(payload, dict): - action_type = payload.get("type") - # Skip non-action types if filtering is enabled - if filter_non_actions and action_type in NON_ACTION_TYPES: - logger.debug(f"[ResponseParser] Skipping non-action type: {action_type}") - return [] - # Single action - if action_type and isinstance(payload.get("payload", {}), dict): - return [payload] - # Batch actions - actions = payload.get("actions") - if isinstance(actions, list): - out: List[Dict[str, Any]] = [] - for item in actions: - if isinstance(item, dict) and "type" in item: - item_type = item.get("type") - if not filter_non_actions or item_type not in NON_ACTION_TYPES: - out.append(item) - return out - return [] diff --git a/backend/app/core/copilot/tool_output_parser.py b/backend/app/core/copilot/tool_output_parser.py deleted file mode 100644 index 8b91dea1a..000000000 --- a/backend/app/core/copilot/tool_output_parser.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Tool Output Parser - Unified parser for extracting action data from tool outputs. - -Handles various tool output formats: -- ToolMessage objects with content attribute -- Direct dict output -- JSON string output -- Plain string with embedded JSON -""" - -import json -from typing import Any, Dict, Optional - -from loguru import logger - - -def parse_tool_output(tool_output_raw: Any, tool_name: Optional[str] = None) -> Optional[Dict[str, Any]]: - """ - Parse tool output to extract action data. - - Handles various output formats: - - ToolMessage objects with content attribute - - Direct dict output - - JSON string output - - Plain string with embedded JSON - - Args: - tool_output_raw: Raw tool output (any type) - tool_name: Optional name of the tool (for logging) - - Returns: - Parsed action data dict, or None if parsing fails - """ - if not tool_output_raw: - return None - - tool_output = None - - # Extract actual content from tool output - if hasattr(tool_output_raw, "content"): - # Object with content attribute (like ToolMessage) - tool_output = tool_output_raw.content - logger.debug(f"[ToolOutputParser] Extracted content from tool output object: {type(tool_output)}") - elif isinstance(tool_output_raw, dict): - # Dict - check if it has content key or is the action directly - if "content" in tool_output_raw: - tool_output = tool_output_raw["content"] - elif "type" in tool_output_raw: - # Already the action dict - tool_output = tool_output_raw - else: - tool_output = tool_output_raw - else: - # Direct output (string or other) - tool_output = tool_output_raw - - # Try to parse tool output to get action - if isinstance(tool_output, dict): - return tool_output - elif isinstance(tool_output, str): - # String output - try to parse as JSON - try: - parsed = json.loads(tool_output) - if isinstance(parsed, dict): - return parsed - except json.JSONDecodeError: - # Try to extract JSON from string using regex - try: - start_idx = tool_output.find('{"type"') - if start_idx == -1: - start_idx = tool_output.find("{'type'") - if start_idx != -1: - # Find matching closing brace - brace_count = 0 - end_idx = start_idx - for i in range(start_idx, len(tool_output)): - if tool_output[i] == "{": - brace_count += 1 - elif tool_output[i] == "}": - brace_count -= 1 - if brace_count == 0: - end_idx = i + 1 - break - - if end_idx > start_idx: - json_str = tool_output[start_idx:end_idx] - # Replace single quotes with double quotes if needed - if json_str.startswith("{'") or "'type'" in json_str: - json_str = json_str.replace("'", '"') - parsed = json.loads(json_str) - if isinstance(parsed, dict): - return parsed - except Exception as e: - logger.debug(f"[ToolOutputParser] Failed to extract JSON from string: {e}") - - if tool_name: - logger.warning(f"[ToolOutputParser] Could not parse tool output. tool={tool_name}, type={type(tool_output)}") - return None diff --git a/backend/app/core/copilot/tools/__init__.py b/backend/app/core/copilot/tools/__init__.py deleted file mode 100644 index 8521c35d9..000000000 --- a/backend/app/core/copilot/tools/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Copilot Tools Module - Exports all tools for graph manipulation. - -This module provides LangChain tools for the Copilot Agent to generate -graph actions (CREATE_NODE, CONNECT_NODES, etc.). -""" - -# Import registry functions -# Import analysis tool -from app.core.copilot.tools.analysis import analyze_workflow - -# Import context management -from app.core.copilot.tools.context import ( - get_current_graph_context, - get_preloaded_models, - set_current_graph_context, - set_preloaded_models, -) - -# Import core tools -from app.core.copilot.tools.core import ( - connect_nodes, - create_node, - delete_node, - update_config, -) - -# Import layout tool -from app.core.copilot.tools.layout import auto_layout - -# Import models tool -from app.core.copilot.tools.models import list_models -from app.core.copilot.tools.registry import ( - NodeIdRegistry, - get_node_registry, - reset_node_registry, -) - -# Import think tool -from app.core.copilot.tools.think import think - -# Import external research tool -from app.core.tools.builtin.research_tools import tavily_search - -# List of all Copilot tools -COPILOT_TOOLS = [ - create_node, - connect_nodes, - delete_node, - update_config, - list_models, - auto_layout, - analyze_workflow, - think, # Self-reflection tool - tavily_search, -] - - -def get_copilot_tools(): - """Get the list of Copilot tools for agent creation.""" - return COPILOT_TOOLS.copy() - - -__all__ = [ - # Registry - "NodeIdRegistry", - "get_node_registry", - "reset_node_registry", - # Context - "set_current_graph_context", - "get_current_graph_context", - "set_preloaded_models", - "get_preloaded_models", - # Core tools - "create_node", - "connect_nodes", - "delete_node", - "update_config", - # Layout - "auto_layout", - # Analysis - "analyze_workflow", - # Models - "list_models", - # Think - "think", - # Research - "tavily_search", - # Main export - "get_copilot_tools", - "COPILOT_TOOLS", -] diff --git a/backend/app/core/copilot/tools/analysis.py b/backend/app/core/copilot/tools/analysis.py deleted file mode 100644 index 23e9a7c3d..000000000 --- a/backend/app/core/copilot/tools/analysis.py +++ /dev/null @@ -1,215 +0,0 @@ -""" -Analysis Tools - Workflow analysis and optimization. - -Provides analyze_workflow tool for analyzing graph structure and providing recommendations. -""" - -import json -from typing import Dict, List - -from langchain.tools import tool -from pydantic import BaseModel, Field - -from app.core.copilot.tools.context import get_current_graph_context - - -class AnalyzeWorkflowInput(BaseModel): - """Input schema for analyze_workflow tool.""" - - analysis_type: str = Field( - default="comprehensive", - description="Type of analysis: 'comprehensive' (all checks), 'bottleneck' (performance issues), 'complexity' (complexity metrics), 'coverage' (missing handlers), 'quality' (best practices)", - ) - reasoning: str = Field(description="Explanation for why analysis is needed") - - -@tool( - args_schema=AnalyzeWorkflowInput, - description="Analyze workflow structure and provide optimization suggestions. Analysis types: comprehensive, bottleneck, complexity, coverage, quality.", -) -def analyze_workflow( - reasoning: str, - analysis_type: str = "comprehensive", -) -> str: - """ - Analyze workflow and provide optimization suggestions. - - Args: - reasoning: Why analysis is needed - analysis_type: 'comprehensive' (default), 'bottleneck', 'complexity', 'coverage', 'quality' - - Returns: - JSON with analysis results, issues, and recommendations. - """ - graph_context = get_current_graph_context() - - nodes = graph_context.get("nodes", []) - edges = graph_context.get("edges", []) - - if not nodes: - return json.dumps( - { - "error": "No nodes in the current graph to analyze", - "suggestion": "Create some nodes first before running analysis", - }, - ensure_ascii=False, - ) - - analysis_result = { - "analysis_type": analysis_type, - "reasoning": reasoning, - "summary": {}, - "issues": [], - "recommendations": [], - "metrics": {}, - } - - # Build adjacency structures - outgoing: Dict[str, List[str]] = {n.get("id"): [] for n in nodes} - incoming: Dict[str, List[str]] = {n.get("id"): [] for n in nodes} - - for edge in edges: - src = edge.get("source") - tgt = edge.get("target") - if src in outgoing and tgt in incoming: - outgoing[src].append(tgt) - incoming[tgt].append(src) - - # Find structural elements - root_nodes = [n for n in nodes if not incoming.get(n.get("id"), [])] - leaf_nodes = [n for n in nodes if not outgoing.get(n.get("id"), [])] - - # Count node types - node_types: Dict[str, int] = {} - for node in nodes: - data = node.get("data", {}) - node_type = data.get("type", "unknown") - node_types[node_type] = node_types.get(node_type, 0) + 1 - - # Basic metrics - analysis_result["metrics"] = { - "total_nodes": len(nodes), - "total_edges": len(edges), - "root_nodes": len(root_nodes), - "leaf_nodes": len(leaf_nodes), - "node_types": node_types, - "avg_connections": round(len(edges) / max(len(nodes), 1), 2), - } - - # Analysis checks - issues = [] - recommendations = [] - - # Check 1: Dead ends (leaf nodes that are not direct_reply or human_input) - for node in leaf_nodes: - data = node.get("data", {}) - node_type = data.get("type", "") - label = data.get("label", node.get("id")) - if node_type not in ["direct_reply", "human_input"]: - issues.append( - { - "type": "dead_end", - "severity": "warning", - "node_id": node.get("id"), - "message": f"Node '{label}' has no outgoing edges - workflow may end unexpectedly", - } - ) - recommendations.append(f"Add an outgoing edge from '{label}' or convert to a terminal node") - - # Check 2: Orphan nodes (no connections at all) - for node in nodes: - node_id = node.get("id") - label = node.get("data", {}).get("label", node_id) - if not incoming.get(node_id) and not outgoing.get(node_id) and len(nodes) > 1: - issues.append( - { - "type": "orphan_node", - "severity": "error", - "node_id": node_id, - "message": f"Node '{label}' is not connected to any other node", - } - ) - recommendations.append(f"Connect '{label}' to the workflow or remove it") - - # Check 3: Multiple root nodes (might be intentional, but worth noting) - if len(root_nodes) > 1: - issues.append( - { - "type": "multiple_entry_points", - "severity": "info", - "message": f"Workflow has {len(root_nodes)} entry points - ensure this is intentional", - } - ) - - # Check 4: Missing systemPrompts for agent nodes - for node in nodes: - data = node.get("data", {}) - if data.get("type") == "agent": - config = data.get("config", {}) - system_prompt = config.get("systemPrompt", "") - label = data.get("label", node.get("id")) - if not system_prompt or len(system_prompt) < 50: - issues.append( - { - "type": "weak_prompt", - "severity": "warning", - "node_id": node.get("id"), - "message": f"Agent '{label}' has a weak or missing systemPrompt", - } - ) - recommendations.append(f"Improve systemPrompt for '{label}' with specific instructions") - - # Check 5: DeepAgents without children - for node in nodes: - data = node.get("data", {}) - config = data.get("config", {}) - if config.get("useDeepAgents"): - node_id = node.get("id") - label = data.get("label", node_id) - children = outgoing.get(node_id, []) - if not children: - issues.append( - { - "type": "deep_agent_no_children", - "severity": "error", - "node_id": node_id, - "message": f"DeepAgent '{label}' has no subagent children", - } - ) - recommendations.append(f"Add subagent nodes connected to '{label}'") - - # Check 6: Condition nodes without both branches - for node in nodes: - data = node.get("data", {}) - if data.get("type") in ["condition", "condition_agent"]: - node_id = node.get("id") - label = data.get("label", node_id) - out_edges = outgoing.get(node_id, []) - if len(out_edges) < 2: - issues.append( - { - "type": "incomplete_branching", - "severity": "warning", - "node_id": node_id, - "message": f"Condition '{label}' has only {len(out_edges)} outgoing edges (expected 2+)", - } - ) - recommendations.append(f"Add missing branch(es) from '{label}'") - - # Summary - error_count = len([i for i in issues if i.get("severity") == "error"]) - warning_count = len([i for i in issues if i.get("severity") == "warning"]) - info_count = len([i for i in issues if i.get("severity") == "info"]) - - analysis_result["summary"] = { - "health_score": max(0, 100 - error_count * 20 - warning_count * 5), - "errors": error_count, - "warnings": warning_count, - "info": info_count, - "status": "healthy" if error_count == 0 else "needs_attention" if warning_count > 0 else "critical", - } - - analysis_result["issues"] = issues # type: ignore[assignment] - analysis_result["recommendations"] = recommendations[:10] # type: ignore[assignment] # Top 10 recommendations - - return json.dumps(analysis_result, ensure_ascii=False, indent=2) diff --git a/backend/app/core/copilot/tools/context.py b/backend/app/core/copilot/tools/context.py deleted file mode 100644 index 8ed069518..000000000 --- a/backend/app/core/copilot/tools/context.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Context Management - Thread-safe context variables for Copilot tools. - -Provides per-request isolation for graph context and preloaded models. -""" - -import contextvars -from typing import Any, Dict, List - -# Thread-safe context variable for graph context (per-request isolation) -_current_graph_context: contextvars.ContextVar[Dict[str, Any]] = contextvars.ContextVar("graph_context", default={}) - - -def set_current_graph_context(context: Dict[str, Any]) -> None: - """Set the current graph context for analysis tools.""" - _current_graph_context.set(context) - - -def get_current_graph_context() -> Dict[str, Any]: - """Get the current graph context from context.""" - return _current_graph_context.get() - - -# Thread-safe context variable for preloaded models (per-request isolation) -_preloaded_models: contextvars.ContextVar[List[Dict[str, Any]]] = contextvars.ContextVar("preloaded_models", default=[]) - - -def set_preloaded_models(models: List[Dict[str, Any]]) -> None: - """Set the preloaded models list. Called by agent.py during initialization.""" - _preloaded_models.set(models) - - -def get_preloaded_models() -> List[Dict[str, Any]]: - """Get the preloaded models list from context.""" - return _preloaded_models.get() diff --git a/backend/app/core/copilot/tools/core.py b/backend/app/core/copilot/tools/core.py deleted file mode 100644 index 91fbc7f07..000000000 --- a/backend/app/core/copilot/tools/core.py +++ /dev/null @@ -1,376 +0,0 @@ -""" -Core Copilot Tools - Node CRUD operations. - -Provides tools for creating, connecting, deleting, and updating nodes in the graph. -""" - -import json -import uuid -from typing import Any, Dict, List, Optional - -from langchain.tools import tool -from pydantic import BaseModel, Field - -from app.core.copilot.tools.registry import get_node_registry -from app.core.model.utils.model_ref import parse_model_ref - -# ==================== Tool Input Schemas ==================== - - -class CreateNodeInput(BaseModel): - """Input schema for create_node tool.""" - - node_type: str = Field( - description="Node type. Must be one of: 'agent', 'condition', 'condition_agent', 'direct_reply', 'human_input', 'http', 'custom_function', 'execute_flow', 'iteration'" - ) - label: str = Field(description="Human-readable label for the node (e.g., 'Support Agent', 'Sentiment Check')") - position_x: float = Field( - description="X position coordinate on canvas. Use nextAvailablePosition.x from context_data for sequential nodes." - ) - position_y: float = Field( - description="Y position coordinate on canvas. Use nextAvailablePosition.y from context_data for sequential nodes." - ) - system_prompt: Optional[str] = Field( - default=None, - description="REQUIRED for agent nodes. Detailed system prompt defining the agent's behavior, role, and task. Be specific!", - ) - model: Optional[str] = Field( - default=None, - description="Optional for agent nodes. Model in 'provider_name:model_name' format (e.g., 'openai:gpt-4o'). Leave empty to use default.", - ) - use_deep_agents: Optional[bool] = Field( - default=False, - description="For agent nodes only. Set to True to enable DeepAgents mode for complex multi-step tasks.", - ) - description: Optional[str] = Field( - default=None, - description="REQUIRED for DeepAgents subagents. Clear, action-oriented description of what this subagent does (e.g., 'Conducts web research and synthesizes findings'). Required when use_deep_agents=True or parent has useDeepAgents=true.", - ) - tools_builtin: Optional[List[str]] = Field( - default=None, - description="For agent nodes only. List of builtin tool names (e.g., ['web_search', 'code_interpreter']).", - ) - tools_mcp: Optional[List[str]] = Field( - default=None, - description="For agent nodes only. List of MCP tool identifiers (e.g., ['server_name::tool_name']).", - ) - reasoning: str = Field(description="Explanation for why this node is being created") - - -class ConnectNodesInput(BaseModel): - """Input schema for connect_nodes tool.""" - - source: str = Field(description="Source node ID") - target: str = Field(description="Target node ID") - reasoning: str = Field(description="Explanation for why these nodes are being connected") - - -class DeleteNodeInput(BaseModel): - """Input schema for delete_node tool.""" - - node_id: str = Field(description="ID of the node to delete") - reasoning: str = Field(description="Explanation for why this node is being deleted") - - -class UpdateConfigInput(BaseModel): - """Input schema for update_config tool.""" - - node_id: str = Field(description="ID of the node to update") - system_prompt: Optional[str] = Field(default=None, description="New system prompt") - model: Optional[str] = Field(default=None, description="New model name") - use_deep_agents: Optional[bool] = Field(default=None, description="Enable/disable DeepAgents mode") - description: Optional[str] = Field(default=None, description="New description for DeepAgents") - reasoning: str = Field(description="Explanation for this configuration update") - - -# ==================== Tool Definitions ==================== - - -@tool( - args_schema=CreateNodeInput, - description="Create a new node in the graph workflow. Returns CREATE_NODE action with generated ID. See system prompt for DeepAgents rules and position calculation.", -) -def create_node( - node_type: str, - label: str, - position_x: float, - position_y: float, - reasoning: str, - system_prompt: Optional[str] = None, - model: Optional[str] = None, - use_deep_agents: Optional[bool] = False, - description: Optional[str] = None, - tools_builtin: Optional[List[str]] = None, - tools_mcp: Optional[List[str]] = None, - expression: Optional[str] = None, - instruction: Optional[str] = None, - options: Optional[List[str]] = None, - template: Optional[str] = None, - prompt: Optional[str] = None, -) -> str: - """ - Create a new node in the graph workflow. - - Args: - node_type: Node type ('agent', 'condition', etc.) - label: Human-readable label - position_x: X coordinate (use pre-calculated positions from system prompt context) - position_y: Y coordinate (use pre-calculated positions from system prompt context) - reasoning: Why this node is being created - system_prompt: Required for agent nodes - defines agent behavior - model: Optional model name for agent nodes - use_deep_agents: Set True for DeepAgents Manager nodes only - description: Required for DeepAgents nodes - action-oriented description - tools_builtin: Optional builtin tools list (not for DeepAgents Manager) - tools_mcp: Optional MCP tools list (not for DeepAgents Manager) - expression: For condition nodes - instruction: For condition_agent nodes - options: For condition_agent nodes - template: For direct_reply nodes - prompt: For human_input nodes - - Returns: - JSON string with CREATE_NODE action containing generated node ID. - Node can be referenced by ID or label (@label:LabelName). - - Note: - - Position values are pre-calculated in system prompt for DeepAgents workflows - - For DeepAgents: Use exact positions from section - - For single nodes: Use nextAvailablePosition from context - - See system prompt for DeepAgents architecture rules and systemPrompt quality standards - """ - try: - # Generate unique node ID - node_id = f"{node_type}_{uuid.uuid4().hex[:8]}" - - # Register in the node registry for semantic reference - registry = get_node_registry() - registry.register(node_id, label, node_type) - - # Build config based on node type - config: Dict[str, Any] = {} - - if node_type == "agent": - if system_prompt: - config["systemPrompt"] = system_prompt - if model: - provider, model_name = parse_model_ref(model) - if provider: - config["provider_name"] = provider - if model_name: - config["model_name"] = model_name - if use_deep_agents: - config["useDeepAgents"] = True - if description: - config["description"] = description - if tools_builtin or tools_mcp: - config["tools"] = { - "builtin": tools_builtin or [], - "mcp": tools_mcp or [], - } - elif node_type == "condition": - if expression: - config["expression"] = expression - elif node_type == "condition_agent": - if instruction: - config["instruction"] = instruction - if options: - config["options"] = options - else: - config["options"] = ["Option A", "Option B"] - elif node_type == "direct_reply": - if template: - config["template"] = template - elif node_type == "human_input": - if prompt: - config["prompt"] = prompt - - action = { - "type": "CREATE_NODE", - "payload": { - "id": node_id, - "type": node_type, - "label": label, - "position": {"x": position_x, "y": position_y}, - "config": config, - }, - "reasoning": reasoning, - "_label_ref": f"@{label}", # Can also reference by label - } - # Return as JSON string for LangChain compatibility - return json.dumps(action, ensure_ascii=False) - except Exception as e: - from loguru import logger - - logger.error(f"create_node failed: {e}") - return json.dumps({"type": "ERROR", "error": str(e), "message": "Failed to create node"}, ensure_ascii=False) - - -@tool( - args_schema=ConnectNodesInput, - description="Connect two nodes with an edge. Nodes can be referenced by ID or label (@label:Name). See system prompt for DeepAgents topology rules.", -) -def connect_nodes( - source: str, - target: str, - reasoning: str, -) -> str: - """ - Connect two nodes with an edge. - - Args: - source: Source node ID or label reference (@label:Name) - target: Target node ID or label reference (@label:Name) - reasoning: Why these nodes are connected - - Returns: - JSON string with CONNECT_NODES action. - - Note: See system prompt for DeepAgents star topology requirements. - """ - try: - # Resolve semantic references to actual IDs - registry = get_node_registry() - resolved_source = registry.resolve(source) - resolved_target = registry.resolve(target) - - action = { - "type": "CONNECT_NODES", - "payload": { - "source": resolved_source, - "target": resolved_target, - }, - "reasoning": reasoning, - } - - # Include original references for debugging if they were resolved - if resolved_source != source or resolved_target != target: - action["_resolved"] = { - "source_ref": source, - "target_ref": target, - "source_id": resolved_source, - "target_id": resolved_target, - } - - return json.dumps(action, ensure_ascii=False) - except Exception as e: - from loguru import logger - - logger.error(f"connect_nodes failed: {e}") - return json.dumps({"type": "ERROR", "error": str(e), "message": "Failed to connect nodes"}, ensure_ascii=False) - - -@tool( - args_schema=DeleteNodeInput, - description="Delete a node from the graph. Node can be referenced by ID or label. Removes node and all connected edges.", -) -def delete_node( - node_id: str, - reasoning: str, -) -> str: - """ - Delete a node from the graph. - - Args: - node_id: Node ID or label reference (@label:Name) - reasoning: Why this node is being deleted - - Returns: - JSON string with DELETE_NODE action. - """ - try: - # Resolve semantic reference to actual ID - registry = get_node_registry() - resolved_id = registry.resolve(node_id) - - action = { - "type": "DELETE_NODE", - "payload": { - "id": resolved_id, - }, - "reasoning": reasoning, - } - return json.dumps(action, ensure_ascii=False) - except Exception as e: - from loguru import logger - - logger.error(f"delete_node failed: {e}") - return json.dumps({"type": "ERROR", "error": str(e), "message": "Failed to delete node"}, ensure_ascii=False) - - -@tool( - args_schema=UpdateConfigInput, - description="Update node configuration. Only include parameters that need to change. Node can be referenced by ID or label.", -) -def update_config( - node_id: str, - reasoning: str, - system_prompt: Optional[str] = None, - model: Optional[str] = None, - use_deep_agents: Optional[bool] = None, - description: Optional[str] = None, - expression: Optional[str] = None, - instruction: Optional[str] = None, - options: Optional[List[str]] = None, - template: Optional[str] = None, -) -> str: - """ - Update node configuration. - - Args: - node_id: Node ID or label reference (@label:Name) - reasoning: Why this update is needed - system_prompt: New system prompt (agent nodes) - model: New model name (agent nodes) - use_deep_agents: Enable/disable DeepAgents mode - description: New description (DeepAgents nodes) - expression: For condition nodes - instruction: For condition_agent nodes - options: For condition_agent nodes - template: For direct_reply nodes - - Returns: - JSON string with UPDATE_CONFIG action. - """ - try: - # Resolve semantic reference to actual ID - registry = get_node_registry() - resolved_id = registry.resolve(node_id) - - config: Dict[str, Any] = {} - - if system_prompt is not None: - config["systemPrompt"] = system_prompt - if model is not None: - provider, model_name = parse_model_ref(model) - if provider: - config["provider_name"] = provider - if model_name: - config["model_name"] = model_name - if use_deep_agents is not None: - config["useDeepAgents"] = use_deep_agents - if description is not None: - config["description"] = description - if expression is not None: - config["expression"] = expression - if instruction is not None: - config["instruction"] = instruction - if options is not None: - config["options"] = options - if template is not None: - config["template"] = template - - action = { - "type": "UPDATE_CONFIG", - "payload": { - "id": resolved_id, - "config": config, - }, - "reasoning": reasoning, - } - return json.dumps(action, ensure_ascii=False) - except Exception as e: - from loguru import logger - - logger.error(f"update_config failed: {e}") - return json.dumps({"type": "ERROR", "error": str(e), "message": "Failed to update config"}, ensure_ascii=False) diff --git a/backend/app/core/copilot/tools/layout.py b/backend/app/core/copilot/tools/layout.py deleted file mode 100644 index 79daa779e..000000000 --- a/backend/app/core/copilot/tools/layout.py +++ /dev/null @@ -1,219 +0,0 @@ -""" -Layout Tools - Automatic node positioning. - -Provides auto_layout tool for arranging nodes in various layouts. -""" - -import json -from typing import Dict, List - -from langchain.tools import tool -from pydantic import BaseModel, Field - -from app.core.copilot.tools.context import get_current_graph_context - - -class AutoLayoutInput(BaseModel): - """Input schema for auto_layout tool.""" - - layout_type: str = Field( - default="horizontal", - description="Layout type: 'horizontal' (left-to-right), 'vertical' (top-to-bottom), 'tree' (hierarchical), 'grid' (grid arrangement)", - ) - node_spacing_x: float = Field(default=300, description="Horizontal spacing between nodes") - node_spacing_y: float = Field(default=150, description="Vertical spacing between nodes") - start_x: float = Field(default=100, description="Starting X position") - start_y: float = Field(default=100, description="Starting Y position") - reasoning: str = Field(description="Explanation for why auto layout is needed") - - -@tool( - args_schema=AutoLayoutInput, - description="Automatically rearrange nodes for better visualization. Layout types: horizontal, vertical, tree, grid.", -) -def auto_layout( - reasoning: str, - layout_type: str = "horizontal", - node_spacing_x: float = 300, - node_spacing_y: float = 150, - start_x: float = 100, - start_y: float = 100, -) -> str: - """ - Automatically rearrange nodes for better visualization. - - Args: - reasoning: Why auto layout is needed - layout_type: 'horizontal' (default), 'vertical', 'tree', 'grid' - node_spacing_x: Horizontal spacing (default: 300) - node_spacing_y: Vertical spacing (default: 150) - start_x: Starting X position (default: 100) - start_y: Starting Y position (default: 100) - - Returns: - JSON with UPDATE_POSITION actions for all nodes. - """ - graph_context = get_current_graph_context() - - nodes = graph_context.get("nodes", []) - edges = graph_context.get("edges", []) - - if not nodes: - return json.dumps( - { - "error": "No nodes in the current graph to layout", - "suggestion": "Create some nodes first before applying auto layout", - }, - ensure_ascii=False, - ) - - actions = [] - - # Build adjacency list for topology analysis - outgoing: Dict[str, List[str]] = {n.get("id"): [] for n in nodes} - incoming: Dict[str, List[str]] = {n.get("id"): [] for n in nodes} - - for edge in edges: - src = edge.get("source") - tgt = edge.get("target") - if src in outgoing and tgt in incoming: - outgoing[src].append(tgt) - incoming[tgt].append(src) - - # Find root nodes (no incoming edges) - root_nodes = [n for n in nodes if not incoming.get(n.get("id"), [])] - - if layout_type == "horizontal": - # BFS from root nodes, assign columns - visited = set() - columns: Dict[str, int] = {} - queue = [(n.get("id"), 0) for n in root_nodes] - - while queue: - node_id, col = queue.pop(0) - if node_id in visited: - continue - visited.add(node_id) - columns[node_id] = col - for child in outgoing.get(node_id, []): - if child not in visited: - queue.append((child, col + 1)) - - # Assign positions by column - col_counts: Dict[int, int] = {} - for node in nodes: - node_id = node.get("id") - col = columns.get(node_id, 0) - row = col_counts.get(col, 0) - col_counts[col] = row + 1 - - new_x = start_x + col * node_spacing_x - new_y = start_y + row * node_spacing_y - - actions.append( - { - "type": "UPDATE_POSITION", - "payload": {"id": node_id, "position": {"x": new_x, "y": new_y}}, - "reasoning": f"Auto-layout: placing node in column {col}, row {row}", - } - ) - - elif layout_type == "vertical": - # Similar to horizontal but swap x and y - visited = set() - rows: Dict[str, int] = {} - queue = [(n.get("id"), 0) for n in root_nodes] - - while queue: - node_id, row = queue.pop(0) - if node_id in visited: - continue - visited.add(node_id) - rows[node_id] = row - for child in outgoing.get(node_id, []): - if child not in visited: - queue.append((child, row + 1)) - - row_counts: Dict[int, int] = {} - for node in nodes: - node_id = node.get("id") - row = rows.get(node_id, 0) - col = row_counts.get(row, 0) - row_counts[row] = col + 1 - - new_x = start_x + col * node_spacing_x - new_y = start_y + row * node_spacing_y - - actions.append( - { - "type": "UPDATE_POSITION", - "payload": {"id": node_id, "position": {"x": new_x, "y": new_y}}, - "reasoning": f"Auto-layout (vertical): placing node in row {row}, column {col}", - } - ) - - elif layout_type == "tree": - # Tree layout with root at top - visited = set() - levels: Dict[str, int] = {} - level_nodes: Dict[int, List[str]] = {} - queue = [(n.get("id"), 0) for n in root_nodes] - - while queue: - node_id, level = queue.pop(0) - if node_id in visited: - continue - visited.add(node_id) - levels[node_id] = level - if level not in level_nodes: - level_nodes[level] = [] - level_nodes[level].append(node_id) - for child in outgoing.get(node_id, []): - if child not in visited: - queue.append((child, level + 1)) - - # Position nodes centered in each level - for level, node_list in level_nodes.items(): - count = len(node_list) - total_width = (count - 1) * node_spacing_x - start_offset = -total_width / 2 - - for i, node_id in enumerate(node_list): - new_x = start_x + 400 + start_offset + i * node_spacing_x # Center around 400 - new_y = start_y + level * node_spacing_y - - actions.append( - { - "type": "UPDATE_POSITION", - "payload": {"id": node_id, "position": {"x": new_x, "y": new_y}}, - "reasoning": f"Auto-layout (tree): level {level}, position {i + 1}/{count}", - } - ) - - elif layout_type == "grid": - # Simple grid arrangement - cols = max(3, int(len(nodes) ** 0.5)) - for i, node in enumerate(nodes): - row = i // cols - col = i % cols - new_x = start_x + col * node_spacing_x - new_y = start_y + row * node_spacing_y - - actions.append( - { - "type": "UPDATE_POSITION", - "payload": {"id": node.get("id"), "position": {"x": new_x, "y": new_y}}, - "reasoning": f"Auto-layout (grid): row {row}, col {col}", - } - ) - - return json.dumps( - { - "layout_type": layout_type, - "nodes_repositioned": len(actions), - "actions": actions, - "reasoning": reasoning, - }, - ensure_ascii=False, - indent=2, - ) diff --git a/backend/app/core/copilot/tools/models.py b/backend/app/core/copilot/tools/models.py deleted file mode 100644 index efbc61f84..000000000 --- a/backend/app/core/copilot/tools/models.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Model Tools - Query available models. - -Provides list_models tool for querying available LLM models. -""" - -import json - -from langchain.tools import tool -from pydantic import BaseModel, Field - -from app.core.copilot.tools.context import get_preloaded_models - - -class ListModelsInput(BaseModel): - """Input schema for list_models tool.""" - - model_type: str = Field( - default="chat", description="Model type to query: 'chat', 'embedding', etc. Currently only 'chat' is preloaded." - ) - - -@tool( - args_schema=ListModelsInput, - description="Query available LLM models for agent nodes. Returns model list with capabilities and availability.", -) -def list_models(model_type: str = "chat") -> str: - """ - Query available models for agent nodes. - - Args: - model_type: Model type to query (default: 'chat') - - Returns: - JSON with model list containing name, display_name, provider, is_available. - - Note: Use before creating agent nodes to verify model names. Prefer 'claude'/'gpt-4' for complex tasks, 'mini'/'fast' for simple tasks. - """ - preloaded_models = get_preloaded_models() - - if not preloaded_models: - return json.dumps( - { - "error": "No models preloaded. Models should be loaded during agent initialization.", - "suggestion": "Check system configuration or contact administrator.", - }, - ensure_ascii=False, - ) - - # Filter by model type if needed (currently all preloaded are chat models) - filtered_models = [ - { - "name": m.get("name"), - "display_name": m.get("display_name"), - "provider_name": m.get("provider_name"), - "description": m.get("description", ""), - "is_available": m.get("is_available", False), - } - for m in preloaded_models - if m.get("is_available", False) # Only return available models - ] - - return json.dumps( - { - "model_type": model_type, - "total_count": len(filtered_models), - "models": filtered_models, - }, - ensure_ascii=False, - indent=2, - ) diff --git a/backend/app/core/copilot/tools/registry.py b/backend/app/core/copilot/tools/registry.py deleted file mode 100644 index 2f06e835d..000000000 --- a/backend/app/core/copilot/tools/registry.py +++ /dev/null @@ -1,116 +0,0 @@ -""" -Node ID Registry - Thread-safe registry for tracking created nodes. - -Provides semantic reference resolution for Copilot tools, allowing nodes -to be referenced by label or sequence number in addition to their actual IDs. -""" - -import contextvars -from typing import Any, Dict, List, Optional - - -class NodeIdRegistry: - """ - Registry to track created nodes and their semantic references. - - Allows the model to reference nodes by: - 1. Actual UUID (e.g., "agent_abc12345") - 2. Label-based reference (e.g., "@Research Manager", "@label:Research Agent") - """ - - def __init__(self): - self._nodes: Dict[str, Dict[str, Any]] = {} # id -> {label, type, seq} - self._by_label: Dict[str, str] = {} # normalized_label -> id - self._by_seq: Dict[int, str] = {} # sequence_number -> id - self._counter: int = 0 - - def register(self, node_id: str, label: str, node_type: str) -> int: - """Register a new node and return its sequence number.""" - self._counter += 1 - seq = self._counter - - self._nodes[node_id] = { - "label": label, - "type": node_type, - "seq": seq, - } - - # Register by normalized label for lookup - normalized_label = label.lower().strip() - self._by_label[normalized_label] = node_id - self._by_seq[seq] = node_id - - return seq - - def resolve(self, ref: str) -> str: - """ - Resolve a reference to an actual node ID. - - Supported formats: - - Actual ID: "agent_abc12345" -> returns as-is - - Label: "@Research Agent", "@label:Support" -> resolves by label - """ - if not ref: - return ref - - ref = ref.strip() - - # Check if it's already an actual ID (contains underscore with hex) - if "_" in ref and not ref.startswith("@"): - return ref - - # Check for label reference: @Label or @label:Label - if ref.startswith("@label:"): - label = ref[7:].lower().strip() - elif ref.startswith("@"): - label = ref[1:].lower().strip() - else: - return ref # Not a reference, return as-is - - if label in self._by_label: - return self._by_label[label] - - # Try partial match - for stored_label, node_id in self._by_label.items(): - if label in stored_label or stored_label in label: - return node_id - - return ref # Return original if not found - - def get_last_id(self) -> Optional[str]: - """Get the ID of the last created node.""" - if self._counter > 0 and self._counter in self._by_seq: - return self._by_seq[self._counter] - return None - - def get_all_ids(self) -> List[str]: - """Get all registered node IDs.""" - return list(self._nodes.keys()) - - def clear(self): - """Clear the registry for a new session.""" - self._nodes.clear() - self._by_label.clear() - self._by_seq.clear() - self._counter = 0 - - -# Thread-safe context variable for node registry (per-request isolation) -_node_registry_context: contextvars.ContextVar[Optional[NodeIdRegistry]] = contextvars.ContextVar( - "node_registry", default=None -) - - -def get_node_registry() -> NodeIdRegistry: - """Get the current node registry from context, creating one if needed.""" - registry = _node_registry_context.get() - if registry is None: - registry = NodeIdRegistry() - _node_registry_context.set(registry) - return registry - - -def reset_node_registry(): - """Reset the node registry for a new request.""" - registry = get_node_registry() - registry.clear() diff --git a/backend/app/core/copilot/tools/think.py b/backend/app/core/copilot/tools/think.py deleted file mode 100644 index 06c21eb5f..000000000 --- a/backend/app/core/copilot/tools/think.py +++ /dev/null @@ -1,319 +0,0 @@ -""" -Think Tool - Self-reflection and validation for DeepAgents workflows. - -Provides think tool for validating DeepAgents workflow structure at different stages. -Simplified: focus on two-step validation - Planning (blueprint stage) and Validation (acceptance stage). - -Enhanced: Validation stage now automatically reads actual nodes and edges from graph_context -to ensure comprehensive detection of all created nodes and connections. -""" - -import json -from typing import Dict, List, Optional - -from langchain.tools import tool -from loguru import logger -from pydantic import BaseModel, Field - -from app.core.copilot.tools.context import get_current_graph_context - - -class ThinkInput(BaseModel): - """Input schema for think tool.""" - - stage: str = Field(description="Validation stage: 'planning' (before creation) or 'validation' (after completion)") - reflection: str = Field(description="Current reasoning or understanding of the task") - nodes: Optional[List[str]] = Field( - default=None, - description="Role list or created node names. In validation stage, auto-read from graph_context if not provided", - ) - connections: Optional[List[str]] = Field( - default=None, - description="Connection relationships, e.g. ['Manager -> Analyst']. In validation stage, auto-read from graph_context if not provided", - ) - - -@tool( - args_schema=ThinkInput, - description="Self-reflection tool for DeepAgents workflow validation. Use at planning (before creating) and validation (after completion) stages. In validation stage, automatically reads actual nodes and edges from graph_context if not provided.", -) -def think( - stage: str, - reflection: str, - nodes: Optional[List[str]] = None, - connections: Optional[List[str]] = None, -) -> str: - """ - Simplified Think Tool: focus on DeepAgents core architecture validation. - - Enhanced: In validation stage, automatically reads actual nodes and edges from graph_context - if not provided, ensuring comprehensive detection of all created nodes and connections. - - Args: - stage: 'planning' (before creation) or 'validation' (after completion) - reflection: current reasoning or understanding of the task - nodes: role list or created node names (optional in validation stage; auto-read from graph_context) - connections: connection relationships, e.g. ['Manager -> Analyst'] (optional in validation stage; auto-read from graph_context) - - Returns: - JSON with validation feedback and recommendations. - - Note: Use planning stage FIRST for DeepAgents workflows. See system prompt for validation criteria. - """ - logger.info( - f"[think] starting validation stage={stage}, nodes_count={len(nodes) if nodes else 0}, connections_count={len(connections) if connections else 0}" - ) - logger.debug( - f"[think] reflection={reflection[:100]}..." if len(reflection) > 100 else f"[think] reflection={reflection}" - ) - - issues = [] - recommendations = [] - consistency_issues = [] - auto_read_used = False # track whether auto-read was used - - # ---------- 0. VALIDATION stage: auto-read actual nodes and edges from graph_context ---------- - if stage == "validation": - logger.debug("[think] validation stage: reading actual nodes and edges from graph_context") - graph_context = get_current_graph_context() - actual_nodes_data = graph_context.get("nodes", []) - actual_edges_data = graph_context.get("edges", []) - - logger.debug( - f"[think] read {len(actual_nodes_data)} nodes and {len(actual_edges_data)} edges from graph_context" - ) - - # extract node names from actual graph_context - actual_node_names = [] - actual_node_id_to_label = {} - for node in actual_nodes_data: - node_id = node.get("id", "") - data = node.get("data", {}) - label = data.get("label", node_id) - actual_node_names.append(label) - actual_node_id_to_label[node_id] = label - - # extract edge relationships from actual graph_context - actual_connections = [] - for edge in actual_edges_data: - source_id = edge.get("source", "") - target_id = edge.get("target", "") - source_label = actual_node_id_to_label.get(source_id, source_id) - target_label = actual_node_id_to_label.get(target_id, target_id) - actual_connections.append(f"{source_label} -> {target_label}") - - # if nodes/connections were provided, perform consistency check - if nodes is not None: - logger.debug( - f"[think] node consistency check: provided {len(nodes)} nodes, actual {len(actual_node_names)} nodes" - ) - # check whether provided nodes exist in actually created nodes - provided_nodes_lower = {n.lower() for n in nodes} - actual_nodes_lower = {n.lower() for n in actual_node_names} - - missing_in_actual = provided_nodes_lower - actual_nodes_lower - missing_in_provided = actual_nodes_lower - provided_nodes_lower - - if missing_in_actual: - logger.warning( - f"[think] consistency check: provided nodes '{', '.join(missing_in_actual)}' not found in actual nodes" - ) - consistency_issues.append(f"Provided nodes '{', '.join(missing_in_actual)}' not found in actual nodes") - if missing_in_provided: - logger.warning( - f"[think] consistency check: actual nodes '{', '.join(missing_in_provided)}' not mentioned in provided params" - ) - consistency_issues.append( - f"Actual nodes '{', '.join(missing_in_provided)}' not mentioned in provided params" - ) - if not missing_in_actual and not missing_in_provided: - logger.debug("[think] node consistency check passed: provided nodes match actual nodes") - else: - # if not provided, use actual nodes - logger.info( - f"[think] nodes param not provided, auto-using {len(actual_node_names)} nodes from graph_context" - ) - nodes = actual_node_names - auto_read_used = True - - # if connections were provided, perform consistency check - if connections is not None: - logger.debug( - f"[think] edge consistency check: provided {len(connections)} edges, actual {len(actual_connections)} edges" - ) - # check whether provided edges exist in actually created edges - provided_conns_lower = {c.lower().strip() for c in connections} - actual_conns_lower = {c.lower().strip() for c in actual_connections} - - missing_in_actual = provided_conns_lower - actual_conns_lower - missing_in_provided = actual_conns_lower - provided_conns_lower - - if missing_in_actual: - logger.warning( - f"[think] consistency check: provided edges '{', '.join(missing_in_actual)}' not found in actual edges" - ) - consistency_issues.append(f"Provided edges '{', '.join(missing_in_actual)}' not found in actual edges") - if missing_in_provided: - logger.warning( - f"[think] consistency check: actual edges '{', '.join(missing_in_provided)}' not mentioned in provided params" - ) - consistency_issues.append( - f"Actual edges '{', '.join(missing_in_provided)}' not mentioned in provided params" - ) - if not missing_in_actual and not missing_in_provided: - logger.debug("[think] edge consistency check passed: provided edges match actual edges") - else: - # if not provided, use actual edges - logger.info( - f"[think] connections param not provided, auto-using {len(actual_connections)} edges from graph_context" - ) - connections = actual_connections - auto_read_used = True - - # if nodes is still None (planning stage with no input), use empty list - if nodes is None: - nodes = [] - - # prepare base data - manager_nodes = [n for n in nodes if "manager" in n.lower() or "coordinator" in n.lower()] - subagents = [n for n in nodes if n not in manager_nodes] - - logger.debug(f"[think] node analysis: Manager={len(manager_nodes)}, SubAgent={len(subagents)}") - - # ---------- 1. PLANNING stage: logic validation ---------- - if stage == "planning": - logger.debug("[think] running planning stage validation") - - # planning stage: if nodes is empty, plan has not been provided yet; skip validation - if not nodes: - logger.info("[think] planning stage: nodes is empty, skipping validation (plan not yet provided)") - # add no issues; return pass - else: - # only validate when nodes is non-empty - logger.debug(f"[think] planning stage: validating {len(nodes)} planned nodes") - - # 1.1 manager check - if not manager_nodes: - issues.append("DeepAgents architecture must include a Manager node") - - # 1.2 count check (3-8 guideline) - if len(subagents) < 3: - issues.append(f"Too few SubAgents ({len(subagents)}), recommend 3-8 for effective collaboration") - elif len(subagents) > 8: - issues.append(f"Too many SubAgents ({len(subagents)}), recommend splitting or merging to 8 or fewer") - - # 1.3 single responsibility check - for node in nodes: - if " and " in node.lower() or "&" in node: - issues.append( - f"Role '{node}' has ambiguous responsibilities, consider splitting into two separate Agents" - ) - - # ---------- 2. VALIDATION stage: topology validation ---------- - elif stage == "validation": - logger.debug("[think] running validation stage checks") - # 2.1 basic completeness - if len(manager_nodes) != 1: - logger.warning(f"[think] Manager count check failed: expected 1, found {len(manager_nodes)}") - issues.append(f"Must have exactly one Manager, found {len(manager_nodes)}") - else: - logger.debug("[think] Manager count check passed: 1") - - # 2.2 star topology check (core) - if connections: - logger.debug(f"[think] starting star topology check, connections={len(connections)}") - conn_map: Dict[str, List[str]] = {n.lower(): [] for n in nodes} - for c in connections: - if "->" in c: - src, tgt = [p.strip().lower() for p in c.split("->")] - if src in conn_map: - conn_map[src].append(tgt) - - # check whether Manager is connected to all SubAgents - if manager_nodes: - mgr_lower = manager_nodes[0].lower() - disconnected_subagents = [] - for sa in subagents: - if sa.lower() not in conn_map.get(mgr_lower, []): - disconnected_subagents.append(sa) - issues.append(f"Disconnected: Manager is not connected to {sa}") - - if disconnected_subagents: - logger.warning( - f"[think] star topology check: Manager not connected to {len(disconnected_subagents)} SubAgents: {', '.join(disconnected_subagents)}" - ) - else: - logger.debug(f"[think] star topology check: Manager connected to all {len(subagents)} SubAgents") - - # check for chain connections between SubAgents (anti-pattern) - subagent_with_children = [] - for sa in subagents: - if conn_map.get(sa.lower()): - subagent_with_children.append(sa) - issues.append( - f"Non-star connection detected: {sa} has downstream nodes, let Manager coordinate instead" - ) - - if subagent_with_children: - logger.warning( - f"[think] star topology check: found {len(subagent_with_children)} SubAgents with downstream nodes (violates star topology): {', '.join(subagent_with_children)}" - ) - else: - logger.debug("[think] star topology check: no inter-SubAgent connections found (star topology OK)") - else: - logger.warning("[think] no connections detected") - issues.append("No connections detected") - - else: - issues.append(f"Unknown validation stage: {stage}. Valid stages are 'planning' or 'validation'") - - # ---------- 3. merge consistency check issues ---------- - if consistency_issues: - logger.info(f"[think] found {len(consistency_issues)} consistency issues") - issues.extend([f"Consistency check: {issue}" for issue in consistency_issues]) - - # ---------- 4. generate feedback ---------- - passed = len(issues) == 0 - if passed: - logger.info( - f"[think] validation passed: stage={stage}, nodes={len(nodes)}, connections={len(connections or [])}" - ) - recommendations = [ - "Structure follows DeepAgents best practices", - "Ready to proceed to next stage" if stage == "planning" else "Ready to deliver results", - ] - else: - logger.warning( - f"[think] validation failed: stage={stage}, issues={len(issues)}, nodes={len(nodes)}, connections={len(connections or [])}" - ) - recommendations = [f"Needs improvement: {i}" for i in issues] - - # build feedback summary - summary_parts = [f"Scanned {len(nodes)} nodes and {len(connections or [])} connections"] - if auto_read_used: - summary_parts.append("(auto-read actual data from graph_context)") - if consistency_issues: - summary_parts.append(f"Found {len(consistency_issues)} consistency issues") - - result = json.dumps( - { - "type": "THINK", - "feedback": { - "stage": stage, - "passed": passed, - "issues_found": len(issues), - "consistency_issues": len(consistency_issues) if consistency_issues else 0, - "recommendations": recommendations, - "summary": " | ".join(summary_parts), - }, - }, - ensure_ascii=False, - indent=2, - ) - - logger.info( - f"[think] validation complete: stage={stage}, passed={passed}, issues={len(issues)}, consistency_issues={len(consistency_issues)}" - ) - logger.debug(f"[think] result summary: {summary_parts[0]}") - - return result diff --git a/backend/app/core/copilot_deepagents/__init__.py b/backend/app/core/copilot_deepagents/__init__.py deleted file mode 100644 index ec5e2126b..000000000 --- a/backend/app/core/copilot_deepagents/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -DeepAgents Copilot - Generate arbitrary Agent workflow graphs using the DeepAgents pattern. - -Architecture: -- Manager Agent: orchestrate sub-agents + call create_node/connect_nodes -- SubAgents: requirements-analyst, workflow-architect, validator - -Features: -- Sub-agent collaboration: analyze -> design -> validate -> generate -- Artifact persistence: analysis.json, blueprint.json, validation.json -- Standard output: GraphAction (fully compatible with existing Copilot) -""" - -from .manager import DEEPAGENTS_AVAILABLE -from .runner import run_copilot_manager, stream_copilot_manager - -__all__ = [ - "stream_copilot_manager", - "run_copilot_manager", - "DEEPAGENTS_AVAILABLE", -] diff --git a/backend/app/core/copilot_deepagents/artifacts.py b/backend/app/core/copilot_deepagents/artifacts.py deleted file mode 100644 index c839a585f..000000000 --- a/backend/app/core/copilot_deepagents/artifacts.py +++ /dev/null @@ -1,311 +0,0 @@ -""" -Artifacts store for DeepAgents Copilot runs. - -Directory layout (per run): - $DEEPAGENTS_ARTIFACTS_DIR/{graph_id}/{run_id}/ - 00_request.json - analysis.json (sub-agent artifact) - blueprint.json (sub-agent artifact) - validation.json (sub-agent artifact) - actions.json (final GraphAction list) - events.sse.jsonl (SSE event stream) - index.json (run index) -""" - -from __future__ import annotations - -import json -import os -import re -import uuid -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from loguru import logger - -from app.utils.datetime import utc_now - - -def _default_artifacts_root() -> Path: - return Path.home() / ".agent-platform" / "deepagents" - - -def resolve_artifacts_root() -> Path: - env = os.getenv("DEEPAGENTS_ARTIFACTS_DIR", "").strip() - if env: - return Path(env).expanduser() - return _default_artifacts_root() - - -def _sanitize_path_component(component: str, default: str = "unknown") -> str: - """ - Sanitize a path component to prevent directory traversal attacks. - - Rules: - 1. Remove all path separators (/, \\) - 2. Remove relative path symbols (.., .) - 3. Remove control characters and special characters - 4. Limit length (prevent excessively long paths) - 5. Fall back to default if sanitized result is empty - - Args: - component: path component to sanitize - default: fallback value if sanitization yields empty string - - Returns: - Sanitized path component safe for filesystem use. - """ - if not component: - return default - - # remove all path separators and relative path symbols - sanitized = re.sub(r"[\\/\.\.]+", "", component) - - # remove control characters, spaces, and special characters (keep letters, digits, underscores, hyphens) - sanitized = re.sub(r"[^\w\-]", "", sanitized) - - # limit length (prevent excessively long paths) - sanitized = sanitized[:100] - - # fall back to default if empty - if not sanitized: - return default - - return sanitized - - -def _sanitize_filename(filename: str) -> str: - """ - Sanitize a filename to prevent directory traversal attacks. - - Only allow letters, digits, underscores, hyphens, and dots; no path separators. - - Args: - filename: filename to sanitize - - Returns: - Sanitized filename safe for filesystem use. - """ - if not filename: - raise ValueError("Filename cannot be empty") - - # remove all path separators - sanitized = filename.replace("/", "").replace("\\", "") - - # remove relative path symbols - sanitized = sanitized.replace("..", "").replace(".", "") - - # keep only letters, digits, underscores, hyphens, dots - sanitized = re.sub(r"[^\w\-\.]", "", sanitized) - - if not sanitized: - raise ValueError(f"Invalid filename after sanitization: {filename}") - - return sanitized - - -@dataclass -class ArtifactStore: - """Manages run directory and writing artifact files.""" - - graph_id: Optional[str] = None - run_id: str = field(default_factory=lambda: f"run_{uuid.uuid4().hex[:12]}") - run_dir: Optional[Path] = None - - def __post_init__(self): - # if run_dir is not specified, build it automatically - if self.run_dir is None: - root = resolve_artifacts_root() - # sanitize graph_id and run_id to prevent directory traversal - graph_dir = _sanitize_path_component(self.graph_id or "unknown_graph", default="unknown_graph") - run_id_sanitized = _sanitize_path_component(self.run_id, default=f"run_{uuid.uuid4().hex[:12]}") - self.run_dir = root / graph_dir / run_id_sanitized - # update run_id to the sanitized value for consistency - self.run_id = run_id_sanitized - else: - # if run_dir is provided, verify it is within the artifacts root - root = resolve_artifacts_root() - try: - # use resolve() to get absolute path, then check containment - resolved_run_dir = Path(self.run_dir).resolve() - resolved_root = root.resolve() - if not str(resolved_run_dir).startswith(str(resolved_root)): - raise ValueError( - f"run_dir must be within artifacts root: {resolved_run_dir} not in {resolved_root}" - ) - except Exception as e: - logger.error(f"[ArtifactStore] Invalid run_dir: {e}") - raise ValueError(f"Invalid run_dir: {e}") from e - - # ensure type is Path - if isinstance(self.run_dir, str): - self.run_dir = Path(self.run_dir) - - def ensure(self) -> None: - """Ensure the run directory exists.""" - if self.run_dir is None: - raise ValueError("run_dir must be set") - self.run_dir.mkdir(parents=True, exist_ok=True) - - # extra validation: ensure directory is within artifacts root - root = resolve_artifacts_root() - try: - resolved_run_dir = self.run_dir.resolve() - resolved_root = root.resolve() - if not str(resolved_run_dir).startswith(str(resolved_root)): - raise ValueError(f"run_dir escaped from artifacts root: {resolved_run_dir} not in {resolved_root}") - except Exception as e: - logger.error(f"[ArtifactStore] Security check failed: {e}") - raise - - def _write_json(self, filename: str, data: Any) -> None: - """Safely write a JSON file.""" - self.ensure() - if self.run_dir is None: - raise ValueError("run_dir must be set") - # sanitize filename to prevent directory traversal - safe_filename = _sanitize_filename(filename) - path = self.run_dir / safe_filename - - # extra validation: ensure final path is still within run_dir (defense in depth) - try: - resolved_path = path.resolve() - resolved_run_dir = self.run_dir.resolve() - if not str(resolved_path).startswith(str(resolved_run_dir)): - raise ValueError(f"Path traversal detected: {resolved_path} not in {resolved_run_dir}") - except Exception as e: - logger.error(f"[ArtifactStore] Path traversal detected in filename: {filename}") - raise ValueError(f"Invalid filename: {filename}") from e - - with path.open("w", encoding="utf-8") as f: - json.dump(data, f, ensure_ascii=False, indent=2, default=str) - - def write_request(self, payload: Dict[str, Any]) -> None: - self._write_json("00_request.json", payload) - - def write_analysis(self, payload: Dict[str, Any]) -> None: - self._write_json("analysis.json", payload) - - def write_blueprint(self, payload: Dict[str, Any]) -> None: - self._write_json("blueprint.json", payload) - - def write_validation(self, payload: Dict[str, Any]) -> None: - self._write_json("validation.json", payload) - - def write_actions(self, payload: Union[List[Dict[str, Any]], Dict[str, Any]]) -> None: - self._write_json("actions.json", payload) - - def write_index(self, payload: Dict[str, Any]) -> None: - """Write the run index.""" - # add timestamp - if "created_at" not in payload: - payload["created_at"] = utc_now().isoformat() - self._write_json("index.json", payload) - - def append_event(self, event: Dict[str, Any]) -> None: - """Append SSE event envelope as jsonl for replay.""" - self.ensure() - if self.run_dir is None: - raise ValueError("run_dir must be set") - # use hardcoded filename; no sanitization needed - path = self.run_dir / "events.sse.jsonl" - - # verify path safety - try: - resolved_path = path.resolve() - resolved_run_dir = self.run_dir.resolve() - if not str(resolved_path).startswith(str(resolved_run_dir)): - raise ValueError("Path traversal detected in append_event") - except Exception as e: - logger.error(f"[ArtifactStore] Security check failed in append_event: {e}") - raise - - with path.open("a", encoding="utf-8") as f: - f.write(json.dumps(event, ensure_ascii=False, default=str) + "\n") - - # ==================== Read Methods ==================== - - def _read_json(self, filename: str) -> Optional[Dict[str, Any]]: - """Safely read a JSON file; return None on failure.""" - if self.run_dir is None: - raise ValueError("run_dir must be set") - # sanitize filename to prevent directory traversal - safe_filename = _sanitize_filename(filename) - path = self.run_dir / safe_filename - - # verify path safety - try: - resolved_path = path.resolve() - resolved_run_dir = self.run_dir.resolve() - if not str(resolved_path).startswith(str(resolved_run_dir)): - logger.warning(f"[ArtifactStore] Path traversal detected in read: {filename}") - return None - except Exception as e: - logger.warning(f"[ArtifactStore] Security check failed in read: {e}") - return None - - if not path.exists(): - return None - try: - with path.open("r", encoding="utf-8") as f: - result = json.load(f) - return result if isinstance(result, dict) else None - except (json.JSONDecodeError, IOError) as e: - logger.warning(f"[ArtifactStore] Failed to read {filename}: {e}") - return None - - def read_analysis(self) -> Optional[Dict[str, Any]]: - """Read the requirement analysis result.""" - return self._read_json("analysis.json") - - def read_blueprint(self) -> Optional[Dict[str, Any]]: - """Read the workflow blueprint.""" - return self._read_json("blueprint.json") - - def read_validation(self) -> Optional[Dict[str, Any]]: - """Read the validation report.""" - return self._read_json("validation.json") - - def read_actions(self) -> Optional[List[Dict[str, Any]]]: - """Read the actions list.""" - data = self._read_json("actions.json") - if isinstance(data, list): - return data - return None - - def read_index(self) -> Optional[Dict[str, Any]]: - """Read the run index.""" - return self._read_json("index.json") - - def file_exists(self, filename: str) -> bool: - """Check whether a file exists.""" - if self.run_dir is None: - raise ValueError("run_dir must be set") - try: - safe_filename = _sanitize_filename(filename) - path = self.run_dir / safe_filename - - # verify path safety - resolved_path = path.resolve() - resolved_run_dir = self.run_dir.resolve() - if not str(resolved_path).startswith(str(resolved_run_dir)): - logger.warning(f"[ArtifactStore] Path traversal detected in file_exists: {filename}") - return False - except Exception as e: - logger.warning(f"[ArtifactStore] Security check failed in file_exists: {e}") - return False - - return path.exists() - - -def new_run_store(graph_id: str) -> ArtifactStore: - """Create a new ArtifactStore instance.""" - # graph_id will be sanitized in __post_init__ - store = ArtifactStore(graph_id=graph_id) - try: - store.ensure() - except Exception as e: - logger.error(f"[ArtifactStore] Failed to create run dir: {e}") - raise - return store diff --git a/backend/app/core/copilot_deepagents/layout.py b/backend/app/core/copilot_deepagents/layout.py deleted file mode 100644 index b23b369c1..000000000 --- a/backend/app/core/copilot_deepagents/layout.py +++ /dev/null @@ -1,295 +0,0 @@ -""" -Auto Layout Engine for DeepAgents Copilot. - -Use networkx to implement a hierarchical layout algorithm, solving the problem -of LLMs being unable to generate tidy ReactFlow coordinates. -Uses topological sort + layered layout to ensure: -- Manager node on the left -- Sub-agent nodes vertically aligned on the right -- Automatic overlap avoidance -- Clean edge connections -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Set - -from loguru import logger - -try: - import networkx as nx - - NETWORKX_AVAILABLE = True -except ImportError: - nx = None - NETWORKX_AVAILABLE = False - logger.warning("[LayoutEngine] networkx not available, using fallback layout") - - -def apply_auto_layout( - blueprint_data: Dict[str, Any], - x_spacing: int = 300, - y_spacing: int = 150, - start_x: int = 100, - start_y: int = 100, -) -> Dict[str, Any]: - """ - Compute layered layout using topological sort, overriding LLM-generated coordinates. - - Args: - blueprint_data: blueprint dict containing nodes and edges - x_spacing: horizontal spacing between nodes - y_spacing: vertical spacing between nodes - start_x: starting X coordinate - start_y: starting Y coordinate - - Returns: - blueprint_data with updated positions. - """ - nodes = blueprint_data.get("nodes", []) - blueprint_data.get("edges", []) - - if not nodes: - return blueprint_data - - if NETWORKX_AVAILABLE: - return _apply_networkx_layout(blueprint_data, x_spacing, y_spacing, start_x, start_y) - else: - return _apply_fallback_layout(blueprint_data, x_spacing, y_spacing, start_x, start_y) - - -def _apply_networkx_layout( - blueprint_data: Dict[str, Any], - x_spacing: int, - y_spacing: int, - start_x: int, - start_y: int, -) -> Dict[str, Any]: - """Implement layered layout using networkx.""" - nodes = blueprint_data.get("nodes", []) - edges = blueprint_data.get("edges", []) - - # build directed graph - G = nx.DiGraph() - node_map = {n["id"]: n for n in nodes} - - for node in nodes: - G.add_node(node["id"]) - for edge in edges: - source, target = edge.get("source"), edge.get("target") - if source in node_map and target in node_map: - G.add_edge(source, target) - - # handle possible cycles (break them for topological sort) - if not nx.is_directed_acyclic_graph(G): - logger.warning("[LayoutEngine] Graph has cycles, attempting to break them") - G = _break_cycles(G) - - # compute each node's layer (via topological sort) - levels: Dict[str, int] = {} - try: - for node_id in nx.topological_sort(G): - predecessors = list(G.predecessors(node_id)) - if predecessors: - levels[node_id] = max(levels.get(p, 0) for p in predecessors) + 1 - else: - levels[node_id] = 0 - except nx.NetworkXError as e: - logger.warning(f"[LayoutEngine] Topological sort failed: {e}, using fallback") - return _apply_fallback_layout(blueprint_data, x_spacing, y_spacing, start_x, start_y) - - # assign orphan nodes (no edges) to layer 0 - for node in nodes: - if node["id"] not in levels: - levels[node["id"]] = 0 - - # group by layer and compute coordinates - level_nodes: Dict[int, List[str]] = {} - for node_id, lvl in levels.items(): - level_nodes.setdefault(lvl, []).append(node_id) - - # sort within each layer (preserve original order stability) - node_order = {n["id"]: i for i, n in enumerate(nodes)} - for lvl in level_nodes: - level_nodes[lvl].sort(key=lambda nid: node_order.get(nid, 999)) - - # assign coordinates - for node in nodes: - node_id = node["id"] - lvl = levels.get(node_id, 0) - level_list = level_nodes.get(lvl, []) - idx_in_level = level_list.index(node_id) if node_id in level_list else 0 - - # compute centering offset (vertically center nodes in the same layer) - level_height = (len(level_list) - 1) * y_spacing - y_offset = -level_height // 2 if len(level_list) > 1 else 0 - - node["position"] = { - "x": start_x + lvl * x_spacing, - "y": start_y + idx_in_level * y_spacing + y_offset + (level_height // 2), - } - - logger.info(f"[LayoutEngine] Applied networkx layout to {len(nodes)} nodes across {len(level_nodes)} levels") - return blueprint_data - - -def _apply_fallback_layout( - blueprint_data: Dict[str, Any], - x_spacing: int, - y_spacing: int, - start_x: int, - start_y: int, -) -> Dict[str, Any]: - """ - Fallback layout: simple layered layout without networkx dependency. - Manually compute layers based on edge relationships. - """ - nodes = blueprint_data.get("nodes", []) - edges = blueprint_data.get("edges", []) - - node_ids = {n["id"] for n in nodes} - - # build adjacency lists - children: Dict[str, List[str]] = {n["id"]: [] for n in nodes} - parents: Dict[str, List[str]] = {n["id"]: [] for n in nodes} - - for edge in edges: - source, target = edge.get("source"), edge.get("target") - if source in node_ids and target in node_ids: - children[source].append(target) - parents[target].append(source) - - # find root nodes (no parents) - roots = [nid for nid in node_ids if not parents[nid]] - if not roots: - # if no root nodes, pick the first node as root - roots = [nodes[0]["id"]] if nodes else [] - - # BFS to compute layers - levels: Dict[str, int] = {} - visited: Set[str] = set() - queue = [(root, 0) for root in roots] - - while queue: - node_id, lvl = queue.pop(0) - if node_id in visited: - continue - visited.add(node_id) - levels[node_id] = max(levels.get(node_id, 0), lvl) - - for child in children.get(node_id, []): - if child not in visited: - queue.append((child, lvl + 1)) - - # handle unvisited nodes (orphan nodes) - for node in nodes: - if node["id"] not in levels: - levels[node["id"]] = 0 - - # group by layer - level_nodes: Dict[int, List[str]] = {} - for node_id, lvl in levels.items(): - level_nodes.setdefault(lvl, []).append(node_id) - - # assign coordinates - node_map = {n["id"]: n for n in nodes} - for lvl, node_list in level_nodes.items(): - for idx, node_id in enumerate(node_list): - node = node_map.get(node_id) - if node: - node["position"] = { - "x": start_x + lvl * x_spacing, - "y": start_y + idx * y_spacing, - } - - logger.info(f"[LayoutEngine] Applied fallback layout to {len(nodes)} nodes") - return blueprint_data - - -def _break_cycles(G: "nx.DiGraph") -> "nx.DiGraph": - """ - Break cycles in the graph to enable topological sort. - Use a simplified feedback arc set approach. - """ - # copy the graph - G_copy = G.copy() - - # find and remove edges in cycles - try: - cycles = list(nx.simple_cycles(G_copy)) - edges_to_remove = set() - - for cycle in cycles: - if len(cycle) >= 2: - # remove the last edge in the cycle - edges_to_remove.add((cycle[-1], cycle[0])) - - for edge in edges_to_remove: - if G_copy.has_edge(*edge): - G_copy.remove_edge(*edge) - logger.debug(f"[LayoutEngine] Removed edge {edge} to break cycle") - except Exception as e: - logger.warning(f"[LayoutEngine] Failed to break cycles: {e}") - - return G_copy - - -def calculate_optimal_spacing( - nodes: List[Dict[str, Any]], - edges: List[Dict[str, Any]], - canvas_width: int = 1200, - canvas_height: int = 800, -) -> tuple[int, int]: - """ - Calculate optimal spacing based on node count and canvas size. - - Returns: - (x_spacing, y_spacing) tuple. - """ - num_nodes = len(nodes) - - if num_nodes <= 2: - return 350, 150 - elif num_nodes <= 5: - return 300, 150 - elif num_nodes <= 10: - return 280, 120 - else: - # large graph: compress spacing - return 250, 100 - - -def center_graph_on_canvas( - blueprint_data: Dict[str, Any], - canvas_width: int = 1200, - canvas_height: int = 800, -) -> Dict[str, Any]: - """ - Center the entire graph on the canvas. - """ - nodes = blueprint_data.get("nodes", []) - if not nodes: - return blueprint_data - - # compute current bounds - min_x = min(n["position"]["x"] for n in nodes) - max_x = max(n["position"]["x"] for n in nodes) - min_y = min(n["position"]["y"] for n in nodes) - max_y = max(n["position"]["y"] for n in nodes) - - graph_width = max_x - min_x - graph_height = max_y - min_y - - # compute offset to center - offset_x = (canvas_width - graph_width) // 2 - min_x - offset_y = (canvas_height - graph_height) // 2 - min_y - - # ensure no negative coordinates - offset_x = max(offset_x, 50 - min_x) - offset_y = max(offset_y, 50 - min_y) - - for node in nodes: - node["position"]["x"] += offset_x - node["position"]["y"] += offset_y - - return blueprint_data diff --git a/backend/app/core/copilot_deepagents/manager.py b/backend/app/core/copilot_deepagents/manager.py deleted file mode 100644 index ab28e2450..000000000 --- a/backend/app/core/copilot_deepagents/manager.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -DeepAgents Copilot Manager. - -Use the DeepAgents Manager + sub-agent pattern to generate arbitrary graph types: -- Manager: orchestrate sub-agents, call create_node/connect_nodes tools to output GraphActions -- Sub-agents: plan/design/validate, producing artifact files -- Final output: standard GraphActions (fully compatible with existing Copilot) -""" - -from __future__ import annotations - -import os -import uuid -from pathlib import Path -from typing import TYPE_CHECKING, Any, List, Optional, Type - -if TYPE_CHECKING: - from deepagents import SubAgent as SubAgentT - from deepagents.backends.filesystem import FilesystemBackend as FilesystemBackendT - -from langchain_core.runnables import Runnable -from loguru import logger - -from app.core.copilot.tools import connect_nodes, create_node, delete_node, update_config - -from .artifacts import ArtifactStore -from .prompts import ( - MANAGER_SYSTEM_PROMPT, - REQUIREMENTS_ANALYST_PROMPT, - VALIDATOR_PROMPT, - WORKFLOW_ARCHITECT_PROMPT, -) - -# ==================== Optional deepagents imports ==================== -# declare as optional first to avoid assigning None in except block (mypy "Cannot assign to a type") -create_deep_agent: Any = None -FilesystemMiddleware: Type[Any] | None = None -FilesystemBackend: Type[Any] | None = None -SubAgent: Type[Any] | None = None -DEEPAGENTS_AVAILABLE = False - -try: - from deepagents import ( - FilesystemMiddleware as _FilesystemMiddleware, - ) - from deepagents import ( - SubAgent as _SubAgent, - ) - from deepagents import ( - create_deep_agent as _create_deep_agent, - ) - from deepagents.backends.filesystem import FilesystemBackend as _FilesystemBackend - - create_deep_agent = _create_deep_agent - FilesystemMiddleware = _FilesystemMiddleware - FilesystemBackend = _FilesystemBackend - SubAgent = _SubAgent - DEEPAGENTS_AVAILABLE = True -except ImportError: - logger.warning("[DeepAgentsCopilot] deepagents library not available") - -# ==================== Manager Factory ==================== - - -def get_artifacts_root() -> Path: - """Return the artifacts root directory.""" - root = os.environ.get("DEEPAGENTS_ARTIFACTS_DIR", "") - if not root: - root = str(Path.home() / ".agent-platform" / "deepagents") - return Path(root) - - -def _build_subagents(backend: "FilesystemBackendT") -> List["SubAgentT"]: - """ - Build the sub-agent list. - - Each sub-agent only has filesystem tools (read/write files); they do not call Copilot tools. - - SubAgent description best practices (per DeepAgents docs): - - specific, action-oriented - - describe "what it does" not "what it is" - - help the Manager select the right sub-agent - - Reference: https://docs.langchain.com/oss/python/deepagents/subagents - """ - return [ - { - "name": "requirements-analyst", - "description": ( - "Analyze the user's agent workflow request and output a structured requirements spec. " - "Used for: 1) determining whether to create a new graph or update an existing one; " - "2) assessing complexity level; " - "3) deciding whether DeepAgents multi-agent collaboration is needed. " - "Outputs /analysis.json containing fields such as goal, mode, complexity, use_deep_agents." - ), - "system_prompt": REQUIREMENTS_ANALYST_PROMPT, - "tools": [], # filesystem tools provided via middleware - }, - { - "name": "workflow-architect", - "description": ( - "Design the complete architecture of an agent workflow based on requirements analysis. " - "Used for: 1) designing node structure and connection relationships; " - "2) writing professional systemPrompts for each agent; " - "3) configuring DeepAgents hierarchy (Manager + sub-agents). " - "Outputs /blueprint.json in ReactFlow-compatible format containing nodes and edges. " - "Also used to fix validation issues by reading the existing blueprint and making targeted modifications." - ), - "system_prompt": WORKFLOW_ARCHITECT_PROMPT, - "tools": [], - }, - { - "name": "validator", - "description": ( - "Validate the structural integrity and quality of the workflow blueprint. " - "Used for: 1) checking required fields and data formats; " - "2) verifying DeepAgents rules (description, hierarchy constraints); " - "3) assessing systemPrompt quality; " - "4) detecting topology issues (orphan nodes, invalid edges). " - "Outputs /validation.json containing is_valid, health_score, and an issues list." - ), - "system_prompt": VALIDATOR_PROMPT, - "tools": [], - }, - ] - - -def create_copilot_manager( - *, - model: Any, - graph_id: Optional[str] = None, - run_id: Optional[str] = None, - user_id: Optional[str] = None, -) -> tuple[Runnable, ArtifactStore]: - """ - Create a DeepAgents Copilot Manager. - - Args: - model: Pre-created LangChain model instance (from ModelResolver) - graph_id: Graph ID for artifact storage - run_id: Optional run ID (auto-generated if not provided) - user_id: User ID for workspace isolation - - Returns: - (manager_agent, artifact_store) - """ - if not DEEPAGENTS_AVAILABLE or create_deep_agent is None or FilesystemBackend is None: - raise RuntimeError("deepagents library not available. Install with: pip install deepagents") - assert create_deep_agent is not None and FilesystemBackend is not None # narrow types for mypy - - # generate run_id - if not run_id: - run_id = f"run_{uuid.uuid4().hex[:12]}" - - # create artifact store - artifacts_root = get_artifacts_root() - graph_dir = graph_id or "unknown_graph" - run_dir = artifacts_root / graph_dir / run_id - run_dir.mkdir(parents=True, exist_ok=True) - - store = ArtifactStore( - graph_id=graph_id, - run_id=run_id, - run_dir=run_dir, - ) - - # create filesystem backend (for sub-agent file I/O) - backend = FilesystemBackend(root_dir=run_dir) - - # Copilot tools (Manager uses these to generate GraphActions) - copilot_tools = [ - create_node, - connect_nodes, - delete_node, - update_config, - ] - - # sub-agent configuration - subagent_specs = _build_subagents(backend) - - # FilesystemMiddleware gives both Agent and sub-agents filesystem tools - # DeepAgents already includes FilesystemMiddleware - - # create DeepAgents Manager - manager = create_deep_agent( - model=model, - system_prompt=MANAGER_SYSTEM_PROMPT, - tools=copilot_tools, - subagents=subagent_specs, - name="copilot-deepagents-manager", - ) - - logger.info(f"[DeepAgentsCopilot] Created manager run_id={run_id} run_dir={run_dir}") - - return manager, store - - -# ==================== Schema Validation Helpers (Moved to .utils) ==================== - -# Re-exporting from .utils if needed, but better to import directly from .utils - -# ==================== Helpers (Moved to .utils) ==================== diff --git a/backend/app/core/copilot_deepagents/prompts.py b/backend/app/core/copilot_deepagents/prompts.py deleted file mode 100644 index c21af7b21..000000000 --- a/backend/app/core/copilot_deepagents/prompts.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -DeepAgents Copilot - Prompt constants for Manager and sub-agents. - -Prompts are maintained as Markdown files under the ``prompts/`` directory -and loaded once at import time. -""" - -from pathlib import Path - -_PROMPT_DIR = Path(__file__).parent / "prompts" - - -def _load(name: str) -> str: - return (_PROMPT_DIR / name).read_text(encoding="utf-8") - - -MANAGER_SYSTEM_PROMPT = _load("manager.md") -REQUIREMENTS_ANALYST_PROMPT = _load("requirements_analyst.md") -WORKFLOW_ARCHITECT_PROMPT = _load("workflow_architect.md") -VALIDATOR_PROMPT = _load("validator.md") diff --git a/backend/app/core/copilot_deepagents/prompts/manager.md b/backend/app/core/copilot_deepagents/prompts/manager.md deleted file mode 100644 index 7bd64fce6..000000000 --- a/backend/app/core/copilot_deepagents/prompts/manager.md +++ /dev/null @@ -1,104 +0,0 @@ -You are an Agent workflow generation expert (DeepAgents Copilot Manager). - -## Core Responsibilities - -You are the top-level coordinator, responsible for: -1. Understanding user intent and decomposing tasks -2. Delegating specialized work to sub-agents (keeping your own context clean) -3. Integrating results and generating the final ReactFlow graph - -**Key principle**: Complex tasks must be delegated to sub-agents — do not handle detailed work yourself. This keeps context clean and improves output quality. - -## Available Sub-Agents - -Use the `task()` tool to delegate work. Each sub-agent focuses on a specific domain: - -| Sub-Agent | Purpose | Output File | -|-----------|---------|-------------| -| requirements-analyst | Analyze requirement complexity, identify patterns, determine DeepAgents applicability | /analysis.json | -| workflow-architect | Design node/edge structure, write systemPrompt, plan topology | /blueprint.json | -| validator | Validate structural integrity, check DeepAgents rules, assess quality | /validation.json | - -## Standard Workflow - -### Phase 1: Requirements Analysis - -Call the requirements-analyst sub-agent: -- task(name="requirements-analyst", description="Analyze user request: , current graph: nodes/ edges") -- Wait for completion, then read /analysis.json -- Obtain key information: mode, complexity, use_deep_agents, etc. - -### Phase 2: Architecture Design - -Call the workflow-architect sub-agent: -- task(name="workflow-architect", description="Design workflow based on requirements analysis: , mode=") -- Wait for completion, then read /blueprint.json - -### Phase 3: Validation Loop (Reflexion Pattern) - -Must execute the validation loop, with up to 3 retries: - -1. Call validator: task(name="validator", description="Validate /blueprint.json") -2. Read /validation.json -3. Check is_valid: - - If true: proceed to Phase 4 - - If false and retry count < 3: call architect to fix, then return to step 1 - - If false and retry count >= 3: force continue - -Warning: Force continue after 3 retries to avoid infinite loops. - -### Phase 4: Generate Graph Elements - -Read the final /blueprint.json, then: - -Create nodes (when mode=create or new nodes are needed): -- create_node(node_type=, label=