feat: moderation v2 core backend engine and pipeline

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Added
+- Moderation: persist structured moderation snapshots (static scan + VT/LLM merged verdict, reason codes, and evidence) on skills and versions (#333) (thanks @ArthurzKV).
 - Moderation: add comment reporting with per-user active report caps, unique reporter/target enforcement, and auto-hide on the 4th unique report.
 - Moderation: add AI-driven comment scam backfill (`commentModeration:*`) with persisted verdict/confidence/explainer metadata and strict auto-ban for `certain_scam` + `high` confidence.
 - Admin: add manual unban for banned users (clears `deletedAt` + `banReason`, audit log entry). Revoked API tokens stay revoked.

convex/_generated/api.d.ts

@@ -56,6 +56,8 @@ import type * as lib_httpHeaders from "../lib/httpHeaders.js";
 import type * as lib_httpRateLimit from "../lib/httpRateLimit.js";
 import type * as lib_leaderboards from "../lib/leaderboards.js";
 import type * as lib_moderation from "../lib/moderation.js";
+import type * as lib_moderationEngine from "../lib/moderationEngine.js";
+import type * as lib_moderationReasonCodes from "../lib/moderationReasonCodes.js";
 import type * as lib_openaiResponse from "../lib/openaiResponse.js";
 import type * as lib_public from "../lib/public.js";
 import type * as lib_reporting from "../lib/reporting.js";
@@ -152,6 +154,8 @@ declare const fullApi: ApiFromModules<{
   "lib/httpRateLimit": typeof lib_httpRateLimit;
   "lib/leaderboards": typeof lib_leaderboards;
   "lib/moderation": typeof lib_moderation;
+  "lib/moderationEngine": typeof lib_moderationEngine;
+  "lib/moderationReasonCodes": typeof lib_moderationReasonCodes;
   "lib/openaiResponse": typeof lib_openaiResponse;
   "lib/public": typeof lib_public;
   "lib/reporting": typeof lib_reporting;

convex/httpApi.handlers.test.ts

@@ -343,6 +343,7 @@ describe('httpApi handlers', () => {
         displayName: 'Cool Skill',
         version: '1.2.3',
         changelog: 'c',
+        acceptLicenseTerms: true,
         files: [{ path: 'SKILL.md', size: 1, storageId: 'id', sha256: 'a' }],
       }),
     })
@@ -365,6 +366,7 @@ describe('httpApi handlers', () => {
         displayName: 'Cool Skill',
         version: '1.2.3',
         changelog: 'c',
+        acceptLicenseTerms: true,
         files: [{ path: 'SKILL.md', size: 1, storageId: 'id', sha256: 'a' }],
       }),
     })

convex/lib/moderationEngine.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from 'vitest'
+import { buildModerationSnapshot, runStaticModerationScan } from './moderationEngine'
+
+describe('moderationEngine', () => {
+  it('does not flag benign token/password docs text alone', () => {
+    const result = runStaticModerationScan({
+      slug: 'demo',
+      displayName: 'Demo',
+      summary: 'A normal integration skill',
+      frontmatter: {},
+      metadata: {},
+      files: [{ path: 'SKILL.md', size: 64 }],
+      fileContents: [
+        {
+          path: 'SKILL.md',
+          content:
+            'This skill requires API token and password from the official provider settings.',
+        },
+      ],
+    })
+
+    expect(result.reasonCodes).toEqual([])
+    expect(result.status).toBe('clean')
+  })
+
+  it('flags dynamic eval usage as suspicious', () => {
+    const result = runStaticModerationScan({
+      slug: 'demo',
+      displayName: 'Demo',
+      summary: 'A normal integration skill',
+      frontmatter: {},
+      metadata: {},
+      files: [{ path: 'index.ts', size: 64 }],
+      fileContents: [{ path: 'index.ts', content: 'const value = eval(code)' }],
+    })
+
+    expect(result.reasonCodes).toContain('suspicious.dynamic_code_execution')
+    expect(result.status).toBe('suspicious')
+  })
+
+  it('upgrades merged verdict to malicious when VT is malicious', () => {
+    const snapshot = buildModerationSnapshot({
+      staticScan: {
+        status: 'suspicious',
+        reasonCodes: ['suspicious.dynamic_code_execution'],
+        findings: [],
+        summary: '',
+        engineVersion: 'v2.0.0',
+        checkedAt: Date.now(),
+      },
+      vtStatus: 'malicious',
+    })
+
+    expect(snapshot.verdict).toBe('malicious')
+    expect(snapshot.reasonCodes).toContain('malicious.vt_malicious')
+  })
+
+  it('rebuilds snapshots from current signals instead of retaining stale scanner codes', () => {
+    const snapshot = buildModerationSnapshot({
+      staticScan: {
+        status: 'clean',
+        reasonCodes: [],
+        findings: [],
+        summary: '',
+        engineVersion: 'v2.0.0',
+        checkedAt: Date.now(),
+      },
+    })
+
+    expect(snapshot.verdict).toBe('clean')
+    expect(snapshot.reasonCodes).toEqual([])
+  })
+})