feat(eval): add eval online-eval CLI commands

src/core/onlineEvalExecutionRole.test.ts

@@ -0,0 +1,86 @@
+import { test, expect } from "bun:test";
+import {
+  executionPolicy,
+  onlineEvalExecutionRoleName,
+  scopePolicyName,
+} from "./onlineEvalExecutionRole";
+
+const REGION = "us-west-2";
+const ACCOUNT = "123456789012";
+const LOG_GROUPS = ["/aws/bedrock-agentcore/runtimes/orders-agent-abc123-DEFAULT"];
+
+function statements(policy: string): { Sid?: string; Action?: unknown; Resource?: unknown }[] {
+  return JSON.parse(policy).Statement;
+}
+
+// The service validates at create time that the role can decrypt any evaluator
+// encrypted with a customer managed key, so the statement has to be present and
+// scoped to exactly those keys.
+test("grants kms:Decrypt scoped to the referenced evaluator keys", () => {
+  const keys = [
+    "arn:aws:kms:us-west-2:123456789012:key/aaaaaaaa-1111",
+    "arn:aws:kms:us-west-2:123456789012:key/bbbbbbbb-2222",
+  ];
+  const decrypt = statements(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, keys)).find(
+    (s) => s.Sid === "DecryptEvaluatorKeys",
+  );
+
+  expect(decrypt).toBeDefined();
+  expect(decrypt?.Action).toEqual(["kms:Decrypt", "kms:DescribeKey"]);
+  expect(decrypt?.Resource).toEqual(keys);
+});
+
+// No wildcard when nothing is encrypted: the builtin evaluators carry no key, so
+// the common case must not widen the role.
+test("omits the KMS statement when no evaluator is encrypted", () => {
+  const sids = statements(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])).map((s) => s.Sid);
+  expect(sids).not.toContain("DecryptEvaluatorKeys");
+});
+
+// Query access is scoped to the runtime prefix, not one endpoint's log group: the
+// service validates at the runtime level and rejects a narrower policy.
+test("scopes trace queries to the runtime prefix and aws/spans", () => {
+  const query = statements(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])).find(
+    (s) => s.Sid === "QuerySampledTraces",
+  );
+  expect(query?.Resource).toEqual([
+    `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:aws/spans*`,
+    `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`,
+  ]);
+});
+
+// IAM caps role names at 64 chars. Truncating alone would let two configs share a
+// role, and provisioning is idempotent by name, so the second create would
+// re-scope the first's policy.
+test("keeps role names within 64 characters and distinct", () => {
+  const a = onlineEvalExecutionRoleName("x".repeat(44) + "AAAA");
+  const b = onlineEvalExecutionRoleName("x".repeat(44) + "BBBB");
+  expect(a.length).toBeLessThanOrEqual(64);
+  expect(b.length).toBeLessThanOrEqual(64);
+  expect(a).not.toBe(b);
+  expect(onlineEvalExecutionRoleName("short")).toBe("AgentCoreOnlineEval-short");
+});
+
+// Each scope must map to its own policy name. Granting a new scope writes a new
+// policy rather than overwriting the current one, which is what lets an update
+// keep the old scope intact until the config change has landed.
+test("gives policies with different contents different names", () => {
+  const orders = scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/orders*"], []));
+  const checkout = scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/checkout*"], []));
+  expect(orders).not.toBe(checkout);
+});
+
+// The name is a fingerprint of the scope, so re-granting an unchanged scope is a
+// no-op rewrite of the same policy rather than an accumulating new one — and the
+// update path can compare names to know whether anything needs revoking.
+test("gives identical policies the same name", () => {
+  const keys = ["arn:aws:kms:us-west-2:123456789012:key/aaaa"];
+  const doc = executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], keys);
+  expect(scopePolicyName(doc)).toBe(
+    scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], keys)),
+  );
+  // A differing KMS key renders a different document, hence a different name.
+  expect(scopePolicyName(doc)).not.toBe(
+    scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])),
+  );
+});

src/core/onlineEvalExecutionRole.tsx

@@ -0,0 +1,256 @@
+import {
+  CreateRoleCommand,
+  DeleteRolePolicyCommand,
+  GetRoleCommand,
+  PutRolePolicyCommand,
+  type IAMClient,
+} from "@aws-sdk/client-iam";
+
+// Default online-evaluation execution role provisioning, mirroring
+// core/executionRole.tsx's pattern for harnesses: CreateOnlineEvaluationConfig
+// requires an IAM role the service assumes to read the target CloudWatch log
+// group, invoke Bedrock models for LLM-as-a-Judge evaluators, and write
+// evaluation results back to CloudWatch. When the caller doesn't bring one,
+// OnlineEvalClient provisions a per-config default here, scoped to the log
+// group(s) being sampled. Idempotent: an existing role is reused.
+//
+// Each scope is stored as its own inline policy, named after a fingerprint of the
+// scope, so granting a new scope never overwrites the policy backing the current
+// one. IAM unions Allows across a role's inline policies, which lets an update
+// grant the new scope before changing the config and drop the old scope only once
+// the change has landed.
+
+const POLICY_PREFIX = "AgentCoreOnlineEvalExecutionPolicy";
+
+const ROLE_NAME_PREFIX = "AgentCoreOnlineEval-";
+const ROLE_NAME_MAX = 64;
+const NAME_HASH_LENGTH = 8;
+
+// onlineEvalExecutionRoleName derives the default role's name from the online eval
+// config name. IAM caps role names at 64 characters, which leaves only 44 for the
+// config name — while config names run to 100 — so a name that would overflow is
+// truncated and given a hash suffix. Truncating alone would let two configs share
+// one role, and because provisioning is idempotent by name the second create would
+// silently re-scope the first's policy to a different runtime.
+export function onlineEvalExecutionRoleName(configName: string): string {
+  const full = `${ROLE_NAME_PREFIX}${configName}`;
+  if (full.length <= ROLE_NAME_MAX) return full;
+
+  const hash = Bun.hash(configName)
+    .toString(16)
+    .padStart(NAME_HASH_LENGTH, "0")
+    .slice(-NAME_HASH_LENGTH);
+  const room = ROLE_NAME_MAX - ROLE_NAME_PREFIX.length - NAME_HASH_LENGTH - 1;
+  return `${ROLE_NAME_PREFIX}${configName.slice(0, room)}-${hash}`;
+}
+
+function trustPolicy(): string {
+  return JSON.stringify({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Principal: { Service: "bedrock-agentcore.amazonaws.com" },
+        Action: "sts:AssumeRole",
+      },
+    ],
+  });
+}
+
+// runtimeLogGroupPrefix strips the trailing `-<endpoint>` qualifier from an
+// AgentCore runtime log group name, yielding the runtime-level prefix the
+// service expects the execution role to be scoped to. A log group that does not
+// follow the runtime naming convention (e.g. a caller-supplied custom group) is
+// returned unchanged.
+function runtimeLogGroupPrefix(logGroupName: string): string {
+  const match = logGroupName.match(/^(\/aws\/bedrock-agentcore\/runtimes\/.+)-[^-]+$/);
+  return match?.[1] ?? logGroupName;
+}
+
+// executionPolicy grants the permissions CreateOnlineEvaluationConfig validates
+// at creation time. Exported for assertion: the policy body is not observable
+// through the recorded IAM fixtures, whose responses are empty.
+//
+// at creation time: Logs Insights query access over the sampled log groups plus
+// the `aws/spans` group that carries the actual trace spans, Bedrock model
+// invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based
+// ones, and permission to write results back to CloudWatch. Modeled on the
+// policy the CDK-deployed online evaluations use, since the service rejects a
+// role that cannot query the log groups it was pointed at.
+export function executionPolicy(
+  region: string,
+  accountId: string,
+  logGroupNames: string[],
+  kmsKeyArns: string[],
+): string {
+  const logs = `arn:aws:logs:${region}:${accountId}:log-group`;
+  const spansArn = `${logs}:aws/spans`;
+  // Scope to the runtime prefix rather than the exact endpoint log group: the
+  // service validates query access at the runtime level (all of a runtime's
+  // endpoints share the `...-<runtimeId>-<endpoint>` naming), and a policy
+  // pinned to one endpoint is rejected as insufficient.
+  const sampledArns = logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`);
+  return JSON.stringify({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Sid: "DiscoverLogGroups",
+        Effect: "Allow",
+        Action: [
+          "cloudwatch:GenerateQuery",
+          "cloudwatch:GenerateQueryResultsSummary",
+          "logs:DescribeLogGroups",
+        ],
+        Resource: "*",
+      },
+      {
+        // Spans live in `aws/spans`; the runtime's own log group carries the
+        // session logs. Both are queried when sampling sessions.
+        Sid: "QuerySampledTraces",
+        Effect: "Allow",
+        Action: [
+          "logs:DescribeLogStreams",
+          "logs:FilterLogEvents",
+          "logs:GetLogEvents",
+          "logs:GetQueryResults",
+          "logs:StartQuery",
+        ],
+        Resource: [`${spansArn}*`, ...sampledArns],
+      },
+      {
+        Sid: "WriteEvaluationResults",
+        Effect: "Allow",
+        Action: [
+          "logs:CreateLogGroup",
+          "logs:CreateLogStream",
+          "logs:DescribeLogStreams",
+          "logs:PutLogEvents",
+        ],
+        Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`,
+      },
+      {
+        Sid: "IndexSpans",
+        Effect: "Allow",
+        Action: ["logs:DescribeIndexPolicies", "logs:PutIndexPolicy"],
+        Resource: spansArn,
+      },
+      {
+        Sid: "BedrockModelInvocation",
+        Effect: "Allow",
+        Action: ["bedrock:InvokeModel", "bedrock:InvokeModelWithResponseStream"],
+        Resource: [
+          "arn:aws:bedrock:*::foundation-model/*",
+          `arn:aws:bedrock:${region}:${accountId}:inference-profile/*`,
+        ],
+      },
+      {
+        // Code-based evaluators are Lambda-backed, so the role that runs an
+        // online evaluation must be able to invoke them.
+        Sid: "InvokeCodeBasedEvaluators",
+        Effect: "Allow",
+        Action: ["lambda:GetFunction", "lambda:InvokeFunction"],
+        Resource: `arn:aws:lambda:${region}:${accountId}:function:*`,
+      },
+      // Evaluators encrypted with a customer managed key need kms:Decrypt on that
+      // key, which the service validates when the config is created. Scoped to the
+      // referenced keys, and omitted when no evaluator is encrypted.
+      ...(kmsKeyArns.length > 0
+        ? [
+            {
+              Sid: "DecryptEvaluatorKeys",
+              Effect: "Allow",
+              Action: ["kms:Decrypt", "kms:DescribeKey"],
+              Resource: kmsKeyArns,
+            },
+          ]
+        : []),
+    ],
+  });
+}
+
+export function accountIdFromRoleArn(arn: string): string {
+  const accountId = arn.split(":")[4];
+  if (!accountId) {
+    throw new Error(`Cannot extract an account id from role ARN "${arn}"`);
+  }
+  return accountId;
+}
+
+// scopePolicyName derives the inline-policy name from a fingerprint of the whole
+// rendered policy document. Keying the name on the policy's exact contents means
+// any change to what the policy grants yields a new name, so writing one scope's
+// policy can never clobber another's — a superseded scope stays intact until it
+// is explicitly revoked.
+export function scopePolicyName(policyDocument: string): string {
+  const fingerprint = Bun.hash(policyDocument)
+    .toString(16)
+    .padStart(NAME_HASH_LENGTH, "0")
+    .slice(-NAME_HASH_LENGTH);
+  return `${POLICY_PREFIX}-${fingerprint}`;
+}
+
+// grantOnlineEvalScope creates the execution role for `configName` if it does not
+// exist and attaches the inline policy for this scope, returning the role ARN and
+// the policy name written. The caller revokes the superseded scope once whatever
+// change prompted the new one has succeeded.
+export async function grantOnlineEvalScope(
+  iam: IAMClient,
+  configName: string,
+  region: string,
+  logGroupNames: string[],
+  kmsKeyArns: string[] = [],
+): Promise<{ roleArn: string; policyName: string }> {
+  const roleName = onlineEvalExecutionRoleName(configName);
+
+  let roleArn: string;
+  try {
+    const existing = await iam.send(new GetRoleCommand({ RoleName: roleName }));
+    roleArn = existing.Role!.Arn!;
+  } catch (error) {
+    if ((error as Error).name !== "NoSuchEntityException") throw error;
+    const created = await iam.send(
+      new CreateRoleCommand({
+        RoleName: roleName,
+        AssumeRolePolicyDocument: trustPolicy(),
+        Description: `Default execution role for the AgentCore online evaluation config "${configName}" (created by the agentcore CLI)`,
+      }),
+    );
+    roleArn = created.Role!.Arn!;
+  }
+
+  const policyDocument = executionPolicy(
+    region,
+    accountIdFromRoleArn(roleArn),
+    logGroupNames,
+    kmsKeyArns,
+  );
+  const policyName = scopePolicyName(policyDocument);
+  await iam.send(
+    new PutRolePolicyCommand({
+      RoleName: roleName,
+      PolicyName: policyName,
+      PolicyDocument: policyDocument,
+    }),
+  );
+
+  return { roleArn, policyName };
+}
+
+// revokeOnlineEvalScope detaches a scope's inline policy, dropping the access it
+// granted. A scope that is already absent is treated as revoked.
+export async function revokeOnlineEvalScope(
+  iam: IAMClient,
+  configName: string,
+  policyName: string,
+): Promise<void> {
+  try {
+    await iam.send(
+      new DeleteRolePolicyCommand({
+        RoleName: onlineEvalExecutionRoleName(configName),
+        PolicyName: policyName,
+      }),
+    );
+  } catch (error) {
+    if ((error as Error).name !== "NoSuchEntityException") throw error;
+  }
+}

src/handlers/eval/index.tsx

@@ -3,9 +3,11 @@ import type { AppIO } from "../../io";
 import type { Core } from "../types";
 import { createHelpDefault } from "../help";
 import { createEvaluatorHandler } from "./evaluator";
+import { createOnlineEvalHandler } from "./online-eval";
 
 export function createEvalHandler(core: Core, io: AppIO): Router {
   return new Router("eval", "evaluate and optimize AgentCore agents")
     .default(createHelpDefault(io))
-    .handler(createEvaluatorHandler(core, io));
+    .handler(createEvaluatorHandler(core, io))
+    .handler(createOnlineEvalHandler(core, io));
 }

src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json

@@ -0,0 +1,14 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl",
+  "createdAt": {
+    "$date": "2026-08-03T21:29:57.112Z"
+  },
+  "status": "CREATING",
+  "executionStatus": "DISABLED",
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-2KNTGuGVDl"
+    }
+  }
+}

src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json

@@ -0,0 +1,14 @@
+{
+  "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5",
+  "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5",
+  "createdAt": {
+    "$date": "2026-08-03T21:29:51.133Z"
+  },
+  "status": "CREATING",
+  "executionStatus": "DISABLED",
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-bG4DUW3Ua5"
+    }
+  }
+}