feat: add telemetry schemas and client (#941)

Author: Hweinstock

package-lock.json

@@ -89,8 +89,11 @@
     "@aws-sdk/credential-providers": "^3.893.0",
     "@aws/agent-inspector": "0.2.1",
     "@commander-js/extra-typings": "^14.0.0",
-    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.214.0",
     "@opentelemetry/otlp-transformer": "^0.213.0",
+    "@opentelemetry/resources": "^2.6.1",
+    "@opentelemetry/sdk-metrics": "^2.6.1",
     "@smithy/shared-ini-file-loader": "^4.4.2",
     "commander": "^14.0.2",
     "dotenv": "^17.2.3",

package.json

@@ -1,5 +1,5 @@
 import { GLOBAL_CONFIG_DIR, GLOBAL_CONFIG_FILE, updateGlobalConfig } from '../../global-config.js';
-import { resolveTelemetryPreference } from '../../telemetry/resolve.js';
+import { resolveTelemetryPreference } from '../../telemetry/config.js';
 
 export async function handleTelemetryDisable(
   configDir = GLOBAL_CONFIG_DIR,

src/cli/commands/telemetry/actions.ts

@@ -0,0 +1,146 @@
+/* eslint-disable @typescript-eslint/require-await */
+import { CANCELLED, TelemetryClient } from '../client';
+import { InMemorySink } from '../sinks/in-memory-sink';
+import { describe, expect, it } from 'vitest';
+
+describe('TelemetryClient', () => {
+  describe('withCommandRun', () => {
+    it('records success with returned attrs', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await client.withCommandRun('update', async () => ({ check_only: true }));
+
+      expect(sink.metrics).toHaveLength(1);
+      expect(sink.metrics[0]!.attrs).toMatchObject({
+        command_group: 'update',
+        command: 'update',
+        exit_reason: 'success',
+        check_only: 'true',
+      });
+    });
+
+    it('accepts sync callbacks', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await client.withCommandRun('telemetry.disable', () => ({}));
+
+      expect(sink.metrics).toHaveLength(1);
+      expect(sink.metrics[0]!.attrs).toMatchObject({ exit_reason: 'success' });
+    });
+
+    it('records failure and re-throws on error', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await expect(
+        client.withCommandRun('deploy', async () => {
+          throw new Error('boom');
+        })
+      ).rejects.toThrow('boom');
+
+      expect(sink.metrics).toHaveLength(1);
+      expect(sink.metrics[0]!.attrs).toMatchObject({
+        command_group: 'deploy',
+        exit_reason: 'failure',
+        error_name: 'UnknownError',
+      });
+    });
+
+    it('classifies PackagingError subclasses', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      class MissingDependencyError extends Error {
+        constructor() {
+          super('missing dep');
+          this.name = 'MissingDependencyError';
+        }
+      }
+
+      await expect(
+        client.withCommandRun('deploy', async () => {
+          throw new MissingDependencyError();
+        })
+      ).rejects.toThrow();
+
+      expect(sink.metrics[0]!.attrs).toMatchObject({
+        error_name: 'PackagingError',
+        is_user_error: 'false',
+      });
+    });
+
+    it('marks credential errors as user errors', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      class AwsCredentialsError extends Error {
+        constructor() {
+          super('creds expired');
+          this.name = 'AwsCredentialsError';
+        }
+      }
+
+      await expect(
+        client.withCommandRun('invoke', async () => {
+          throw new AwsCredentialsError();
+        })
+      ).rejects.toThrow();
+
+      expect(sink.metrics[0]!.attrs).toMatchObject({
+        error_name: 'CredentialsError',
+        is_user_error: 'true',
+      });
+    });
+
+    it('records duration as a non-negative integer', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await client.withCommandRun('telemetry.disable', async () => {
+        await new Promise(r => globalThis.setTimeout(r, 5));
+        return {};
+      });
+
+      expect(sink.metrics[0]!.value).toBeGreaterThanOrEqual(0);
+      expect(Number.isInteger(sink.metrics[0]!.value)).toBe(true);
+    });
+
+    it('converts boolean attrs to strings', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await client.withCommandRun('update', async () => ({ check_only: true }));
+
+      expect(sink.metrics[0]!.attrs.check_only).toBe('true');
+    });
+
+    it('silently drops invalid success payloads', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      // Missing required attrs for 'create' — should silently drop
+      await client.withCommandRun(
+        'create',
+        // @ts-expect-error — intentionally incomplete
+        async () => ({ language: 'python' })
+      );
+
+      expect(sink.metrics).toHaveLength(0);
+    });
+
+    it('records cancel when callback returns CANCELLED', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await client.withCommandRun('deploy', () => CANCELLED);
+
+      expect(sink.metrics).toHaveLength(1);
+      expect(sink.metrics[0]!.attrs).toMatchObject({
+        command_group: 'deploy',
+        exit_reason: 'cancel',
+      });
+    });
+  });
+});

src/cli/telemetry/__tests__/client.test.ts

@@ -0,0 +1,60 @@
+import { InMemorySink } from '../sinks/in-memory-sink';
+import { CompositeSink, type MetricSink } from '../sinks/metric-sink';
+import { describe, expect, it, vi } from 'vitest';
+
+describe('CompositeSink', () => {
+  it('fans out records to all sinks', () => {
+    const a = new InMemorySink();
+    const b = new InMemorySink();
+    const composite = new CompositeSink([a, b]);
+
+    composite.record(100, { command: 'deploy' });
+
+    expect(a.metrics).toHaveLength(1);
+    expect(b.metrics).toHaveLength(1);
+    expect(a.metrics[0]!.attrs.command).toBe('deploy');
+  });
+
+  it('isolates errors — one sink throwing does not affect others', () => {
+    const bad: MetricSink = {
+      record: vi.fn(() => {
+        throw new Error('sink failed');
+      }),
+      flush: vi.fn().mockResolvedValue(undefined),
+      shutdown: vi.fn().mockResolvedValue(undefined),
+    };
+    const good = new InMemorySink();
+    const composite = new CompositeSink([bad, good]);
+
+    composite.record(100, { command: 'deploy' });
+
+    expect(good.metrics).toHaveLength(1);
+  });
+
+  it('flushes all sinks in parallel', async () => {
+    const a = new InMemorySink();
+    const b = new InMemorySink();
+    const flushA = vi.spyOn(a, 'flush');
+    const flushB = vi.spyOn(b, 'flush');
+    const composite = new CompositeSink([a, b]);
+
+    await composite.flush(5000);
+
+    expect(flushA).toHaveBeenCalledWith(5000);
+    expect(flushB).toHaveBeenCalledWith(5000);
+  });
+
+  it('flush settles even if one sink rejects', async () => {
+    const bad: MetricSink = {
+      record: vi.fn(),
+      flush: vi.fn().mockRejectedValue(new Error('flush failed')),
+      shutdown: vi.fn().mockResolvedValue(undefined),
+    };
+    const good = new InMemorySink();
+    const flushGood = vi.spyOn(good, 'flush');
+    const composite = new CompositeSink([bad, good]);
+
+    await expect(composite.flush()).resolves.toBeUndefined();
+    expect(flushGood).toHaveBeenCalled();
+  });
+});

src/cli/telemetry/__tests__/composite-sink.test.ts

@@ -0,0 +1,63 @@
+import { classifyError, isUserError } from '../error-classification';
+import { describe, expect, it } from 'vitest';
+
+function errorWithName(name: string): Error {
+  const err = new Error('test');
+  err.name = name;
+  return err;
+}
+
+describe('classifyError', () => {
+  it.each([
+    ['ConfigValidationError', 'ConfigError'],
+    ['ConfigNotFoundError', 'ConfigError'],
+    ['ConfigReadError', 'ConfigError'],
+    ['ConfigWriteError', 'ConfigError'],
+    ['ConfigParseError', 'ConfigError'],
+    ['AwsCredentialsError', 'CredentialsError'],
+    ['AccessDeniedException', 'CredentialsError'],
+    ['ExpiredToken', 'CredentialsError'],
+    ['PackagingError', 'PackagingError'],
+    ['MissingDependencyError', 'PackagingError'],
+    ['ArtifactSizeError', 'PackagingError'],
+    ['NoProjectError', 'ProjectError'],
+    ['AgentAlreadyExistsError', 'ProjectError'],
+    ['ResourceNotFoundException', 'ServiceError'],
+    ['ValidationException', 'ServiceError'],
+    ['ConflictException', 'ServiceError'],
+    ['ConnectionError', 'ConnectionError'],
+    ['ServerError', 'ConnectionError'],
+  ] as const)('%s → %s', (errorName, expected) => {
+    expect(classifyError(errorWithName(errorName))).toBe(expected);
+  });
+
+  it('returns UnknownError for unrecognized errors', () => {
+    expect(classifyError(new Error('something'))).toBe('UnknownError');
+  });
+
+  it('returns UnknownError for non-Error values', () => {
+    expect(classifyError('string')).toBe('UnknownError');
+    expect(classifyError(null)).toBe('UnknownError');
+    expect(classifyError(undefined)).toBe('UnknownError');
+  });
+
+  it('uses err.name when constructor.name is Error (SDK pattern)', () => {
+    // AWS SDK errors often: new Error(); err.name = 'ValidationException'
+    expect(classifyError(errorWithName('ValidationException'))).toBe('ServiceError');
+  });
+});
+
+describe('isUserError', () => {
+  it('returns true for user-fixable categories', () => {
+    expect(isUserError(errorWithName('ConfigValidationError'))).toBe(true);
+    expect(isUserError(errorWithName('AwsCredentialsError'))).toBe(true);
+    expect(isUserError(errorWithName('NoProjectError'))).toBe(true);
+  });
+
+  it('returns false for system categories', () => {
+    expect(isUserError(errorWithName('PackagingError'))).toBe(false);
+    expect(isUserError(errorWithName('ResourceNotFoundException'))).toBe(false);
+    expect(isUserError(errorWithName('ConnectionError'))).toBe(false);
+    expect(isUserError(new Error('unknown'))).toBe(false);
+  });
+});

src/cli/telemetry/__tests__/error-classification.test.ts

@@ -1,5 +1,5 @@
 import { createTempConfig } from '../../__tests__/helpers/temp-config';
-import { resolveTelemetryPreference } from '../resolve';
+import { resolveTelemetryPreference } from '../config';
 import { writeFile } from 'fs/promises';
 import { join } from 'node:path';
 import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest';

src/cli/telemetry/__tests__/resolve.test.ts

@@ -0,0 +1,50 @@
+import { resolveResourceAttributes } from '../config';
+import { ResourceAttributesSchema } from '../schemas/common-attributes';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+const ORIGINAL_ENV = process.env.AGENTCORE_CONFIG_DIR;
+
+describe('resolveResourceAttributes', () => {
+  beforeEach(() => {
+    process.env.AGENTCORE_CONFIG_DIR = '/tmp/telemetry-test-' + Date.now();
+  });
+
+  afterEach(() => {
+    if (ORIGINAL_ENV === undefined) {
+      delete process.env.AGENTCORE_CONFIG_DIR;
+    } else {
+      process.env.AGENTCORE_CONFIG_DIR = ORIGINAL_ENV;
+    }
+  });
+
+  it('returns attributes that pass schema validation', async () => {
+    const attrs = await resolveResourceAttributes('cli');
+    expect(() => ResourceAttributesSchema.parse(attrs)).not.toThrow();
+  });
+
+  it('sets service.name to agentcore-cli', async () => {
+    const attrs = await resolveResourceAttributes('cli');
+    expect(attrs['service.name']).toBe('agentcore-cli');
+  });
+
+  it('generates unique session_id per call', async () => {
+    const a = await resolveResourceAttributes('cli');
+    const b = await resolveResourceAttributes('cli');
+    expect(a['agentcore-cli.session_id']).not.toBe(b['agentcore-cli.session_id']);
+  });
+
+  it('reflects the mode parameter', async () => {
+    const cli = await resolveResourceAttributes('cli');
+    const tui = await resolveResourceAttributes('tui');
+    expect(cli['agentcore-cli.mode']).toBe('cli');
+    expect(tui['agentcore-cli.mode']).toBe('tui');
+  });
+
+  it('populates os and node fields', async () => {
+    const attrs = await resolveResourceAttributes('cli');
+    expect(attrs['os.type']).toBeTruthy();
+    expect(attrs['os.version']).toBeTruthy();
+    expect(attrs['host.arch']).toBeTruthy();
+    expect(attrs['node.version']).toMatch(/^v\d+/);
+  });
+});