feat: Add Vercel AI Gateway as a Provider

packages/app/server/src/env.ts

@@ -45,6 +45,7 @@ export const env = createEnv({
     GROQ_API_KEY: z.string().optional(),
     XAI_API_KEY: z.string().optional(),
     OPENROUTER_API_KEY: z.string().optional(),
+    VERCEL_AI_GATEWAY_API_KEY: z.string().optional(),
     TAVILY_API_KEY: z.string().optional(),
     E2B_API_KEY: z.string().optional(),
     GOOGLE_SERVICE_ACCOUNT_KEY_ENCODED: z.string().optional(),

packages/app/server/src/providers/BaseProvider.ts

@@ -13,6 +13,7 @@ export abstract class BaseProvider {
   protected readonly GEMINI_GPT_BASE_URL =
     'https://generativelanguage.googleapis.com/v1beta/openai';
   protected readonly OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
+  protected readonly VERCEL_AI_GATEWAY_BASE_URL = 'https://ai-gateway.vercel.sh/v1';
 
   private echoControlService: EchoControlService | undefined;
   private readonly isStream: boolean;

packages/app/server/src/providers/ProviderFactory.ts

@@ -22,6 +22,7 @@ import { OpenAIResponsesProvider } from './OpenAIResponsesProvider';
 import { OpenRouterProvider } from './OpenRouterProvider';
 import { ProviderType } from './ProviderType';
 import { XAIProvider } from './XAIProvider';
+import { VercelAIGatewayProvider } from './VercelAIGatewayProvider';
 import {
   VertexAIProvider,
   PROXY_PASSTHROUGH_ONLY_MODEL as VertexAIProxyPassthroughOnlyModel,
@@ -58,6 +59,9 @@ const createChatModelToProviderMapping = (): Record<string, ProviderType> => {
         case 'Xai':
           mapping[modelConfig.model_id] = ProviderType.XAI;
           break;
+        case 'Vercel':
+          mapping[modelConfig.model_id] = ProviderType.VERCEL_AI_GATEWAY;
+          break;
         // Add other providers as needed
         default:
           // Skip models with unsupported providers
@@ -192,6 +196,8 @@ export const getProvider = (
       return new GroqProvider(stream, model);
     case ProviderType.XAI:
       return new XAIProvider(stream, model);
+    case ProviderType.VERCEL_AI_GATEWAY:
+      return new VercelAIGatewayProvider(stream, model);
     default:
       throw new Error(`Unknown provider type: ${type}`);
   }

packages/app/server/src/providers/ProviderType.ts

@@ -12,4 +12,5 @@ export enum ProviderType {
   OPENAI_VIDEOS = 'OPENAI_VIDEOS',
   GROQ = 'GROQ',
   XAI = 'XAI',
+  VERCEL_AI_GATEWAY = 'VERCEL_AI_GATEWAY',
 }

packages/app/server/src/providers/VercelAIGatewayProvider.ts

@@ -0,0 +1,124 @@
+import { LlmTransactionMetadata, Transaction } from '../types';
+import { BaseProvider } from './BaseProvider';
+import { ProviderType } from './ProviderType';
+import { getCostPerToken } from '../services/AccountingService';
+import logger from '../logger';
+import { env } from '../env';
+
+interface CompletionStateBody {
+  id: string;
+  usage: {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+  };
+}
+
+interface StreamingChunkBody {
+  id: string;
+  choices: {
+    index: number;
+    delta: {
+      content?: string;
+    };
+    finish_reason: string | null;
+  }[];
+  usage: {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+  } | null;
+}
+
+const parseSSEGPTFormat = (data: string): StreamingChunkBody[] => {
+  // Split by double newlines to separate events
+  const events = data.split('\n\n');
+  const chunks: StreamingChunkBody[] = [];
+
+  for (const event of events) {
+    if (!event.trim()) continue;
+
+    // Each event should start with 'data: '
+    if (event.startsWith('data: ')) {
+      const jsonStr = event.slice(6); // Remove 'data: ' prefix
+
+      // Skip [DONE] marker
+      if (jsonStr.trim() === '[DONE]') continue;
+
+      try {
+        const parsed = JSON.parse(jsonStr);
+        chunks.push(parsed);
+      } catch (error) {
+        logger.error(`Error parsing SSE chunk: ${error}`);
+      }
+    }
+  }
+
+  return chunks;
+};
+
+export class VercelAIGatewayProvider extends BaseProvider {
+  getType(): ProviderType {
+    return ProviderType.VERCEL_AI_GATEWAY;
+  }
+
+  getBaseUrl(): string {
+    return this.VERCEL_AI_GATEWAY_BASE_URL;
+  }
+
+  getApiKey(): string | undefined {
+    return env.VERCEL_AI_GATEWAY_API_KEY;
+  }
+
+  async handleBody(data: string): Promise<Transaction> {
+    try {
+      let prompt_tokens = 0;
+      let completion_tokens = 0;
+      let total_tokens = 0;
+      let providerId = 'null';
+
+      if (this.getIsStream()) {
+        const chunks = parseSSEGPTFormat(data);
+
+        for (const chunk of chunks) {
+          if (chunk.usage && chunk.usage !== null) {
+            prompt_tokens += chunk.usage.prompt_tokens;
+            completion_tokens += chunk.usage.completion_tokens;
+            total_tokens += chunk.usage.total_tokens;
+          }
+          providerId = chunk.id || 'null';
+        }
+      } else {
+        const parsed = JSON.parse(data) as CompletionStateBody;
+        prompt_tokens += parsed.usage.prompt_tokens;
+        completion_tokens += parsed.usage.completion_tokens;
+        total_tokens += parsed.usage.total_tokens;
+        providerId = parsed.id || 'null';
+      }
+
+      const cost = getCostPerToken(
+        this.getModel(),
+        prompt_tokens,
+        completion_tokens
+      );
+
+      const metadata: LlmTransactionMetadata = {
+        providerId: providerId,
+        provider: this.getType(),
+        model: this.getModel(),
+        inputTokens: prompt_tokens,
+        outputTokens: completion_tokens,
+        totalTokens: total_tokens,
+      };
+
+      return {
+        metadata: metadata,
+        rawTransactionCost: cost,
+        status: 'success',
+      };
+    } catch (error) {
+      logger.error(`Error processing data: ${error}`);
+      throw error;
+    }
+  }
+}

packages/app/server/src/services/AccountingService.ts

@@ -10,6 +10,7 @@ import {
   SupportedImageModel,
   SupportedVideoModel,
   XAIModels,
+  VercelAIGatewayModels,
 } from '@merit-systems/echo-typescript-sdk';
 
 import { Decimal } from '@prisma/client/runtime/library';
@@ -30,6 +31,7 @@ export const ALL_SUPPORTED_MODELS: SupportedModel[] = [
   ...OpenRouterModels,
   ...GroqModels,
   ...XAIModels,
+  ...VercelAIGatewayModels,
 ];
 
 // Handle image models separately since they have different pricing structure

packages/sdk/ts/src/index.ts

@@ -49,6 +49,8 @@ export { GroqModels } from './supported-models/chat/groq';
 export type { GroqModel } from './supported-models/chat/groq';
 export { XAIModels } from './supported-models/chat/xai';
 export type { XAIModel } from './supported-models/chat/xai';
+export { VercelAIGatewayModels } from './supported-models/chat/vercel-ai-gateway';
+export type { VercelAIGatewayModel } from './supported-models/chat/vercel-ai-gateway';
 export { OpenAIImageModels } from './supported-models/image/openai';
 export type { OpenAIImageModel } from './supported-models/image/openai';
 export { GeminiVideoModels } from './supported-models/video/gemini';

packages/sdk/ts/src/supported-models/chat/vercel-ai-gateway.ts

@@ -0,0 +1,140 @@
+import { SupportedModel } from '../types';
+
+// Union type of all valid Vercel AI Gateway model IDs
+// The Vercel AI Gateway provides access to models from multiple providers
+// through a single OpenAI-compatible API endpoint
+export type VercelAIGatewayModel =
+  | 'gpt-4o'
+  | 'gpt-4o-mini'
+  | 'gpt-4-turbo'
+  | 'gpt-4'
+  | 'gpt-3.5-turbo'
+  | 'claude-3-opus-20240229'
+  | 'claude-3-sonnet-20240229'
+  | 'claude-3-haiku-20240307'
+  | 'claude-3-5-sonnet-20240620'
+  | 'gemini-1.5-pro'
+  | 'gemini-1.5-flash'
+  | 'gemini-1.0-pro'
+  | 'mistral-large-latest'
+  | 'mistral-medium-latest'
+  | 'mistral-small-latest'
+  | 'llama-3.1-405b-instruct'
+  | 'llama-3.1-70b-instruct'
+  | 'llama-3.1-8b-instruct';
+
+export const VercelAIGatewayModels: SupportedModel[] = [
+  // OpenAI models via Vercel AI Gateway
+  {
+    model_id: 'gpt-4o',
+    input_cost_per_token: 0.000005,
+    output_cost_per_token: 0.000015,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'gpt-4o-mini',
+    input_cost_per_token: 1.5e-7,
+    output_cost_per_token: 6e-7,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'gpt-4-turbo',
+    input_cost_per_token: 0.00001,
+    output_cost_per_token: 0.00003,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'gpt-4',
+    input_cost_per_token: 0.00003,
+    output_cost_per_token: 0.00006,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'gpt-3.5-turbo',
+    input_cost_per_token: 5e-7,
+    output_cost_per_token: 0.0000015,
+    provider: 'Vercel',
+  },
+  // Anthropic models via Vercel AI Gateway
+  {
+    model_id: 'claude-3-opus-20240229',
+    input_cost_per_token: 0.000015,
+    output_cost_per_token: 0.000075,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'claude-3-sonnet-20240229',
+    input_cost_per_token: 0.000003,
+    output_cost_per_token: 0.000015,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'claude-3-haiku-20240307',
+    input_cost_per_token: 2.5e-7,
+    output_cost_per_token: 0.00000125,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'claude-3-5-sonnet-20240620',
+    input_cost_per_token: 0.000003,
+    output_cost_per_token: 0.000015,
+    provider: 'Vercel',
+  },
+  // Google models via Vercel AI Gateway
+  {
+    model_id: 'gemini-1.5-pro',
+    input_cost_per_token: 0.0000035,
+    output_cost_per_token: 0.0000105,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'gemini-1.5-flash',
+    input_cost_per_token: 3.5e-7,
+    output_cost_per_token: 0.00000105,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'gemini-1.0-pro',
+    input_cost_per_token: 5e-7,
+    output_cost_per_token: 0.0000015,
+    provider: 'Vercel',
+  },
+  // Mistral models via Vercel AI Gateway
+  {
+    model_id: 'mistral-large-latest',
+    input_cost_per_token: 0.000004,
+    output_cost_per_token: 0.000012,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'mistral-medium-latest',
+    input_cost_per_token: 0.0000027,
+    output_cost_per_token: 0.0000081,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'mistral-small-latest',
+    input_cost_per_token: 0.000001,
+    output_cost_per_token: 0.000003,
+    provider: 'Vercel',
+  },
+  // Meta Llama models via Vercel AI Gateway
+  {
+    model_id: 'llama-3.1-405b-instruct',
+    input_cost_per_token: 0.000005,
+    output_cost_per_token: 0.000016,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'llama-3.1-70b-instruct',
+    input_cost_per_token: 9e-7,
+    output_cost_per_token: 0.0000009,
+    provider: 'Vercel',
+  },
+  {
+    model_id: 'llama-3.1-8b-instruct',
+    input_cost_per_token: 2e-7,
+    output_cost_per_token: 2e-7,
+    provider: 'Vercel',
+  },
+];