diff --git a/.changeset/huge-pens-lay.md b/.changeset/huge-pens-lay.md new file mode 100644 index 000000000..4029ce577 --- /dev/null +++ b/.changeset/huge-pens-lay.md @@ -0,0 +1,5 @@ +--- +'@walkeros/collector': minor +--- + +code destination diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index 04e2aebc9..000000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(git mv:*)", - "Bash(cp:*)", - "Bash(ls:*)", - "Bash(grep:*)", - "Bash(mkdir:*)", - "Bash(rm:*)", - "Bash(mv:*)", - "Bash(find:*)", - "Bash(rg:*)", - "Bash(git rm:*)", - "Bash(npm run build:*)", - "Bash(node:*)", - "Bash(npm install)", - "Bash(npm run lint)", - "Bash(npm run clean:*)", - "Bash(npm run test:*)", - "Bash(npm test)", - "Bash(for:*)", - "Bash(do sed -i 's/Walkerjs/webCollector/g' \"$file\")", - "Bash(done)", - "Bash(npm start)", - "Bash(npm run lint:*)", - "WebSearch", - "Bash(npm test:*)" - ], - "deny": [] - } -} diff --git a/.claude/skills/create-destination/SKILL.md b/.claude/skills/create-destination/SKILL.md new file mode 100644 index 000000000..4d3a0320a --- /dev/null +++ b/.claude/skills/create-destination/SKILL.md @@ -0,0 +1,15 @@ +--- +name: create-destination +description: + Use when creating a new walkerOS destination (web or server). Step-by-step + workflow from research to documentation. (project) +--- + +# Create a New Destination + +The actual content is maintained in: + +Read @skills/create-destination/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/create-source/SKILL.md b/.claude/skills/create-source/SKILL.md new file mode 100644 index 000000000..b71419186 --- /dev/null +++ b/.claude/skills/create-source/SKILL.md @@ -0,0 +1,15 @@ +--- +name: create-source +description: + Use when creating a new walkerOS source (web or server). Step-by-step workflow + for capturing events from new platforms. (project) +--- + +# Create a New Source + +The actual content is maintained in: + +Read @skills/create-source/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/debugging/SKILL.md b/.claude/skills/debugging/SKILL.md new file mode 100644 index 000000000..efb2aa704 --- /dev/null +++ b/.claude/skills/debugging/SKILL.md @@ -0,0 +1,16 @@ +--- +name: debugging +description: + Use when events aren't reaching destinations, debugging event flow, or + troubleshooting mapping issues. Covers common problems and debugging + strategies. (project) +--- + +# Debugging walkerOS Events + +The actual content is maintained in: + +Read @skills/debugging/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/mapping-configuration/SKILL.md b/.claude/skills/mapping-configuration/SKILL.md new file mode 100644 index 000000000..3806c969f --- /dev/null +++ b/.claude/skills/mapping-configuration/SKILL.md @@ -0,0 +1,15 @@ +--- +name: mapping-configuration +description: + Use when configuring event mappings for specific use cases. Provides recipes + for GA4, Meta, custom APIs, and common transformation patterns. (project) +--- + +# Mapping Configuration Recipes + +The actual content is maintained in: + +Read @skills/mapping-configuration/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/testing-strategy/SKILL.md b/.claude/skills/testing-strategy/SKILL.md new file mode 100644 index 000000000..69e068c26 --- /dev/null +++ b/.claude/skills/testing-strategy/SKILL.md @@ -0,0 +1,16 @@ +--- +name: testing-strategy +description: + Use when writing tests, reviewing test code, or discussing testing approach + for walkerOS packages. Covers env pattern, dev examples, and package-specific + strategies. (project) +--- + +# walkerOS Testing Strategy + +The actual testing strategy is maintained in: + +Read @skills/testing-strategy/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/understanding-destinations/SKILL.md b/.claude/skills/understanding-destinations/SKILL.md new file mode 100644 index 000000000..34edfd8a9 --- /dev/null +++ b/.claude/skills/understanding-destinations/SKILL.md @@ -0,0 +1,16 @@ +--- +name: understanding-destinations +description: + Use when working with destinations, understanding the destination interface, + or learning about env pattern and configuration. Covers interface, lifecycle, + env mocking, and paths. (project) +--- + +# Understanding walkerOS Destinations + +The actual content is maintained in: + +Read @skills/understanding-destinations/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/understanding-development/SKILL.md b/.claude/skills/understanding-development/SKILL.md new file mode 100644 index 000000000..827a94fe7 --- /dev/null +++ b/.claude/skills/understanding-development/SKILL.md @@ -0,0 +1,16 @@ +--- +name: understanding-development +description: + Use when contributing to walkerOS, before writing code, or when unsure about + project conventions. Covers build/test/lint workflow, XP principles, folder + structure, and package usage. (project) +--- + +# Understanding walkerOS Development + +The actual content is maintained in: + +Read @skills/understanding-development/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/understanding-events/SKILL.md b/.claude/skills/understanding-events/SKILL.md new file mode 100644 index 000000000..f917fb058 --- /dev/null +++ b/.claude/skills/understanding-events/SKILL.md @@ -0,0 +1,16 @@ +--- +name: understanding-events +description: + Use when creating events, understanding event structure, or working with event + properties. Covers entity-action naming, event properties, statelessness, and + vendor-agnostic design. (project) +--- + +# Understanding walkerOS Events + +The actual content is maintained in: + +Read @skills/understanding-events/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/understanding-flow/SKILL.md b/.claude/skills/understanding-flow/SKILL.md new file mode 100644 index 000000000..7f876fd64 --- /dev/null +++ b/.claude/skills/understanding-flow/SKILL.md @@ -0,0 +1,16 @@ +--- +name: understanding-flow +description: + Use when learning walkerOS architecture, understanding data flow, or designing + composable event pipelines. Covers Source→Collector→Destination pattern and + separation of concerns. (project) +--- + +# Understanding walkerOS Flow + +The actual content is maintained in: + +Read @skills/understanding-flow/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/understanding-mapping/SKILL.md b/.claude/skills/understanding-mapping/SKILL.md new file mode 100644 index 000000000..dfdab30ee --- /dev/null +++ b/.claude/skills/understanding-mapping/SKILL.md @@ -0,0 +1,16 @@ +--- +name: understanding-mapping +description: + Use when transforming events at any point in the flow (source→collector or + collector→destination), configuring data/map/loop/condition, or understanding + value extraction. Covers all mapping strategies. (project) +--- + +# Understanding walkerOS Mapping + +The actual content is maintained in: + +Read @skills/understanding-mapping/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/understanding-sources/SKILL.md b/.claude/skills/understanding-sources/SKILL.md new file mode 100644 index 000000000..654675bc8 --- /dev/null +++ b/.claude/skills/understanding-sources/SKILL.md @@ -0,0 +1,16 @@ +--- +name: understanding-sources +description: + Use when working with sources, understanding event capture, or learning about + the push interface. Covers browser, dataLayer, and server source patterns. + (project) +--- + +# Understanding walkerOS Sources + +The actual content is maintained in: + +Read @skills/understanding-sources/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/using-logger/SKILL.md b/.claude/skills/using-logger/SKILL.md new file mode 100644 index 000000000..ecfdebb44 --- /dev/null +++ b/.claude/skills/using-logger/SKILL.md @@ -0,0 +1,16 @@ +--- +name: using-logger +description: + Use when working with sources/destinations to understand standard logging + patterns, replace console.log, or add logging to external API calls. Covers + DRY principles, when to log, and migration patterns. +--- + +# Using the walkerOS Logger + +The actual content is maintained in: + +Read @skills/using-logger/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.claude/skills/writing-documentation/SKILL.md b/.claude/skills/writing-documentation/SKILL.md new file mode 100644 index 000000000..361a4b34b --- /dev/null +++ b/.claude/skills/writing-documentation/SKILL.md @@ -0,0 +1,16 @@ +--- +name: writing-documentation +description: + Use when writing or updating any documentation - README, website docs, or + skills. Covers quality standards, example validation, and DRY patterns. + (project) +--- + +# Writing Documentation + +The actual content is maintained in: + +Read @skills/writing-documentation/SKILL.md + +This reference ensures Claude Code can discover this skill while maintaining the +primary content in the tool-agnostic location. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cd8870025..f2459c46c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,10 +6,14 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, + + // Persist Claude Code data across container rebuilds (uses existing .claude folder in repo) + "containerEnv": { + "CLAUDE_CONFIG_DIR": "/workspaces/walkerOS/.claude/data" + }, "customizations": { "vscode": { "extensions": [ - "unifiedjs.vscode-mdx", "esbenp.prettier-vscode", "streetsidesoftware.code-spell-checker", "chakrounanas.turbo-console-log", @@ -35,8 +39,11 @@ // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], + // Resource limits for the container + "runArgs": ["--memory=8g", "--memory-swap=10g", "--cpus=4"], + // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "npm install && npm install -g @anthropic-ai/claude-code @openai/codex && npm run build" + "postCreateCommand": "npm install && npm install -g @anthropic-ai/claude-code && npm run build" // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cac33514a..de8bf86a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,3 +48,5 @@ jobs: packages/*/.tmp/ retention-days: 7 - run: npm run lint + - name: Validate documentation + run: npm run validate:links diff --git a/.gitignore b/.gitignore index 9d76059dc..993822720 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,9 @@ typings/ # Turbobuild .turbo +# Claude Code data (conversations, cache) +.claude/data/ + .DS_Store .npm-cache .tmp diff --git a/AGENT.md b/AGENT.md index a4d508891..167d54da1 100644 --- a/AGENT.md +++ b/AGENT.md @@ -1,543 +1,105 @@ -# CLAUDE.md +# AGENT.md - walkerOS Developer Hub -This file provides guidance to Claude Code (claude.ai/code) when working with -code in this repository. +> walkerOS is a privacy-first event data collection and tag management solution. +> Architecture: **Source → Collector → Destination(s)** -## Quick Start Commands - -### Development - -```bash -# Install dependencies -npm install - -# Run development mode for all packages -npm run dev - -# Run development for specific package -cd packages/[package-name] && npm run dev - -# Start website documentation locally -cd website && npm start -``` - -### Building +## Quick Start ```bash -# Build all packages -npm run build - -# Build specific package -cd packages/[package-name] && npm run build +npm install # Install dependencies +npm run dev # Watch mode +npm run build # Build all packages +npm run test # Run tests +npm run lint # Check code quality ``` -### Testing - -```bash -# Run all tests -npm run test - -# Run tests for specific package -cd packages/[package-name] && npm run test - -# Watch mode for development -cd packages/[package-name] && npm run dev -``` +## Understanding walkerOS -### Code Quality +Learn the concepts before coding: -```bash -# Run linting across all packages -npm run lint - -# Format code -npm run format - -# Clean all build artifacts -npm run clean -``` - -### Publishing - -```bash -# Publish packages (includes build, lint, test) -npm run publish-packages -``` - -## Architecture Overview - -walkerOS is a **privacy-first event data collection and tag management solution -as code**. It follows a **Source → Collector → Destination(s)** architecture -pattern for event processing. - -### Core Data Flow - -``` -Sources → Collector → Destinations -(Data Creation) (Processing) (Third-party Tools) -``` +| Skill | What You'll Learn | +| ------------------------------------------------------------------------ | ----------------------------------------------- | +| [understanding-development](skills/understanding-development/SKILL.md) | Build workflow, XP principles, folder structure | +| [understanding-flow](skills/understanding-flow/SKILL.md) | Architecture, composability, data flow | +| [understanding-events](skills/understanding-events/SKILL.md) | Event model, entity-action naming, properties | +| [understanding-mapping](skills/understanding-mapping/SKILL.md) | Event transformation, data/map/loop | +| [understanding-destinations](skills/understanding-destinations/SKILL.md) | Destination interface, env pattern | +| [understanding-sources](skills/understanding-sources/SKILL.md) | Source interface, capture patterns | +| [using-logger](skills/using-logger/SKILL.md) | Logger access, DRY principles, when to log | -### Key Components +## Creating Things -- **Sources**: Capture events from various sources (browser DOM, dataLayer, - server) -- **Collector**: Central event processing engine with consent management -- **Destinations**: Transform and send events to analytics/marketing tools -- **Mapping System**: Flexible event transformation and routing +| Task | Skill | +| ------------------- | -------------------------------------------------------------- | +| Write tests | [testing-strategy](skills/testing-strategy/SKILL.md) | +| Create destination | [create-destination](skills/create-destination/SKILL.md) | +| Create source | [create-source](skills/create-source/SKILL.md) | +| Configure mappings | [mapping-configuration](skills/mapping-configuration/SKILL.md) | +| Debug event flow | [debugging](skills/debugging/SKILL.md) | +| Write documentation | [writing-documentation](skills/writing-documentation/SKILL.md) | -### Package Structure +## Package Navigation -``` +```text packages/ -├── core/ # Platform-agnostic types and utilities -├── collector/ # Central event collection and processing -├── config/ # Shared configuration (eslint, jest, tsconfig, tsup) -├── web/ -│ ├── core/ # Web-specific utilities -│ ├── sources/ # Data sources (browser DOM, dataLayer) -│ └── destinations/ # Web destinations (gtag, meta, api, piwikpro, plausible) -└── server/ - ├── core/ # Server-specific utilities - ├── sources/ # Server sources (gcp) - └── destinations/ # Server destinations (aws, gcp, meta) -``` - -### Applications - -- **apps/walkerjs**: Ready-to-use browser bundle -- **apps/quickstart**: Code examples and getting started templates -- **website**: Documentation site built with Docusaurus - -## Critical Event Model Rules - -### 1. Entity-Action Event Naming - -**STRICT REQUIREMENT**: All events MUST follow the "ENTITY ACTION" format with -space separation: - -- ✅ Correct: `"page view"`, `"product add"`, `"order complete"`, - `"button click"` -- ❌ Wrong: `"page_view"`, `"purchase"`, `"add_to_cart"`, `"pageview"` - -The event name is parsed as: `const [entity, action] = event.split(' ')` - -### 2. Universal Push Interface Standard - -**CRITICAL**: All walkerOS components communicate via `push` functions: - -- **Sources**: `source.push()` - Interface to external world (HTTP, DOM events, - etc.) -- **Collector**: `collector.push()` - Central event processing -- **Destinations**: `destination.push()` - Receive processed events -- **ELB**: `elb()` - Alias for collector.push, used for component wiring - -**Source Push Signatures by Type**: +├── core/ # Types, utilities, schemas (@walkeros/core) +├── collector/ # Event processing engine +├── config/ # Shared tooling config +├── web/ # Browser: sources/, destinations/ +└── server/ # Node.js: sources/, destinations/ -- Cloud Functions: `push(req, res) => Promise` (HTTP handler) -- Browser: `push(event, data) => Promise` (Walker events) -- DataLayer: `push(event, data) => Promise` (Walker events) - -**Key Principle**: Source `push` IS the handler - no wrappers needed. Example: -`http('handler', source.push)` for direct deployment. - -### 3. Event Structure - -All events follow this consistent structure: - -```typescript -{ - name: 'product view', // ENTITY ACTION format - data: { // Entity-specific properties - id: 'P123', - name: 'Laptop', - price: 999 - }, - context: { // State/environment information - stage: ['shopping', 1], - test: ['variant-A', 0] - }, - globals: { // Global properties - language: 'en', - currency: 'USD' - }, - user: { // User identification - id: 'user123', - device: 'device456' - }, - nested: [ // Related entities - { type: 'category', data: { name: 'Electronics' } } - ], - consent: { functional: true, marketing: true }, - // System-generated fields: - id: '1647261462000-01b5e2-2', - timestamp: 1647261462000, - entity: 'product', - action: 'view' -} +apps/ +├── quickstart/ # Validated examples (source of truth) +├── walkerjs/ # Browser bundle +└── demos/ # Demo applications ``` -## Mapping System Architecture +## Key Files -### Core Mapping Functions +| What | Where | +| --------------------- | -------------------------------------------------------------------------------- | +| Event types | [packages/core/src/types/event.ts](packages/core/src/types/event.ts) | +| Mapping functions | [packages/core/src/mapping.ts](packages/core/src/mapping.ts) | +| Flow type | [packages/collector/src/types/flow.ts](packages/collector/src/types/flow.ts) | +| Destination interface | [packages/core/src/types/destination.ts](packages/core/src/types/destination.ts) | +| Source interface | [packages/core/src/types/source.ts](packages/core/src/types/source.ts) | +| Validated examples | [apps/quickstart/](apps/quickstart/) | -From `/workspaces/walkerOS/packages/core/src/mapping.ts` +## Non-Negotiables -- **`getMappingEvent(event, mappingRules)`**: Finds the appropriate mapping - configuration for an event -- **`getMappingValue(value, mappingConfig)`**: Transforms values using flexible - mapping strategies +- **Event naming:** `"entity action"` format with space (`"page view"`, not + `"page_view"`) +- **No `any`:** Never in production code +- **XP principles:** DRY, KISS, YAGNI, TDD +- **Test first:** Watch it fail before implementing +- **Verify:** Run tests before claiming complete -### Event Mapping Patterns +## Creating/Maintaining Skills -```typescript -const mapping = { - // Exact entity-action match - product: { - view: { name: 'view_item' }, - add: { name: 'add_to_cart' }, - }, - - // Wildcard patterns - product: { - '*': { name: 'product_interaction' }, // Matches any action - }, - '*': { - click: { name: 'generic_click' }, // Matches any entity - }, - - // Conditional mappings - order: { - complete: [ - { - condition: (event) => event.data?.value > 100, - name: 'high_value_purchase', - }, - { name: 'purchase' }, // Fallback - ], - }, -}; -``` +Skills live in two locations that must stay in sync: -### Value Mapping Strategies +| Location | Purpose | +| -------------------------------- | ----------------------------------------- | +| `skills/[name]/SKILL.md` | Primary content (tool-agnostic) | +| `.claude/skills/[name]/SKILL.md` | Claude Code reference (points to primary) | -```typescript -// String key mapping -'user.id' // Extracts nested property +**To create a new skill:** -// Static values -{ value: 'USD' } +1. Create primary content: `skills/[name]/SKILL.md` +2. Create Claude reference: `.claude/skills/[name]/SKILL.md` with: -// Custom functions -{ fn: (event) => event.user.email.split('@')[1] } +```markdown +--- +name: [name] +description: [When to use - shown in Claude Code skill list] +--- -// Object transformation -{ - map: { - item_id: 'data.id', - item_name: 'data.name', - currency: { value: 'USD' } - } -} +# [Title] -// Array processing -{ - loop: [ - 'nested', // Source array - { map: { item_id: 'data.id' } } // Transform each item - ] -} +The actual content is maintained in: -// Consent-based mapping -{ - key: 'user.email', - consent: { marketing: true } // Only return if consent granted -} +Read @skills/[name]/SKILL.md ``` -## Destination Interface - -### Standard Destination Structure - -```typescript -interface Destination { - type?: string; - init?: (context: Context) => Promise; - push: (event: WalkerOS.Event, context: PushContext) => Promise; - pushBatch?: (batch: Batch, context: PushBatchContext) => void; - config: { - settings?: Settings; // Destination-specific config - mapping?: MappingRules; // Event transformation rules - data?: MappingValue; // Global data mapping - consent?: Consent; // Required consent states - policy?: Policy; // Processing rules - queue?: boolean; // Event queuing - dryRun?: boolean; // Test mode - }; -} -``` - -### Validated Destination Example - -From `apps/quickstart/src/web-destinations/ga4-complete.ts` (working, tested -code): - -```typescript -export async function setupGA4Complete(): Promise<{ - collector: Collector.Instance; - elb: WalkerOS.Elb; -}> { - const { collector, elb } = await startFlow({ - destinations: { - gtag: { - ...destinationGtag, - config: { - settings: { - ga4: { measurementId: 'G-XXXXXXXXXX' }, - }, - mapping: { - product: { - view: { - name: 'view_item', - data: { - map: { - currency: { value: 'USD' }, - value: 'data.price', - }, - }, - }, - }, - }, - }, - }, - }, - }); - return { collector, elb }; -} -``` - -## Elb Function Interfaces - -### Collector Elb Interface - -The collector provides the core `WalkerOS.Elb` interface (from -`@walkeros/core`): - -```typescript -// From apps/quickstart/src/collector/basic.ts (validated example) -export async function trackPageView(elb: WalkerOS.Elb): Promise { - await elb('page view', { - title: 'Home Page', - path: '/', - }); -} - -export async function trackUserAction(elb: WalkerOS.Elb): Promise { - await elb('button click', { - id: 'cta-button', - text: 'Get Started', - }); -} -``` - -### Browser Source Elb Interface - -The browser source extends the collector interface with `BrowserPush` (from -`@walkeros/web-source-browser/types/elb.ts`): - -- **Additional Commands**: `walker init` for DOM scoping -- **Flexible Arguments**: More permissive argument patterns -- **Browser-Specific Types**: Different destination and context types - -**Key Distinction**: Browser sources return their own elb interface that extends -but differs from the collector's elb. - -## Development Rules - -### 0. Feature Development Protocol - -**CRITICAL**: Do NOT add unwanted or non-discussed features, code, or -functionality. - -- Plan and discuss before developing any new features -- Only implement explicitly requested functionality -- Ideas and suggestions are welcome AFTER iterations, not during implementation -- Focus on the specific task at hand - avoid scope creep - -### 1. TypeScript Strictness - -- **NEVER use `any` type** - always use proper TypeScript types -- Use strict type definitions: - - `@walkeros/core` for shared types - - Package-specific types from individual packages - - Internal types for package-specific functionality - -### 2. Event Naming Compliance - -- Always use space-separated "ENTITY ACTION" format -- Entity should be a noun (page, product, user, order) -- Action should be a verb (view, click, login, complete) - -### 3. Mapping Validation - -- Reference validated examples from - `apps/quickstart/src/mappings/custom-functions.ts` -- Use specific mappings over wildcards when possible -- Implement proper consent checking in sensitive mappings - -### 4. Test-Driven Documentation - -**CRITICAL**: Before documenting code patterns, create functional, executable -tests to validate them. - -- Use `/workspaces/walkerOS/apps/quickstart` as the source of truth for working - examples -- Reference existing tests for validated code patterns -- Follow DRY principle - avoid repeating information - -### 5. Development Guidelines - -- **No Backward Compatibility Code**: Use migrations and documentation for - breaking changes -- **Clean Git History**: No inline comments about changes - Git tracks history -- **Rewrite When Needed**: It's OK to refactor/rewrite - just discuss first -- **Build System**: Uses `tsup` with multiple output formats (CJS, ESM, browser - bundles) -- **Testing**: Jest with environment-specific configurations (jsdom/node) -- **Orchestration**: Turborepo handles parallel builds and dependency ordering - -## Monorepo Workflow - -1. **Root-level commands**: Use Turborepo for parallel operations across - packages -2. **Package-specific work**: Navigate to individual packages for focused - development -3. **Dependency management**: Changes in core packages affect multiple consumers -4. **Testing strategy**: Test changes across all affected packages -5. **Version coordination**: All packages are versioned and published together - -## Critical Integration Points - -- **Collector is central**: All event flow goes through the collector package -- **Shared types**: Web and server packages use the same core event model -- **Mapping system**: Critical for destination integration and data - transformation -- **Consent management**: Built into the core architecture, not an afterthought -- **Source flexibility**: Multiple sources can feed into the same collector - instance - -## Consent Handling - -Consent is OPTIONAL and configurable at multiple levels: - -```typescript -// 1. Destination-level -config: { - consent: { marketing: true } -} - -// 2. Event mapping level -mapping: { - user: { - login: { - consent: { functional: true }, - name: 'user_login' - } - } -} - -// 3. Value mapping level -data: { - map: { - email: { - key: 'user.email', - consent: { marketing: true } - } - } -} - -// 4. Policy level -config: { - policy: { - 'consent.marketing': true // Process only if marketing consent - } -} -``` - -Without consent requirements, events process normally. With requirements, events -queue until consent is granted. - -## Testing Strategy - -**Component-Level Integration Tests**: Test each component by mocking external -APIs and using the wrap utility: - -- **Destination Testing**: Mock external APIs (gtag, fbq, etc.), verify they're - called correctly -- **Collector Testing**: Mock destinations and sources, test event processing -- **Source Testing**: Mock collector interface, test event capture and - transformation - -Example patterns: - -```typescript -// Test destination by mocking external API -it('sends correct data to gtag', async () => { - const mockGtag = jest.fn(); - global.gtag = mockGtag; - - const destination = createDestination(config); - const { wrap } = context; - - // Use wrap to intercept calls - const wrappedPush = wrap(destination.push); - await wrappedPush(mockEvent, context); - - // Verify external API was called correctly - expect(mockGtag).toHaveBeenCalledWith('event', 'view_item', { - item_id: 'P123', - value: 99.99, - }); -}); - -// Test collector with mocked boundaries -it('processes events correctly', async () => { - const mockDestination = { push: jest.fn() }; - const collector = startFlow({ - destinations: { test: mockDestination }, - }); - await collector.push('page view', {}); - expect(mockDestination.push).toHaveBeenCalledWith( - expect.objectContaining({ name: 'page view' }), - expect.any(Object), - ); -}); -``` - -Focus on verifying external API calls, not return values. - -## Code Principles - -- **Use Core Functions**: Always leverage existing utilities from - `@walkeros/core`: - - `getEvent()`, `createEvent()` for event creation - - `getMappingEvent()`, `getMappingValue()` for transformations - - `isString()`, `isObject()`, `isDefined()` for type checking - - `assign()`, `clone()` for object operations - - `tryCatch()`, `tryCatchAsync()` for error handling -- **Smart Abstractions**: Use Higher-Order Functions (HOF) and avoid redundancy -- **Clean Code**: Keep implementations lean and performant -- **No Inline Change Comments**: Git tracks changes - avoid comments like "// - Fixed for version X" or "// Changed due to Y" -- **Forward-Looking**: OK to rewrite code - use migrations/docs for breaking - changes instead of maintaining backward compatibility in code -- **Type Safety**: Use TypeScript strictly without runtime overhead - -## Common Pitfalls - -- **Event Naming**: Must use space separator: `"page view"` not `"page_view"` -- **Consent**: Not required by default - only when explicitly configured -- **Testing Scope**: Test components individually, not entire chains -- **Core Functions**: Use existing utilities instead of reimplementing -- **Type Imports**: Use `import type` for type-only imports -- **Mock External APIs**: Test destinations by mocking gtag, fbq, etc., not by - checking returns +3. Add to AGENT.md tables above (Understanding or Creating section) +4. Add to [skills/README.md](skills/README.md) index diff --git a/README.md b/README.md index 12b62d0b6..fa9fb26a0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- - + +

@@ -286,30 +286,30 @@ analytics platforms, marketing tools, and data warehouses. #### Web Destinations -- **[API](https://www.elbwalker.com/docs/destinations/web/api)** - Send events - to your own endpoints -- **[Google (gtag)](https://www.elbwalker.com/docs/destinations/web/gtag/)** - +- **[API](https://www.walkeros.io/docs/destinations/web/api)** - Send events to + your own endpoints +- **[Google (gtag)](https://www.walkeros.io/docs/destinations/web/gtag/)** - GA4, Google Ads, and GTM integration -- **[Meta Pixel](https://www.elbwalker.com/docs/destinations/web/meta-pixel)** - +- **[Meta Pixel](https://www.walkeros.io/docs/destinations/web/meta-pixel)** - Facebook and Instagram advertising -- **[Plausible Analytics](https://www.elbwalker.com/docs/destinations/web/plausible)** - +- **[Plausible Analytics](https://www.walkeros.io/docs/destinations/web/plausible)** - Privacy-focused web analytics -- **[Piwik PRO](https://www.elbwalker.com/docs/destinations/web/piwikpro)** - +- **[Piwik PRO](https://www.walkeros.io/docs/destinations/web/piwikpro)** - Privacy-focused analytics platform #### Server Destinations -- **[AWS Firehose](https://www.elbwalker.com/docs/destinations/server/aws)** - +- **[AWS Firehose](https://www.walkeros.io/docs/destinations/server/aws)** - Amazon cloud services integration -- **[GCP BigQuery](https://www.elbwalker.com/docs/destinations/server/gcp)** - - GCP services and BigQuery -- **[Meta Conversions API](https://www.elbwalker.com/docs/destinations/server/meta-capi)** - +- **[GCP BigQuery](https://www.walkeros.io/docs/destinations/server/gcp)** - GCP + services and BigQuery +- **[Meta Conversions API](https://www.walkeros.io/docs/destinations/server/meta-capi)** - Server-side Facebook/Instagram tracking ## Contributing ⭐️ Help us grow and star us. See our -[Contributing Guidelines](https://www.elbwalker.com/docs/contributing) to get +[Contributing Guidelines](https://www.walkeros.io/docs/contributing) to get involved. ## Support diff --git a/apps/demos/destination/README.md b/apps/demos/destination/README.md index db2f280e1..8a6dcc05c 100644 --- a/apps/demos/destination/README.md +++ b/apps/demos/destination/README.md @@ -1,6 +1,6 @@ diff --git a/apps/demos/flows/.gitignore b/apps/demos/flows/.gitignore new file mode 100644 index 000000000..0441d73e6 --- /dev/null +++ b/apps/demos/flows/.gitignore @@ -0,0 +1,3 @@ +dist/ +shared/credentials/*.json +!shared/credentials/.gitkeep diff --git a/apps/demos/flows/README.md b/apps/demos/flows/README.md new file mode 100644 index 000000000..6f099aedb --- /dev/null +++ b/apps/demos/flows/README.md @@ -0,0 +1,38 @@ +# Browser to BigQuery Demo + +Complete event pipeline: Browser → Server → BigQuery + +## Setup + +1. Add credentials: + + ```bash + cp ~/sa-bigquery.json shared/credentials/sa-bigquery.json + ``` + +2. Bundle flows: + + ```bash + walkeros bundle browser-to-bigquery.json --flow server + walkeros bundle browser-to-bigquery.json --flow web + ``` + +3. Run server (Terminal 1): + + ```bash + walkeros run collect dist/bundle.mjs --port 8080 + ``` + +4. Serve web (Terminal 2): + + ```bash + npx serve . -p 3000 + ``` + +5. Open http://localhost:3000 and click buttons + +## Verify in BigQuery + +```sql +SELECT * FROM walkerOS.events ORDER BY timestamp DESC LIMIT 10 +``` diff --git a/apps/demos/flows/browser-to-bigquery.json b/apps/demos/flows/browser-to-bigquery.json new file mode 100644 index 000000000..065084a89 --- /dev/null +++ b/apps/demos/flows/browser-to-bigquery.json @@ -0,0 +1,108 @@ +{ + "version": 1, + "variables": { + "COLLECT_URL": "http://localhost:8080/collect", + "GCP_PROJECT_ID": "playground-388912" + }, + "flows": { + "web": { + "web": {}, + "packages": { + "@walkeros/collector": { "imports": ["startFlow"] }, + "@walkeros/web-source-browser": {}, + "@walkeros/web-destination-api": {}, + "@walkeros/destination-demo": {} + }, + "sources": { + "browser": { + "package": "@walkeros/web-source-browser", + "code": "sourceBrowser", + "config": { + "settings": { + "pageview": true, + "session": true + } + } + } + }, + "destinations": { + "api": { + "package": "@walkeros/web-destination-api", + "code": "destinationAPI", + "config": { + "settings": { + "url": "${COLLECT_URL}" + } + } + }, + "demo": { + "package": "@walkeros/destination-demo", + "code": "destinationDemo", + "config": { + "settings": { + "name": "Browser Demo" + } + } + } + }, + "collector": { + "run": true + } + }, + "server": { + "server": {}, + "packages": { + "@walkeros/collector": { "imports": ["startFlow"] }, + "@walkeros/server-source-express": {}, + "@walkeros/destination-demo": {}, + "@walkeros/server-destination-gcp": {} + }, + "sources": { + "express": { + "package": "@walkeros/server-source-express", + "code": "sourceExpress", + "config": { + "settings": { + "path": "/collect", + "port": 8080, + "status": true + } + } + } + }, + "destinations": { + "demo": { + "package": "@walkeros/destination-demo", + "code": "destinationDemo", + "config": { + "settings": { + "name": "Server Demo", + "values": ["name", "data", "timestamp"] + } + } + }, + "bigquery": { + "package": "@walkeros/server-destination-gcp", + "code": "destinationBigQuery", + "config": { + "settings": { + "projectId": "${GCP_PROJECT_ID}", + "datasetId": "walkerOS", + "tableId": "events", + "location": "EU", + "bigquery": { + "keyFilename": "./shared/credentials/sa-bigquery.json" + } + } + } + } + }, + "collector": { + "run": true, + "logger": { + "level": "DEBUG" + } + } + } + } +} diff --git a/apps/demos/flows/index.html b/apps/demos/flows/index.html new file mode 100644 index 000000000..fec953275 --- /dev/null +++ b/apps/demos/flows/index.html @@ -0,0 +1,28 @@ + + + + Browser to BigQuery Demo + + + +

Browser to BigQuery Demo

+

Click buttons to send events through the pipeline:

+ +
+ + + +
+ +
+ Event Flow:
+ Browser → API Destination → Express Source → BigQuery +
+ + + + diff --git a/apps/demos/flows/shared/credentials/.gitkeep b/apps/demos/flows/shared/credentials/.gitkeep new file mode 100644 index 000000000..9292dae54 --- /dev/null +++ b/apps/demos/flows/shared/credentials/.gitkeep @@ -0,0 +1 @@ +# Place sa-bigquery.json here diff --git a/apps/demos/source/README.md b/apps/demos/source/README.md index d1d68d647..96e2e25c0 100644 --- a/apps/demos/source/README.md +++ b/apps/demos/source/README.md @@ -1,6 +1,6 @@ diff --git a/apps/demos/storybook/README.md b/apps/demos/storybook/README.md index 88e5b93fd..02f19abc8 100644 --- a/apps/demos/storybook/README.md +++ b/apps/demos/storybook/README.md @@ -127,6 +127,6 @@ npm run build # Build demo components ## Learn More -- [walkerOS Documentation](https://docs.elbwalker.com) +- [walkerOS Documentation](https://www.walkeros.io/docs) - [Storybook Addon Source](../../../storybook-addon/) - [walkerOS GitHub](https://github.com/elbwalker/walkerOS) diff --git a/apps/scripts/validate-docs.ts b/apps/scripts/validate-docs.ts new file mode 100644 index 000000000..558f6f024 --- /dev/null +++ b/apps/scripts/validate-docs.ts @@ -0,0 +1,128 @@ +#!/usr/bin/env npx tsx +/** + * Validates documentation quality standards + * Usage: npx tsx scripts/validate-docs.ts + */ + +import { readFileSync } from 'fs'; +import { glob } from 'glob'; +import { relative } from 'path'; + +interface Issue { + file: string; + line?: number; + severity: 'error' | 'warning'; + message: string; +} + +const ROOT = process.cwd(); +const issues: Issue[] = []; + +// Check for legacy API patterns +function checkLegacyPatterns(file: string, content: string): void { + const lines = content.split('\n'); + + lines.forEach((line, index) => { + // Legacy destination pattern (only in code examples, not explanatory text) + if ( + line.includes("elb('walker destination'") && + !line.includes('//') && + !line.includes('legacy') + ) { + issues.push({ + file: relative(ROOT, file), + line: index + 1, + severity: 'warning', + message: + 'Uses legacy elb("walker destination") pattern - prefer startFlow()', + }); + } + }); +} + +// Check for old domain references +function checkDomainRefs(file: string, content: string): void { + if (content.includes('elbwalker.com') && !content.includes('img/elbwalker')) { + issues.push({ + file: relative(ROOT, file), + severity: 'warning', + message: 'Contains old domain reference (elbwalker.com)', + }); + } +} + +// Check README structure +function checkReadmeStructure(file: string, content: string): void { + if (!file.endsWith('README.md')) return; + + const hasInstallation = /^##?\s+Installation/im.test(content); + const hasUsage = /^##?\s+(Usage|Quick Start)/im.test(content); + + if (!hasInstallation) { + issues.push({ + file: relative(ROOT, file), + severity: 'warning', + message: 'README missing Installation section', + }); + } + + if (!hasUsage) { + issues.push({ + file: relative(ROOT, file), + severity: 'warning', + message: 'README missing Usage/Quick Start section', + }); + } +} + +async function main() { + console.log('📋 Validating documentation standards...\n'); + + const patterns = [ + 'packages/**/README.md', + 'website/docs/**/*.mdx', + 'skills/**/*.md', + ]; + + const files = await glob(patterns, { + cwd: ROOT, + ignore: ['**/node_modules/**', '**/dist/**'], + absolute: true, + }); + + for (const file of files) { + const content = readFileSync(file, 'utf-8'); + checkLegacyPatterns(file, content); + checkDomainRefs(file, content); + checkReadmeStructure(file, content); + } + + const errors = issues.filter((i) => i.severity === 'error'); + const warnings = issues.filter((i) => i.severity === 'warning'); + + if (issues.length === 0) { + console.log('✅ All documentation passes quality checks!\n'); + process.exit(0); + } + + if (errors.length > 0) { + console.log(`❌ Found ${errors.length} errors:\n`); + for (const issue of errors) { + console.log(` ${issue.file}${issue.line ? `:${issue.line}` : ''}`); + console.log(` ${issue.message}\n`); + } + } + + if (warnings.length > 0) { + console.log(`⚠️ Found ${warnings.length} warnings:\n`); + for (const issue of warnings) { + console.log(` ${issue.file}${issue.line ? `:${issue.line}` : ''}`); + console.log(` ${issue.message}\n`); + } + } + + // Exit with error only for errors, not warnings + process.exit(errors.length > 0 ? 1 : 0); +} + +main().catch(console.error); diff --git a/apps/scripts/validate-links.ts b/apps/scripts/validate-links.ts new file mode 100644 index 000000000..abb79d510 --- /dev/null +++ b/apps/scripts/validate-links.ts @@ -0,0 +1,161 @@ +#!/usr/bin/env npx tsx +/** + * Validates internal links in documentation files + * Usage: npx tsx apps/scripts/validate-links.ts + */ + +import { readFileSync, existsSync } from 'fs'; +import { glob } from 'glob'; +import { resolve, dirname, relative } from 'path'; + +interface LinkError { + file: string; + line: number; + link: string; + reason: string; +} + +const ROOT = process.cwd(); +const errors: LinkError[] = []; + +// Patterns to check +const PATTERNS = [ + 'packages/**/README.md', + 'website/docs/**/*.mdx', + 'skills/**/*.md', +]; + +// Skip these directories (contain templates/examples, not real docs) +const SKIP_PATTERNS = [ + 'docs/**/*.md', // Internal planning docs with example links + '**/dist/**', // Built output +]; + +// Links that are clearly examples/templates (not real paths) +const EXAMPLE_LINK_PATTERNS = [ + /^link$/, // Just "link" + /^\.\/src\/types/, // Template examples + /skill-name/, // Template placeholder + /path\/to\//, // Example path + /\.\.\/skill\//, // Template skill reference + /\.\.\./, // Ellipsis in path (template placeholder like /docs/...) +]; + +// Extract markdown links from content +function extractLinks(content: string): Array<{ link: string; line: number }> { + const links: Array<{ link: string; line: number }> = []; + const lines = content.split('\n'); + + lines.forEach((line, index) => { + // Match [text](link) but not external URLs + const regex = /\[([^\]]*)\]\(([^)]+)\)/g; + let match; + while ((match = regex.exec(line)) !== null) { + const link = match[2]; + // Skip external URLs, anchors only, mailto, and example patterns + if ( + !link.startsWith('http://') && + !link.startsWith('https://') && + !link.startsWith('#') && + !link.startsWith('mailto:') && + !EXAMPLE_LINK_PATTERNS.some((pattern) => pattern.test(link)) + ) { + links.push({ link, line: index + 1 }); + } + } + }); + + return links; +} + +// Validate a single link +function validateLink( + file: string, + link: string, + line: number, +): LinkError | null { + // Remove anchor + const [path] = link.split('#'); + if (!path) return null; // Anchor only + + let resolvedPath: string; + + // Handle Docusaurus root-relative links (/docs/...) + if (path.startsWith('/docs/')) { + // /docs/sources/ -> website/docs/sources/ + resolvedPath = resolve(ROOT, 'website', path.slice(1)); + } else if (path.startsWith('/')) { + // Other root-relative paths (like /diagrams/) + resolvedPath = resolve(ROOT, 'website/static', path.slice(1)); + } else { + // Relative paths - resolve from file's directory + const fileDir = dirname(file); + resolvedPath = resolve(ROOT, fileDir, path); + } + + // Check if file/directory exists + if (!existsSync(resolvedPath)) { + // Try with common extensions + const extensions = [ + '.md', + '.mdx', + '.ts', + '.tsx', + '/index.mdx', + '/index.md', + '.png', + '.jpg', + '.svg', + ]; + const found = extensions.some((ext) => existsSync(resolvedPath + ext)); + + if (!found) { + return { + file: relative(ROOT, file), + line, + link, + reason: `Target not found: ${relative(ROOT, resolvedPath)}`, + }; + } + } + + return null; +} + +async function main() { + console.log('🔗 Validating internal links...\n'); + + const files = await glob(PATTERNS, { + cwd: ROOT, + ignore: ['**/node_modules/**', ...SKIP_PATTERNS], + absolute: true, + }); + + for (const file of files) { + const content = readFileSync(file, 'utf-8'); + const links = extractLinks(content); + + for (const { link, line } of links) { + const error = validateLink(file, link, line); + if (error) { + errors.push(error); + } + } + } + + if (errors.length === 0) { + console.log('✅ All internal links are valid!\n'); + process.exit(0); + } + + console.log(`❌ Found ${errors.length} broken links:\n`); + for (const error of errors) { + console.log(` ${error.file}:${error.line}`); + console.log(` Link: ${error.link}`); + console.log(` ${error.reason}\n`); + } + + process.exit(1); +} + +main().catch(console.error); diff --git a/apps/storybook-addon/README.md b/apps/storybook-addon/README.md index 34dad526c..b6d2df815 100644 --- a/apps/storybook-addon/README.md +++ b/apps/storybook-addon/README.md @@ -183,7 +183,7 @@ export default { ## License -MIT © [elbwalker GmbH](https://www.elbwalker.com) +MIT © [elbwalker GmbH](https://www.walkeros.io) ## Links diff --git a/apps/storybook-addon/package.json b/apps/storybook-addon/package.json index 40f9e1bd0..25c547979 100644 --- a/apps/storybook-addon/package.json +++ b/apps/storybook-addon/package.json @@ -121,6 +121,6 @@ "web-components", "html" ], - "icon": "https://www.elbwalker.com/img/elbwalker_logo.png" + "icon": "https://www.walkeros.io/img/elbwalker_logo.png" } } diff --git a/apps/walkerjs/README.md b/apps/walkerjs/README.md index 8cd833d5d..d55139b73 100644 --- a/apps/walkerjs/README.md +++ b/apps/walkerjs/README.md @@ -91,14 +91,14 @@ Walker.js supports multiple configuration approaches with different priorities: ### Settings -| Name | Type | Default | Description | -| :---------- | :------------------ | :------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | -| `elb` | `string` | `"elb"` | Global function name for event tracking | -| `name` | `string` | `"walkerjs"` | Global instance name | -| `run` | `boolean` | `true` | Auto-initialize walker.js on load | -| `browser` | `object \| boolean` | `{ run: true, session: true, scope: document.body, pageview: true }` | [Browser source configuration](https://www.elbwalker.com/docs/sources/web/browser/) | -| `dataLayer` | `object \| boolean` | `false` | [DataLayer source configuration](https://www.elbwalker.com/docs/sources/web/dataLayer) | -| `collector` | `object` | `{}` | [Collector configuration](https://www.elbwalker.com/docs/collector/) including destinations and consent settings | +| Name | Type | Default | Description | +| :---------- | :------------------ | :------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------- | +| `elb` | `string` | `"elb"` | Global function name for event tracking | +| `name` | `string` | `"walkerjs"` | Global instance name | +| `run` | `boolean` | `true` | Auto-initialize walker.js on load | +| `browser` | `object \| boolean` | `{ run: true, session: true, scope: document.body, pageview: true }` | [Browser source configuration](https://www.walkeros.io/docs/sources/web/browser/) | +| `dataLayer` | `object \| boolean` | `false` | [DataLayer source configuration](https://www.walkeros.io/docs/sources/web/dataLayer) | +| `collector` | `object` | `{}` | [Collector configuration](https://www.walkeros.io/docs/collector/) including destinations and consent settings | #### Browser Source Settings @@ -119,10 +119,10 @@ Walker.js supports multiple configuration approaches with different priorities: #### Collector Settings -| Name | Type | Default | Description | -| :----------------------- | :------- | :--------------------- | :------------------------------------------------------------------------ | -| `collector.consent` | `object` | `{ functional: true }` | Default consent state | -| `collector.destinations` | `object` | `{}` | [Destination configurations](https://www.elbwalker.com/docs/destinations) | +| Name | Type | Default | Description | +| :----------------------- | :------- | :--------------------- | :---------------------------------------------------------------------- | +| `collector.consent` | `object` | `{ functional: true }` | Default consent state | +| `collector.destinations` | `object` | `{}` | [Destination configurations](https://www.walkeros.io/docs/destinations) | ### Full Configuration Object @@ -212,7 +212,7 @@ Walker.js automatically tracks events based on HTML data attributes: ``` For detailed information on data attributes, see the -[Browser Source documentation](https://www.elbwalker.com/docs/sources/web/browser/tagging). +[Browser Source documentation](https://www.walkeros.io/docs/sources/web/browser/tagging). ### Manual Event Tracking @@ -324,7 +324,7 @@ window.elbConfig = { ``` For comprehensive destination options, see the -[Destinations documentation](https://www.elbwalker.com/docs/destinations/). +[Destinations documentation](https://www.walkeros.io/docs/destinations/). ## Troubleshooting @@ -391,13 +391,13 @@ npm run preview # Serve examples on localhost:3333 ## Related Documentation -- **[Browser Source](https://www.elbwalker.com/docs/sources/web/browser/)** - +- **[Browser Source](https://www.walkeros.io/docs/sources/web/browser/)** - Detailed DOM tracking capabilities -- **[Collector](https://www.elbwalker.com/docs/collector/)** - Event processing +- **[Collector](https://www.walkeros.io/docs/collector/)** - Event processing and routing -- **[Destinations](https://www.elbwalker.com/docs/destinations/)** - Available +- **[Destinations](https://www.walkeros.io/docs/destinations/)** - Available destination options -- **[DataLayer Source](https://www.elbwalker.com/docs/sources/web/dataLayer/)** - +- **[DataLayer Source](https://www.walkeros.io/docs/sources/web/dataLayer/)** - DataLayer integration details Walker.js combines all these components into a single, easy-to-use package diff --git a/package-lock.json b/package-lock.json index 982d26088..7132aa08d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12231,6 +12231,13 @@ "license": "MIT", "peer": true }, + "node_modules/@types/aws-lambda": { + "version": "8.10.159", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.159.tgz", + "integrity": "sha512-SAP22WSGNN12OQ8PlCzGzRCZ7QDCwI85dQZbmpz7+mAk+L7j+wI7qnvmdKh+o7A5LaOp6QnOZ2NJphAZQTTHQg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -13757,10 +13764,18 @@ "resolved": "packages/server/destinations/meta", "link": true }, + "node_modules/@walkeros/server-source-aws": { + "resolved": "packages/server/sources/aws", + "link": true + }, "node_modules/@walkeros/server-source-express": { "resolved": "packages/server/sources/express", "link": true }, + "node_modules/@walkeros/server-source-fetch": { + "resolved": "packages/server/sources/fetch", + "link": true + }, "node_modules/@walkeros/server-source-gcp": { "resolved": "packages/server/sources/gcp", "link": true @@ -21054,6 +21069,7 @@ "version": "4.7.8", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "dev": true, "license": "MIT", "dependencies": { "minimist": "^1.2.5", @@ -21075,6 +21091,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" @@ -38739,6 +38756,7 @@ "version": "3.19.3", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "dev": true, "license": "BSD-2-Clause", "optional": true, "bin": { @@ -41013,6 +41031,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true, "license": "MIT" }, "node_modules/wordwrapjs": { @@ -41408,7 +41427,6 @@ "commander": "^11.0.0", "esbuild": "^0.19.0", "fs-extra": "^11.0.0", - "handlebars": "^4.7.8", "jsdom": "^22.1.0", "pacote": "^17.0.0" }, @@ -41469,6 +41487,7 @@ "version": "0.2.2", "license": "MIT", "dependencies": { + "@walkeros/core": "*", "cors": "^2.8.5", "express": "^4.18.2" }, @@ -41579,6 +41598,26 @@ }, "devDependencies": {} }, + "packages/server/sources/aws": { + "name": "@walkeros/server-source-aws", + "version": "0.4.2", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/elbwalker" + } + ], + "license": "MIT", + "dependencies": { + "@walkeros/core": "0.4.2" + }, + "devDependencies": { + "@types/aws-lambda": "^8.10.145" + }, + "peerDependencies": { + "@types/aws-lambda": "^8.10.0" + } + }, "packages/server/sources/express": { "name": "@walkeros/server-source-express", "version": "0.4.1", @@ -41599,6 +41638,15 @@ "@types/express": "^4.17.21" } }, + "packages/server/sources/fetch": { + "name": "@walkeros/server-source-fetch", + "version": "0.4.2", + "license": "MIT", + "dependencies": { + "@walkeros/core": "*" + }, + "devDependencies": {} + }, "packages/server/sources/gcp": { "name": "@walkeros/server-source-gcp", "version": "0.4.2", @@ -41617,6 +41665,27 @@ "@google-cloud/functions-framework": "^3.0.0" } }, + "packages/server/sources/lambda": { + "name": "@walkeros/server-source-lambda", + "version": "0.4.2", + "extraneous": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/elbwalker" + } + ], + "license": "MIT", + "dependencies": { + "@walkeros/core": "0.4.2" + }, + "devDependencies": { + "@types/aws-lambda": "^8.10.145" + }, + "peerDependencies": { + "@types/aws-lambda": "^8.10.0" + } + }, "packages/web/core": { "name": "@walkeros/web-core", "version": "0.4.2", diff --git a/package.json b/package.json index 626aed021..434ceceae 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,10 @@ "publish-packages": "rm -rf .turbo/cache && npm run build && npm run lint && npm run test && changeset version && changeset publish", "publish-docker": "npm run -w @walkeros/cli docker:publish && npm run -w @walkeros/docker docker:publish", "test": "turbo run test --filter=!@walkeros/website --output-logs=errors-only", - "prepare": "husky || true" + "prepare": "husky || true", + "validate:links": "npx tsx apps/scripts/validate-links.ts", + "validate:docs": "npx tsx apps/scripts/validate-docs.ts", + "validate": "npm run validate:links && npm run validate:docs" }, "devDependencies": { "@changesets/cli": "^2.28.1", diff --git a/packages/cli/README.md b/packages/cli/README.md index 2433486e8..8124d57ac 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -24,6 +24,22 @@ npm install -g @walkeros/cli npm install @walkeros/cli ``` +## Quick Start + +```bash +# Bundle a flow configuration +walkeros bundle flow.json + +# Test with simulated events (no real API calls) +walkeros simulate flow.json --event '{"name":"page view"}' + +# Push real events to destinations +walkeros push flow.json --event '{"name":"page view"}' + +# Run a collection server locally +walkeros run collect flow.json --port 3000 +``` + ## Commands ### bundle @@ -86,6 +102,49 @@ walkeros simulate \ --json ``` +### push + +Execute your flow with real API calls to configured destinations. Unlike +`simulate` which mocks API calls, `push` performs actual HTTP requests. + +```bash +walkeros push --event '' [options] +``` + +**Options:** + +- `-e, --event ` - Event to push (JSON string, file path, or URL) + **Required** +- `--flow ` - Flow name (for multi-flow configs) +- `--json` - Output results as JSON +- `-v, --verbose` - Verbose output +- `-s, --silent` - Suppress output (for CI/CD) +- `--local` - Execute locally without Docker + +**Event input formats:** + +```bash +# Inline JSON +walkeros push flow.json --event '{"name":"page view","data":{"title":"Home"}}' + +# File path +walkeros push flow.json --event ./events/order.json + +# URL +walkeros push flow.json --event https://example.com/sample-event.json +``` + +**Push vs Simulate:** + +| Feature | `push` | `simulate` | +| ------------ | ----------------------------------- | ------------------ | +| API Calls | Real HTTP requests | Mocked (captured) | +| Use Case | Integration testing | Safe local testing | +| Side Effects | Full (writes to DBs, sends to APIs) | None | + +Use `simulate` first to validate configuration safely, then `push` to verify +real integrations. + ### run Run flows locally using @walkeros/docker as a library (no Docker daemon @@ -176,12 +235,12 @@ Flow configs use the `Flow.Setup` format with `version` and `flows`: "server": {}, "packages": { "@walkeros/collector": { "imports": ["startFlow"] }, - "@walkeros/server-source-express": { "imports": ["sourceExpress"] }, - "@walkeros/destination-demo": { "imports": ["destinationDemo"] } + "@walkeros/server-source-express": {}, + "@walkeros/destination-demo": {} }, "sources": { "http": { - "code": "sourceExpress", + "package": "@walkeros/server-source-express", "config": { "settings": { "path": "/collect", "port": 8080 } } @@ -189,7 +248,7 @@ Flow configs use the `Flow.Setup` format with `version` and `flows`: }, "destinations": { "demo": { - "code": "destinationDemo", + "package": "@walkeros/destination-demo", "config": { "settings": { "name": "Demo" } } @@ -203,6 +262,76 @@ Flow configs use the `Flow.Setup` format with `version` and `flows`: Platform is determined by the `web: {}` or `server: {}` key presence. +### Package Configuration Patterns + +The CLI automatically resolves imports based on how you configure packages: + +**1. Default exports (recommended for single-export packages):** + +```json +{ + "packages": { + "@walkeros/server-destination-api": {} + }, + "destinations": { + "api": { + "package": "@walkeros/server-destination-api" + } + } +} +``` + +The CLI generates: +`import _walkerosServerDestinationApi from '@walkeros/server-destination-api';` + +**2. Named exports (for multi-export packages):** + +```json +{ + "packages": { + "@walkeros/server-destination-gcp": {} + }, + "destinations": { + "bigquery": { + "package": "@walkeros/server-destination-gcp", + "code": "destinationBigQuery" + }, + "analytics": { + "package": "@walkeros/server-destination-gcp", + "code": "destinationAnalytics" + } + } +} +``` + +The CLI generates: +`import { destinationBigQuery, destinationAnalytics } from '@walkeros/server-destination-gcp';` + +**3. Utility imports (for helper functions):** + +```json +{ + "packages": { + "lodash": { "imports": ["get", "set"] } + }, + "mappings": { + "custom": { + "data": "({ data }) => get(data, 'user.email')" + } + } +} +``` + +The CLI generates: `import { get, set } from 'lodash';` + +**Key points:** + +- Omit `packages.imports` for destinations/sources - the default export is used + automatically +- Only specify `code` when using a specific named export from a multi-export + package +- Use `packages.imports` only for utilities needed in mappings or custom code + ### Local Packages Use local packages instead of npm for development or testing unpublished @@ -364,13 +493,15 @@ walkeros bundle config.json - **Node.js**: 18+ or 22+ - **Docker**: Not required for CLI (only for production deployment) -## Documentation +## Type Definitions + +See [src/types.ts](./src/types.ts) for TypeScript interfaces. -Detailed guides in [docs/](./docs/): +## Related -- [RUN_COMMAND.md](./docs/RUN_COMMAND.md) - Run command details -- [PUBLISHING.md](./docs/PUBLISHING.md) - Publishing guide -- [MANUAL_TESTING_GUIDE.md](./docs/MANUAL_TESTING_GUIDE.md) - Testing guide +- [Website Documentation](https://www.walkeros.io/docs/cli/) +- [Flow Configuration](https://www.walkeros.io/docs/getting-started/flow/) +- [Docker Package](../docker/) - Production runtime ## License diff --git a/packages/cli/demos/README.md b/packages/cli/demos/README.md index f80b484cb..ac5355e00 100644 --- a/packages/cli/demos/README.md +++ b/packages/cli/demos/README.md @@ -1,5 +1,16 @@ # WalkerOS CLI & Docker Demos +## Installation + +These are demo configurations - see the main [CLI README](../README.md) for +installation. + +## Usage + +Run `./quick-demo.sh` for a 5-minute interactive demo of the complete workflow. + +--- + This directory contains practical demonstrations of walkerOS CLI and Docker functionality. @@ -90,20 +101,10 @@ node packages/cli/dist/index.js run serve static-walker.js \ --no-pull ``` -## Testing - -For comprehensive testing instructions, see: - -- [Manual Testing Guide](../docs/MANUAL_TESTING_GUIDE.md) - ## Documentation -For detailed documentation: - - [CLI Documentation](../README.md) -- [Run Command Guide](../docs/RUN_COMMAND.md) -- [Publishing Guide](../docs/PUBLISHING.md) -- [Docker Build Strategy](../../docker/docs/BUILD_STRATEGY.md) +- [Docker Package](../../docker/README.md) ## Support diff --git a/packages/cli/examples/README.md b/packages/cli/examples/README.md index d949efdb0..de8929c66 100644 --- a/packages/cli/examples/README.md +++ b/packages/cli/examples/README.md @@ -1,5 +1,17 @@ # walkerOS Flow Examples +## Installation + +These are example configurations - see the main [CLI README](../README.md) for +installation. + +## Usage + +Use `walkeros bundle .json` to build any example, then `walkeros run` +to execute. + +--- + This directory contains example flow configurations demonstrating various walkerOS use cases. @@ -340,4 +352,4 @@ The event name is parsed as: `const [entity, action] = event.split(' ')` 1. Try each example with `walkeros bundle` and `walkeros simulate` 2. Modify examples to match your tracking requirements 3. Create custom flow files for your use case -4. Deploy to production (see [PUBLISHING.md](../docs/PUBLISHING.md)) +4. Deploy to production diff --git a/packages/cli/examples/flow-order-complete.json b/packages/cli/examples/flow-order-complete.json index b22edac57..e14f451d4 100644 --- a/packages/cli/examples/flow-order-complete.json +++ b/packages/cli/examples/flow-order-complete.json @@ -9,8 +9,7 @@ "imports": ["startFlow"] }, "@walkeros/destination-demo": { - "version": "latest", - "imports": ["destinationDemo"] + "version": "latest" } }, "destinations": { diff --git a/packages/cli/examples/flow-simple.json b/packages/cli/examples/flow-simple.json index e5423b9cd..e68a9aa97 100644 --- a/packages/cli/examples/flow-simple.json +++ b/packages/cli/examples/flow-simple.json @@ -9,8 +9,7 @@ "imports": ["startFlow"] }, "@walkeros/destination-demo": { - "version": "latest", - "imports": ["destinationDemo"] + "version": "latest" } }, "destinations": { diff --git a/packages/cli/examples/server-collect.json b/packages/cli/examples/server-collect.json index 1b469a1e4..a4f33fa78 100644 --- a/packages/cli/examples/server-collect.json +++ b/packages/cli/examples/server-collect.json @@ -9,17 +9,16 @@ "imports": ["startFlow"] }, "@walkeros/server-source-express": { - "version": "latest", - "imports": ["sourceExpress"] + "version": "latest" }, "@walkeros/destination-demo": { - "version": "latest", - "imports": ["destinationDemo"] + "version": "latest" } }, "sources": { "http": { "package": "@walkeros/server-source-express", + "code": "sourceExpress", "config": { "settings": { "path": "/collect", @@ -33,6 +32,7 @@ "destinations": { "demo": { "package": "@walkeros/destination-demo", + "code": "destinationDemo", "config": { "settings": { "name": "Server Collection Demo", diff --git a/packages/cli/package.json b/packages/cli/package.json index 17e854583..bb6539544 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -17,7 +17,6 @@ }, "files": [ "dist/**", - "templates/**", "examples/**", "README.md", "CHANGELOG.md" @@ -41,7 +40,6 @@ "commander": "^11.0.0", "esbuild": "^0.19.0", "fs-extra": "^11.0.0", - "handlebars": "^4.7.8", "jsdom": "^22.1.0", "pacote": "^17.0.0" }, diff --git a/packages/cli/src/__tests__/bundle/bundler-helpers.test.ts b/packages/cli/src/__tests__/bundle/bundler-helpers.test.ts index 24fca3d8a..43baedfc7 100644 --- a/packages/cli/src/__tests__/bundle/bundler-helpers.test.ts +++ b/packages/cli/src/__tests__/bundle/bundler-helpers.test.ts @@ -135,171 +135,6 @@ describe('Bundler Helper Functions', () => { }); }); - describe('processTemplate', () => { - it('should return code directly when no template', async () => { - const flowConfig: Flow.Config = {} as Flow.Config; - const buildOptions = { - code: 'console.log("test");', - template: undefined, - } as unknown as BuildOptions; - - // Expected: returns code as-is - expect(buildOptions.template).toBeUndefined(); - expect(buildOptions.code).toBe('console.log("test");'); - }); - - it('should process template when configured', async () => { - const flowConfig: Flow.Config = { - sources: {}, - destinations: {}, - collector: {}, - } as unknown as Flow.Config; - - const buildOptions = { - code: 'custom code', - template: 'flow-template', - } as unknown as BuildOptions; - - // Expected: calls TemplateEngine.process with all parameters - expect(buildOptions.template).toBe('flow-template'); - }); - - it('should handle empty code', async () => { - const flowConfig: Flow.Config = {} as Flow.Config; - const buildOptions = { - code: undefined, - template: undefined, - } as unknown as BuildOptions; - - // Expected: returns empty string - expect(buildOptions.code).toBeUndefined(); - }); - }); - - describe('wrapCodeForFormat', () => { - it('should not wrap code when template is used', () => { - const code = 'const flow = startFlow();'; - const format = 'esm'; - const hasTemplate = true; - - // Expected: returns code as-is - expect(hasTemplate).toBe(true); - }); - - it('should wrap ESM code without exports', () => { - const code = 'const flow = startFlow();'; - const format = 'esm'; - const hasTemplate = false; - - // Expected: export default const flow = startFlow(); - expect(format).toBe('esm'); - expect(hasTemplate).toBe(false); - expect(code).not.toMatch(/^\s*export\s/m); - }); - - it('should not wrap ESM code that already has exports', () => { - const code = 'export const flow = startFlow();'; - const format = 'esm'; - const hasTemplate = false; - - // Expected: returns code as-is (already has export) - expect(code).toMatch(/^\s*export\s/m); - }); - - it('should not wrap non-ESM formats', () => { - const code = 'const flow = startFlow();'; - const format = 'cjs'; - const hasTemplate = false; - - // Expected: returns code as-is - expect(format).toBe('cjs'); - }); - - it('should not wrap IIFE format', () => { - const code = 'const flow = startFlow();'; - const format = 'iife'; - const hasTemplate = false; - - // Expected: returns code as-is - expect(format).toBe('iife'); - }); - }); - - describe('assembleFinalCode', () => { - it('should combine imports, examples, and code', () => { - const importStatements = [ - "import { getId } from '@walkeros/core';", - "import { startFlow } from '@walkeros/collector';", - ]; - const examplesObject = - 'const examples = {\n gtag: gtag_examples\n};\n\n'; - const wrappedCode = 'export default startFlow();'; - const format = 'esm'; - - // Expected: imports + examples + code - expect(importStatements).toHaveLength(2); - expect(examplesObject).toContain('const examples'); - expect(wrappedCode).toContain('export default'); - }); - - it('should handle empty imports', () => { - const importStatements: string[] = []; - const examplesObject = ''; - const wrappedCode = 'export default startFlow();'; - const format = 'esm'; - - // Expected: just the code - expect(importStatements).toHaveLength(0); - }); - - it('should add examples export for ESM with examples', () => { - const importStatements = ["import { getId } from '@walkeros/core';"]; - const examplesObject = - 'const examples = {\n gtag: gtag_examples\n};\n\n'; - const wrappedCode = 'export default startFlow();'; - const format = 'esm'; - - // Expected to include: export { examples }; - expect(format).toBe('esm'); - expect(examplesObject).toContain('const examples'); - }); - - it('should not add examples export for non-ESM formats', () => { - const importStatements = ["import { getId } from '@walkeros/core';"]; - const examplesObject = - 'const examples = {\n gtag: gtag_examples\n};\n\n'; - const wrappedCode = 'startFlow();'; - const format = 'cjs'; - - // Should not include: export { examples }; - expect(format).not.toBe('esm'); - }); - - it('should not add examples export when no examples', () => { - const importStatements = ["import { getId } from '@walkeros/core';"]; - const examplesObject = ''; - const wrappedCode = 'export default startFlow();'; - const format = 'esm'; - - // Should not include: export { examples }; - expect(examplesObject).toBe(''); - }); - - it('should handle multiple import statements', () => { - const importStatements = [ - "import walkerCore from '@walkeros/core';", - "import { getId, trim } from '@walkeros/core';", - "import { startFlow } from '@walkeros/collector';", - ]; - const examplesObject = ''; - const wrappedCode = 'export default startFlow();'; - const format = 'esm'; - - // Expected: all imports joined with newlines - expect(importStatements).toHaveLength(3); - }); - }); - describe('Integration: createEntryPoint refactored behavior', () => { it('should maintain backward compatibility with original implementation', () => { // This test documents that the refactored version should produce @@ -324,7 +159,6 @@ describe('Bundler Helper Functions', () => { }, }, code: 'export const flow = startFlow();', - template: undefined, format: 'esm' as const, platform: 'browser' as const, } as unknown as BuildOptions; diff --git a/packages/cli/src/__tests__/bundle/bundler.test.ts b/packages/cli/src/__tests__/bundle/bundler.test.ts index 99483e715..6263326bc 100644 --- a/packages/cli/src/__tests__/bundle/bundler.test.ts +++ b/packages/cli/src/__tests__/bundle/bundler.test.ts @@ -1,6 +1,11 @@ import fs from 'fs-extra'; import path from 'path'; -import { bundleCore as bundle } from '../../commands/bundle/bundler.js'; +import { + bundleCore as bundle, + buildConfigObject, + generatePlatformWrapper, + createEntryPoint, +} from '../../commands/bundle/bundler.js'; import { loadBundleConfig } from '../../config/index.js'; import { createLogger, type Logger } from '../../core/index.js'; import { getId, type Flow } from '@walkeros/core'; @@ -95,7 +100,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: 'export const test = getId(8);', - template: '', platform: 'browser', format: 'esm', output: path.join(testOutputDir, 'minimal.js'), @@ -119,7 +123,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: 'export default { processText: (text) => trim(text) };', - template: '', platform: 'node', format: 'esm', output: path.join(testOutputDir, 'server-bundle.mjs'), @@ -143,7 +146,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: "export function processData(data) {\n return data.map(item => ({\n ...item,\n id: getId(8),\n timestamp: new Date().toISOString().split('T')[0],\n processed: true\n }));\n}\n\nexport function extractNestedValues(data, path) {\n return data.map(item => getByPath(item, path, null)).filter(val => val !== null);\n}\n\nexport function deepCloneData(data) {\n return clone(data);\n}\n\nexport function cleanStringData(data) {\n return data.map(item => ({\n ...item,\n name: typeof item.name === 'string' ? trim(item.name) : item.name\n }));\n}\n\n// Re-export walkerOS utilities\nexport { getId, getByPath, clone, trim, isObject };", - template: '', platform: 'browser', format: 'esm', target: 'es2020', @@ -171,7 +173,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: 'export const test = getId(8);', - template: '', format: 'esm', output: path.join(testOutputDir, 'stats-test.js'), }); @@ -195,7 +196,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: 'import * as walkerCore from "@walkeros/core";\nexport const test = walkerCore.getId;', - template: '', format: 'esm', output: path.join(testOutputDir, 'test.js'), }); @@ -218,7 +218,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: 'export const test = getId(8);', - template: '', format: 'esm', output: path.join(testOutputDir, 'no-stats.js'), }); @@ -229,74 +228,6 @@ describe('Bundler', () => { }); }); - describe('Template System', () => { - it('should handle template configuration', async () => { - // Create a test template file - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile(templatePath, '{{{CODE}}}\n// Template footer'); - - const flowConfig: Flow.Config = { - web: {}, - packages: { - '@walkeros/core': { imports: ['getId'] }, - }, - }; - - const buildOptions = createBuildOptions({ - packages: flowConfig.packages || {}, - code: 'export const generateId = () => getId(8);', - template: templatePath, - output: path.join(testOutputDir, 'template-test.js'), - }); - - await expect( - bundle(flowConfig, buildOptions, logger), - ).resolves.not.toThrow(); - }); - - it('should handle missing template variables gracefully', async () => { - const flowConfig: Flow.Config = { - web: {}, - packages: { - '@walkeros/core': { imports: ['trim'] }, - }, - }; - - const buildOptions = createBuildOptions({ - packages: flowConfig.packages || {}, - code: 'export const test = trim("hello");', - template: '', - format: 'esm', - output: path.join(testOutputDir, 'missing-vars.js'), - }); - - await expect( - bundle(flowConfig, buildOptions, logger), - ).resolves.not.toThrow(); - }); - - it('should append bundle code when placeholder not found', async () => { - const flowConfig: Flow.Config = { - web: {}, - packages: { - '@walkeros/core': { imports: ['getId'] }, - }, - }; - - const buildOptions = createBuildOptions({ - packages: flowConfig.packages || {}, - code: 'export const test = getId(6);', - template: '', - format: 'esm', - output: path.join(testOutputDir, 'append-test.js'), - }); - - await expect( - bundle(flowConfig, buildOptions, logger), - ).resolves.not.toThrow(); - }); - }); - describe('Configuration Scenarios', () => { it('should handle custom temp directory configuration', async () => { const flowConfig: Flow.Config = { @@ -309,7 +240,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: 'export const test = getId();', - template: '', format: 'esm', tempDir: '/tmp/my-custom-bundler-temp', output: path.join(testOutputDir, 'custom-temp-example.js'), @@ -331,7 +261,6 @@ describe('Bundler', () => { const buildOptions = createBuildOptions({ packages: flowConfig.packages || {}, code: '// Test version pinning\nexport const test = getId();', - template: '', platform: 'browser', format: 'esm', target: 'es2020', @@ -362,4 +291,158 @@ describe('Bundler', () => { }).toThrow(/Invalid configuration/); }); }); + + describe('buildConfigObject', () => { + it('uses explicit code for named imports', () => { + const flowConfig: Flow.Config = { + server: {}, + sources: { + http: { + package: '@walkeros/server-source-express', + code: 'sourceExpress', + config: { settings: { port: 8080 } }, + }, + }, + destinations: { + demo: { + package: '@walkeros/destination-demo', + code: 'destinationDemo', + config: { settings: { name: 'Test' } }, + }, + }, + }; + + const explicitCodeImports = new Map([ + ['@walkeros/server-source-express', new Set(['sourceExpress'])], + ['@walkeros/destination-demo', new Set(['destinationDemo'])], + ]); + + const result = buildConfigObject(flowConfig, explicitCodeImports); + + expect(result).toContain('code: sourceExpress'); + expect(result).toContain('code: destinationDemo'); + expect(result).toContain('"port": 8080'); + expect(result).toContain('"name": "Test"'); + }); + + it('uses default import variable when no explicit code', () => { + const flowConfig: Flow.Config = { + server: {}, + sources: { + http: { + package: '@walkeros/server-source-express', + config: { settings: { port: 8080 } }, + }, + }, + destinations: {}, + }; + + const explicitCodeImports = new Map(); // No explicit imports + + const result = buildConfigObject(flowConfig, explicitCodeImports); + + expect(result).toContain('code: _walkerosServerSourceExpress'); + }); + }); + + describe('generatePlatformWrapper', () => { + it('generates web IIFE wrapper', () => { + const config = '{ sources: {}, destinations: {} }'; + const userCode = 'console.log("custom code");'; + const buildOptions = { + platform: 'browser', + windowCollector: 'collector', + windowElb: 'elb', + }; + + const result = generatePlatformWrapper(config, userCode, buildOptions); + + expect(result).toContain('(async () => {'); + expect(result).toContain( + 'const config = { sources: {}, destinations: {} };', + ); + expect(result).toContain('console.log("custom code");'); + expect(result).toContain('await startFlow(config)'); + expect(result).toContain("window['collector'] = collector"); + expect(result).toContain("window['elb'] = elb"); + }); + + it('generates server export default wrapper', () => { + const config = '{ sources: {}, destinations: {} }'; + const userCode = ''; + const buildOptions = { platform: 'node' }; + + const result = generatePlatformWrapper(config, userCode, buildOptions); + + expect(result).toContain('export default async function'); + expect(result).toContain( + 'const config = { sources: {}, destinations: {} };', + ); + expect(result).toContain('return await startFlow(config)'); + expect(result).not.toContain('window'); + }); + }); + + describe('createEntryPoint integration', () => { + it('generates complete entry point with explicit code', async () => { + const flowConfig: Flow.Config = { + server: {}, + packages: { + '@walkeros/collector': { imports: ['startFlow'] }, + '@walkeros/server-source-express': {}, + '@walkeros/destination-demo': {}, + }, + sources: { + http: { + package: '@walkeros/server-source-express', + code: 'sourceExpress', + config: { settings: { port: 8080 } }, + }, + }, + destinations: { + demo: { + package: '@walkeros/destination-demo', + code: 'destinationDemo', + config: { settings: { name: 'Test' } }, + }, + }, + }; + + const buildOptions = { + platform: 'node', + format: 'esm', + packages: { + '@walkeros/collector': { imports: ['startFlow'] }, + '@walkeros/server-source-express': {}, + '@walkeros/destination-demo': {}, + }, + output: './dist/bundle.mjs', + code: '', + }; + + const result = await createEntryPoint( + flowConfig, + buildOptions as BuildOptions, + new Map(), + ); + + // Should have named imports + expect(result).toContain( + "import { startFlow } from '@walkeros/collector'", + ); + expect(result).toContain( + "import { sourceExpress } from '@walkeros/server-source-express'", + ); + expect(result).toContain( + "import { destinationDemo } from '@walkeros/destination-demo'", + ); + + // Should use those imports in config + expect(result).toContain('code: sourceExpress'); + expect(result).toContain('code: destinationDemo'); + + // Should have server wrapper + expect(result).toContain('export default async function'); + }); + }); }); diff --git a/packages/cli/src/__tests__/bundle/programmatic.test.ts b/packages/cli/src/__tests__/bundle/programmatic.test.ts index 4654e8856..770cb68df 100644 --- a/packages/cli/src/__tests__/bundle/programmatic.test.ts +++ b/packages/cli/src/__tests__/bundle/programmatic.test.ts @@ -176,13 +176,11 @@ describe('Programmatic Bundle API', () => { expect(webDefaults.format).toBe('iife'); expect(webDefaults.platform).toBe('browser'); expect(webDefaults.target).toBe('es2020'); - expect(webDefaults.template).toBe('web.hbs'); const serverDefaults = getBuildDefaults('server'); expect(serverDefaults.format).toBe('esm'); expect(serverDefaults.platform).toBe('node'); expect(serverDefaults.target).toBe('node20'); - expect(serverDefaults.template).toBe('server.hbs'); }); it('should use convention-based output paths', () => { diff --git a/packages/cli/src/__tests__/bundle/serializer.test.ts b/packages/cli/src/__tests__/bundle/serializer.test.ts deleted file mode 100644 index bb64c6af4..000000000 --- a/packages/cli/src/__tests__/bundle/serializer.test.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { - serializeToJS, - serializeConfig, - processTemplateVariables, -} from '../../commands/bundle/serializer.js'; - -describe('Serializer', () => { - describe('serializeToJS', () => { - it('should serialize primitives correctly', () => { - expect(serializeToJS(null)).toBe('null'); - expect(serializeToJS(undefined)).toBe('undefined'); - expect(serializeToJS(true)).toBe('true'); - expect(serializeToJS(false)).toBe('false'); - expect(serializeToJS(42)).toBe('42'); - expect(serializeToJS('hello')).toBe('"hello"'); - }); - - it('should serialize strings with single quotes when specified', () => { - expect(serializeToJS('test', { singleQuotes: true })).toBe("'test'"); - }); - - it('should detect and preserve arrow functions', () => { - const fn1 = '(x) => x * 2'; - const fn2 = 'entity => entity.type === "product"'; - const fn3 = '() => { return true; }'; - - expect(serializeToJS(fn1)).toBe(fn1); - expect(serializeToJS(fn2)).toBe(fn2); - expect(serializeToJS(fn3)).toBe(fn3); - }); - - it('should serialize arrays correctly', () => { - expect(serializeToJS([])).toBe('[]'); - expect(serializeToJS([1, 2, 3])).toBe('[\n 1,\n 2,\n 3\n]'); - expect(serializeToJS(['a', 'b'])).toBe('[\n "a",\n "b"\n]'); - }); - - it('should serialize objects correctly', () => { - expect(serializeToJS({})).toBe('{}'); - - const simple = { name: 'test', value: 42 }; - expect(serializeToJS(simple)).toBe('{\n name: "test",\n value: 42\n}'); - }); - - it('should handle nested objects and arrays', () => { - const complex = { - settings: { - ga4: { - measurementId: 'G-123', - enabled: true, - }, - }, - items: [ - { id: 1, name: 'item1' }, - { id: 2, name: 'item2' }, - ], - }; - - const result = serializeToJS(complex); - expect(result).toContain('settings: {'); - expect(result).toContain('ga4: {'); - expect(result).toContain('measurementId: "G-123"'); - expect(result).toContain('items: ['); - }); - - it('should handle keys that need quotes', () => { - const obj = { - 'key-with-dash': 'value1', - '123numeric': 'value2', - 'key with space': 'value3', - normalKey: 'value4', - }; - - const result = serializeToJS(obj); - expect(result).toContain('"key-with-dash": "value1"'); - expect(result).toContain('"123numeric": "value2"'); - expect(result).toContain('"key with space": "value3"'); - expect(result).toContain('normalKey: "value4"'); - }); - }); - - describe('serializeConfig', () => { - it('should handle empty config', () => { - expect(serializeConfig({})).toBe('{}'); - }); - - it('should serialize config with single quotes', () => { - const config = { - settings: { - endpoint: 'https://api.example.com', - }, - }; - - const result = serializeConfig(config); - expect(result).toContain("endpoint: 'https://api.example.com'"); - }); - - it('should handle complex mapping configurations', () => { - const config = { - mapping: { - order: { - complete: { - name: 'purchase', - data: { - map: { - transaction_id: 'data.id', - condition: '(entity) => entity.entity === "product"', - }, - }, - }, - }, - }, - }; - - const result = serializeConfig(config); - expect(result).toContain("name: 'purchase'"); - expect(result).toContain("transaction_id: 'data.id'"); - expect(result).toContain('(entity) => entity.entity === "product"'); - }); - }); - - describe('processTemplateVariables', () => { - it('should process sources and destinations config objects', () => { - const variables = { - sources: { - browser: { - code: 'sourceBrowser', - config: { debug: true }, - }, - }, - destinations: { - gtag: { - code: 'destinationGtag', - config: { - settings: { - ga4: { measurementId: 'G-123' }, - }, - }, - }, - }, - }; - - const result = processTemplateVariables(variables); - expect(result.sources?.browser?.config).toBe('{\n debug: true\n}'); - expect(result.destinations?.gtag?.config).toContain( - "measurementId: 'G-123'", - ); - }); - - it('should handle string configs (pass through)', () => { - const variables = { - sources: { - test: { - code: 'source', - config: '{ debug: true }', - }, - }, - }; - - const result = processTemplateVariables(variables); - expect(result.sources?.test?.config).toBe('{ debug: true }'); - }); - - it('should handle undefined env values', () => { - const variables = { - sources: { - test: { - code: 'source', - config: {}, - env: undefined, - }, - }, - }; - - const result = processTemplateVariables(variables); - expect(result.sources?.test?.env).toBeUndefined(); - expect('env' in result.sources!.test).toBe(false); - }); - - it('should handle collector configuration', () => { - const variables = { - collector: { - settings: { - debug: true, - }, - }, - }; - - const result = processTemplateVariables(variables); - expect(result.collector).toContain('debug: true'); - }); - - it('should preserve other variables', () => { - const variables = { - title: 'My App', - version: '1.0.0', - }; - - const result = processTemplateVariables(variables); - expect(result.title).toBe('My App'); - expect(result.version).toBe('1.0.0'); - }); - }); -}); diff --git a/packages/cli/src/__tests__/bundle/template-engine.test.ts b/packages/cli/src/__tests__/bundle/template-engine.test.ts deleted file mode 100644 index 111480b17..000000000 --- a/packages/cli/src/__tests__/bundle/template-engine.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -import fs from 'fs-extra'; -import path from 'path'; -import { TemplateEngine } from '../../commands/bundle/template-engine.js'; -import { SourceDestinationItem } from '../../types/template.js'; -import { getId } from '@walkeros/core'; - -describe('TemplateEngine', () => { - const testOutputDir = path.join('.tmp', `template-${Date.now()}-${getId()}`); - const engine = new TemplateEngine(); - - beforeEach(async () => { - await fs.ensureDir(testOutputDir); - }); - - afterEach(async () => { - await fs.remove(testOutputDir); - }); - - describe('Templates', () => { - it('should process template with sources and destinations', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile( - templatePath, - '{{{CODE}}}\nSources: {{#each sources}}{{@key}} {{/each}}\nDestinations: {{#each destinations}}{{@key}} {{/each}}', - ); - - const sources = { - browser: { code: 'sourceBrowser', config: {} } as SourceDestinationItem, - }; - const destinations = { - gtag: { code: 'destinationGtag', config: {} } as SourceDestinationItem, - }; - const collector = {}; - - const result = await engine.process( - templatePath, - 'console.log("test");', - sources, - destinations, - collector, - {}, // build config - ); - - expect(result).toContain('console.log("test");'); - expect(result).toContain('Sources: browser'); - expect(result).toContain('Destinations: gtag'); - }); - - it('should replace bundle placeholder', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile(templatePath, 'Start\n{{{CODE}}}\nEnd'); - - const result = await engine.process( - templatePath, - 'const x = 42;', - {}, - {}, - {}, - {}, // build config - ); - - expect(result).toBe('Start\nconst x = 42;\nEnd'); - }); - - it('should handle missing variables gracefully', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile(templatePath, '{{{CODE}}} - {{MISSING}}'); - - const result = await engine.process(templatePath, 'code', {}, {}, {}, {}); - - expect(result).toBe('code - '); - }); - - it('should handle config serialization', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile( - templatePath, - '{{{CODE}}}\n{{#each sources}}Code: {{{code}}}, Config: {{{config}}}\n{{/each}}', - ); - - const sources = { - browser: { - code: 'sourceBrowser', - config: { settings: { prefix: 'data-elb' } }, - } as SourceDestinationItem, - }; - - const result = await engine.process( - templatePath, - 'const test = true;', - sources, - {}, - {}, - {}, // build config - ); - - expect(result).toContain('Code: sourceBrowser'); - expect(result).toContain('Config: {'); - expect(result).toContain('prefix'); - }); - }); - - describe('Object Iteration', () => { - it('should iterate over sources and destinations objects', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile( - templatePath, - '{{#each sources}}Source {{@key}}: {{code}}\n{{/each}}{{#each destinations}}Destination {{@key}}: {{code}}\n{{/each}}{{{CODE}}}', - ); - - const sources = { - browser: { code: 'sourceBrowser' } as SourceDestinationItem, - dataLayer: { code: 'sourceDataLayer' } as SourceDestinationItem, - }; - const destinations = { - gtag: { code: 'destinationGtag' } as SourceDestinationItem, - api: { code: 'destinationAPI' } as SourceDestinationItem, - }; - - const result = await engine.process( - templatePath, - 'const bundle = true;', - sources, - destinations, - {}, - ); - - expect(result).toContain('Source browser: sourceBrowser'); - expect(result).toContain('Source dataLayer: sourceDataLayer'); - expect(result).toContain('Destination gtag: destinationGtag'); - expect(result).toContain('Destination api: destinationAPI'); - expect(result).toContain('const bundle = true;'); - }); - - it('should handle collector configuration', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile( - templatePath, - '{{{CODE}}}{{#if collector}}\nCollector: {{{collector}}}{{/if}}', - ); - - const collector = { settings: { debug: true } }; - - const result = await engine.process( - templatePath, - 'const code = 1;', - {}, - {}, - collector, - {}, // build config - ); - - expect(result).toContain('const code = 1;'); - expect(result).toContain('Collector: {'); - expect(result).toContain('debug'); - }); - }); - - describe('Edge Cases', () => { - it('should handle template without CODE placeholder', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile(templatePath, 'Header only'); - - const result = await engine.process( - templatePath, - 'const code = true;', - {}, - {}, - {}, - {}, // build config - ); - - expect(result).toBe('Header only'); - }); - - it('should handle minimal template content', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile(templatePath, '{{{CODE}}}'); - - const result = await engine.process( - templatePath, - 'just code', - {}, - {}, - {}, - {}, // build config - ); - - expect(result).toBe('just code'); - }); - - it('should handle empty bundle code', async () => { - const templatePath = path.join(testOutputDir, 'test.hbs'); - await fs.writeFile(templatePath, 'Template: {{{CODE}}}'); - - const result = await engine.process(templatePath, '', {}, {}, {}, {}); - - expect(result).toBe('Template: '); - }); - }); -}); diff --git a/packages/cli/src/__tests__/config-loader.test.ts b/packages/cli/src/__tests__/config-loader.test.ts index 4b817ba5a..51b7ba42a 100644 --- a/packages/cli/src/__tests__/config-loader.test.ts +++ b/packages/cli/src/__tests__/config-loader.test.ts @@ -71,7 +71,6 @@ describe('Config Loader', () => { expect(result.buildOptions.format).toBe('iife'); expect(result.buildOptions.target).toBe('es2020'); expect(result.buildOptions.minify).toBe(true); - expect(result.buildOptions.template).toBe('web.hbs'); // Output path is resolved relative to config file directory expect(result.buildOptions.output).toBe('/test/dist/walker.js'); }); @@ -95,7 +94,6 @@ describe('Config Loader', () => { expect(result.buildOptions.format).toBe('esm'); expect(result.buildOptions.target).toBe('node20'); expect(result.buildOptions.minify).toBe(true); - expect(result.buildOptions.template).toBe('server.hbs'); // Output path is resolved relative to config file directory expect(result.buildOptions.output).toBe('/test/dist/bundle.mjs'); }); diff --git a/packages/cli/src/__tests__/push/fixtures/configs/server-basic.json b/packages/cli/src/__tests__/push/fixtures/configs/server-basic.json index f73f94e83..6bde9de2b 100644 --- a/packages/cli/src/__tests__/push/fixtures/configs/server-basic.json +++ b/packages/cli/src/__tests__/push/fixtures/configs/server-basic.json @@ -7,9 +7,7 @@ "@walkeros/collector": { "imports": ["startFlow"] }, - "@walkeros/destination-demo": { - "imports": ["destinationDemo"] - } + "@walkeros/destination-demo": {} }, "destinations": { "demo": { diff --git a/packages/cli/src/__tests__/push/fixtures/configs/web-basic.json b/packages/cli/src/__tests__/push/fixtures/configs/web-basic.json index 1992f3b6a..0ef904d90 100644 --- a/packages/cli/src/__tests__/push/fixtures/configs/web-basic.json +++ b/packages/cli/src/__tests__/push/fixtures/configs/web-basic.json @@ -7,9 +7,7 @@ "@walkeros/collector": { "imports": ["startFlow"] }, - "@walkeros/destination-demo": { - "imports": ["destinationDemo"] - } + "@walkeros/destination-demo": {} }, "destinations": { "demo": { diff --git a/packages/cli/src/__tests__/simulate/server-simulate.integration.test.ts b/packages/cli/src/__tests__/simulate/server-simulate.integration.test.ts index 1f1fb5c21..05131c76a 100644 --- a/packages/cli/src/__tests__/simulate/server-simulate.integration.test.ts +++ b/packages/cli/src/__tests__/simulate/server-simulate.integration.test.ts @@ -27,7 +27,6 @@ describe('Server Simulation Integration', () => { }, '@walkeros/destination-demo': { version: 'latest', - imports: ['destinationDemo'], }, }, destinations: { diff --git a/packages/cli/src/commands/bundle/bundler.ts b/packages/cli/src/commands/bundle/bundler.ts index fa43add5a..0ffbf3272 100644 --- a/packages/cli/src/commands/bundle/bundler.ts +++ b/packages/cli/src/commands/bundle/bundler.ts @@ -2,10 +2,9 @@ import esbuild from 'esbuild'; import path from 'path'; import fs from 'fs-extra'; import type { Flow } from '@walkeros/core'; +import { packageNameToVariable } from '@walkeros/core'; import type { BuildOptions } from '../../types/bundle.js'; -import type { SourceDestinationItem } from '../../types/template.js'; import { downloadPackages } from './package-manager.js'; -import { TemplateEngine } from './template-engine.js'; import type { Logger } from '../../core/index.js'; import { getTempDir } from '../../config/index.js'; import { @@ -403,18 +402,6 @@ function createEsbuildOptions( return baseOptions; } -// Helper function to convert package name to JS variable name -function packageNameToVariable(packageName: string): string { - return packageName - .replace('@', '_') - .replace(/[/-]/g, '_') - .split('_') - .map((part, i) => - i === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1), - ) - .join(''); -} - /** * Detects destination packages from flow configuration. * Extracts package names from destinations that have explicit 'package' field. @@ -443,6 +430,101 @@ function detectDestinationPackages(flowConfig: Flow.Config): Set { return destinationPackages; } +/** + * Detects source packages from flow configuration. + * Extracts package names from sources that have explicit 'package' field. + */ +function detectSourcePackages(flowConfig: Flow.Config): Set { + const sourcePackages = new Set(); + const sources = ( + flowConfig as unknown as { sources?: Record } + ).sources; + + if (sources) { + for (const [sourceKey, sourceConfig] of Object.entries(sources)) { + // Require explicit package field - no inference for any packages + if ( + typeof sourceConfig === 'object' && + sourceConfig !== null && + 'package' in sourceConfig && + typeof sourceConfig.package === 'string' + ) { + sourcePackages.add(sourceConfig.package); + } + } + } + + return sourcePackages; +} + +/** + * Detects explicit code imports from destinations and sources. + * Returns a map of package names to sets of export names. + */ +function detectExplicitCodeImports( + flowConfig: Flow.Config, +): Map> { + const explicitCodeImports = new Map>(); + + // Check destinations + const destinations = ( + flowConfig as unknown as { destinations?: Record } + ).destinations; + + if (destinations) { + for (const [destKey, destConfig] of Object.entries(destinations)) { + if ( + typeof destConfig === 'object' && + destConfig !== null && + 'package' in destConfig && + typeof destConfig.package === 'string' && + 'code' in destConfig && + typeof destConfig.code === 'string' + ) { + // Only treat as explicit if code doesn't match auto-generated pattern + // Auto-generated code starts with '_' (from packageNameToVariable) + const isAutoGenerated = destConfig.code.startsWith('_'); + if (!isAutoGenerated) { + if (!explicitCodeImports.has(destConfig.package)) { + explicitCodeImports.set(destConfig.package, new Set()); + } + explicitCodeImports.get(destConfig.package)!.add(destConfig.code); + } + } + } + } + + // Check sources + const sources = ( + flowConfig as unknown as { sources?: Record } + ).sources; + + if (sources) { + for (const [sourceKey, sourceConfig] of Object.entries(sources)) { + if ( + typeof sourceConfig === 'object' && + sourceConfig !== null && + 'package' in sourceConfig && + typeof sourceConfig.package === 'string' && + 'code' in sourceConfig && + typeof sourceConfig.code === 'string' + ) { + // Only treat as explicit if code doesn't match auto-generated pattern + // Auto-generated code starts with '_' (from packageNameToVariable) + const isAutoGenerated = sourceConfig.code.startsWith('_'); + if (!isAutoGenerated) { + if (!explicitCodeImports.has(sourceConfig.package)) { + explicitCodeImports.set(sourceConfig.package, new Set()); + } + explicitCodeImports.get(sourceConfig.package)!.add(sourceConfig.code); + } + } + } + } + + return explicitCodeImports; +} + interface ImportGenerationResult { importStatements: string[]; examplesMappings: string[]; @@ -450,17 +532,24 @@ interface ImportGenerationResult { /** * Generates import statements and examples mappings from build packages. - * Handles explicit imports, namespace imports, and auto-imports for destination packages. + * Handles explicit imports, default imports for destinations/sources, and utility imports. */ function generateImportStatements( packages: BuildOptions['packages'], destinationPackages: Set, + sourcePackages: Set, + explicitCodeImports: Map>, ): ImportGenerationResult { const importStatements: string[] = []; const examplesMappings: string[] = []; + const usedPackages = new Set([...destinationPackages, ...sourcePackages]); for (const [packageName, packageConfig] of Object.entries(packages)) { + const isUsedByDestOrSource = usedPackages.has(packageName); + const hasExplicitCode = explicitCodeImports.has(packageName); + if (packageConfig.imports && packageConfig.imports.length > 0) { + // Explicit imports (utilities) - existing logic // Remove duplicates within the same package const uniqueImports = [...new Set(packageConfig.imports)]; @@ -511,14 +600,20 @@ function generateImportStatements( ); } } - } else { - // No imports specified - import as namespace with a warning comment - // User should specify explicit imports for better tree-shaking - const varName = packageNameToVariable(packageName); + } else if (hasExplicitCode) { + // Package with explicit code specified in destinations/sources + // → Generate named imports + const codes = Array.from(explicitCodeImports.get(packageName)!); importStatements.push( - `import * as ${varName} from '${packageName}'; // Consider specifying explicit imports`, + `import { ${codes.join(', ')} } from '${packageName}';`, ); + } else if (isUsedByDestOrSource) { + // Package used by destination/source but no explicit imports or code + // → Generate default import + const varName = packageNameToVariable(packageName); + importStatements.push(`import ${varName} from '${packageName}';`); } + // If package declared but not used by any dest/source, skip import // Examples are no longer auto-imported - simulator loads them dynamically } @@ -527,127 +622,52 @@ function generateImportStatements( } /** - * Processes template if configured, otherwise returns code directly. - * Applies TemplateEngine to transform code with flow configuration. + * Creates the entry point code for the bundle. + * Generates imports, config object, and platform-specific wrapper programmatically. */ -async function processTemplate( - flowConfig: Flow.Config, - buildOptions: BuildOptions, -): Promise { - if (buildOptions.template) { - const templateEngine = new TemplateEngine(); - const flowWithProps = flowConfig as unknown as { - sources?: Record; - destinations?: Record; - collector?: Record; - }; - return await templateEngine.process( - buildOptions.template, - buildOptions.code || '', // Pass user code as parameter (empty if undefined) - (flowWithProps.sources || {}) as unknown as Record< - string, - SourceDestinationItem - >, - (flowWithProps.destinations || {}) as unknown as Record< - string, - SourceDestinationItem - >, - (flowWithProps.collector || {}) as unknown as Record, - buildOptions as unknown as Record, // Pass build config to template - ); - } else { - // No template - just use the code directly - return buildOptions.code || ''; - } -} - -/** - * Wraps code for specific output formats. - * Adds export wrapper for ESM without template when no export exists. - */ -function wrapCodeForFormat( - code: string, - format: BuildOptions['format'], - hasTemplate: boolean, -): string { - // Template outputs ready-to-use code - no wrapping needed - if (hasTemplate) { - return code; - } - - // Only add export wrapper for server ESM without template - if (format === 'esm') { - // Check if code already has export statements - const hasExport = /^\s*export\s/m.test(code); - - if (!hasExport) { - // Raw code without export - wrap as default export - return `export default ${code}`; - } - } - - return code; -} - -/** - * Assembles the final entry point code from all components. - * Combines imports, examples object, and wrapped code with format-specific exports. - */ -function assembleFinalCode( - importStatements: string[], - examplesObject: string, - wrappedCode: string, - format: BuildOptions['format'], -): string { - const importsCode = importStatements.join('\n'); - - let finalCode = importsCode - ? `${importsCode}\n\n${examplesObject}${wrappedCode}` - : `${examplesObject}${wrappedCode}`; - - // Make examples available for ESM - if (examplesObject && format === 'esm') { - // ESM: export as named export - finalCode += `\n\nexport { examples };`; - } - - return finalCode; -} - -async function createEntryPoint( +export async function createEntryPoint( flowConfig: Flow.Config, buildOptions: BuildOptions, packagePaths: Map, ): Promise { - // Detect destination packages for auto-importing examples + // Detect packages used by destinations and sources const destinationPackages = detectDestinationPackages(flowConfig); + const sourcePackages = detectSourcePackages(flowConfig); + const explicitCodeImports = detectExplicitCodeImports(flowConfig); - // Generate import statements (examples generation removed) + // Generate import statements const { importStatements } = generateImportStatements( buildOptions.packages, destinationPackages, + sourcePackages, + explicitCodeImports, ); - // No longer generate examples in bundles - simulator loads them dynamically - const examplesObject = ''; + const importsCode = importStatements.join('\n'); + const hasFlow = destinationPackages.size > 0 || sourcePackages.size > 0; - // Process template or use code directly - const templatedCode = await processTemplate(flowConfig, buildOptions); + // If no sources/destinations, just return user code with imports (no flow wrapper) + if (!hasFlow) { + const userCode = buildOptions.code || ''; + return importsCode ? `${importsCode}\n\n${userCode}` : userCode; + } - // Wrap code for specific output format - const wrappedCode = wrapCodeForFormat( - templatedCode, - buildOptions.format, - !!buildOptions.template, + // Build config object programmatically (DRY - single source of truth) + const configObject = buildConfigObject(flowConfig, explicitCodeImports); + + // Generate platform-specific wrapper + const wrappedCode = generatePlatformWrapper( + configObject, + buildOptions.code || '', + buildOptions as { + platform: string; + windowCollector?: string; + windowElb?: string; + }, ); - // Assemble final entry point code - return assembleFinalCode( - importStatements, - examplesObject, - wrappedCode, - buildOptions.format, - ); + // Assemble final code + return importsCode ? `${importsCode}\n\n${wrappedCode}` : wrappedCode; } interface EsbuildError { @@ -693,3 +713,129 @@ function createBuildError(buildError: EsbuildError, code: string): Error { : ''), ); } + +/** + * Build config object string from flow configuration. + * Respects import strategy decisions from detectExplicitCodeImports. + */ +export function buildConfigObject( + flowConfig: Flow.Config, + explicitCodeImports: Map>, +): string { + const flowWithProps = flowConfig as unknown as { + sources?: Record< + string, + { package: string; code?: string; config?: unknown; env?: unknown } + >; + destinations?: Record< + string, + { package: string; code?: string; config?: unknown; env?: unknown } + >; + collector?: unknown; + }; + + const sources = flowWithProps.sources || {}; + const destinations = flowWithProps.destinations || {}; + + // Build sources + const sourcesEntries = Object.entries(sources).map(([key, source]) => { + const hasExplicitCode = + source.code && explicitCodeImports.has(source.package); + const codeVar = hasExplicitCode + ? source.code + : packageNameToVariable(source.package); + + const configStr = source.config ? processConfigValue(source.config) : '{}'; + const envStr = source.env + ? `,\n env: ${processConfigValue(source.env)}` + : ''; + + return ` ${key}: {\n code: ${codeVar},\n config: ${configStr}${envStr}\n }`; + }); + + // Build destinations + const destinationsEntries = Object.entries(destinations).map( + ([key, dest]) => { + const hasExplicitCode = + dest.code && explicitCodeImports.has(dest.package); + const codeVar = hasExplicitCode + ? dest.code + : packageNameToVariable(dest.package); + + const configStr = dest.config ? processConfigValue(dest.config) : '{}'; + const envStr = dest.env + ? `,\n env: ${processConfigValue(dest.env)}` + : ''; + + return ` ${key}: {\n code: ${codeVar},\n config: ${configStr}${envStr}\n }`; + }, + ); + + // Build collector + const collectorStr = flowWithProps.collector + ? `,\n ...${processConfigValue(flowWithProps.collector)}` + : ''; + + return `{ + sources: { +${sourcesEntries.join(',\n')} + }, + destinations: { +${destinationsEntries.join(',\n')} + }${collectorStr} +}`; +} + +/** + * Process config value for serialization. + * Uses existing serializer utilities. + */ +function processConfigValue(value: unknown): string { + return JSON.stringify(value, null, 2); +} + +/** + * Generate platform-specific wrapper code. + */ +export function generatePlatformWrapper( + configObject: string, + userCode: string, + buildOptions: { + platform: string; + windowCollector?: string; + windowElb?: string; + }, +): string { + if (buildOptions.platform === 'browser') { + // Web platform: IIFE with browser globals + const windowAssignments = []; + if (buildOptions.windowCollector) { + windowAssignments.push( + ` if (typeof window !== 'undefined') window['${buildOptions.windowCollector}'] = collector;`, + ); + } + if (buildOptions.windowElb) { + windowAssignments.push( + ` if (typeof window !== 'undefined') window['${buildOptions.windowElb}'] = elb;`, + ); + } + const assignments = + windowAssignments.length > 0 ? '\n' + windowAssignments.join('\n') : ''; + + return `(async () => { + const config = ${configObject}; + + ${userCode} + + const { collector, elb } = await startFlow(config);${assignments} +})();`; + } else { + // Server platform: Export default function + const codeSection = userCode ? `\n ${userCode}\n` : ''; + + return `export default async function(context = {}) { + const config = ${configObject};${codeSection} + return await startFlow(config); +}`; + } +} diff --git a/packages/cli/src/commands/bundle/serializer.ts b/packages/cli/src/commands/bundle/serializer.ts deleted file mode 100644 index e1146a37d..000000000 --- a/packages/cli/src/commands/bundle/serializer.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * JSON to JavaScript serializer for config objects - * Converts JSON objects to valid JavaScript code for use in templates - */ - -import { isObject } from '../../config/index.js'; -import type { - TemplateSource, - TemplateDestination, - ProcessedTemplateVariables, -} from '../../types/template.js'; - -export interface SerializerOptions { - indent?: number; - singleQuotes?: boolean; -} - -/** - * Serialize a value to JavaScript code - */ -export function serializeToJS( - value: unknown, - options: SerializerOptions = {}, -): string { - const { indent = 2, singleQuotes = false } = options; - const quote = singleQuotes ? "'" : '"'; - - function serialize(val: unknown, currentIndent = 0): string { - if (val === null) return 'null'; - if (val === undefined) return 'undefined'; - - if (typeof val === 'boolean' || typeof val === 'number') { - return String(val); - } - - if (typeof val === 'string') { - // Check if string contains arrow function syntax - if (val.includes('=>')) { - // More comprehensive check for arrow function patterns - const arrowPatterns = [ - /^\s*\([^)]*\)\s*=>/, // (param) => or () => - /^\s*\w+\s*=>/, // param => - /^\s*\([^)]*\)\s*=>\s*\{/, // (param) => { - /^\s*\w+\s*=>\s*\{/, // param => { - ]; - - if (arrowPatterns.some((pattern) => pattern.test(val))) { - // Likely a function - return as-is without quotes - return val; - } - } - // Regular string - escape and quote - return ( - quote + - val.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/"/g, '\\"') + - quote - ); - } - - if (Array.isArray(val)) { - if (val.length === 0) return '[]'; - - const nextIndent = currentIndent + indent; - const spacing = ' '.repeat(nextIndent); - const items = val - .map((item) => spacing + serialize(item, nextIndent)) - .join(',\n'); - - return `[\n${items}\n${' '.repeat(currentIndent)}]`; - } - - if (isObject(val)) { - const entries = Object.entries(val); - if (entries.length === 0) return '{}'; - - const nextIndent = currentIndent + indent; - const spacing = ' '.repeat(nextIndent); - - const props = entries - .map(([key, value]) => { - // Check if key needs quotes (contains special characters or starts with number) - const needsQuotes = /[^a-zA-Z0-9_$]/.test(key) || /^[0-9]/.test(key); - const keyStr = needsQuotes ? quote + key + quote : key; - - return spacing + keyStr + ': ' + serialize(value, nextIndent); - }) - .join(',\n'); - - return `{\n${props}\n${' '.repeat(currentIndent)}}`; - } - - // Fallback for other types - return String(val); - } - - return serialize(value); -} - -/** - * Serialize config object for template usage - * Handles special cases for walkerOS configurations - */ -export function serializeConfig(config: Record): string { - // Handle empty config - if (!config || Object.keys(config).length === 0) { - return '{}'; - } - - return serializeToJS(config, { indent: 2, singleQuotes: true }); -} - -/** - * Process template variables to serialize config objects - */ -export function processTemplateVariables( - variables: Record, -): ProcessedTemplateVariables { - const processed = { ...variables }; - - // Process sources object - if (isObject(processed.sources)) { - const sourcesObj = processed.sources as Record; - const processedSources: Record = {}; - - for (const [name, source] of Object.entries(sourcesObj)) { - const typedSource = source as TemplateSource; - const { env: _, ...sourceWithoutEnv } = typedSource; - processedSources[name] = { - ...sourceWithoutEnv, - config: isObject(typedSource.config) - ? serializeConfig(typedSource.config) - : typedSource.config, // Pass through string configs unchanged - ...(typedSource.env !== undefined && { env: typedSource.env }), - }; - } - - processed.sources = processedSources; - } - - // Process destinations object - if (isObject(processed.destinations)) { - const destinationsObj = processed.destinations as Record; - const processedDestinations: Record = {}; - - for (const [name, dest] of Object.entries(destinationsObj)) { - const typedDest = dest as TemplateDestination; - const { env: _, ...destWithoutEnv } = typedDest; - processedDestinations[name] = { - ...destWithoutEnv, - config: isObject(typedDest.config) - ? serializeConfig(typedDest.config) - : typedDest.config, - ...(typedDest.env !== undefined && { env: typedDest.env }), - }; - } - - processed.destinations = processedDestinations; - } - - // Process collector object (if present) - if (isObject(processed.collector)) { - processed.collector = serializeConfig( - processed.collector as Record, - ); - } - - return processed; -} diff --git a/packages/cli/src/commands/bundle/template-engine.ts b/packages/cli/src/commands/bundle/template-engine.ts deleted file mode 100644 index 6a66c511c..000000000 --- a/packages/cli/src/commands/bundle/template-engine.ts +++ /dev/null @@ -1,85 +0,0 @@ -import fs from 'fs-extra'; -import Handlebars from 'handlebars'; -import type { SourceDestinationItem } from '../../types/template.js'; -import { processTemplateVariables } from './serializer.js'; -import { resolveAsset } from '../../core/asset-resolver.js'; - -export class TemplateEngine { - private handlebars: typeof Handlebars; - - constructor() { - // Create a new Handlebars instance - this.handlebars = Handlebars.create(); - - // Register equality helper for conditional window assignment - this.handlebars.registerHelper('eq', (a: unknown, b: unknown) => a === b); - } - - /** - * Load template content from file path - * - * @param templatePath - Template path (bare name, relative, or absolute) - */ - async loadTemplate(templatePath: string): Promise { - // Use unified asset resolver (works in both local and Docker) - const resolvedPath = resolveAsset(templatePath, 'template'); - - if (!(await fs.pathExists(resolvedPath))) { - throw new Error(`Template file not found: ${resolvedPath}`); - } - - return await fs.readFile(resolvedPath, 'utf-8'); - } - - /** - * Apply template with user code and variable substitution - */ - applyTemplate( - template: string, - userCode: string, - sources: Record, - destinations: Record, - collector: Record, - build?: Record, - ): string { - // Process template variables to serialize config objects - const processedVariables = processTemplateVariables({ - sources, - destinations, - collector, - }); - - // Prepare template data for Handlebars - const templateData: Record = { - CODE: userCode, - build: build || {}, - ...processedVariables, - }; - - // Compile and execute the template - const compiledTemplate = this.handlebars.compile(template); - return compiledTemplate(templateData); - } - - /** - * Process template with user code - */ - async process( - templatePath: string, - userCode: string, - sources: Record, - destinations: Record, - collector: Record, - build?: Record, - ): Promise { - const template = await this.loadTemplate(templatePath); - return this.applyTemplate( - template, - userCode, - sources, - destinations, - collector, - build, - ); - } -} diff --git a/packages/cli/src/commands/run/execution.ts b/packages/cli/src/commands/run/execution.ts index cf0afcafa..a20a56e6a 100644 --- a/packages/cli/src/commands/run/execution.ts +++ b/packages/cli/src/commands/run/execution.ts @@ -1,6 +1,11 @@ +import { createLogger, Level } from '@walkeros/core'; import type { RuntimeConfig, ServeConfig } from '@walkeros/docker'; import { runFlow, runServeMode } from '@walkeros/docker'; +// Create logger for local execution - DEBUG level when VERBOSE, otherwise INFO +const logLevel = process.env.VERBOSE === 'true' ? Level.DEBUG : Level.INFO; +const logger = createLogger({ level: logLevel }); + /** * Execute run command locally * @@ -27,7 +32,7 @@ export async function executeRunLocal( port: options.port, host: options.host, }; - await runFlow(flowPath, config); + await runFlow(flowPath, config, logger.scope('runner')); break; } @@ -39,7 +44,7 @@ export async function executeRunLocal( servePath: options.servePath, filePath: flowPath || undefined, }; - await runServeMode(config); + await runServeMode(config, logger.scope('serve')); break; } diff --git a/packages/cli/src/config/build-defaults.ts b/packages/cli/src/config/build-defaults.ts index 19e0588ec..aa32a7db0 100644 --- a/packages/cli/src/config/build-defaults.ts +++ b/packages/cli/src/config/build-defaults.ts @@ -17,7 +17,6 @@ export const WEB_BUILD_DEFAULTS: Omit = { format: 'iife', platform: 'browser', target: 'es2020', - template: 'web.hbs', minify: true, sourcemap: false, cache: true, @@ -37,7 +36,6 @@ export const SERVER_BUILD_DEFAULTS: Omit = format: 'esm', platform: 'node', target: 'node20', - template: 'server.hbs', minify: true, sourcemap: false, cache: true, diff --git a/packages/cli/src/core/asset-resolver.ts b/packages/cli/src/core/asset-resolver.ts index 86e085992..4dee5027b 100644 --- a/packages/cli/src/core/asset-resolver.ts +++ b/packages/cli/src/core/asset-resolver.ts @@ -1,7 +1,7 @@ /** * Asset Resolver * - * Unified path resolution for package assets (templates, examples) and user assets. + * Unified path resolution for package assets (examples) and user assets. * Assets are always siblings to the CLI entry point (in dist/ for production). */ @@ -15,7 +15,7 @@ import path from 'path'; let cachedAssetDir: string | undefined; /** - * Get the directory containing CLI assets (templates, examples). + * Get the directory containing CLI assets (examples). * * In production: assets are in dist/ alongside the bundled CLI * In development: assets are at package root @@ -28,9 +28,9 @@ export function getAssetDir(): string { const currentFile = fileURLToPath(import.meta.url); let dir = path.dirname(currentFile); - // Walk up until we find a directory with templates/ sibling + // Walk up until we find a directory with examples/ sibling while (dir !== path.dirname(dir)) { - if (existsSync(path.join(dir, 'templates'))) { + if (existsSync(path.join(dir, 'examples'))) { cachedAssetDir = dir; return dir; } @@ -45,13 +45,13 @@ export function getAssetDir(): string { /** * Asset type for resolution strategy */ -export type AssetType = 'template' | 'config' | 'bundle'; +export type AssetType = 'config' | 'bundle'; /** * Resolve asset path using unified strategy * * Resolution rules: - * 1. Bare names (no / or \) → Package asset (templates or examples) + * 1. Bare names (no / or \) → Package asset (examples) * 2. Relative paths (./ or ../) → User asset relative to base directory * 3. Absolute paths → Use as-is * @@ -65,13 +65,9 @@ export function resolveAsset( assetType: AssetType, baseDir?: string, ): string { - // Bare name → package asset + // Bare name → package asset (examples directory) if (!assetPath.includes('/') && !assetPath.includes('\\')) { const assetDir = getAssetDir(); - if (assetType === 'template') { - return path.join(assetDir, 'templates', assetPath); - } - // config or bundle → examples directory return path.join(assetDir, 'examples', assetPath); } diff --git a/packages/cli/src/core/docker.ts b/packages/cli/src/core/docker.ts index b9a53e4dc..237bfe77b 100644 --- a/packages/cli/src/core/docker.ts +++ b/packages/cli/src/core/docker.ts @@ -166,7 +166,9 @@ export async function executeInDocker( if (code === 0) { resolve(); } else { - reject(new Error(`Docker command exited with code ${code}`)); + // Docker already logged the error via stdio inherit + // Just exit with same code - no duplicate message + process.exit(code || 1); } }); }); @@ -323,7 +325,9 @@ export async function executeRunInDocker( if (code === 0) { resolve(); } else { - reject(new Error(`Docker command exited with code ${code}`)); + // Docker already logged the error via stdio inherit + // Just exit with same code - no duplicate message + process.exit(code || 1); } }); }); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 6bb50b9bc..c8f08a402 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2,6 +2,7 @@ import { Command } from 'commander'; import { readFileSync } from 'fs'; import { fileURLToPath } from 'url'; import { dirname, join } from 'path'; +import { VERSION as DOCKER_VERSION } from '@walkeros/docker'; import { bundleCommand } from './commands/bundle/index.js'; import { simulateCommand } from './commands/simulate/index.js'; import { pushCommand } from './commands/push/index.js'; @@ -37,13 +38,6 @@ export type { } from './types/bundle.js'; export type { BundleStats } from './commands/bundle/bundler.js'; export type { SimulationResult } from './commands/simulate/types.js'; -export type { - SourceDestinationItem, - TemplateVariables, - ProcessedTemplateVariables, - TemplateSource, - TemplateDestination, -} from './types/template.js'; export type { RunMode, RunCommandOptions, @@ -59,6 +53,16 @@ program .description('walkerOS CLI - Bundle and deploy walkerOS components') .version(VERSION); +// Display startup banner before any command runs +program.hook('preAction', (thisCommand, actionCommand) => { + const options = actionCommand.opts(); + // Skip banner for --silent, --json, or --help flags + if (!options.silent && !options.json) { + console.log(`🚀 walkerOS CLI v${VERSION}`); + console.log(`🐳 Using Docker runtime: walkeros/docker:${DOCKER_VERSION}`); + } +}); + // Bundle command program .command('bundle [file]') diff --git a/packages/cli/src/types/bundle.ts b/packages/cli/src/types/bundle.ts index 7984af2cc..f7acd9790 100644 --- a/packages/cli/src/types/bundle.ts +++ b/packages/cli/src/types/bundle.ts @@ -41,12 +41,6 @@ export interface CLIBuildOptions */ tempDir?: string; - /** - * Custom template file path. - * @default undefined (uses platform-specific default) - */ - template?: string; - /** * Enable package caching. * @default true diff --git a/packages/cli/src/types/template.ts b/packages/cli/src/types/template.ts deleted file mode 100644 index e5895476e..000000000 --- a/packages/cli/src/types/template.ts +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Template Configuration Types - * - * Type definitions for template processing and serialization. - * Used by the template engine and serializer. - * - * @packageDocumentation - */ - -/** - * Source or Destination configuration item for templates. - * - * @remarks - * This type is used in template processing where config objects - * are serialized to JavaScript code. - */ -export interface SourceDestinationItem { - /** - * JavaScript code reference (variable name or expression) - */ - code: string; - - /** - * Configuration object for the source/destination - */ - config?: unknown; - - /** - * Environment-specific variables - */ - env?: unknown; - - /** - * Allow additional properties for extensibility - */ - [key: string]: unknown; -} - -/** - * Template variables that can be used in Handlebars templates. - * - * @remarks - * These variables are available in template files for customization. - */ -export interface TemplateVariables { - /** - * Serialized sources configuration - */ - sources: string; - - /** - * Serialized destinations configuration - */ - destinations: string; - - /** - * Serialized collector configuration - */ - collector: string; - - /** - * User-provided code to be inserted - */ - CODE: string; - - /** - * Build configuration (optional) - */ - build?: Record; -} - -/** - * Processed template variables after serialization. - * - * @remarks - * Internal type used by the serializer to represent processed configs. - */ -export interface ProcessedTemplateVariables { - /** - * Processed sources with serialized configs - */ - sources?: Record; - - /** - * Processed destinations with serialized configs - */ - destinations?: Record; - - /** - * Processed collector configuration - */ - collector?: Record | string; - - /** - * Allow additional properties - */ - [key: string]: unknown; -} - -/** - * Template source after processing. - * - * @internal - */ -export interface TemplateSource { - code: string; - config?: unknown | string; // Can be object or serialized string - env?: unknown; - [key: string]: unknown; -} - -/** - * Template destination after processing. - * - * @internal - */ -export interface TemplateDestination { - code: string; - config?: unknown | string; // Can be object or serialized string - env?: unknown; - [key: string]: unknown; -} diff --git a/packages/cli/templates/server.hbs b/packages/cli/templates/server.hbs deleted file mode 100644 index 42df9c44a..000000000 --- a/packages/cli/templates/server.hbs +++ /dev/null @@ -1,29 +0,0 @@ -export default async function(context = {}) { - const config = { - sources: { - {{#each sources}} - {{@key}}: { - code: {{{code}}}, - config: {{{config}}}{{#if env}}, - env: {{{env}}}{{/if}} - }, - {{/each}} - }, - destinations: { - {{#each destinations}} - {{@key}}: { - code: {{{code}}}, - config: {{{config}}}{{#if env}}, - env: {{{env}}}{{/if}} - }, - {{/each}} - }{{#if collector}}, - ...{{{collector}}}{{/if}} - }; - - {{{CODE}}} - - const result = await startFlow(config); - - return result; -} diff --git a/packages/cli/templates/web.hbs b/packages/cli/templates/web.hbs deleted file mode 100644 index 249e3bf35..000000000 --- a/packages/cli/templates/web.hbs +++ /dev/null @@ -1,45 +0,0 @@ -(async () => { - // Check if we're in a browser environment - const window = typeof globalThis.window !== 'undefined' ? globalThis.window : undefined; - const document = typeof globalThis.document !== 'undefined' ? globalThis.document : undefined; - - const config = { - sources: { - {{#each sources}} - {{@key}}: { - code: {{{code}}}, - config: {{{config}}}{{#unless config}}{}{{/unless}}{{#if env}}, - env: { - window, - document, - ...{{{env}}} - }{{/if}} - }{{#unless @last}},{{/unless}} - {{/each}} - }, - destinations: { - {{#each destinations}} - {{@key}}: { - code: {{{code}}}, - config: {{{config}}}{{#unless config}}{}{{/unless}}{{#if env}}, - env: { - window, - document, - ...{{{env}}} - }{{/if}} - }{{#unless @last}},{{/unless}} - {{/each}} - }{{#if collector}}, - ...{{{collector}}}{{/if}} - }; - - {{{CODE}}} - - const { collector, elb } = await startFlow(config); - - if (typeof window !== 'undefined') { - {{#if build.windowCollector}}window['{{build.windowCollector}}'] = collector; - {{/if}}{{#if build.windowElb}}window['{{build.windowElb}}'] = elb; - {{/if}} - } -})(); diff --git a/packages/collector/README.md b/packages/collector/README.md index 1c5f46854..736b7f12b 100644 --- a/packages/collector/README.md +++ b/packages/collector/README.md @@ -1,6 +1,6 @@

- - + +

@@ -75,7 +75,7 @@ action by space the collector won't process it. npm install @walkeros/collector ``` -## Setup +## Quick Start ### Basic setup @@ -108,27 +108,38 @@ const { collector, elb } = await startFlow({ destinations: [ // add your event destinations ], - verbose: true, - onError: (error: unknown) => { - console.error('Collector error:', error); - }, - onLog: (message: string, level: 'debug' | 'info' | 'warn' | 'error') => { - console.log(`[${level}] ${message}`); + logger: { + level: 'debug', // 'debug' | 'info' | 'warn' | 'error' + handler: (message, level) => { + console.log(`[${level}] ${message}`); + }, }, }); ``` ## Configuration -| Name | Type | Description | Required | Example | -| -------------- | ---------- | -------------------------------------------------------------- | -------- | ------------------------------------------ | -| `run` | `boolean` | Automatically start the collector pipeline on initialization | No | `true` | -| `sources` | `array` | Configurations for sources providing events to the collector | No | `[{ source, config }]` | -| `destinations` | `array` | Configurations for destinations receiving processed events | No | `[{ destination, config }]` | -| `consent` | `object` | Initial consent state to control routing of events | No | `{ analytics: true, marketing: false }` | -| `verbose` | `boolean` | Enable verbose logging for debugging | No | `false` | -| `onError` | `function` | Error handler triggered when the collector encounters failures | No | `(error) => console.error(error)` | -| `onLog` | `function` | Custom log handler for collector messages | No | `(message, level) => console.log(message)` | +| Name | Type | Description | Required | Example | +| -------------- | --------- | ------------------------------------------------------------ | -------- | --------------------------------------- | +| `run` | `boolean` | Automatically start the collector pipeline on initialization | No | `true` | +| `sources` | `array` | Configurations for sources providing events to the collector | No | `[{ source, config }]` | +| `destinations` | `array` | Configurations for destinations receiving processed events | No | `[{ destination, config }]` | +| `consent` | `object` | Initial consent state to control routing of events | No | `{ analytics: true, marketing: false }` | +| `logger` | `object` | Logger configuration with level and custom handler | No | `{ level: 'info', handler: fn }` | + +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces: + +- [flow.ts](./src/types/flow.ts) - Flow configuration +- [collector.ts](./src/types/collector.ts) - Collector interface + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/getting-started/flow/) +- [Core Package](../core/) - Types and utilities +- [Web Sources](../web/sources/) - Browser event sources +- [Server Sources](../server/sources/) - Node.js event sources ## Contribute diff --git a/packages/collector/src/__tests__/destination-code.test.ts b/packages/collector/src/__tests__/destination-code.test.ts new file mode 100644 index 000000000..6c63f94a3 --- /dev/null +++ b/packages/collector/src/__tests__/destination-code.test.ts @@ -0,0 +1,389 @@ +import type { Collector, Destination } from '@walkeros/core'; +import { createEvent, createMockLogger } from '@walkeros/core'; +import { destinationCode } from '../destination-code'; +import { initDestinations } from '../destination'; +import type { + Settings, + CodeMapping, + InitContext, + PushContext, + PushBatchContext, + Context, +} from '../types/code'; + +describe('destinationCode', () => { + const createMockCollector = (): Collector.Instance => + ({ + consent: {}, + destinations: {}, + sources: {}, + queue: [], + hooks: {}, + on: {}, + globals: {}, + user: {}, + allowed: true, + config: {}, + count: 0, + logger: createMockLogger(), + push: jest.fn(), + }) as unknown as Collector.Instance; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('basic properties', () => { + it('should have correct type', () => { + expect(destinationCode.type).toBe('code'); + }); + + it('should have empty default config', () => { + expect(destinationCode.config).toEqual({}); + }); + }); + + describe('init', () => { + it('executes init code string', () => { + const mockLogger = createMockLogger(); + const context: InitContext = { + collector: createMockCollector(), + config: { + settings: { + init: "context.logger.info('initialized')", + }, + }, + env: {}, + logger: mockLogger, + }; + + destinationCode.init!(context); + + expect(mockLogger.info).toHaveBeenCalledWith('initialized'); + }); + + it('handles missing init code gracefully', () => { + const context: InitContext = { + collector: createMockCollector(), + config: { settings: {} }, + env: {}, + logger: createMockLogger(), + }; + + expect(() => destinationCode.init!(context)).not.toThrow(); + }); + + it('catches and logs errors in init code', () => { + const mockLogger = createMockLogger(); + const context: InitContext = { + collector: createMockCollector(), + config: { + settings: { + init: "throw new Error('test error')", + }, + }, + env: {}, + logger: mockLogger, + }; + + destinationCode.init!(context); + + expect(mockLogger.error).toHaveBeenCalled(); + }); + }); + + describe('push', () => { + it('executes push code from mapping', () => { + const mockLogger = createMockLogger(); + const context: PushContext = { + collector: createMockCollector(), + config: {}, + data: { transformed: true }, + env: {}, + logger: mockLogger, + mapping: { + push: 'context.logger.info(event.name, context.data)', + } as CodeMapping, + }; + + destinationCode.push(createEvent({ name: 'product view' }), context); + + expect(mockLogger.info).toHaveBeenCalledWith('product view', { + transformed: true, + }); + }); + + it('falls back to settings.push when mapping.push is missing', () => { + const mockLogger = createMockLogger(); + const context: PushContext = { + collector: createMockCollector(), + config: { + settings: { + push: "context.logger.info('settings fallback')", + } as Settings, + }, + data: {}, + env: {}, + logger: mockLogger, + mapping: {}, + }; + + destinationCode.push(createEvent({ name: 'product view' }), context); + + expect(mockLogger.info).toHaveBeenCalledWith('settings fallback'); + }); + + it('prefers mapping.push over settings.push', () => { + const mockLogger = createMockLogger(); + const context: PushContext = { + collector: createMockCollector(), + config: { + settings: { + push: "context.logger.info('from settings')", + } as Settings, + }, + data: {}, + env: {}, + logger: mockLogger, + mapping: { + push: "context.logger.info('from mapping')", + } as CodeMapping, + }; + + destinationCode.push(createEvent({ name: 'product view' }), context); + + expect(mockLogger.info).toHaveBeenCalledWith('from mapping'); + expect(mockLogger.info).not.toHaveBeenCalledWith('from settings'); + }); + + it('handles missing push code gracefully', () => { + const context: PushContext = { + collector: createMockCollector(), + config: {}, + env: {}, + logger: createMockLogger(), + mapping: {}, + data: {}, + }; + + expect(() => + destinationCode.push(createEvent({ name: 'product view' }), context), + ).not.toThrow(); + }); + + it('catches and logs errors in push code', () => { + const mockLogger = createMockLogger(); + const context: PushContext = { + collector: createMockCollector(), + config: {}, + env: {}, + logger: mockLogger, + mapping: { + push: "throw new Error('test error')", + } as CodeMapping, + data: {}, + }; + + destinationCode.push(createEvent({ name: 'product view' }), context); + + expect(mockLogger.error).toHaveBeenCalled(); + }); + }); + + describe('pushBatch', () => { + it('executes pushBatch code from mapping', () => { + const mockLogger = createMockLogger(); + const batch: Destination.Batch = { + key: 'product view', + events: [ + createEvent({ name: 'product view', id: '1' }), + createEvent({ name: 'product view', id: '2' }), + ], + data: [{ id: '1' }, { id: '2' }], + }; + + const context: PushBatchContext = { + collector: createMockCollector(), + config: {}, + env: {}, + logger: mockLogger, + mapping: { + pushBatch: "context.logger.info('batch size:', batch.events.length)", + } as CodeMapping, + }; + + destinationCode.pushBatch!(batch, context); + + expect(mockLogger.info).toHaveBeenCalledWith('batch size:', 2); + }); + + it('falls back to settings.pushBatch when mapping.pushBatch is missing', () => { + const mockLogger = createMockLogger(); + const batch: Destination.Batch = { + key: 'test', + events: [], + data: [], + }; + + const context: PushBatchContext = { + collector: createMockCollector(), + config: { + settings: { + pushBatch: "context.logger.info('batch settings fallback')", + } as Settings, + }, + env: {}, + logger: mockLogger, + mapping: {}, + }; + + destinationCode.pushBatch!(batch, context); + + expect(mockLogger.info).toHaveBeenCalledWith('batch settings fallback'); + }); + + it('handles missing pushBatch code gracefully', () => { + const batch: Destination.Batch = { + key: 'test', + events: [], + data: [], + }; + + const context: PushBatchContext = { + collector: createMockCollector(), + config: {}, + env: {}, + logger: createMockLogger(), + mapping: {}, + }; + + expect(() => destinationCode.pushBatch!(batch, context)).not.toThrow(); + }); + + it('catches and logs errors in pushBatch code', () => { + const mockLogger = createMockLogger(); + const batch: Destination.Batch = { + key: 'test', + events: [], + data: [], + }; + + const context: PushBatchContext = { + collector: createMockCollector(), + config: {}, + env: {}, + logger: mockLogger, + mapping: { + pushBatch: "throw new Error('test error')", + } as CodeMapping, + }; + + destinationCode.pushBatch!(batch, context); + + expect(mockLogger.error).toHaveBeenCalled(); + }); + }); + + describe('on', () => { + it('executes on code string', () => { + const mockLogger = createMockLogger(); + const context: Context = { + collector: createMockCollector(), + config: { + settings: { + on: "if (type === 'consent') context.logger.info('consent:', context.data)", + } as Settings, + }, + data: { marketing: true }, + env: {}, + logger: mockLogger, + }; + + destinationCode.on!('consent', context); + + expect(mockLogger.info).toHaveBeenCalledWith('consent:', { + marketing: true, + }); + }); + + it('handles missing on code gracefully', () => { + const context: Context = { + collector: createMockCollector(), + config: { settings: {} }, + env: {}, + logger: createMockLogger(), + }; + + expect(() => destinationCode.on!('consent', context)).not.toThrow(); + }); + + it('catches and logs errors in on code', () => { + const mockLogger = createMockLogger(); + const context: Context = { + collector: createMockCollector(), + config: { + settings: { + on: "throw new Error('test error')", + } as Settings, + }, + env: {}, + logger: mockLogger, + }; + + destinationCode.on!('consent', context); + + expect(mockLogger.error).toHaveBeenCalled(); + }); + }); +}); + +describe('code: true initialization', () => { + it('uses built-in destinationCode when code is true', async () => { + const collector = { + logger: createMockLogger(), + } as unknown as Collector.Instance; + + const destinations = await initDestinations(collector, { + myCodeDest: { + code: true as unknown as Destination.Instance, + config: { + settings: { + init: "context.logger.info('ready')", + }, + }, + }, + }); + + expect(destinations.myCodeDest).toBeDefined(); + expect(destinations.myCodeDest.type).toBe('code'); + expect(destinations.myCodeDest.init).toBeDefined(); + expect(destinations.myCodeDest.push).toBeDefined(); + }); + + it('preserves provided config with code: true', async () => { + const collector = { + logger: createMockLogger(), + } as unknown as Collector.Instance; + + const destinations = await initDestinations(collector, { + myCodeDest: { + code: true as unknown as Destination.Instance, + config: { + settings: { + init: "context.logger.info('custom init')", + push: "context.logger.info('custom push')", + }, + consent: { functional: true }, + }, + }, + }); + + expect(destinations.myCodeDest.config.settings).toEqual({ + init: "context.logger.info('custom init')", + push: "context.logger.info('custom push')", + }); + expect(destinations.myCodeDest.config.consent).toEqual({ + functional: true, + }); + }); +}); diff --git a/packages/collector/src/__tests__/destination.test.ts b/packages/collector/src/__tests__/destination.test.ts index db84c172a..a47b96ad3 100644 --- a/packages/collector/src/__tests__/destination.test.ts +++ b/packages/collector/src/__tests__/destination.test.ts @@ -41,12 +41,17 @@ describe('Destination', () => { ): Collector.Instance { const defaultConfig = createTestConfig(); + // Create mock logger with proper scope chaining + const mockLogger = createMockLogger(); + const scopedMockLogger = createMockLogger(); + mockLogger.scope = jest.fn().mockReturnValue(scopedMockLogger); + return { allowed: true, destinations: { foo: destination }, globals: {}, hooks: {}, - logger: createMockLogger(), + logger: mockLogger, user: {}, consent: {}, queue: [], @@ -115,6 +120,48 @@ describe('Destination', () => { expect(destination.config.init).toBeFalsy(); }); + test('logs init lifecycle', async () => { + const collector = createWalkerjs(); + + await pushToDestinations(collector, event); + + expect(mockInit).toHaveBeenCalledTimes(1); + expect(mockPush).toHaveBeenCalledTimes(1); + + // Verify logger.scope was called with destination type + expect(collector.logger.scope).toHaveBeenCalledWith('unknown'); + + // Get the scoped logger instance + const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] + .value; + + // Verify init lifecycle logs + expect(scopedLogger.debug).toHaveBeenCalledWith('init'); + expect(scopedLogger.debug).toHaveBeenCalledWith('init done'); + }); + + test('logs push lifecycle', async () => { + const collector = createWalkerjs(); + destination.config.init = true; // Skip init for this test + + await pushToDestinations(collector, event); + + expect(mockPush).toHaveBeenCalledTimes(1); + + // Verify logger.scope was called with destination type + expect(collector.logger.scope).toHaveBeenCalledWith('unknown'); + + // Get the scoped logger instance + const scopedLogger = (collector.logger.scope as jest.Mock).mock.results[0] + .value; + + // Verify push lifecycle logs + expect(scopedLogger.debug).toHaveBeenCalledWith('push', { + event: event.name, + }); + expect(scopedLogger.debug).toHaveBeenCalledWith('push done'); + }); + test('DLQ', async () => { const event = createEvent(); // Simulate a failing push @@ -155,7 +202,16 @@ describe('Destination', () => { await elb('walker consent', { marketing: true }); // Verify the destination's on method was called with consent context - expect(mockOnMethod).toHaveBeenCalledWith('consent', { marketing: true }); + expect(mockOnMethod).toHaveBeenCalledWith( + 'consent', + expect.objectContaining({ + data: { marketing: true }, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); it('should call destination on method when session event is triggered', async () => { @@ -178,7 +234,16 @@ describe('Destination', () => { await elb('walker session'); // Verify the destination's on method was called with session context - expect(mockOnMethod).toHaveBeenCalledWith('session', collector.session); + expect(mockOnMethod).toHaveBeenCalledWith( + 'session', + expect.objectContaining({ + data: collector.session, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); it('should call destination on method when ready event is triggered', async () => { @@ -195,7 +260,16 @@ describe('Destination', () => { await elb('walker ready'); // Verify the destination's on method was called - expect(mockOnMethod).toHaveBeenCalledWith('ready', undefined); + expect(mockOnMethod).toHaveBeenCalledWith( + 'ready', + expect.objectContaining({ + data: undefined, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); it('should call destination on method when run event is triggered', async () => { @@ -212,7 +286,16 @@ describe('Destination', () => { await elb('walker run'); // Verify the destination's on method was called - expect(mockOnMethod).toHaveBeenCalledWith('run', undefined); + expect(mockOnMethod).toHaveBeenCalledWith( + 'run', + expect.objectContaining({ + data: undefined, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); it('should not fail if destination does not have on method', async () => { @@ -248,9 +331,16 @@ describe('Destination', () => { await elb('walker consent', { marketing: true }); // Verify the async destination's on method was called - expect(asyncOnMethod).toHaveBeenCalledWith('consent', { - marketing: true, - }); + expect(asyncOnMethod).toHaveBeenCalledWith( + 'consent', + expect.objectContaining({ + data: { marketing: true }, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); it('should call on method for multiple destinations', async () => { @@ -278,8 +368,26 @@ describe('Destination', () => { await elb('walker consent', { marketing: true }); // Both destinations should receive the event - expect(mockOn1).toHaveBeenCalledWith('consent', { marketing: true }); - expect(mockOn2).toHaveBeenCalledWith('consent', { marketing: true }); + expect(mockOn1).toHaveBeenCalledWith( + 'consent', + expect.objectContaining({ + data: { marketing: true }, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); + expect(mockOn2).toHaveBeenCalledWith( + 'consent', + expect.objectContaining({ + data: { marketing: true }, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); it('should handle on method errors gracefully', async () => { @@ -302,9 +410,16 @@ describe('Destination', () => { }).not.toThrow(); // On method should still have been called - expect(errorOnMethod).toHaveBeenCalledWith('consent', { - marketing: true, - }); + expect(errorOnMethod).toHaveBeenCalledWith( + 'consent', + expect.objectContaining({ + data: { marketing: true }, + collector: expect.any(Object), + config: expect.any(Object), + env: expect.any(Object), + logger: expect.any(Object), + }), + ); }); }); }); diff --git a/packages/collector/src/destination-code.ts b/packages/collector/src/destination-code.ts new file mode 100644 index 000000000..9b56cc841 --- /dev/null +++ b/packages/collector/src/destination-code.ts @@ -0,0 +1,61 @@ +import type { Destination } from '@walkeros/core'; +import type { CodeMapping, Settings } from './types/code'; + +export const destinationCode: Destination.Instance = { + type: 'code', + config: {}, + + init(context) { + const { config, logger } = context; + const initCode = (config.settings as Settings | undefined)?.init; + if (!initCode) return; + try { + const fn = new Function('context', initCode); + fn(context); + } catch (e) { + logger.error('Code destination init error:', e); + } + }, + + push(event, context) { + const { mapping, config, logger } = context; + const pushCode = + (mapping as CodeMapping | undefined)?.push ?? + (config.settings as Settings | undefined)?.push; + if (!pushCode) return; + try { + const fn = new Function('event', 'context', pushCode); + fn(event, context); + } catch (e) { + logger.error('Code destination push error:', e); + } + }, + + pushBatch(batch, context) { + const { mapping, config, logger } = context; + const pushBatchCode = + (mapping as CodeMapping | undefined)?.pushBatch ?? + (config.settings as Settings | undefined)?.pushBatch; + if (!pushBatchCode) return; + try { + const fn = new Function('batch', 'context', pushBatchCode); + fn(batch, context); + } catch (e) { + logger.error('Code destination pushBatch error:', e); + } + }, + + on(type, context) { + const { config, logger } = context; + const onCode = (config.settings as Settings | undefined)?.on; + if (!onCode) return; + try { + const fn = new Function('type', 'context', onCode); + fn(type, context); + } catch (e) { + logger.error('Code destination on error:', e); + } + }, +}; + +export default destinationCode; diff --git a/packages/collector/src/destination.ts b/packages/collector/src/destination.ts index 2fee45c34..4d86899d5 100644 --- a/packages/collector/src/destination.ts +++ b/packages/collector/src/destination.ts @@ -11,6 +11,11 @@ import { tryCatchAsync, useHooks, } from '@walkeros/core'; +import { destinationCode } from './destination-code'; + +function resolveCode(code: Destination.Instance | true): Destination.Instance { + return code === true ? destinationCode : code; +} /** * Adds a new destination to the collector. @@ -28,10 +33,11 @@ export async function addDestination( const { code, config: dataConfig = {}, env = {} } = data; const config = options || dataConfig || { init: false }; + const resolved = resolveCode(code); const destination: Destination.Instance = { - ...code, + ...resolved, config, - env: mergeEnvironments(code.env, env), + env: mergeEnvironments(resolved.env, env), }; let id = destination.config.id; // Use given id @@ -226,6 +232,8 @@ export async function destinationInit( logger: destLogger, } as Destination.InitContext; + destLogger.debug('init'); + const configResult = await useHooks( destination.init, 'DestinationInit', @@ -240,6 +248,8 @@ export async function destinationInit( ...(configResult || destination.config), init: true, // Remember that the destination was initialized }; + + destLogger.debug('init done'); } return true; // Destination is ready to push @@ -304,12 +314,18 @@ export async function destinationPush( logger: batchLogger, }; + batchLogger.debug('push batch', { + events: batched.events.length, + }); + useHooks( destination.pushBatch!, 'DestinationPushBatch', (collector as Collector.Instance).hooks, )(batched, batchContext); + batchLogger.debug('push batch done'); + batched.events = []; batched.data = []; }, eventMapping.batch); @@ -317,12 +333,16 @@ export async function destinationPush( eventMapping.batched = batched; eventMapping.batchFn?.(destination, collector); } else { + destLogger.debug('push', { event: processed.event.name }); + // It's time to go to the destination's side now await useHooks( destination.push, 'DestinationPush', collector.hooks, )(processed.event, context); + + destLogger.debug('push done'); } return true; @@ -364,19 +384,17 @@ export async function initDestinations( for (const [name, destinationDef] of Object.entries(destinations)) { const { code, config = {}, env = {} } = destinationDef; + const resolved = resolveCode(code); - // Merge config: destination default + provided config const mergedConfig = { - ...code.config, + ...resolved.config, ...config, }; - // Merge environment: destination default + provided env - const mergedEnv = mergeEnvironments(code.env, env); + const mergedEnv = mergeEnvironments(resolved.env, env); - // Create destination instance by spreading code and overriding config/env result[name] = { - ...code, + ...resolved, config: mergedConfig, env: mergedEnv, }; @@ -389,7 +407,7 @@ export async function initDestinations( * Merges destination environment with config environment * Config env takes precedence over destination env for overrides */ -function mergeEnvironments( +export function mergeEnvironments( destinationEnv?: Destination.Env, configEnv?: Destination.Env, ): Destination.Env { diff --git a/packages/collector/src/index.ts b/packages/collector/src/index.ts index fba86d3ca..51fe9c9e9 100644 --- a/packages/collector/src/index.ts +++ b/packages/collector/src/index.ts @@ -6,6 +6,7 @@ export * from './consent'; export * from './flow'; export * from './push'; export * from './destination'; +export * from './destination-code'; export * from './handle'; export * from './on'; export * from './source'; diff --git a/packages/collector/src/on.ts b/packages/collector/src/on.ts index c1ac2da8f..9f98b0d0a 100644 --- a/packages/collector/src/on.ts +++ b/packages/collector/src/on.ts @@ -1,7 +1,8 @@ -import type { Collector, On, WalkerOS } from '@walkeros/core'; +import type { Collector, On, WalkerOS, Destination } from '@walkeros/core'; import { isArray } from '@walkeros/core'; import { Const } from './constants'; import { tryCatch } from '@walkeros/core'; +import { mergeEnvironments } from './destination'; /** * Registers a callback for a specific event type. @@ -77,9 +78,21 @@ export function onApply( Object.values(collector.destinations).forEach((destination) => { if (destination.on) { - // Cast to runtime-compatible version for type safety - const onFn = destination.on as On.OnFnRuntime; - tryCatch(onFn)(type, contextData as On.AnyEventContext); + const destType = destination.type || 'unknown'; + const destLogger = collector.logger + .scope(destType) + .scope('on') + .scope(type); + + const context: Destination.Context = { + collector, + config: destination.config, + data: contextData as Destination.Data, + env: mergeEnvironments(destination.env, destination.config.env), + logger: destLogger, + }; + + tryCatch(destination.on)(type, context); } }); diff --git a/packages/collector/src/types/code.ts b/packages/collector/src/types/code.ts new file mode 100644 index 000000000..c1f12c08d --- /dev/null +++ b/packages/collector/src/types/code.ts @@ -0,0 +1,28 @@ +import type { Destination, Mapping, On, WalkerOS } from '@walkeros/core'; + +export interface Settings { + init?: string; + on?: string; + push?: string; + pushBatch?: string; +} + +export interface CodeMapping extends Mapping.Rule { + push?: string; + pushBatch?: string; +} + +export type Types = Destination.Types; +export type Config = Destination.Config; +export type Context = Destination.Context; +export type InitContext = Destination.InitContext; +export type PushContext = Destination.PushContext; +export type PushBatchContext = Destination.PushBatchContext; + +export type InitFn = (context: InitContext) => void; +export type OnFn = (type: On.Types, context: Context) => void; +export type PushFn = (event: WalkerOS.Event, context: PushContext) => void; +export type PushBatchFn = ( + batch: Destination.Batch, + context: PushBatchContext, +) => void; diff --git a/packages/collector/src/types/index.ts b/packages/collector/src/types/index.ts index 6c918961e..c3c6d634a 100644 --- a/packages/collector/src/types/index.ts +++ b/packages/collector/src/types/index.ts @@ -1 +1,2 @@ export * from './collector'; +export * as Code from './code'; diff --git a/packages/config/jest/index.mjs b/packages/config/jest/index.mjs index a9ce80545..174c40e34 100644 --- a/packages/config/jest/index.mjs +++ b/packages/config/jest/index.mjs @@ -97,10 +97,10 @@ const config = { moduleNameMapper: getModuleMapper(), globals: getGlobals(), - // Performance settings - fixed values for consistent behavior - maxWorkers: 4, + // Performance settings - reduced for devcontainer memory constraints + maxWorkers: 2, testTimeout: 30000, - forceExit: true, + // forceExit disabled to allow proper cleanup and detect handle leaks clearMocks: true, restoreMocks: true, detectOpenHandles: true, diff --git a/packages/core/README.md b/packages/core/README.md index 8ceaa07c2..6d0866c5d 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -1,6 +1,6 @@

- - + +

@@ -15,7 +15,35 @@ data manipulation, validation, mapping, and more. ## Installation -Import the core utilities directly from the `@walkeros/core` package: +```bash +npm install @walkeros/core +``` + +## Quick Start + +The core package provides types and utilities used across walkerOS. In a Flow +configuration: + +```json +{ + "version": 1, + "flows": { + "default": { + "web": {}, + "destinations": { + "api": { + "package": "@walkeros/web-destination-api", + "config": { + "url": "https://collect.example.com/events" + } + } + } + } + } +} +``` + +Import utilities directly: ```ts import { assign, anonymizeIP, getMappingValue } from '@walkeros/core'; @@ -119,7 +147,7 @@ getId(10); // Returns 10-character string `getMappingValue(event: WalkerOS.Event, mapping: Mapping.Data, options?: Mapping.Options): Promise` extracts values from events using -[mapping configurations](https://www.elbwalker.com/docs/destinations/event-mapping). +[mapping configurations](https://www.walkeros.io/docs/destinations/event-mapping). ```ts // Simple path mapping @@ -300,13 +328,22 @@ validates event structure and throws on invalid events. Validates that values conform to walkerOS property types. ---- +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces: + +- [event.ts](./src/types/event.ts) - Event structure +- [destination.ts](./src/types/destination.ts) - Destination interface +- [source.ts](./src/types/source.ts) - Source interface +- [mapping.ts](./src/types/mapping.ts) - Mapping configuration -For platform-specific utilities, see: +## Related -- [Web Core](https://www.elbwalker.com/docs/core/web) - Browser-specific +- [Website Documentation](https://www.walkeros.io/docs/) +- [Collector Package](../collector/) - Event processing engine +- [Web Core](https://www.walkeros.io/docs/sources/web/) - Browser-specific functions -- [Server Core](https://www.elbwalker.com/docs/core/server) - Node.js server +- [Server Core](https://www.walkeros.io/docs/sources/server/) - Node.js server functions ## Contribute diff --git a/packages/core/src/__tests__/eventGenerator.test.ts b/packages/core/src/__tests__/eventGenerator.test.ts index 0ac788df9..cbff0951a 100644 --- a/packages/core/src/__tests__/eventGenerator.test.ts +++ b/packages/core/src/__tests__/eventGenerator.test.ts @@ -58,7 +58,7 @@ describe('createEvent', () => { data: { domain: 'www.example.com', title: 'walkerOS documentation', - referrer: 'https://www.elbwalker.com/', + referrer: 'https://www.walkeros.io/', search: '?foo=bar', hash: '#hash', id: '/docs/', diff --git a/packages/core/src/__tests__/flow.test.ts b/packages/core/src/__tests__/flow.test.ts index 945116921..8cc21c5d9 100644 --- a/packages/core/src/__tests__/flow.test.ts +++ b/packages/core/src/__tests__/flow.test.ts @@ -14,7 +14,7 @@ import { sourceReferenceJsonSchema, destinationReferenceJsonSchema, } from '../schemas/flow'; -import { getFlowConfig, getPlatform } from '../flow'; +import { getFlowConfig, getPlatform, packageNameToVariable } from '../flow'; describe('Flow Schemas', () => { // ======================================== @@ -1125,79 +1125,6 @@ describe('Flow Schemas', () => { describe('getFlowConfig', () => { describe('code resolution from package', () => { - test('resolves code from package for sources', () => { - const setup = { - version: 1 as const, - flows: { - default: { - server: {}, - packages: { - '@walkeros/server-source-express': { - imports: ['sourceExpress'], - }, - }, - sources: { - http: { - package: '@walkeros/server-source-express', - config: { port: 8080 }, - }, - }, - }, - }, - }; - const config = getFlowConfig(setup as any); - expect(config.sources?.http.code).toBe('sourceExpress'); - }); - - test('resolves code from package for destinations', () => { - const setup = { - version: 1 as const, - flows: { - default: { - web: {}, - packages: { - '@walkeros/web-destination-gtag': { - imports: ['destinationGtag'], - }, - }, - destinations: { - ga4: { - package: '@walkeros/web-destination-gtag', - config: { measurementId: 'G-123' }, - }, - }, - }, - }, - }; - const config = getFlowConfig(setup as any); - expect(config.destinations?.ga4.code).toBe('destinationGtag'); - }); - - test('preserves explicit code if provided', () => { - const setup = { - version: 1 as const, - flows: { - default: { - server: {}, - packages: { - '@walkeros/server-source-express': { - imports: ['sourceExpress'], - }, - }, - sources: { - http: { - package: '@walkeros/server-source-express', - code: 'customCode', // Explicit code should be preserved - config: {}, - }, - }, - }, - }, - }; - const config = getFlowConfig(setup as any); - expect(config.sources?.http.code).toBe('customCode'); - }); - test('does not set code if package not in packages config', () => { const setup = { version: 1 as const, @@ -1218,7 +1145,7 @@ describe('getFlowConfig', () => { expect(config.sources?.http.code).toBeUndefined(); }); - test('does not set code if package has no imports', () => { + test('auto-generates code when not provided', () => { const setup = { version: 1 as const, flows: { @@ -1227,7 +1154,6 @@ describe('getFlowConfig', () => { packages: { '@walkeros/server-source-express': { version: 'latest', - // No imports array }, }, sources: { @@ -1240,25 +1166,19 @@ describe('getFlowConfig', () => { }, }; const config = getFlowConfig(setup as any); - expect(config.sources?.http.code).toBeUndefined(); + expect(config.sources?.http.code).toBe('_walkerosServerSourceExpress'); }); - test('resolves code for multiple sources and destinations', () => { + test('auto-generates code for multiple sources and destinations when not provided', () => { const setup = { version: 1 as const, flows: { default: { server: {}, packages: { - '@walkeros/server-source-express': { - imports: ['sourceExpress'], - }, - '@walkeros/destination-demo': { - imports: ['destinationDemo'], - }, - '@walkeros/server-destination-gcp': { - imports: ['destinationBigQuery'], - }, + '@walkeros/server-source-express': {}, + '@walkeros/destination-demo': {}, + '@walkeros/server-destination-gcp': {}, }, sources: { http: { @@ -1280,9 +1200,11 @@ describe('getFlowConfig', () => { }, }; const config = getFlowConfig(setup as any); - expect(config.sources?.http.code).toBe('sourceExpress'); - expect(config.destinations?.demo.code).toBe('destinationDemo'); - expect(config.destinations?.bigquery.code).toBe('destinationBigQuery'); + expect(config.sources?.http.code).toBe('_walkerosServerSourceExpress'); + expect(config.destinations?.demo.code).toBe('_walkerosDestinationDemo'); + expect(config.destinations?.bigquery.code).toBe( + '_walkerosServerDestinationGcp', + ); }); }); @@ -1342,3 +1264,133 @@ describe('getPlatform', () => { ); }); }); + +// ======================================== +// packageNameToVariable Tests +// ======================================== + +describe('packageNameToVariable', () => { + test('converts scoped package names to valid variable names', () => { + expect(packageNameToVariable('@walkeros/server-destination-api')).toBe( + '_walkerosServerDestinationApi', + ); + }); + + test('converts unscoped package names', () => { + expect(packageNameToVariable('lodash')).toBe('lodash'); + }); + + test('handles multiple hyphens and slashes', () => { + expect(packageNameToVariable('@custom/my-helper')).toBe('_customMyHelper'); + expect(packageNameToVariable('@scope/package-name-test')).toBe( + '_scopePackageNameTest', + ); + }); + + test('handles package names with numbers', () => { + expect(packageNameToVariable('@walkeros/web-destination-gtag')).toBe( + '_walkerosWebDestinationGtag', + ); + }); +}); + +// ======================================== +// resolveCodeFromPackage with default exports +// ======================================== + +describe('resolveCodeFromPackage - default export fallback', () => { + test('auto-generates code when not provided', () => { + const setup = { + version: 1 as const, + flows: { + default: { + server: {}, + packages: { + '@walkeros/server-destination-api': {}, // No imports specified + }, + destinations: { + api: { + package: '@walkeros/server-destination-api', + config: {}, + }, + }, + }, + }, + }; + const config = getFlowConfig(setup as any); + expect(config.destinations?.api.code).toBe('_walkerosServerDestinationApi'); + }); + + test('uses explicit code when provided', () => { + const setup = { + version: 1 as const, + flows: { + default: { + server: {}, + packages: { + '@walkeros/server-destination-api': {}, + }, + destinations: { + api: { + package: '@walkeros/server-destination-api', + code: 'myCustomCode', // Explicit code should be preserved + config: {}, + }, + }, + }, + }, + }; + const config = getFlowConfig(setup as any); + expect(config.destinations?.api.code).toBe('myCustomCode'); + }); + + test('uses explicit code for named exports', () => { + const setup = { + version: 1 as const, + flows: { + default: { + server: {}, + packages: { + '@walkeros/server-destination-gcp': {}, + }, + destinations: { + bq: { + package: '@walkeros/server-destination-gcp', + code: 'destinationBigQuery', + config: {}, + }, + }, + }, + }, + }; + const config = getFlowConfig(setup as any); + expect(config.destinations?.bq.code).toBe('destinationBigQuery'); + }); + + test('auto-generates code for multiple destinations with same package', () => { + const setup = { + version: 1 as const, + flows: { + default: { + web: {}, + packages: { + '@walkeros/web-destination-api': {}, + }, + destinations: { + api1: { + package: '@walkeros/web-destination-api', + config: { endpoint: 'https://api1.example.com' }, + }, + api2: { + package: '@walkeros/web-destination-api', + config: { endpoint: 'https://api2.example.com' }, + }, + }, + }, + }, + }; + const config = getFlowConfig(setup as any); + expect(config.destinations?.api1.code).toBe('_walkerosWebDestinationApi'); + expect(config.destinations?.api2.code).toBe('_walkerosWebDestinationApi'); + }); +}); diff --git a/packages/core/src/__tests__/getMarketingParameters.test.ts b/packages/core/src/__tests__/getMarketingParameters.test.ts index 1ff1eb68a..61415ab64 100644 --- a/packages/core/src/__tests__/getMarketingParameters.test.ts +++ b/packages/core/src/__tests__/getMarketingParameters.test.ts @@ -2,7 +2,7 @@ import { getMarketingParameters } from '..'; describe('getMarketingParameters', () => { test('marketing parameters', () => { - const url = 'https://www.elbwalker.com/?'; + const url = 'https://www.walkeros.io/?'; expect(getMarketingParameters(new URL(url))).toStrictEqual({}); expect( diff --git a/packages/core/src/dev.ts b/packages/core/src/dev.ts index f405abd13..0c99ddf26 100644 --- a/packages/core/src/dev.ts +++ b/packages/core/src/dev.ts @@ -1,3 +1,2 @@ export * as schemas from './schemas'; - -export { zodToSchema, z, type JSONSchema } from './schemas'; +export { z, zodToSchema, type JSONSchema } from './schemas'; diff --git a/packages/core/src/eventGenerator.ts b/packages/core/src/eventGenerator.ts index ba4effae7..abfb971b5 100644 --- a/packages/core/src/eventGenerator.ts +++ b/packages/core/src/eventGenerator.ts @@ -192,7 +192,7 @@ export function getEvent( data: { domain: 'www.example.com', title: 'walkerOS documentation', - referrer: 'https://www.elbwalker.com/', + referrer: 'https://www.walkeros.io/', search: '?foo=bar', hash: '#hash', id: '/docs/', diff --git a/packages/core/src/flow.ts b/packages/core/src/flow.ts index 7fc51466d..011bcf37f 100644 --- a/packages/core/src/flow.ts +++ b/packages/core/src/flow.ts @@ -88,28 +88,47 @@ function interpolateVariables( return value; } +/** + * Convert package name to valid JavaScript variable name. + * Used for deterministic default import naming. + * @example + * packageNameToVariable('@walkeros/server-destination-api') + * // → '_walkerosServerDestinationApi' + */ +export function packageNameToVariable(packageName: string): string { + const hasScope = packageName.startsWith('@'); + const normalized = packageName + .replace('@', '') + .replace(/[/-]/g, '_') + .split('_') + .filter((part) => part.length > 0) + .map((part, i) => + i === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1), + ) + .join(''); + + return hasScope ? '_' + normalized : normalized; +} + /** * Resolve code from package reference. - * Looks up the import name from the packages configuration. + * Preserves explicit code fields, or auto-generates from package name. */ function resolveCodeFromPackage( packageName: string | undefined, existingCode: string | undefined, packages: Flow.Packages | undefined, ): string | undefined { - // If code already exists, preserve it + // Preserve explicit code first if (existingCode) return existingCode; - // If no package or packages config, nothing to resolve + // Auto-generate code from package name if package exists if (!packageName || !packages) return undefined; - // Look up the package in packages config const pkgConfig = packages[packageName]; - if (pkgConfig?.imports?.[0]) { - return pkgConfig.imports[0]; - } + if (!pkgConfig) return undefined; - return undefined; + return packageNameToVariable(packageName); } /** diff --git a/packages/core/src/schemas/README.md b/packages/core/src/schemas/README.md index 4084d3e08..eedb15bc1 100644 --- a/packages/core/src/schemas/README.md +++ b/packages/core/src/schemas/README.md @@ -155,8 +155,7 @@ Source configuration schemas: #### [`utilities.ts`](./utilities.ts) -Mirrors [`types/storage.ts`](../types/storage.ts) + -[`types/handler.ts`](../types/handler.ts) +Mirrors [`types/storage.ts`](../types/storage.ts) Utility type schemas: diff --git a/packages/core/src/schemas/flow.ts b/packages/core/src/schemas/flow.ts index 53fbac853..422df3827 100644 --- a/packages/core/src/schemas/flow.ts +++ b/packages/core/src/schemas/flow.ts @@ -112,6 +112,12 @@ export const SourceReferenceSchema = z .describe( 'Package specifier with optional version (e.g., "@walkeros/web-source-browser@2.0.0")', ), + code: z + .string() + .optional() + .describe( + 'Named export to use from the package (e.g., "sourceExpress"). If omitted, uses default export.', + ), config: z .unknown() .optional() @@ -151,6 +157,12 @@ export const DestinationReferenceSchema = z .describe( 'Package specifier with optional version (e.g., "@walkeros/web-destination-gtag@2.0.0")', ), + code: z + .string() + .optional() + .describe( + 'Named export to use from the package (e.g., "destinationAnalytics"). If omitted, uses default export.', + ), config: z .unknown() .optional() diff --git a/packages/core/src/types/destination.ts b/packages/core/src/types/destination.ts index 1ac6d1277..63158da64 100644 --- a/packages/core/src/types/destination.ts +++ b/packages/core/src/types/destination.ts @@ -94,8 +94,10 @@ export interface Policy { [key: string]: WalkerOSMapping.Value; } +export type Code = Instance | true; + export type Init = { - code: Instance; + code: Code; config?: Partial>; env?: Partial>; }; diff --git a/packages/core/src/types/on.ts b/packages/core/src/types/on.ts index 03cd5f083..0552f5222 100644 --- a/packages/core/src/types/on.ts +++ b/packages/core/src/types/on.ts @@ -1,4 +1,4 @@ -import type { Collector, WalkerOS } from './'; +import type { Collector, Destination, WalkerOS } from './'; // collector state for the on actions export type Config = { @@ -72,13 +72,13 @@ export interface OnConfig { } // Destination on function type with automatic type inference -export type OnFn = ( - event: T, - context: EventContextMap[T], +export type OnFn = ( + type: Types, + context: Destination.Context, ) => WalkerOS.PromiseOrValue; // Runtime-compatible version for internal usage export type OnFnRuntime = ( - event: Types, - context: AnyEventContext, + type: Types, + context: Destination.Context, ) => WalkerOS.PromiseOrValue; diff --git a/packages/docker/README.md b/packages/docker/README.md index 689329fbd..2cb64dab1 100644 --- a/packages/docker/README.md +++ b/packages/docker/README.md @@ -3,6 +3,12 @@ Runtime Docker container for walkerOS - executes pre-built flow bundles with instant startup and includes working demos for quick testing. +## Installation + +```bash +docker pull walkeros/docker:latest +``` + ## Overview This is a **demo-enabled runtime container** designed for both testing and @@ -33,8 +39,7 @@ flow.mjs ──────────────→ Running collector **What's included:** Express server, flow executor, graceful shutdown, demo bundles **What's NOT included:** CLI, bundler, npm, build tools -See [docs/CAPABILITIES.md](./docs/CAPABILITIES.md) for detailed architecture -documentation. +This is a minimal runtime image optimized for production deployments. ## Quick Start @@ -327,13 +332,6 @@ docker run -p 8080:8080 \ npm test ``` -See comprehensive guides in [docs/](./docs/): - -- **[CAPABILITIES.md](./docs/CAPABILITIES.md)** - Architecture and capabilities -- **[LOCAL-TESTING.md](./docs/LOCAL-TESTING.md)** - Testing Docker images - locally -- **[DOCKER-HUB.md](./docs/DOCKER-HUB.md)** - Publishing to Docker Hub - ## Library Usage The Docker package can be imported as a library (used by @walkeros/cli): diff --git a/packages/docker/demos/README.md b/packages/docker/demos/README.md index 0b65e7034..18410dd35 100644 --- a/packages/docker/demos/README.md +++ b/packages/docker/demos/README.md @@ -1,5 +1,18 @@ # walkerOS Docker Demos +## Installation + +These are demo configurations - see the main [Docker README](../README.md) for +installation. + +## Usage + +Run +`docker run -p 8080:8080 -e MODE=collect -e FLOW=/app/demos/demo-collect.mjs walkeros/docker` +for instant testing. + +--- + This directory contains **pre-built demo bundles** ready to use immediately with the walkerOS Docker image. No CLI setup or bundling required - just run and test! diff --git a/packages/docker/docker-compose.bundle.yml b/packages/docker/docker-compose.bundle.yml deleted file mode 100644 index 5567b6f8e..000000000 --- a/packages/docker/docker-compose.bundle.yml +++ /dev/null @@ -1,38 +0,0 @@ -# docker-compose for walkerOS Bundle Mode -# Generates a static JavaScript bundle and exits - -version: '3.8' - -services: - walkeros-bundle: - image: walkeros/docker:latest - container_name: walkeros-bundle - - environment: - # Required: Set operational mode - MODE: bundle - - # Required: Flow configuration file path (built-in flows at /app/flows/) - FLOW: /app/flows/bundle-web.json - - # Optional: Override output path - # OUTPUT_PATH: /app/dist/walker.js - - # Optional: Enable debug logging - # DEBUG: "true" - - volumes: - # Mount output directory (bundle will be written here) - - ./dist:/app/dist - - # Optional: Mount custom flow config - # - ./my-flow.json:/app/custom.json:ro - # Then set FLOW: /app/custom.json - - # Bundle mode exits after completion, so no restart - restart: "no" - -# Usage: -# 1. Ensure dist/ directory exists: mkdir -p dist -# 2. Run: docker-compose -f docker-compose.bundle.yml up -# 3. Check output: ls -l dist/walker.js diff --git a/packages/docker/docker-compose.collect.yml b/packages/docker/docker-compose.collect.yml deleted file mode 100644 index 223374488..000000000 --- a/packages/docker/docker-compose.collect.yml +++ /dev/null @@ -1,54 +0,0 @@ -# docker-compose for walkerOS Collect Mode -# Runs an HTTP event collection server - -version: '3.8' - -services: - walkeros-collect: - image: walkeros/docker:latest - container_name: walkeros-collect - - environment: - # Required: Set operational mode - MODE: collect - - # Required: Flow configuration file path (built-in flows at /app/flows/) - FLOW: /app/flows/collect-console.json - - # Optional: Override port from config - PORT: 8080 - - # Optional: Override host - HOST: 0.0.0.0 - - # Optional: Enable debug logging - # DEBUG: "true" - - ports: - - "8080:8080" - - # volumes: - # Optional: Mount custom flow config - # - ./my-flow.json:/app/custom.json:ro - # Then set FLOW: /app/custom.json - - # Health check - healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://localhost:8080/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1))"] - interval: 30s - timeout: 3s - start_period: 5s - retries: 3 - - # Restart policy - restart: unless-stopped - - # Resource limits (adjust as needed) - deploy: - resources: - limits: - cpus: '1.0' - memory: 512M - reservations: - cpus: '0.5' - memory: 256M diff --git a/packages/docker/docker-compose.serve.yml b/packages/docker/docker-compose.serve.yml deleted file mode 100644 index 5ccfd256e..000000000 --- a/packages/docker/docker-compose.serve.yml +++ /dev/null @@ -1,67 +0,0 @@ -# docker-compose for walkerOS Serve Mode -# Serves static files (typically generated bundles) via HTTP - -version: '3.8' - -services: - walkeros-serve: - image: walkeros/docker:latest - container_name: walkeros-serve - - environment: - # Required: Set operational mode - MODE: serve - - # Required: Flow configuration file path (built-in flows at /app/flows/) - FLOW: /app/flows/serve.json - - # Optional: Override port from config - PORT: 8080 - - # Optional: Override host - HOST: 0.0.0.0 - - # Optional: Override static directory - # STATIC_DIR: /app/static - - ports: - - "8080:8080" - - volumes: - # Mount static files directory - - ./dist:/app/dist:ro - - # Optional: Mount different static directory - # - ./static:/app/static:ro - - # Optional: Mount custom flow config - # - ./my-flow.json:/app/custom.json:ro - # Then set FLOW: /app/custom.json - - # Health check - healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://localhost:8080/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1))"] - interval: 30s - timeout: 3s - start_period: 5s - retries: 3 - - # Restart policy - restart: unless-stopped - - # Resource limits (adjust as needed) - deploy: - resources: - limits: - cpus: '0.5' - memory: 256M - reservations: - cpus: '0.25' - memory: 128M - -# Usage: -# 1. Generate bundle first: -# docker-compose -f docker-compose.bundle.yml up -# 2. Serve the bundle: -# docker-compose -f docker-compose.serve.yml up -# 3. Access bundle: curl http://localhost:8080/walker.js diff --git a/packages/docker/package.json b/packages/docker/package.json index f2067575b..2e2dd067e 100644 --- a/packages/docker/package.json +++ b/packages/docker/package.json @@ -28,6 +28,7 @@ "docker:publish": "bash scripts/publish.sh" }, "dependencies": { + "@walkeros/core": "*", "cors": "^2.8.5", "express": "^4.18.2" }, diff --git a/packages/docker/src/index.ts b/packages/docker/src/index.ts index 823fabaa4..616ea3541 100644 --- a/packages/docker/src/index.ts +++ b/packages/docker/src/index.ts @@ -1,22 +1,17 @@ #!/usr/bin/env node -import { readFileSync } from 'fs'; -import { fileURLToPath } from 'url'; -import path from 'path'; +import { createLogger, Level } from '@walkeros/core'; +import type { Logger } from '@walkeros/core'; import { runFlow } from './services/runner'; import { runServeMode } from './services/serve'; +import { VERSION } from './version'; -// Read version from package.json (ESM-compatible) -const moduleFilename = fileURLToPath(import.meta.url); -const moduleDir = path.dirname(moduleFilename); -const packageJson = JSON.parse( - readFileSync(path.join(moduleDir, '../package.json'), 'utf-8'), -) as { version: string }; +// Re-export version for external consumers +export { VERSION } from './version'; -/** - * Package version - exported for use by @walkeros/cli - */ -export const VERSION = packageJson.version; +// Create logger - DEBUG level when VERBOSE, otherwise INFO +const logLevel = process.env.VERBOSE === 'true' ? Level.DEBUG : Level.INFO; +const logger = createLogger({ level: logLevel }); /** * walkerOS Docker Container @@ -32,23 +27,25 @@ async function main() { const mode = process.env.MODE; if (!mode) { - console.error('❌ Error: MODE environment variable required'); - console.error(' Valid modes: collect | serve'); - console.error(' Example: MODE=collect FLOW=/app/flow.mjs'); + logger.error('MODE environment variable required'); + logger.info('Valid modes: collect | serve'); + logger.info('Example: MODE=collect FLOW=/app/flow.mjs'); process.exit(1); } if (!['collect', 'serve'].includes(mode)) { - console.error(`❌ Error: Invalid MODE="${mode}"`); - console.error(' Valid modes: collect | serve'); - console.error(' Note: Build flows with @walkeros/cli first'); + logger.error(`Invalid MODE="${mode}"`); + logger.info('Valid modes: collect | serve'); + logger.info('Note: Build flows with @walkeros/cli first'); process.exit(1); } + // Display banner (always shown, not through logger) console.log('╔════════════════════════════════════════╗'); console.log('║ walkerOS Docker Container ║'); + console.log(`║ v${VERSION.padStart(6)} ║`); console.log('╚════════════════════════════════════════╝\n'); - console.log(`Mode: ${mode.toUpperCase()}\n`); + logger.info(`Mode: ${mode.toUpperCase()}`); try { // Run the appropriate mode @@ -57,10 +54,10 @@ async function main() { const flowPath = process.env.FLOW; if (!flowPath) { - throw new Error( - 'FLOW environment variable required. ' + - 'Example: FLOW=/app/flow.mjs', + logger.throw( + 'FLOW environment variable required. Example: FLOW=/app/flow.mjs', ); + return; // TypeScript narrowing (never reached) } // Extract port from environment if set @@ -69,7 +66,7 @@ async function main() { : undefined; const host = process.env.HOST; - await runFlow(flowPath, { port, host }); + await runFlow(flowPath, { port, host }, logger.scope('runner')); break; } @@ -81,18 +78,18 @@ async function main() { staticDir: process.env.STATIC_DIR || '/app/dist', }; - await runServeMode(config); + await runServeMode(config, logger.scope('serve')); break; } default: - throw new Error(`Unhandled mode: ${mode}`); + logger.throw(`Unhandled mode: ${mode}`); } } catch (error) { - console.error('\n❌ Fatal error:', error); + const message = error instanceof Error ? error.message : String(error); + logger.error(`Fatal error: ${message}`); if (error instanceof Error && error.stack) { - console.error('\nStack trace:'); - console.error(error.stack); + logger.debug('Stack trace:', { stack: error.stack }); } process.exit(1); } @@ -100,13 +97,17 @@ async function main() { // Handle uncaught errors process.on('uncaughtException', (error) => { - console.error('\n❌ Uncaught exception:', error); + logger.error(`Uncaught exception: ${error.message}`); + logger.debug('Stack trace:', { stack: error.stack }); process.exit(1); }); -process.on('unhandledRejection', (reason, promise) => { - console.error('\n❌ Unhandled rejection at:', promise); - console.error('Reason:', reason); +process.on('unhandledRejection', (reason) => { + const message = reason instanceof Error ? reason.message : String(reason); + logger.error(`Unhandled rejection: ${message}`); + if (reason instanceof Error && reason.stack) { + logger.debug('Stack trace:', { stack: reason.stack }); + } process.exit(1); }); diff --git a/packages/docker/src/services/runner.ts b/packages/docker/src/services/runner.ts index b2da71368..f74570abe 100644 --- a/packages/docker/src/services/runner.ts +++ b/packages/docker/src/services/runner.ts @@ -7,6 +7,7 @@ import { pathToFileURL } from 'url'; import { resolve, dirname } from 'path'; +import type { Logger } from '@walkeros/core'; export interface RuntimeConfig { port?: number; @@ -18,12 +19,14 @@ export interface RuntimeConfig { * * @param flowPath - Absolute path to pre-built .mjs flow file * @param config - Optional runtime configuration + * @param logger - Logger instance for output */ export async function runFlow( flowPath: string, - config?: RuntimeConfig, + config: RuntimeConfig | undefined, + logger: Logger.Instance, ): Promise { - console.log(`🚀 Loading flow from ${flowPath}`); + logger.info(`Loading flow from ${flowPath}`); try { const absolutePath = resolve(flowPath); @@ -38,7 +41,7 @@ export async function runFlow( const module = await import(fileUrl); if (!module.default || typeof module.default !== 'function') { - throw new Error( + logger.throw( `Invalid flow bundle: ${flowPath} must export a default function`, ); } @@ -47,31 +50,32 @@ export async function runFlow( const result = await module.default(config); if (!result || !result.collector) { - throw new Error( + logger.throw( `Invalid flow bundle: ${flowPath} must return { collector }`, ); } const { collector } = result; - console.log('✅ Flow running'); + logger.info('Flow running'); if (config?.port) { - console.log(` Port: ${config.port}`); + logger.info(`Port: ${config.port}`); } // Graceful shutdown handler const shutdown = async (signal: string) => { - console.log(`\n📡 Received ${signal}, shutting down gracefully...`); + logger.info(`Received ${signal}, shutting down gracefully...`); try { // Use collector's shutdown command if available if (collector.command) { await collector.command('shutdown'); } - console.log('✅ Shutdown complete'); + logger.info('Shutdown complete'); process.exit(0); } catch (error) { - console.error('❌ Error during shutdown:', error); + const message = error instanceof Error ? error.message : String(error); + logger.error(`Error during shutdown: ${message}`); process.exit(1); } }; @@ -83,10 +87,10 @@ export async function runFlow( // Keep process alive await new Promise(() => {}); } catch (error) { - console.error('❌ Failed to run flow:', error); + const message = error instanceof Error ? error.message : String(error); + logger.error(`Failed to run flow: ${message}`); if (error instanceof Error && error.stack) { - console.error('\nStack trace:'); - console.error(error.stack); + logger.debug('Stack trace:', { stack: error.stack }); } throw error; } diff --git a/packages/docker/src/services/serve.ts b/packages/docker/src/services/serve.ts index 6f92dc7d1..2010f3e20 100644 --- a/packages/docker/src/services/serve.ts +++ b/packages/docker/src/services/serve.ts @@ -1,5 +1,6 @@ import express from 'express'; -import path from 'path'; +import type { Logger } from '@walkeros/core'; +import { VERSION } from '../version'; export interface ServeConfig { port?: number; @@ -11,8 +12,14 @@ export interface ServeConfig { /** * Run serve mode - serve single file (typically generated bundle) + * + * @param config - Server configuration + * @param logger - Logger instance for output */ -export async function runServeMode(config?: ServeConfig): Promise { +export async function runServeMode( + config: ServeConfig | undefined, + logger: Logger.Instance, +): Promise { // Port priority: ENV variable > config > default const port = process.env.PORT ? parseInt(process.env.PORT, 10) @@ -34,9 +41,9 @@ export async function runServeMode(config?: ServeConfig): Promise { // Build full URL path const urlPath = servePath ? `/${servePath}/${serveName}` : `/${serveName}`; - console.log('📁 Serve mode: Starting single-file server...'); - console.log(` File: ${filePath}`); - console.log(` URL: http://${host}:${port}${urlPath}`); + logger.info('Starting single-file server...'); + logger.info(`File: ${filePath}`); + logger.info(`URL: http://${host}:${port}${urlPath}`); try { const app = express(); @@ -45,6 +52,7 @@ export async function runServeMode(config?: ServeConfig): Promise { app.get('/health', (req, res) => { res.json({ status: 'ok', + version: VERSION, timestamp: Date.now(), mode: 'serve', file: filePath, @@ -59,27 +67,28 @@ export async function runServeMode(config?: ServeConfig): Promise { // Start server const server = app.listen(port, host, () => { - console.log(`✅ Server listening on http://${host}:${port}`); - console.log(` GET ${urlPath} - Bundle file`); - console.log(` GET /health - Health check`); + logger.info(`Server listening on http://${host}:${port}`); + logger.info(`GET ${urlPath} - Bundle file`); + logger.info(`GET /health - Health check`); }); // Graceful shutdown const shutdownHandler = (signal: string) => { - console.log(`\n⏹️ Received ${signal}, shutting down...`); + logger.info(`Received ${signal}, shutting down...`); server.close(() => { - console.log('✅ Server closed'); + logger.info('Server closed'); process.exit(0); }); }; - process.on('SIGTERM', shutdownHandler); - process.on('SIGINT', shutdownHandler); + process.on('SIGTERM', () => shutdownHandler('SIGTERM')); + process.on('SIGINT', () => shutdownHandler('SIGINT')); // Keep process alive await new Promise(() => {}); } catch (error) { - console.error('❌ Server failed:', error); + const message = error instanceof Error ? error.message : String(error); + logger.error(`Server failed: ${message}`); process.exit(1); } } diff --git a/packages/docker/src/version.ts b/packages/docker/src/version.ts new file mode 100644 index 000000000..4596fd182 --- /dev/null +++ b/packages/docker/src/version.ts @@ -0,0 +1,8 @@ +// Version injected at build time via tsup define (from buildModules) +declare const __VERSION__: string; + +/** + * Package version - exported for use by @walkeros/cli and internal services + */ +export const VERSION = + typeof __VERSION__ !== 'undefined' ? __VERSION__ : '0.0.0'; diff --git a/packages/server/core/README.md b/packages/server/core/README.md index 1e4e35dc1..71340c92f 100644 --- a/packages/server/core/README.md +++ b/packages/server/core/README.md @@ -1,6 +1,6 @@

- - + +

@@ -240,7 +240,7 @@ await sendServer('https://secure-api.example.com/events', data, { ## Integration with Core Server utilities work seamlessly with -[Core Utilities](https://www.elbwalker.com/docs/core): +[Core Utilities](https://www.walkeros.io/docs/core): ```js import { getMappingValue, anonymizeIP } from '@walkeros/core'; @@ -265,7 +265,7 @@ async function processServerSideEvent(rawEvent, clientIP) { --- For platform-agnostic utilities, see -[Core Utilities](https://www.elbwalker.com/docs/core). +[Core Utilities](https://www.walkeros.io/docs/core). ## Contribute diff --git a/packages/server/destinations/aws/README.md b/packages/server/destinations/aws/README.md index b7cf5b58b..b88f11dda 100644 --- a/packages/server/destinations/aws/README.md +++ b/packages/server/destinations/aws/README.md @@ -1,6 +1,6 @@

- - + +

@@ -22,27 +22,60 @@ and analysis. npm install @walkeros/server-destination-aws ``` -## Usage +## Quick Start + +Configure in your Flow JSON: + +```json +{ + "version": 1, + "flows": { + "default": { + "server": {}, + "destinations": { + "firehose": { + "package": "@walkeros/server-destination-aws", + "config": { + "settings": { + "firehose": { + "streamName": "your-firehose-stream-name", + "region": "eu-central-1" + } + } + } + } + } + } + } +} +``` -Here's a basic example of how to use the AWS Firehose destination: +Or programmatically: ```typescript -import { elb } from '@walkeros/collector'; +import { startFlow } from '@walkeros/collector'; import { destinationFirehose } from '@walkeros/server-destination-aws'; -elb('walker destination', destinationFirehose, { - settings: { - firehose: { - streamName: 'your-firehose-stream-name', - region: 'eu-central-1', +const { elb } = await startFlow({ + destinations: [ + { + destination: destinationFirehose, config: { - credentials: { - accessKeyId: 'your-access-key-id', - secretAccessKey: 'your-secret-access-key', + settings: { + firehose: { + streamName: 'your-firehose-stream-name', + region: 'eu-central-1', + config: { + credentials: { + accessKeyId: 'your-access-key-id', + secretAccessKey: 'your-secret-access-key', + }, + }, + }, }, }, }, - }, + ], }); ``` @@ -63,6 +96,15 @@ The `firehose` object has the following properties: | `region` | `string` | AWS region for the Firehose service | No | `'us-east-1'` | | `config` | `FirehoseClientConfig` | AWS SDK client configuration options | No | `{ credentials: awsCredentials }` | +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces. + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/destinations/server/aws/) +- [Destination Interface](../../../core/src/types/destination.ts) + ## Contribute Feel free to contribute by submitting an diff --git a/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts b/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts index 96fd198d9..03baabe6a 100644 --- a/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts +++ b/packages/server/destinations/aws/src/firehose/__tests__/firehose.test.ts @@ -1,6 +1,6 @@ import type { Config, Settings, Destination, Env } from '../types'; -import type { WalkerOS, Collector } from '@walkeros/core'; -import { createEvent, mockEnv, createMockLogger } from '@walkeros/core'; +import type { Collector } from '@walkeros/core'; +import { createEvent, createMockLogger } from '@walkeros/core'; import * as examples from '../examples'; const { env } = examples; diff --git a/packages/server/destinations/aws/src/firehose/lib/firehose.ts b/packages/server/destinations/aws/src/firehose/lib/firehose.ts index 64b706d3e..490928011 100644 --- a/packages/server/destinations/aws/src/firehose/lib/firehose.ts +++ b/packages/server/destinations/aws/src/firehose/lib/firehose.ts @@ -1,4 +1,4 @@ -import type { Destination } from '@walkeros/core'; +import type { Destination, Logger } from '@walkeros/core'; import type { FirehoseConfig, Env } from '../types'; import { throwError } from '@walkeros/core'; @@ -38,9 +38,10 @@ export function getConfigFirehose( export async function pushFirehose( pushEvents: Destination.PushEvents, config: FirehoseConfig, - env?: unknown, + context: Destination.PushContext, ) { const { client, streamName } = config; + const { env, logger } = context; if (!client) return { queue: pushEvents }; @@ -49,6 +50,11 @@ export async function pushFirehose( Data: Buffer.from(JSON.stringify(event)), })); + logger.debug('Calling AWS Firehose API', { + stream: streamName, + recordCount: records.length, + }); + // Use environment-injected SDK command or fall back to direct import if (isAWSEnvironment(env)) { await client.send( @@ -67,4 +73,6 @@ export async function pushFirehose( }), ); } + + logger?.debug('AWS Firehose API response', { ok: true }); } diff --git a/packages/server/destinations/aws/src/firehose/push.ts b/packages/server/destinations/aws/src/firehose/push.ts index 49434d584..43c04010d 100644 --- a/packages/server/destinations/aws/src/firehose/push.ts +++ b/packages/server/destinations/aws/src/firehose/push.ts @@ -1,10 +1,10 @@ import type { PushFn } from './types'; import { pushFirehose } from './lib/firehose'; -export const push: PushFn = async function (event, { config, collector, env }) { - const { firehose } = config.settings || {}; +export const push: PushFn = async function (event, context) { + const { firehose } = context.config.settings || {}; - if (firehose) pushFirehose([{ event }], firehose, env); + if (firehose) pushFirehose([{ event }], firehose, context); return; }; diff --git a/packages/server/destinations/datamanager/README.md b/packages/server/destinations/datamanager/README.md index 2f2355f19..af1ec8be0 100644 --- a/packages/server/destinations/datamanager/README.md +++ b/packages/server/destinations/datamanager/README.md @@ -791,12 +791,21 @@ authentication** | `eventName` | string | 40 | Event name (required for GA4) | | `eventSource` | string | | WEB, APP, IN_STORE, PHONE, OTHER | +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces. + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/destinations/server/datamanager/) +- [Destination Interface](../../../core/src/types/destination.ts) + ## Resources - [Google Data Manager API Documentation](https://developers.google.com/data-manager/api) - [Data Formatting Guidelines](https://developers.google.com/data-manager/api/devguides/concepts/formatting) - [DMA Compliance](https://developers.google.com/data-manager/api/devguides/concepts/dma) -- [walkerOS Documentation](https://www.elbwalker.com/docs/) +- [walkerOS Documentation](https://www.walkeros.io/docs/) ## License @@ -807,5 +816,5 @@ MIT For issues and questions: - [GitHub Issues](https://github.com/elbwalker/walkerOS/issues) -- [walkerOS Documentation](https://www.elbwalker.com/docs/) +- [walkerOS Documentation](https://www.walkeros.io/docs/) - [Google Data Manager Support](https://developers.google.com/data-manager/api/support/contact) diff --git a/packages/server/destinations/datamanager/src/index.ts b/packages/server/destinations/datamanager/src/index.ts index 1d46953fb..4e39ac35b 100644 --- a/packages/server/destinations/datamanager/src/index.ts +++ b/packages/server/destinations/datamanager/src/index.ts @@ -11,22 +11,12 @@ export const destinationDataManager: DestinationInterface = { config: {}, async init({ config: partialConfig, env, logger }) { - logger.debug('Data Manager init started'); - logger.info('Data Manager initializing...'); - // getConfig validates required fields and returns ValidatedConfig const config = getConfig(partialConfig, logger); - logger.debug('Settings validated', { - validateOnly: config.settings.validateOnly, - destinationCount: config.settings.destinations.length, - eventSource: config.settings.eventSource, - }); - try { - logger.debug('Creating auth client...'); const authClient = await createAuthClient(config.settings); - logger.debug('Auth client created successfully'); + logger.debug('Auth client created'); return { ...config, diff --git a/packages/server/destinations/datamanager/src/push.ts b/packages/server/destinations/datamanager/src/push.ts index ffdbd5816..936dd9ee1 100644 --- a/packages/server/destinations/datamanager/src/push.ts +++ b/packages/server/destinations/datamanager/src/push.ts @@ -85,12 +85,6 @@ export const push: PushFn = async function ( // Format event for Data Manager API const dataManagerEvent = await formatEvent(event, finalData); - logger.debug('Processing event', { - name: event.name, - id: event.id, - timestamp: event.timestamp, - }); - // Apply event source from settings (required) if (!dataManagerEvent.eventSource) { dataManagerEvent.eventSource = eventSource; @@ -186,8 +180,4 @@ export const push: PushFn = async function ( `Validation errors: ${JSON.stringify(result.validationErrors)}`, ); } - - logger.info('Event processed successfully', { - requestId: result.requestId, - }); }; diff --git a/packages/server/destinations/datamanager/src/schemas/settings.ts b/packages/server/destinations/datamanager/src/schemas/settings.ts index 90b04db4c..ebae2077d 100644 --- a/packages/server/destinations/datamanager/src/schemas/settings.ts +++ b/packages/server/destinations/datamanager/src/schemas/settings.ts @@ -5,12 +5,32 @@ import { ConsentSchema, } from './primitives'; -export const SettingsSchema = z.object({ - accessToken: z +/** + * Service account credentials schema + */ +const CredentialsSchema = z.object({ + client_email: z.string().email().describe('Service account email'), + private_key: z .string() .min(1) + .describe('Service account private key (PEM format)'), +}); + +export const SettingsSchema = z.object({ + credentials: CredentialsSchema.optional().describe( + 'Service account credentials (client_email + private_key). Recommended for serverless environments.', + ), + keyFilename: z + .string() + .optional() + .describe( + 'Path to service account JSON file. For local development or environments with filesystem access.', + ), + scopes: z + .array(z.string()) + .optional() .describe( - 'OAuth 2.0 access token with datamanager scope (like ya29.c.xxx)', + 'OAuth scopes for Data Manager API. Defaults to datamanager scope.', ), destinations: z .array(DestinationSchema) diff --git a/packages/server/destinations/gcp/README.md b/packages/server/destinations/gcp/README.md index f4649844d..d26f813d8 100644 --- a/packages/server/destinations/gcp/README.md +++ b/packages/server/destinations/gcp/README.md @@ -1,6 +1,6 @@

- - + +

@@ -21,20 +21,52 @@ Google Cloud's powerful data processing and machine learning capabilities. npm install @walkeros/server-destination-gcp ``` -## Usage +## Quick Start + +Configure in your Flow JSON: + +```json +{ + "version": 1, + "flows": { + "default": { + "server": {}, + "destinations": { + "bigquery": { + "package": "@walkeros/server-destination-gcp", + "config": { + "settings": { + "projectId": "YOUR_PROJECT_ID", + "datasetId": "YOUR_DATASET_ID", + "tableId": "YOUR_TABLE_ID" + } + } + } + } + } + } +} +``` -Here's a basic example of how to use the GCP BigQuery destination: +Or programmatically: ```typescript -import { elb } from '@walkeros/collector'; +import { startFlow } from '@walkeros/collector'; import { destinationBigQuery } from '@walkeros/server-destination-gcp'; -elb('walker destination', destinationBigQuery, { - settings: { - projectId: 'YOUR_PROJECT_ID', - datasetId: 'YOUR_DATASET_ID', - tableId: 'YOUR_TABLE_ID', - }, +const { elb } = await startFlow({ + destinations: [ + { + destination: destinationBigQuery, + config: { + settings: { + projectId: 'YOUR_PROJECT_ID', + datasetId: 'YOUR_DATASET_ID', + tableId: 'YOUR_TABLE_ID', + }, + }, + }, + ], }); ``` @@ -80,7 +112,16 @@ CREATE TABLE IF NOT EXISTS `YOUR_PROJECT.walkeros.events` ( Object and array fields (`data`, `context`, `globals`, etc.) are JSON stringified. For custom schemas using the `data` mapping config, see the -[full documentation](https://www.elbwalker.com/docs/destinations/server/gcp). +[full documentation](https://www.walkeros.io/docs/destinations/server/gcp). + +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces. + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/destinations/server/gcp/) +- [Destination Interface](../../../core/src/types/destination.ts) ## Contribute diff --git a/packages/server/destinations/gcp/src/bigquery/push.ts b/packages/server/destinations/gcp/src/bigquery/push.ts index 00e7515fd..ac27acba9 100644 --- a/packages/server/destinations/gcp/src/bigquery/push.ts +++ b/packages/server/destinations/gcp/src/bigquery/push.ts @@ -4,15 +4,13 @@ import { isObject, isArray } from '@walkeros/core'; export const push: PushFn = async function ( event, - { config, mapping: _mapping, data }, + { config, mapping: _mapping, data, logger }, ) { const { client, datasetId, tableId } = config.settings!; - if (!client || !datasetId || !tableId) { - throw new Error( - 'Missing required BigQuery configuration. Ensure init() was called successfully.', - ); - } + if (!client) return logger.throw('client is missing'); + if (!datasetId) return logger.throw('datasetId is missing'); + if (!tableId) return logger.throw('tableId is missing'); let row: WalkerOS.AnyObject | undefined; @@ -29,8 +27,16 @@ export const push: PushFn = async function ( const rows = [mapEvent(row)]; + logger.debug('Calling BigQuery API', { + dataset: datasetId, + table: tableId, + rowCount: rows.length, + }); + await client.dataset(datasetId).table(tableId).insert(rows); + logger.debug('BigQuery API response', { ok: true }); + return; }; diff --git a/packages/server/destinations/meta/README.md b/packages/server/destinations/meta/README.md index c17062cc7..ab4d08293 100644 --- a/packages/server/destinations/meta/README.md +++ b/packages/server/destinations/meta/README.md @@ -1,6 +1,6 @@

- - + +

@@ -27,13 +27,20 @@ npm install @walkeros/server-destination-meta Here's a basic example of how to use the Meta CAPI destination: ```typescript -import { elb } from '@walkeros/collector'; +import { startFlow } from '@walkeros/collector'; import { destinationMeta } from '@walkeros/server-destination-meta'; -elb('walker destination', destinationMeta, { - settings: { - accessToken: 'YOUR_ACCESS_TOKEN', - pixelId: 'YOUR_PIXEL_ID', +await startFlow({ + destinations: { + meta: { + code: destinationMeta, + config: { + settings: { + accessToken: 'YOUR_ACCESS_TOKEN', + pixelId: 'YOUR_PIXEL_ID', + }, + }, + }, }, }); ``` diff --git a/packages/server/destinations/meta/src/push.ts b/packages/server/destinations/meta/src/push.ts index e8b320777..f77e08f90 100644 --- a/packages/server/destinations/meta/src/push.ts +++ b/packages/server/destinations/meta/src/push.ts @@ -12,7 +12,7 @@ import { hashEvent } from './hash'; export const push: PushFn = async function ( event, - { config, mapping, data, collector, env }, + { config, mapping, data, collector, env, logger }, ) { const { accessToken, @@ -69,14 +69,27 @@ export const push: PushFn = async function ( // Test event code if (test_event_code) body.test_event_code = test_event_code; + const endpoint = `${url}${pixelId}/events`; + logger.debug('Calling Meta API', { + endpoint, + method: 'POST', + eventName: serverEvent.event_name, + eventId: serverEvent.event_id, + }); + const sendServerFn = env?.sendServer || sendServer; const result = await sendServerFn( - `${url}${pixelId}/events?access_token=${accessToken}`, + `${endpoint}?access_token=${accessToken}`, JSON.stringify(body), ); - if (isObject(result) && result.ok === false) - throw new Error(JSON.stringify(result)); + logger.debug('Meta API response', { + ok: isObject(result) ? result.ok : true, + }); + + if (isObject(result) && result.ok === false) { + logger.throw(`Meta API error: ${JSON.stringify(result)}`); + } }; function formatClickId(clickId: unknown, time?: number): string | undefined { diff --git a/packages/server/sources/aws/CHANGELOG.md b/packages/server/sources/aws/CHANGELOG.md new file mode 100644 index 000000000..612f53dd8 --- /dev/null +++ b/packages/server/sources/aws/CHANGELOG.md @@ -0,0 +1,19 @@ +# @walkeros/server-source-aws + +## 0.4.2 + +### Patch Changes + +- Initial release of AWS Lambda source +- Support for API Gateway v1, v2, and Function URLs +- CORS configuration (default and custom) +- GET pixel tracking with 1x1 GIF +- POST event ingestion +- Health check endpoint +- Request ID tracking +- Logging integration +- Zod schema validation +- Base64 body decoding +- TypeScript type safety +- Updated dependencies + - @walkeros/core@0.4.2 diff --git a/packages/server/sources/aws/README.md b/packages/server/sources/aws/README.md new file mode 100644 index 000000000..d62d1bdc0 --- /dev/null +++ b/packages/server/sources/aws/README.md @@ -0,0 +1,321 @@ +# @walkeros/server-source-aws + +AWS server sources for walkerOS - lightweight, single-purpose runtime adapters +for AWS services. + +## Installation + +```bash +npm install @walkeros/server-source-aws @types/aws-lambda +``` + +## Usage + +```typescript +import { sourceLambda, type SourceLambda } from '@walkeros/server-source-aws'; +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + sources: { lambda: { code: sourceLambda } }, +}); + +export const handler = elb; +``` + +--- + +## Lambda Source + +The Lambda source provides an HTTP handler that receives walker events and +forwards them to the walkerOS collector. Works with API Gateway v1 (REST API), +v2 (HTTP API), and Lambda Function URLs. + +### Basic Usage + +```typescript +import { sourceLambda, type SourceLambda } from '@walkeros/server-source-aws'; +import { startFlow } from '@walkeros/collector'; + +// Handler singleton - reused across warm invocations +let handler: SourceLambda.Push; + +async function setup() { + if (handler) return handler; + + const { elb } = await startFlow({ + sources: { + lambda: { + code: sourceLambda, + config: { + settings: { + cors: true, + healthPath: '/health', + }, + }, + }, + }, + destinations: { + // Your destinations + }, + }); + + handler = elb; + return handler; +} + +export const main: SourceLambda.Push = async (event, context) => { + const h = await setup(); + return h(event, context); +}; + +// Export for Lambda runtime +export { main as handler }; +``` + +### Deployment + +#### Lambda Function URL (Simplest) + +```typescript +// No API Gateway needed +import * as lambda from 'aws-cdk-lib/aws-lambda'; + +const fn = new lambda.Function(this, 'Walker', { + runtime: lambda.Runtime.NODEJS_20_X, + handler: 'index.handler', + code: lambda.Code.fromAsset('./dist'), +}); + +fn.addFunctionUrl({ + authType: lambda.FunctionUrlAuthType.NONE, + cors: { + allowedOrigins: ['*'], + allowedMethods: [lambda.HttpMethod.GET, lambda.HttpMethod.POST], + }, +}); +``` + +#### API Gateway HTTP API (v2) - Recommended + +```yaml +# serverless.yml +service: walkeros-collector + +provider: + name: aws + runtime: nodejs20.x + memorySize: 256 + timeout: 30 + +functions: + collector: + handler: dist/index.handler + events: + - httpApi: + path: /collect + method: post + - httpApi: + path: /collect + method: get + - httpApi: + path: /health + method: get +``` + +#### API Gateway REST API (v1) + +```yaml +# template.yaml (AWS SAM) +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Resources: + WalkerFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./dist + Handler: index.handler + Runtime: nodejs20.x + Architectures: [arm64] + Events: + CollectPost: + Type: Api + Properties: + Path: /collect + Method: POST + CollectGet: + Type: Api + Properties: + Path: /collect + Method: GET + Health: + Type: Api + Properties: + Path: /health + Method: GET +``` + +### Configuration Options + +```typescript +interface Settings { + cors?: boolean | CorsOptions; // Enable CORS (default: true) + timeout?: number; // Request timeout (default: 30000ms, max: 900000ms) + enablePixelTracking?: boolean; // Enable GET tracking (default: true) + healthPath?: string; // Health check path (default: '/health') +} + +interface CorsOptions { + origin?: string | string[]; // Allowed origins + methods?: string[]; // Allowed methods + headers?: string[]; // Allowed headers + credentials?: boolean; // Allow credentials + maxAge?: number; // Preflight cache time +} +``` + +### Request Format + +**POST - Single Event:** + +```json +{ + "event": "page view", + "data": { + "title": "Home Page", + "path": "/" + }, + "context": { + "stage": ["prod", 1] + }, + "user": { + "id": "user-123" + } +} +``` + +**GET - Pixel Tracking:** + +``` +GET /collect?event=page%20view&data[title]=Home&data[path]=/ +``` + +### Response Format + +**Success:** + +```json +{ + "success": true, + "id": "event-id-123", + "requestId": "aws-request-id" +} +``` + +**Error:** + +```json +{ + "success": false, + "error": "Invalid request format", + "requestId": "aws-request-id" +} +``` + +**Health Check:** + +```json +{ + "status": "ok", + "timestamp": 1733328000000, + "source": "lambda", + "requestId": "aws-request-id" +} +``` + +### Supported Platforms + +- ✅ AWS API Gateway REST API (v1) +- ✅ AWS API Gateway HTTP API (v2) +- ✅ Lambda Function URLs +- ✅ Direct Lambda invocation + +### Features + +- **Auto-detection**: Automatically detects API Gateway version +- **CORS**: Configurable CORS with defaults +- **Pixel Tracking**: Optional GET requests with 1x1 GIF response +- **Base64 Decoding**: Handles base64-encoded request bodies +- **Health Checks**: Built-in health check endpoint +- **Request IDs**: AWS request ID in all responses and logs +- **Logging**: Integrated with walkerOS logger +- **Type-Safe**: Full TypeScript support + +### Production Considerations + +#### Cold Starts + +Use handler singleton pattern (shown in Basic Usage) to reuse source instance +across warm invocations. + +#### Logging + +The source integrates with the walkerOS logger from `env.logger`. Configure +CloudWatch Logs: + +```typescript +import { createLogger } from '@walkeros/core'; + +const logger = createLogger({ + level: 'info', + // CloudWatch-friendly JSON output + format: (level, message, meta) => + JSON.stringify({ level, message, ...meta, timestamp: Date.now() }), +}); +``` + +#### Error Handling + +All errors include request IDs for tracing. Configure CloudWatch Insights +queries: + +``` +fields @timestamp, level, message, requestId, error +| filter level = "error" +| sort @timestamp desc +``` + +#### Monitoring + +Key metrics to track: + +- Lambda Duration (p50, p99) +- Lambda Errors +- Lambda Throttles +- API Gateway 4xx/5xx responses + +#### Security + +- Use API Gateway with API keys or AWS IAM for authentication +- Enable AWS WAF for DDoS protection +- Set Lambda reserved concurrency to prevent runaway costs +- Validate CORS origins in production (don't use `cors: true`) + +### Examples + +See [examples directory](./examples/) for: + +- SAM deployment +- Serverless Framework deployment +- CDK deployment +- Local testing with SAM CLI + +## License + +MIT + +## Support + +- [Documentation](https://www.walkeros.io/docs) +- [GitHub Issues](https://github.com/elbwalker/walkerOS/issues) +- [Discord Community](https://discord.gg/elbwalker) diff --git a/packages/server/sources/aws/examples/README.md b/packages/server/sources/aws/examples/README.md new file mode 100644 index 000000000..904b73e0a --- /dev/null +++ b/packages/server/sources/aws/examples/README.md @@ -0,0 +1,113 @@ +# AWS Lambda Deployment Examples + +## Installation + +These are example configurations - see the main +[AWS Source README](../README.md) for installation. + +## Usage + +Copy an example to your project, configure destinations, and deploy with SAM, +Serverless, or CDK. + +--- + +This directory contains working deployment examples for the walkerOS AWS Lambda +source. + +## Files + +- **`basic-handler.ts`** - Recommended Lambda handler with singleton pattern +- **`sam-template.yaml`** - AWS SAM deployment template +- **`serverless.yml`** - Serverless Framework configuration +- **`cdk-stack.ts`** - AWS CDK TypeScript stack + +## Getting Started + +### 1. Build Your Handler + +```bash +# Use the basic handler as a starting point +cp examples/basic-handler.ts src/index.ts + +# Add your destinations +# Edit src/index.ts and add destinations to startFlow() + +# Build +npm run build +``` + +### 2. Deploy with SAM + +```bash +# Copy template +cp examples/sam-template.yaml template.yaml + +# Deploy +sam build +sam deploy --guided +``` + +### 3. Test + +```bash +# Health check +curl https://your-api-id.execute-api.us-east-1.amazonaws.com/Prod/health + +# Send event +curl -X POST https://your-api-id.execute-api.us-east-1.amazonaws.com/Prod/collect \ + -H "Content-Type: application/json" \ + -d '{"event": "page view", "data": {"title": "Test"}}' +``` + +## Local Testing + +### SAM CLI + +```bash +sam local start-api +curl http://localhost:3000/collect +``` + +### Serverless Offline + +```bash +npm install --save-dev serverless-offline +serverless offline +``` + +## Architecture Choices + +### Lambda Function URL + +- **Simplest**: No API Gateway needed +- **Lowest cost**: No API Gateway charges +- **Best for**: Internal tools, MVPs +- **Limitations**: Less control over routing/throttling + +### API Gateway HTTP API (v2) + +- **Recommended**: Good balance of features and cost +- **Lower cost**: ~70% cheaper than REST API +- **Best for**: Production deployments +- **Features**: JWT authorization, CORS, throttling + +### API Gateway REST API (v1) + +- **Most features**: WAF, API keys, usage plans +- **Higher cost**: More expensive than HTTP API +- **Best for**: Complex requirements, enterprise +- **Features**: Everything in HTTP API + more + +## Production Checklist + +- [ ] Configure CloudWatch log retention +- [ ] Set Lambda reserved concurrency +- [ ] Enable X-Ray tracing +- [ ] Configure CloudWatch alarms +- [ ] Set up API Gateway throttling +- [ ] Use custom domain name +- [ ] Enable AWS WAF (if using REST API) +- [ ] Configure VPC (if accessing private resources) +- [ ] Set up proper IAM roles for destinations +- [ ] Enable CloudTrail for API calls diff --git a/packages/server/sources/aws/examples/basic-handler.ts b/packages/server/sources/aws/examples/basic-handler.ts new file mode 100644 index 000000000..7ad643feb --- /dev/null +++ b/packages/server/sources/aws/examples/basic-handler.ts @@ -0,0 +1,54 @@ +/** + * Basic Lambda Handler Example + * + * This demonstrates the recommended singleton pattern for Lambda functions + * to maximize warm start performance. + */ + +import { sourceLambda, type SourceLambda } from '@walkeros/server-source-aws'; +import { startFlow } from '@walkeros/collector'; + +// Handler singleton - reused across warm invocations +let handler: SourceLambda.Push; + +/** + * Initialize the Lambda source and collector + * Only runs once per Lambda container lifecycle + */ +async function setup() { + if (handler) return handler; + + const { elb } = await startFlow({ + sources: { + lambda: { + code: sourceLambda, + config: { + settings: { + cors: true, + enablePixelTracking: true, + healthPath: '/health', + }, + }, + }, + }, + destinations: { + // Add your destinations here + // Example: AWS Kinesis, S3, CloudWatch, etc. + }, + }); + + handler = elb; + return handler; +} + +/** + * Lambda handler entry point + * AWS invokes this function for each request + */ +export const main: SourceLambda.Push = async (event, context) => { + const h = await setup(); + return h(event, context); +}; + +// Export for Lambda runtime +export { main as handler }; diff --git a/packages/server/sources/aws/examples/cdk-stack.ts b/packages/server/sources/aws/examples/cdk-stack.ts new file mode 100644 index 000000000..1bccc364d --- /dev/null +++ b/packages/server/sources/aws/examples/cdk-stack.ts @@ -0,0 +1,90 @@ +import * as cdk from 'aws-cdk-lib'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as apigateway from 'aws-cdk-lib/aws-apigateway'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import { Construct } from 'constructs'; + +export class WalkerOSStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + // Lambda function + const collectorFn = new lambda.Function(this, 'WalkerCollector', { + runtime: lambda.Runtime.NODEJS_20_X, + handler: 'index.handler', + code: lambda.Code.fromAsset('./dist'), + architecture: lambda.Architecture.ARM_64, + memorySize: 256, + timeout: cdk.Duration.seconds(30), + environment: { + NODE_ENV: 'production', + }, + logRetention: logs.RetentionDays.ONE_WEEK, + }); + + // API Gateway + const api = new apigateway.RestApi(this, 'WalkerAPI', { + restApiName: 'walkerOS Collector', + description: 'walkerOS event collection API', + defaultCorsPreflightOptions: { + allowOrigins: apigateway.Cors.ALL_ORIGINS, + allowMethods: apigateway.Cors.ALL_METHODS, + }, + }); + + // /collect endpoint + const collect = api.root.addResource('collect'); + const integration = new apigateway.LambdaIntegration(collectorFn); + collect.addMethod('POST', integration); + collect.addMethod('GET', integration); + + // /health endpoint + const health = api.root.addResource('health'); + health.addMethod('GET', integration); + + // Outputs + new cdk.CfnOutput(this, 'ApiUrl', { + value: api.url, + description: 'API Gateway URL', + }); + + new cdk.CfnOutput(this, 'CollectEndpoint', { + value: `${api.url}collect`, + description: 'Collection endpoint', + }); + + new cdk.CfnOutput(this, 'HealthEndpoint', { + value: `${api.url}health`, + description: 'Health check endpoint', + }); + } +} + +// Alternative: Lambda Function URL (no API Gateway) +export class WalkerOSFunctionUrlStack extends cdk.Stack { + constructor(scope: Construct, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + const collectorFn = new lambda.Function(this, 'WalkerCollector', { + runtime: lambda.Runtime.NODEJS_20_X, + handler: 'index.handler', + code: lambda.Code.fromAsset('./dist'), + architecture: lambda.Architecture.ARM_64, + }); + + // Function URL (simpler, lower cost than API Gateway) + const fnUrl = collectorFn.addFunctionUrl({ + authType: lambda.FunctionUrlAuthType.NONE, + cors: { + allowedOrigins: ['*'], + allowedMethods: [lambda.HttpMethod.GET, lambda.HttpMethod.POST], + allowedHeaders: ['Content-Type', 'Authorization'], + }, + }); + + new cdk.CfnOutput(this, 'FunctionUrl', { + value: fnUrl.url, + description: 'Lambda Function URL', + }); + } +} diff --git a/packages/server/sources/aws/examples/sam-template.yaml b/packages/server/sources/aws/examples/sam-template.yaml new file mode 100644 index 000000000..914769c38 --- /dev/null +++ b/packages/server/sources/aws/examples/sam-template.yaml @@ -0,0 +1,58 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: walkerOS AWS Lambda Collector + +Globals: + Function: + Timeout: 30 + Runtime: nodejs20.x + MemorySize: 256 + Architectures: + - arm64 + Environment: + Variables: + NODE_ENV: production + +Resources: + WalkerOSFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./dist + Handler: index.handler + Events: + # POST endpoint for event collection + CollectPost: + Type: Api + Properties: + Path: /collect + Method: POST + + # GET endpoint for pixel tracking + CollectGet: + Type: Api + Properties: + Path: /collect + Method: GET + + # Health check endpoint + Health: + Type: Api + Properties: + Path: /health + Method: GET + + # CloudWatch Log Group with retention + WalkerOSLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /aws/lambda/${WalkerOSFunction} + RetentionInDays: 7 + +Outputs: + ApiEndpoint: + Description: API Gateway endpoint URL + Value: !Sub 'https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/collect' + + HealthEndpoint: + Description: Health check endpoint URL + Value: !Sub 'https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/health' diff --git a/packages/server/sources/aws/examples/serverless.yml b/packages/server/sources/aws/examples/serverless.yml new file mode 100644 index 000000000..500149344 --- /dev/null +++ b/packages/server/sources/aws/examples/serverless.yml @@ -0,0 +1,49 @@ +service: walkeros-collector + +provider: + name: aws + runtime: nodejs20.x + memorySize: 256 + timeout: 30 + region: us-east-1 + environment: + NODE_ENV: production + + # IAM permissions for destinations + # iam: + # role: + # statements: + # - Effect: Allow + # Action: + # - kinesis:PutRecord + # - s3:PutObject + # Resource: "*" + +functions: + collector: + handler: dist/index.handler + events: + # HTTP API (v2) - recommended for lower cost + - httpApi: + path: /collect + method: post + - httpApi: + path: /collect + method: get + - httpApi: + path: /health + method: get + + # CloudWatch Logs + logRetentionInDays: 7 + + # Reserved concurrency to prevent runaway costs + # reservedConcurrency: 10 + +plugins: + - serverless-plugin-typescript + +custom: + # Build configuration + serverless-plugin-typescript: + tsConfigFileLocation: './tsconfig.json' diff --git a/packages/server/sources/aws/jest.config.mjs b/packages/server/sources/aws/jest.config.mjs new file mode 100644 index 000000000..9408b288a --- /dev/null +++ b/packages/server/sources/aws/jest.config.mjs @@ -0,0 +1,3 @@ +import config from '@walkeros/config/jest'; + +export default config; diff --git a/packages/server/sources/aws/package.json b/packages/server/sources/aws/package.json new file mode 100644 index 000000000..19a0ec9ea --- /dev/null +++ b/packages/server/sources/aws/package.json @@ -0,0 +1,67 @@ +{ + "name": "@walkeros/server-source-aws", + "description": "AWS server sources for walkerOS (Lambda, API Gateway, Function URLs)", + "version": "0.4.2", + "license": "MIT", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "files": [ + "dist/**", + "README.md", + "CHANGELOG.md" + ], + "scripts": { + "build": "tsup --silent", + "clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist", + "dev": "jest --watchAll --colors", + "lint": "tsc && eslint \"**/*.ts*\"", + "test": "jest", + "update": "npx npm-check-updates -u && npm update" + }, + "dependencies": { + "@walkeros/core": "0.4.2" + }, + "peerDependencies": { + "@types/aws-lambda": "^8.10.0" + }, + "devDependencies": { + "@types/aws-lambda": "^8.10.145" + }, + "repository": { + "url": "git+https://github.com/elbwalker/walkerOS.git", + "directory": "packages/server/sources/aws" + }, + "author": "elbwalker ", + "homepage": "https://github.com/elbwalker/walkerOS#readme", + "bugs": { + "url": "https://github.com/elbwalker/walkerOS/issues" + }, + "keywords": [ + "walker", + "walkerOS", + "source", + "server", + "aws", + "lambda", + "api gateway" + ], + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/elbwalker" + } + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "./dev": { + "types": "./dist/dev.d.ts", + "import": "./dist/dev.mjs", + "require": "./dist/dev.js" + } + } +} diff --git a/packages/server/sources/aws/src/dev.ts b/packages/server/sources/aws/src/dev.ts new file mode 100644 index 000000000..607bc0c6f --- /dev/null +++ b/packages/server/sources/aws/src/dev.ts @@ -0,0 +1 @@ +export * as lambda from './lambda'; diff --git a/packages/server/sources/aws/src/index.ts b/packages/server/sources/aws/src/index.ts new file mode 100644 index 000000000..4944eae22 --- /dev/null +++ b/packages/server/sources/aws/src/index.ts @@ -0,0 +1,2 @@ +export * from './lambda'; +export { default as sourceLambda } from './lambda'; diff --git a/packages/server/sources/aws/src/lambda/__tests__/index.test.ts b/packages/server/sources/aws/src/lambda/__tests__/index.test.ts new file mode 100644 index 000000000..1ede56963 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/__tests__/index.test.ts @@ -0,0 +1,932 @@ +import { sourceLambda } from '../index'; +import type { EventRequest, LambdaEvent, LambdaContext, Types } from '../types'; +import type { APIGatewayProxyEvent, APIGatewayProxyEventV2 } from 'aws-lambda'; +import type { Source } from '@walkeros/core'; +import { createMockLogger } from '@walkeros/core'; +import * as examples from '../examples'; + +// Mock API Gateway v1 event +function createMockEventV1( + method = 'POST', + body?: string, + queryStringParameters?: Record, + headers?: Record, +): APIGatewayProxyEvent { + return { + httpMethod: method, + body: body ?? null, + queryStringParameters: queryStringParameters ?? null, + headers: headers ?? {}, + isBase64Encoded: false, + path: '/collect', + resource: '/collect', + pathParameters: null, + stageVariables: null, + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + protocol: 'HTTP/1.1', + httpMethod: method, + path: '/collect', + stage: 'prod', + requestId: 'request-id', + requestTimeEpoch: Date.now(), + resourceId: 'resource-id', + resourcePath: '/collect', + identity: { + sourceIp: '127.0.0.1', + userAgent: 'test', + accessKey: null, + accountId: null, + apiKey: null, + apiKeyId: null, + caller: null, + clientCert: null, + cognitoAuthenticationProvider: null, + cognitoAuthenticationType: null, + cognitoIdentityId: null, + cognitoIdentityPoolId: null, + principalOrgId: null, + user: null, + userArn: null, + }, + authorizer: null, + }, + multiValueHeaders: {}, + multiValueQueryStringParameters: null, + }; +} + +// Mock API Gateway v2 event +function createMockEventV2( + method = 'POST', + body?: string, + rawQueryString?: string, + headers?: Record, +): APIGatewayProxyEventV2 { + return { + version: '2.0', + routeKey: '$default', + rawPath: '/collect', + rawQueryString: rawQueryString ?? '', + headers: headers ?? {}, + body: body ?? undefined, + isBase64Encoded: false, + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + domainName: 'api.example.com', + domainPrefix: 'api', + http: { + method, + path: '/collect', + protocol: 'HTTP/1.1', + sourceIp: '127.0.0.1', + userAgent: 'test', + }, + requestId: 'request-id', + routeKey: '$default', + stage: 'prod', + time: new Date().toISOString(), + timeEpoch: Date.now(), + }, + }; +} + +// Mock Lambda context +function createMockContext(): LambdaContext { + return { + callbackWaitsForEmptyEventLoop: false, + functionName: 'test-function', + functionVersion: '1', + invokedFunctionArn: 'arn:aws:lambda:us-east-1:123456789012:function:test', + memoryLimitInMB: '256', + awsRequestId: 'request-id', + logGroupName: '/aws/lambda/test', + logStreamName: '2024/01/01/[$LATEST]abcd', + getRemainingTimeInMillis: () => 3000, + done: () => {}, + fail: () => {}, + succeed: () => {}, + }; +} + +describe('sourceLambda', () => { + let mockPush: jest.MockedFunction<(...args: unknown[]) => unknown>; + let mockCommand: jest.MockedFunction<(...args: unknown[]) => unknown>; + let mockLogger: ReturnType; + + beforeEach(() => { + mockPush = jest.fn().mockResolvedValue({ + event: { id: 'test-id' }, + ok: true, + successful: [], + queued: [], + failed: [], + }); + mockCommand = jest.fn().mockResolvedValue({ + ok: true, + successful: [], + queued: [], + failed: [], + }); + mockLogger = createMockLogger(); + }); + + describe('initialization', () => { + it('should initialize with default settings', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + expect(source.type).toBe('lambda'); + expect(source.config.settings).toEqual({ + cors: true, + timeout: 30000, + enablePixelTracking: true, + healthPath: '/health', + }); + expect(typeof source.push).toBe('function'); + }); + + it('should merge custom settings with defaults', async () => { + const config: Partial> = { + settings: { + cors: false, + enablePixelTracking: false, + }, + }; + + const source = await sourceLambda(config, { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }); + + expect(source.config.settings).toEqual({ + cors: false, + timeout: 30000, + enablePixelTracking: false, + healthPath: '/health', + }); + }); + }); + + describe('API Gateway v1 events', () => { + it('should handle POST request with event data (v1)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'page view', + data: { title: 'Test Page' }, + }; + + const event = createMockEventV1('POST', JSON.stringify(eventRequest)); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(mockPush).toHaveBeenCalledWith({ + name: 'page view', + data: { title: 'Test Page' }, + context: undefined, + user: undefined, + globals: undefined, + consent: undefined, + }); + expect(result.statusCode).toBe(200); + expect(JSON.parse(result.body)).toMatchObject({ + success: true, + id: 'test-id', + requestId: expect.any(String), + }); + }); + + it('should handle OPTIONS request (v1)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('OPTIONS'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(204); + expect(mockPush).not.toHaveBeenCalled(); + }); + + it('should handle GET request with pixel tracking (v1)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('GET', undefined, { + event: 'page view', + 'data[title]': 'Test', + }); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(mockPush).toHaveBeenCalled(); + expect(result.statusCode).toBe(200); + expect(result.headers?.['Content-Type']).toBe('image/gif'); + expect(result.isBase64Encoded).toBe(true); + }); + + it('should reject GET when pixel tracking disabled (v1)', async () => { + const source = await sourceLambda( + { settings: { enablePixelTracking: false } }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('GET'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(405); + expect(JSON.parse(result.body)).toMatchObject({ + success: false, + error: 'GET not allowed', + requestId: expect.any(String), + }); + }); + }); + + describe('API Gateway v2 events', () => { + it('should handle POST request with event data (v2)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'product view', + data: { id: 'P123', name: 'Product' }, + }; + + const event = createMockEventV2('POST', JSON.stringify(eventRequest)); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(mockPush).toHaveBeenCalledWith({ + name: 'product view', + data: { id: 'P123', name: 'Product' }, + context: undefined, + user: undefined, + globals: undefined, + consent: undefined, + }); + expect(result.statusCode).toBe(200); + expect(JSON.parse(result.body)).toMatchObject({ + success: true, + id: 'test-id', + requestId: expect.any(String), + }); + }); + + it('should handle OPTIONS request (v2)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV2('OPTIONS'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(204); + expect(mockPush).not.toHaveBeenCalled(); + }); + + it('should handle GET request with query string (v2)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV2( + 'GET', + undefined, + 'event=page%20view&data[title]=Test', + ); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(mockPush).toHaveBeenCalled(); + expect(result.statusCode).toBe(200); + expect(result.headers?.['Content-Type']).toBe('image/gif'); + }); + }); + + describe('error handling', () => { + it('should require request body for POST', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('POST', null as unknown as string); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body)).toMatchObject({ + success: false, + error: 'Request body is required', + requestId: expect.any(String), + }); + }); + + it('should reject invalid event body', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1( + 'POST', + JSON.stringify({ invalid: 'format' }), + ); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body)).toMatchObject({ + success: false, + error: 'Invalid request format', + requestId: expect.any(String), + }); + }); + + it('should handle event processing errors', async () => { + const errorPush = jest + .fn() + .mockRejectedValue(new Error('Processing failed')); + const source = await sourceLambda( + {}, + { + push: errorPush, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'error event', + data: { test: true }, + }; + + const event = createMockEventV1('POST', JSON.stringify(eventRequest)); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(400); + expect(JSON.parse(result.body)).toMatchObject({ + success: false, + error: 'Processing failed', + requestId: expect.any(String), + }); + }); + + it('should reject unsupported methods', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('PUT'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(405); + expect(JSON.parse(result.body)).toMatchObject({ + success: false, + error: 'Method not allowed', + requestId: expect.any(String), + }); + }); + }); + + describe('CORS handling', () => { + it('should set default CORS headers when enabled', async () => { + const source = await sourceLambda( + { settings: { cors: true } }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('OPTIONS'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.headers?.['Access-Control-Allow-Origin']).toBe('*'); + expect(result.headers?.['Access-Control-Allow-Methods']).toBe( + 'GET, POST, OPTIONS', + ); + }); + + it('should set custom CORS headers', async () => { + const source = await sourceLambda( + { + settings: { + cors: { + origin: ['https://example.com'], + methods: ['POST'], + headers: ['Content-Type'], + credentials: true, + maxAge: 7200, + }, + }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('OPTIONS'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.headers?.['Access-Control-Allow-Origin']).toBe( + 'https://example.com', + ); + expect(result.headers?.['Access-Control-Allow-Methods']).toBe('POST'); + expect(result.headers?.['Access-Control-Allow-Credentials']).toBe('true'); + expect(result.headers?.['Access-Control-Max-Age']).toBe('7200'); + }); + + it('should not set CORS headers when disabled', async () => { + const source = await sourceLambda( + { settings: { cors: false } }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('OPTIONS'); + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.headers?.['Access-Control-Allow-Origin']).toBeUndefined(); + }); + }); + + describe('base64 encoding', () => { + it('should decode base64 encoded body', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'test event', + data: { encoded: true }, + }; + + const encodedBody = Buffer.from(JSON.stringify(eventRequest)).toString( + 'base64', + ); + const event = createMockEventV1('POST', encodedBody); + event.isBase64Encoded = true; + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(mockPush).toHaveBeenCalledWith({ + name: 'test event', + data: { encoded: true }, + context: undefined, + user: undefined, + globals: undefined, + consent: undefined, + }); + expect(result.statusCode).toBe(200); + }); + }); + + describe('example-based tests', () => { + let env: Parameters[1]; + + beforeEach(() => { + env = { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }; + }); + + it('processes v2 POST event correctly', async () => { + const source = await sourceLambda({}, env); + const context = createMockContext(); + + const result = await source.push( + examples.inputs.apiGatewayV2PostEvent, + context, + ); + + expect(result.statusCode).toBe(200); + expect(result.headers?.['Content-Type']).toBe('application/json'); + expect(result.headers?.['Access-Control-Allow-Origin']).toBe('*'); + const body = JSON.parse(result.body); + expect(body.success).toBe(true); + }); + + it('handles GET pixel tracking', async () => { + const source = await sourceLambda({}, env); + const context = createMockContext(); + + const result = await source.push( + examples.inputs.apiGatewayV2GetEvent, + context, + ); + + expect(result.statusCode).toBe(200); + expect(result.headers?.['Content-Type']).toBe('image/gif'); + expect(result.isBase64Encoded).toBe(true); + }); + + it('handles v1 POST event', async () => { + const source = await sourceLambda({}, env); + const context = createMockContext(); + + const result = await source.push( + examples.inputs.apiGatewayV1PostEvent, + context, + ); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.success).toBe(true); + }); + }); + + describe('health check endpoint', () => { + it('should respond to health check on configured path (v2)', async () => { + const source = await sourceLambda( + { settings: { healthPath: '/health' } }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV2('GET'); + event.rawPath = '/health'; + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body).toMatchObject({ + status: 'ok', + source: 'lambda', + }); + expect(body.timestamp).toBeGreaterThan(0); + expect(mockPush).not.toHaveBeenCalled(); + }); + + it('should respond to health check on configured path (v1)', async () => { + const source = await sourceLambda( + { settings: { healthPath: '/health' } }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('GET'); + event.path = '/health'; + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body).toMatchObject({ + status: 'ok', + source: 'lambda', + }); + }); + + it('should not trigger health check on different path', async () => { + const source = await sourceLambda( + { settings: { healthPath: '/health' } }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV2('GET'); + event.rawPath = '/collect'; + event.rawQueryString = 'event=test'; + const context = createMockContext(); + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(200); + expect(result.headers?.['Content-Type']).toBe('image/gif'); + }); + }); + + describe('logging', () => { + it('should NOT log for successful requests (collector handles)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'page view', + data: { title: 'Test' }, + }; + + const event = createMockEventV1('POST', JSON.stringify(eventRequest)); + const context = createMockContext(); + + await source.push(event, context); + + // NO logging for normal operations + expect(mockLogger.debug).not.toHaveBeenCalled(); + expect(mockLogger.info).not.toHaveBeenCalled(); + }); + + it('should log processing errors with context', async () => { + const errorPush = jest + .fn() + .mockRejectedValue(new Error('Collector failed')); + const source = await sourceLambda( + {}, + { + push: errorPush, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'test event', + data: { test: true }, + }; + + const event = createMockEventV1('POST', JSON.stringify(eventRequest)); + const context = createMockContext(); + context.awsRequestId = 'req-123'; + + await source.push(event, context); + + expect(mockLogger.error).toHaveBeenCalledWith( + 'Event processing failed', + expect.objectContaining({ + error: expect.any(Error), + eventName: 'test event', + requestId: 'req-123', + }), + ); + }); + + it('should log handler errors with context', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + // Completely invalid event structure that will cause parseEvent to throw + const badEvent = null as any; + const context = createMockContext(); + context.awsRequestId = 'req-456'; + + await source.push(badEvent, context); + + expect(mockLogger.error).toHaveBeenCalledWith( + 'Lambda handler error', + expect.objectContaining({ + error: expect.any(Error), + requestId: 'req-456', + }), + ); + }); + }); + + describe('request ID tracking', () => { + it('should include request ID in successful response (v2)', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const eventRequest: EventRequest = { + event: 'product view', + data: { id: 'P123' }, + }; + + const event = createMockEventV2('POST', JSON.stringify(eventRequest)); + const context = createMockContext(); + context.awsRequestId = 'test-request-id-123'; + + const result = await source.push(event, context); + + expect(result.statusCode).toBe(200); + expect(result.headers?.['X-Request-ID']).toBe('test-request-id-123'); + const body = JSON.parse(result.body); + expect(body.requestId).toBe('test-request-id-123'); + }); + + it('should include request ID in error responses', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const event = createMockEventV1('POST', null as unknown as string); + const context = createMockContext(); + context.awsRequestId = 'error-request-id'; + + const result = await source.push(event, context); + + expect(result.headers?.['X-Request-ID']).toBe('error-request-id'); + }); + }); + + describe('settings validation', () => { + it('should reject invalid timeout value', async () => { + await expect( + sourceLambda( + { + settings: { + timeout: 999999999, // Exceeds 900000ms Lambda limit + }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ), + ).rejects.toThrow(); + }); + + it('should use default settings when none provided', async () => { + const source = await sourceLambda( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + expect(source.config.settings).toEqual({ + cors: true, + timeout: 30000, + enablePixelTracking: true, + healthPath: '/health', + }); + }); + + it('should merge partial settings with defaults', async () => { + const source = await sourceLambda( + { + settings: { + cors: false, + }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + expect(source.config.settings).toEqual({ + cors: false, + timeout: 30000, + enablePixelTracking: true, + healthPath: '/health', + }); + }); + }); +}); diff --git a/packages/server/sources/aws/src/lambda/examples/env.ts b/packages/server/sources/aws/src/lambda/examples/env.ts new file mode 100644 index 000000000..9617b8795 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/examples/env.ts @@ -0,0 +1,52 @@ +import type { Env } from '../types'; +import type { Elb, Logger } from '@walkeros/core'; + +/** + * Example environment configurations for AWS Lambda source + * + * These environments provide standardized mock structures for testing + * Lambda event handling without requiring actual Lambda deployment. + */ + +// Create a properly typed elb/push/command function that returns a promise with PushResult +const createMockElbFn = (): Elb.Fn => { + const fn = (() => + Promise.resolve({ + ok: true, + successful: [], + queued: [], + failed: [], + })) as Elb.Fn; + return fn; +}; + +// Simple no-op logger for demo purposes +const noopFn = () => {}; +const noopLogger: Logger.Instance = { + error: noopFn, + info: noopFn, + debug: noopFn, + throw: (message: string | Error) => { + throw typeof message === 'string' ? new Error(message) : message; + }, + scope: () => noopLogger, +}; + +/** + * Standard mock environment for testing Lambda source + * + * Use this for testing Lambda event ingestion and request/response handling + * without requiring a real AWS Lambda environment. + */ +export const push: Env = { + get push() { + return createMockElbFn(); + }, + get command() { + return createMockElbFn(); + }, + get elb() { + return createMockElbFn(); + }, + logger: noopLogger, +}; diff --git a/packages/server/sources/aws/src/lambda/examples/events.ts b/packages/server/sources/aws/src/lambda/examples/events.ts new file mode 100644 index 000000000..bc33763dc --- /dev/null +++ b/packages/server/sources/aws/src/lambda/examples/events.ts @@ -0,0 +1,25 @@ +import type { WalkerOS } from '@walkeros/core'; + +/** + * Expected walkerOS events from Lambda inputs. + * These are what processEvent should produce. + */ + +// From apiGatewayV2PostEvent +export const pageViewEvent: Partial = { + name: 'page view', + data: { title: 'Home Page', path: '/' }, + user: { id: 'user-123' }, +}; + +// From apiGatewayV2GetEvent +export const buttonClickEvent: Partial = { + name: 'button click', + data: { id: 'cta', text: 'Sign Up' }, +}; + +// From apiGatewayV1PostEvent +export const productAddEvent: Partial = { + name: 'product add', + data: { id: 'P123', name: 'Laptop', price: 999 }, +}; diff --git a/packages/server/sources/aws/src/lambda/examples/index.ts b/packages/server/sources/aws/src/lambda/examples/index.ts new file mode 100644 index 000000000..6ae56492c --- /dev/null +++ b/packages/server/sources/aws/src/lambda/examples/index.ts @@ -0,0 +1,4 @@ +export * as env from './env'; +export * as inputs from './inputs'; +export * as events from './events'; +export * as outputs from './outputs'; diff --git a/packages/server/sources/aws/src/lambda/examples/inputs.ts b/packages/server/sources/aws/src/lambda/examples/inputs.ts new file mode 100644 index 000000000..035b73735 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/examples/inputs.ts @@ -0,0 +1,144 @@ +import type { APIGatewayProxyEvent, APIGatewayProxyEventV2 } from 'aws-lambda'; + +/** + * Real examples of Lambda events this source will receive. + * These define the CONTRACT - implementation must handle these inputs. + */ + +// API Gateway v2 (HTTP API) - POST with walker event +export const apiGatewayV2PostEvent: APIGatewayProxyEventV2 = { + version: '2.0', + routeKey: '$default', + rawPath: '/collect', + rawQueryString: '', + headers: { + 'content-type': 'application/json', + 'user-agent': 'Mozilla/5.0', + }, + body: JSON.stringify({ + event: 'page view', + data: { title: 'Home Page', path: '/' }, + user: { id: 'user-123' }, + }), + isBase64Encoded: false, + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + domainName: 'api.example.com', + domainPrefix: 'api', + http: { + method: 'POST', + path: '/collect', + protocol: 'HTTP/1.1', + sourceIp: '1.2.3.4', + userAgent: 'Mozilla/5.0', + }, + requestId: 'request-123', + routeKey: '$default', + stage: 'prod', + time: '01/Jan/2024:00:00:00 +0000', + timeEpoch: 1704067200000, + }, +}; + +// API Gateway v2 - GET with query params (pixel tracking) +export const apiGatewayV2GetEvent: APIGatewayProxyEventV2 = { + version: '2.0', + routeKey: '$default', + rawPath: '/collect', + rawQueryString: 'event=button%20click&data[id]=cta&data[text]=Sign%20Up', + headers: {}, + isBase64Encoded: false, + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + domainName: 'api.example.com', + domainPrefix: 'api', + http: { + method: 'GET', + path: '/collect', + protocol: 'HTTP/1.1', + sourceIp: '1.2.3.4', + userAgent: 'Mozilla/5.0', + }, + requestId: 'request-456', + routeKey: '$default', + stage: 'prod', + time: '01/Jan/2024:00:00:01 +0000', + timeEpoch: 1704067201000, + }, +}; + +// API Gateway v1 (REST API) - POST with walker event +export const apiGatewayV1PostEvent: APIGatewayProxyEvent = { + httpMethod: 'POST', + path: '/collect', + body: JSON.stringify({ + event: 'product add', + data: { id: 'P123', name: 'Laptop', price: 999 }, + }), + headers: { 'content-type': 'application/json' }, + multiValueHeaders: {}, + isBase64Encoded: false, + pathParameters: null, + queryStringParameters: null, + multiValueQueryStringParameters: null, + stageVariables: null, + resource: '/collect', + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + protocol: 'HTTP/1.1', + httpMethod: 'POST', + path: '/collect', + stage: 'prod', + requestId: 'request-789', + requestTimeEpoch: 1704067202000, + resourceId: 'resource-id', + resourcePath: '/collect', + identity: { + sourceIp: '1.2.3.4', + userAgent: 'Mozilla/5.0', + accessKey: null, + accountId: null, + apiKey: null, + apiKeyId: null, + caller: null, + clientCert: null, + cognitoAuthenticationProvider: null, + cognitoAuthenticationType: null, + cognitoIdentityId: null, + cognitoIdentityPoolId: null, + principalOrgId: null, + user: null, + userArn: null, + }, + authorizer: null, + }, +}; + +// Health check request +export const healthCheckEvent: APIGatewayProxyEventV2 = { + ...apiGatewayV2GetEvent, + rawPath: '/health', + rawQueryString: '', + requestContext: { + ...apiGatewayV2GetEvent.requestContext, + http: { + ...apiGatewayV2GetEvent.requestContext.http, + path: '/health', + }, + }, +}; + +// Invalid event - malformed JSON +export const invalidJsonEvent: APIGatewayProxyEventV2 = { + ...apiGatewayV2PostEvent, + body: '{invalid json', +}; + +// Missing event field +export const missingEventField: APIGatewayProxyEventV2 = { + ...apiGatewayV2PostEvent, + body: JSON.stringify({ data: { foo: 'bar' } }), +}; diff --git a/packages/server/sources/aws/src/lambda/examples/outputs.ts b/packages/server/sources/aws/src/lambda/examples/outputs.ts new file mode 100644 index 000000000..b6a4a04c6 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/examples/outputs.ts @@ -0,0 +1,37 @@ +import type { APIGatewayProxyResult } from 'aws-lambda'; + +/** + * Expected Lambda response outputs. + * Tests verify implementation produces these. + */ + +// Successful event processing +export const successResponse: Partial = { + statusCode: 200, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }, + body: expect.stringContaining('"success":true'), +}; + +// Health check response +export const healthResponse: Partial = { + statusCode: 200, + body: expect.stringContaining('"status":"ok"'), +}; + +// Pixel tracking response +export const pixelResponse: Partial = { + statusCode: 200, + headers: expect.objectContaining({ + 'Content-Type': 'image/gif', + }), + isBase64Encoded: true, +}; + +// Error responses +export const invalidBodyResponse: Partial = { + statusCode: 400, + body: expect.stringContaining('"success":false'), +}; diff --git a/packages/server/sources/aws/src/lambda/index.ts b/packages/server/sources/aws/src/lambda/index.ts new file mode 100644 index 000000000..b04829c53 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/index.ts @@ -0,0 +1,172 @@ +import type { LambdaSource, Env, Settings, EventRequest, Types } from './types'; +import type { Source } from '@walkeros/core'; +import { requestToData } from '@walkeros/core'; +import { + parseEvent, + parseBody, + isEventRequest, + getCorsHeaders, + createResponse, + createPixelResponse, + getPath, +} from './utils'; +import { processEvent } from './push'; +import { SettingsSchema } from './schemas/settings'; + +export * as SourceLambda from './types'; +export * as schemas from './schemas'; + +// Export examples +export * as examples from './examples'; + +export const sourceLambda = async ( + config: Partial> = {}, + env: Env, +): Promise => { + const { push: envPush } = env; + + const settings = SettingsSchema.parse(config.settings || {}); + + const fullConfig: Source.Config = { + ...config, + settings, + }; + + const push: Types['push'] = async (event, context) => { + const requestId = context.awsRequestId; + let parsed; + + try { + const corsHeaders = getCorsHeaders(settings.cors || false); + parsed = parseEvent(event); + const path = getPath(event); + + // Health check + if (settings.healthPath && path === settings.healthPath) { + return createResponse( + 200, + { + status: 'ok', + timestamp: Date.now(), + source: 'lambda', + requestId, + }, + corsHeaders, + requestId, + ); + } + + // Handle OPTIONS for CORS preflight + if (parsed.method === 'OPTIONS') { + return createResponse(204, '', corsHeaders, requestId); + } + + // Handle GET for pixel tracking + if (parsed.method === 'GET') { + if (!settings.enablePixelTracking) { + return createResponse( + 405, + { success: false, error: 'GET not allowed', requestId }, + corsHeaders, + requestId, + ); + } + if (parsed.queryString) { + const parsedData = requestToData(parsed.queryString); + if (parsedData && typeof parsedData === 'object') { + await envPush(parsedData); + } + } + return createPixelResponse(corsHeaders, requestId); + } + + // Handle POST for event data + if (parsed.method === 'POST') { + if (!parsed.body) { + return createResponse( + 400, + { success: false, error: 'Request body is required', requestId }, + corsHeaders, + requestId, + ); + } + + const body = parseBody(parsed.body, parsed.isBase64Encoded); + + if (!body || typeof body !== 'object') { + return createResponse( + 400, + { success: false, error: 'Invalid event body', requestId }, + corsHeaders, + requestId, + ); + } + + if (isEventRequest(body)) { + const result = await processEvent( + body as EventRequest, + envPush, + env.logger, + requestId, + ); + + if (result.error) { + return createResponse( + 400, + { success: false, error: result.error, requestId }, + corsHeaders, + requestId, + ); + } + + return createResponse( + 200, + { success: true, id: result.id, requestId }, + corsHeaders, + requestId, + ); + } + + return createResponse( + 400, + { success: false, error: 'Invalid request format', requestId }, + corsHeaders, + requestId, + ); + } + + return createResponse( + 405, + { success: false, error: 'Method not allowed', requestId }, + corsHeaders, + requestId, + ); + } catch (error) { + // Log handler errors with context - per using-logger skill + env.logger?.error('Lambda handler error', { + error, + requestId, + method: parsed?.method, + }); + return createResponse( + 500, + { + success: false, + error: + error instanceof Error ? error.message : 'Internal server error', + requestId, + }, + {}, + requestId, + ); + } + }; + + return { + type: 'lambda', + config: fullConfig, + push, + }; +}; + +export default sourceLambda; diff --git a/packages/server/sources/aws/src/lambda/push.ts b/packages/server/sources/aws/src/lambda/push.ts new file mode 100644 index 000000000..362bdb269 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/push.ts @@ -0,0 +1,30 @@ +import type { Collector, WalkerOS, Logger } from '@walkeros/core'; +import type { EventRequest } from './types'; + +export async function processEvent( + eventReq: EventRequest, + push: Collector.PushFn, + logger?: Logger.Instance, + requestId?: string, +): Promise<{ id?: string; error?: string }> { + try { + const result = await push({ + name: eventReq.event, + data: (eventReq.data || {}) as WalkerOS.Properties, + context: eventReq.context as WalkerOS.OrderedProperties | undefined, + user: eventReq.user as WalkerOS.User | undefined, + globals: eventReq.globals as WalkerOS.Properties | undefined, + consent: eventReq.consent as WalkerOS.Consent | undefined, + }); + + return { id: result?.event?.id }; + } catch (error) { + // Log with structured context - per using-logger skill + logger?.error('Event processing failed', { + error, + eventName: eventReq.event, + requestId, + }); + return { error: error instanceof Error ? error.message : 'Unknown error' }; + } +} diff --git a/packages/server/sources/aws/src/lambda/schemas/index.ts b/packages/server/sources/aws/src/lambda/schemas/index.ts new file mode 100644 index 000000000..838c4c8cb --- /dev/null +++ b/packages/server/sources/aws/src/lambda/schemas/index.ts @@ -0,0 +1,11 @@ +import { zodToSchema } from '@walkeros/core/dev'; +import { SettingsSchema } from './settings'; + +// Export primitives +export * from './primitives'; + +// Export Zod schemas and types +export { SettingsSchema, type Settings } from './settings'; + +// JSON Schema exports (for website PropertyTable and documentation tools) +export const settings = zodToSchema(SettingsSchema); diff --git a/packages/server/sources/aws/src/lambda/schemas/primitives.ts b/packages/server/sources/aws/src/lambda/schemas/primitives.ts new file mode 100644 index 000000000..34ae09605 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/schemas/primitives.ts @@ -0,0 +1,55 @@ +import { z } from '@walkeros/core/dev'; + +/** + * HTTP methods enum + */ +export const HttpMethod = z.enum([ + 'GET', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', + 'OPTIONS', + 'HEAD', +]); + +/** + * CORS origin configuration + * Accepts: + * - '*' for all origins + * - Single URL string + * - Array of URL strings + */ +export const CorsOrigin = z.union([ + z.string(), + z.array(z.string()), + z.literal('*'), +]); + +/** + * CORS options schema + * Configuration for Cross-Origin Resource Sharing + */ +export const CorsOptionsSchema = z.object({ + origin: CorsOrigin.describe( + 'Allowed origins (* for all, URL string, or array of URLs)', + ).optional(), + + methods: z.array(HttpMethod).describe('Allowed HTTP methods').optional(), + + headers: z.array(z.string()).describe('Allowed request headers').optional(), + + credentials: z + .boolean() + .describe('Allow credentials (cookies, authorization headers)') + .optional(), + + maxAge: z + .number() + .int() + .positive() + .describe('Preflight cache duration in seconds') + .optional(), +}); + +export type CorsOptions = z.infer; diff --git a/packages/server/sources/aws/src/lambda/schemas/settings.ts b/packages/server/sources/aws/src/lambda/schemas/settings.ts new file mode 100644 index 000000000..ab334f1b5 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/schemas/settings.ts @@ -0,0 +1,36 @@ +import { z } from '@walkeros/core/dev'; +import { CorsOptionsSchema } from './primitives'; + +/** + * AWS Lambda source settings schema + */ +export const SettingsSchema = z.object({ + cors: z + .union([z.boolean(), CorsOptionsSchema]) + .describe( + 'CORS configuration: false = disabled, true = allow all origins, object = custom configuration', + ) + .default(true), + + timeout: z + .number() + .int() + .positive() + .max(900000) // AWS Lambda max timeout: 15 minutes + .describe('Request timeout in milliseconds (max: 900000 for Lambda)') + .default(30000), + + enablePixelTracking: z + .boolean() + .describe( + 'Enable GET requests with 1x1 transparent GIF response for pixel tracking', + ) + .default(true), + + healthPath: z + .string() + .describe('Health check endpoint path (e.g., /health)') + .default('/health'), +}); + +export type Settings = z.infer; diff --git a/packages/server/sources/aws/src/lambda/types.ts b/packages/server/sources/aws/src/lambda/types.ts new file mode 100644 index 000000000..4caf8e49b --- /dev/null +++ b/packages/server/sources/aws/src/lambda/types.ts @@ -0,0 +1,82 @@ +import type { WalkerOS, Source as CoreSource } from '@walkeros/core'; +import type { + APIGatewayProxyEvent, + APIGatewayProxyEventV2, + APIGatewayProxyResult, + Context, +} from 'aws-lambda'; +import type { SettingsSchema, CorsOptionsSchema } from './schemas'; +import { z } from '@walkeros/core/dev'; + +// Lambda event types +export type LambdaEvent = APIGatewayProxyEvent | APIGatewayProxyEventV2; +export type LambdaResult = APIGatewayProxyResult; +export type LambdaContext = Context; + +// Types inferred from Zod schemas +export type Settings = z.infer; +export type CorsOptions = z.infer; + +// InitSettings: user input (all optional) +export type InitSettings = Partial; + +export interface Mapping { + // Custom source event mapping properties +} + +// Lambda-specific push type +export type Push = ( + event: LambdaEvent, + context: LambdaContext, +) => Promise; + +export interface Env extends CoreSource.Env { + lambdaEvent?: LambdaEvent; + lambdaContext?: LambdaContext; +} + +// Type bundle (must be after Settings, Mapping, Push, Env are defined) +export type Types = CoreSource.Types< + Settings, + Mapping, + Push, + Env, + InitSettings +>; + +export interface LambdaSource extends Omit, 'push'> { + push: Push; +} + +// Convenience Config type +export type Config = CoreSource.Config; +export type PartialConfig = CoreSource.PartialConfig; + +// Lambda source doesn't follow standard Source.Init pattern due to Lambda handler interface + +export interface EventRequest { + event: string; + data?: WalkerOS.AnyObject; + context?: WalkerOS.AnyObject; + user?: WalkerOS.AnyObject; + globals?: WalkerOS.AnyObject; + consent?: WalkerOS.AnyObject; +} + +export interface EventResponse { + success: boolean; + id?: string; + error?: string; +} + +export type RequestBody = EventRequest; +export type ResponseBody = EventResponse; + +// Parsed request data structure +export interface ParsedRequest { + method: string; + body: unknown; + queryString: string | null; + headers: Record; + isBase64Encoded: boolean; +} diff --git a/packages/server/sources/aws/src/lambda/utils.ts b/packages/server/sources/aws/src/lambda/utils.ts new file mode 100644 index 000000000..807a024a6 --- /dev/null +++ b/packages/server/sources/aws/src/lambda/utils.ts @@ -0,0 +1,166 @@ +import type { APIGatewayProxyEventV2, APIGatewayProxyResult } from 'aws-lambda'; +import type { + LambdaEvent, + ParsedRequest, + CorsOptions, + RequestBody, + EventRequest, +} from './types'; + +export function isAPIGatewayV2( + event: LambdaEvent, +): event is APIGatewayProxyEventV2 { + return 'version' in event && event.version === '2.0'; +} + +export function parseEvent(event: LambdaEvent): ParsedRequest { + if (isAPIGatewayV2(event)) { + const headers: Record = {}; + if (event.headers) { + Object.entries(event.headers).forEach(([key, value]) => { + if (value) headers[key.toLowerCase()] = value; + }); + } + return { + method: event.requestContext.http.method, + body: event.body, + queryString: event.rawQueryString || null, + headers, + isBase64Encoded: event.isBase64Encoded || false, + }; + } else { + const headers: Record = {}; + if (event.headers) { + Object.entries(event.headers).forEach(([key, value]) => { + if (value) headers[key.toLowerCase()] = value; + }); + } + let queryString: string | null = null; + if (event.queryStringParameters) { + const params = new URLSearchParams(); + Object.entries(event.queryStringParameters).forEach(([key, value]) => { + if (value) params.append(key, value); + }); + queryString = params.toString() || null; + } + return { + method: event.httpMethod, + body: event.body, + queryString, + headers, + isBase64Encoded: event.isBase64Encoded || false, + }; + } +} + +export function getPath(event: LambdaEvent): string { + if (isAPIGatewayV2(event)) { + return event.rawPath; + } else { + return event.path; + } +} + +export function parseBody(body: unknown, isBase64Encoded: boolean): unknown { + if (!body || typeof body !== 'string') return body; + try { + const decoded = isBase64Encoded + ? Buffer.from(body, 'base64').toString('utf8') + : body; + return JSON.parse(decoded); + } catch { + return body; + } +} + +export function isEventRequest(body: unknown): body is EventRequest { + return ( + typeof body === 'object' && + body !== null && + 'event' in body && + typeof (body as EventRequest).event === 'string' + ); +} + +export function getCorsHeaders( + corsOptions: boolean | CorsOptions, +): Record { + if (!corsOptions) return {}; + if (corsOptions === true) { + return { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization', + 'Access-Control-Max-Age': '3600', + }; + } + + const headers: Record = {}; + + if (corsOptions.origin) { + const origin = Array.isArray(corsOptions.origin) + ? corsOptions.origin.join(', ') + : corsOptions.origin; + headers['Access-Control-Allow-Origin'] = origin; + } + if (corsOptions.methods) { + headers['Access-Control-Allow-Methods'] = corsOptions.methods.join(', '); + } + if (corsOptions.headers) { + headers['Access-Control-Allow-Headers'] = corsOptions.headers.join(', '); + } + if (corsOptions.credentials) { + headers['Access-Control-Allow-Credentials'] = 'true'; + } + if (corsOptions.maxAge !== undefined) { + headers['Access-Control-Max-Age'] = corsOptions.maxAge.toString(); + } + + return headers; +} + +export function createResponse( + statusCode: number, + body: unknown, + headers: Record = {}, + requestId?: string, +): APIGatewayProxyResult { + const responseHeaders: Record = { + 'Content-Type': + typeof body === 'object' ? 'application/json' : 'text/plain', + ...headers, + }; + + if (requestId) { + responseHeaders['X-Request-ID'] = requestId; + } + + return { + statusCode, + headers: responseHeaders, + body: typeof body === 'object' ? JSON.stringify(body) : String(body), + isBase64Encoded: false, + }; +} + +export function createPixelResponse( + headers: Record = {}, + requestId?: string, +): APIGatewayProxyResult { + const responseHeaders: Record = { + 'Content-Type': 'image/gif', + 'Cache-Control': 'no-cache, no-store, must-revalidate', + ...headers, + }; + + if (requestId) { + responseHeaders['X-Request-ID'] = requestId; + } + + return { + statusCode: 200, + headers: responseHeaders, + body: 'R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', + isBase64Encoded: true, + }; +} diff --git a/packages/server/sources/aws/tsconfig.json b/packages/server/sources/aws/tsconfig.json new file mode 100644 index 000000000..b1c75c589 --- /dev/null +++ b/packages/server/sources/aws/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "@walkeros/config/tsconfig/node.json", + "compilerOptions": { + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/server/sources/aws/tsup.config.ts b/packages/server/sources/aws/tsup.config.ts new file mode 100644 index 000000000..f781dc6ca --- /dev/null +++ b/packages/server/sources/aws/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig, buildModules } from '@walkeros/config/tsup'; + +export default defineConfig([ + buildModules({ + terserOptions: {}, // Don't mangle here + }), +]); diff --git a/packages/server/sources/fetch/README.md b/packages/server/sources/fetch/README.md new file mode 100644 index 000000000..5b24a9823 --- /dev/null +++ b/packages/server/sources/fetch/README.md @@ -0,0 +1,258 @@ +# @walkeros/server-source-fetch + +> Web Standard Fetch API source for walkerOS - Deploy to any modern +> edge/serverless platform + +## What This Source Does + +**Accepts** walkerOS events via HTTP (Fetch API) **Forwards** events to +collector for processing **Returns** HTTP responses with CORS support + +This is an HTTP transport layer - it accepts events in walkerOS format and +forwards them to the collector. Not a transformation source. + +## Features + +- ✅ **Web Standard Fetch API** - Native `(Request) => Response` signature +- ✅ **Platform Agnostic** - Cloudflare Workers, Vercel Edge, Deno, Bun, Node.js + 18+ +- ✅ **Event Validation** - Zod schema validation with detailed error messages +- ✅ **Batch Processing** - Handle multiple events in single request +- ✅ **CORS Support** - Configurable cross-origin resource sharing +- ✅ **Pixel Tracking** - 1x1 transparent GIF for GET requests +- ✅ **Request Limits** - Configurable size and batch limits +- ✅ **Health Checks** - Built-in `/health` endpoint + +## Installation + +```bash +npm install @walkeros/server-source-fetch @walkeros/collector @walkeros/core +``` + +## Quick Start + +```typescript +import { sourceFetch, type SourceFetch } from '@walkeros/server-source-fetch'; +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + sources: { api: { code: sourceFetch } }, +}); + +export default { fetch: elb }; +``` + +## Platform Deployment + +### Cloudflare Workers + +```typescript +import { sourceFetch, type SourceFetch } from '@walkeros/server-source-fetch'; +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + sources: { + api: { + code: sourceFetch, + config: { settings: { cors: true } }, + }, + }, + destinations: { + // Your destinations + }, +}); + +export default { fetch: elb }; +``` + +**Deploy:** `wrangler deploy` + +### Vercel Edge Functions + +```typescript +// api/collect.ts +export const config = { runtime: 'edge' }; + +import { sourceFetch, type SourceFetch } from '@walkeros/server-source-fetch'; +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + sources: { api: { code: sourceFetch } }, +}); + +export default elb; +``` + +### Deno Deploy + +```typescript +import { sourceFetch, type SourceFetch } from '@walkeros/server-source-fetch'; +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + sources: { api: { code: sourceFetch } }, +}); + +Deno.serve(elb); +``` + +### Bun + +```typescript +import { sourceFetch, type SourceFetch } from '@walkeros/server-source-fetch'; +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + sources: { api: { code: sourceFetch } }, +}); + +Bun.serve({ fetch: elb, port: 3000 }); +``` + +## Usage Examples + +### Single Event (POST) + +```javascript +fetch('https://your-endpoint.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: 'page view', + data: { title: 'Home', path: '/' }, + user: { id: 'user-123' }, + globals: { language: 'en' }, + }), +}); +``` + +### Batch Events (POST) + +```javascript +fetch('https://your-endpoint.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + batch: [ + { name: 'page view', data: { title: 'Home' } }, + { name: 'button click', data: { id: 'cta' } }, + { name: 'form submit', data: { formId: 'contact' } }, + ], + }), +}); +``` + +### Pixel Tracking (GET) + +```html + +``` + +### Health Check + +```bash +curl https://your-endpoint.com/health +# {"status":"ok","timestamp":1234567890,"source":"fetch"} +``` + +## Configuration + +```typescript +interface Settings { + path: string; // Collection path (default: '/collect') + cors: boolean | CorsOptions; // CORS config (default: true) + healthPath: string; // Health check path (default: '/health') + maxRequestSize: number; // Max bytes (default: 102400 = 100KB) + maxBatchSize: number; // Max events per batch (default: 100) +} + +interface CorsOptions { + origin?: string | string[] | '*'; + methods?: string[]; + headers?: string[]; + credentials?: boolean; + maxAge?: number; +} +``` + +## Error Responses + +### Validation Error + +```json +{ + "success": false, + "error": "Event validation failed", + "validationErrors": [ + { "path": "name", "message": "Event name is required" }, + { "path": "nested.0.entity", "message": "Required" } + ] +} +``` + +### Batch Partial Failure (207 Multi-Status) + +```json +{ + "success": false, + "processed": 2, + "failed": 1, + "errors": [ + { "index": 1, "error": "Validation failed: Event name is required" } + ] +} +``` + +## Input Format + +Accepts standard walkerOS events. See +[@walkeros/core Event documentation](../../../core#event-structure). + +Required field: + +- `name` (string) - Event name in "entity action" format (e.g., "page view") + +Optional fields: + +- `data` - Event-specific properties +- `user` - User identification +- `context` - Ordered context properties +- `globals` - Global properties +- `custom` - Custom properties +- `nested` - Nested entities +- `consent` - Consent flags + +## Testing + +```bash +npm test # Run tests +npm run dev # Watch mode +npm run lint # Type check + lint +npm run build # Build package +``` + +## Development + +Follows walkerOS XP principles: + +- **DRY** - Uses @walkeros/core utilities +- **KISS** - Minimal HTTP wrapper +- **TDD** - Example-driven tests +- **No `any`** - Strict TypeScript + +See [AGENT.md](../../../../AGENT.md) for walkerOS development guide. + +## Related + +- [understanding-sources skill](../../../../skills/understanding-sources/SKILL.md) +- [using-logger skill](../../../../skills/using-logger/SKILL.md) +- [@walkeros/server-source-express](../express/) - Alternative for Express.js +- [@walkeros/core](../../../../packages/core/) - Core utilities + +## License + +MIT diff --git a/packages/server/sources/fetch/jest.config.mjs b/packages/server/sources/fetch/jest.config.mjs new file mode 100644 index 000000000..9408b288a --- /dev/null +++ b/packages/server/sources/fetch/jest.config.mjs @@ -0,0 +1,3 @@ +import config from '@walkeros/config/jest'; + +export default config; diff --git a/packages/server/sources/fetch/package.json b/packages/server/sources/fetch/package.json new file mode 100644 index 000000000..42382d858 --- /dev/null +++ b/packages/server/sources/fetch/package.json @@ -0,0 +1,57 @@ +{ + "name": "@walkeros/server-source-fetch", + "description": "Web Standard Fetch API source for walkerOS (Cloudflare Workers, Vercel Edge, Deno, Bun)", + "version": "0.4.2", + "license": "MIT", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "tsup --silent", + "clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist", + "dev": "jest --watchAll --colors", + "lint": "tsc && eslint \"**/*.ts*\"", + "test": "jest", + "update": "npx npm-check-updates -u && npm update" + }, + "dependencies": { + "@walkeros/core": "*" + }, + "devDependencies": {}, + "repository": { + "url": "git+https://github.com/elbwalker/walkerOS.git", + "directory": "packages/server/sources/fetch" + }, + "author": "elbwalker ", + "homepage": "https://github.com/elbwalker/walkerOS#readme", + "bugs": { + "url": "https://github.com/elbwalker/walkerOS/issues" + }, + "keywords": [ + "walker", + "walkerOS", + "source", + "server", + "fetch", + "edge", + "cloudflare", + "vercel", + "deno", + "bun" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "./dev": { + "types": "./dist/dev.d.ts", + "import": "./dist/dev.mjs", + "require": "./dist/dev.js" + } + } +} diff --git a/packages/server/sources/fetch/src/__tests__/index.test.ts b/packages/server/sources/fetch/src/__tests__/index.test.ts new file mode 100644 index 000000000..b59833769 --- /dev/null +++ b/packages/server/sources/fetch/src/__tests__/index.test.ts @@ -0,0 +1,490 @@ +import { sourceFetch } from '../index'; +import type { WalkerOS } from '@walkeros/core'; +import { createMockLogger } from '@walkeros/core'; +import { examples } from '../dev'; + +describe('sourceFetch', () => { + let mockPush: jest.MockedFunction<(...args: unknown[]) => unknown>; + let mockCommand: jest.MockedFunction<(...args: unknown[]) => unknown>; + + beforeEach(() => { + mockPush = jest.fn().mockResolvedValue({ + event: { id: 'test-id' }, + ok: true, + successful: [], + queued: [], + failed: [], + }); + mockCommand = jest.fn().mockResolvedValue({ + ok: true, + successful: [], + queued: [], + failed: [], + }); + }); + + describe('initialization', () => { + it('should initialize with default settings', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + expect(source.type).toBe('fetch'); + expect(source.config.settings).toEqual({ + path: '/collect', + cors: true, + healthPath: '/health', + maxRequestSize: 102400, + maxBatchSize: 100, + }); + expect(typeof source.push).toBe('function'); + }); + + it('should merge custom settings with defaults', async () => { + const source = await sourceFetch( + { + settings: { + path: '/events', + cors: false, + healthPath: '/status', + }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + expect(source.config.settings).toEqual({ + path: '/events', + cors: false, + healthPath: '/status', + maxRequestSize: 102400, + maxBatchSize: 100, + }); + }); + }); + + describe('POST request handling', () => { + it('should process valid single event', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const event: WalkerOS.DeepPartialEvent = { + name: 'page view', + data: { title: 'Home' }, + }; + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(event), + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(200); + expect(responseBody).toMatchObject({ + success: true, + timestamp: expect.any(Number), + }); + expect(mockPush).toHaveBeenCalledWith(event); + }); + + it('should reject POST with invalid JSON body', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: 'invalid json', + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(400); + expect(responseBody).toMatchObject({ + success: false, + error: expect.stringContaining('Invalid JSON'), + }); + expect(mockPush).not.toHaveBeenCalled(); + }); + + it('should reject POST with non-object body', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify('invalid'), + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(400); + expect(responseBody).toMatchObject({ + success: false, + error: expect.stringContaining('Invalid event'), + }); + }); + + it('should process complete event with all properties', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const completeEvent: WalkerOS.DeepPartialEvent = { + name: 'product add', + data: { id: 'P123', name: 'Laptop', price: 999 }, + context: { stage: ['shopping', 1] }, + globals: { language: 'en', currency: 'USD' }, + custom: { campaignId: 'summer-sale' }, + user: { id: 'user123', email: 'user@example.com' }, + nested: [{ entity: 'category', data: { name: 'Electronics' } }], + consent: { functional: true, marketing: true }, + trigger: 'click', + group: 'ecommerce', + }; + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(completeEvent), + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(200); + expect(responseBody).toMatchObject({ + success: true, + timestamp: expect.any(Number), + }); + expect(mockPush).toHaveBeenCalledWith(completeEvent); + }); + + it('should handle collector errors', async () => { + const errorPush = jest + .fn() + .mockRejectedValue(new Error('Collector error')); + + const source = await sourceFetch( + {}, + { + push: errorPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: 'page view' }), + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(400); + expect(responseBody).toMatchObject({ + success: false, + error: 'Collector error', + }); + }); + }); + + describe('GET request handling (pixel tracking)', () => { + it('should process event from query parameters', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request( + 'https://example.com/collect?event=page%20view&data[title]=Home&user[id]=user123', + { method: 'GET' }, + ); + + const response = await source.push(request); + + expect(response.status).toBe(200); + expect(response.headers.get('Content-Type')).toBe('image/gif'); + expect(response.headers.get('Cache-Control')).toContain('no-cache'); + expect(mockPush).toHaveBeenCalledWith({ + event: 'page view', + data: { title: 'Home' }, + user: { id: 'user123' }, + }); + }); + + it('should return 1x1 GIF for pixel tracking', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request( + 'https://example.com/collect.gif?event=page%20view', + { method: 'GET' }, + ); + + const response = await source.push(request); + const buffer = await response.arrayBuffer(); + + expect(response.headers.get('Content-Type')).toBe('image/gif'); + expect(buffer.byteLength).toBeGreaterThan(0); + }); + }); + + describe('OPTIONS request handling (CORS)', () => { + it('should handle CORS preflight with default settings', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'OPTIONS', + headers: { Origin: 'https://example.com' }, + }); + + const response = await source.push(request); + + expect(response.status).toBe(204); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*'); + expect(mockPush).not.toHaveBeenCalled(); + }); + + it('should handle CORS preflight with custom settings', async () => { + const source = await sourceFetch( + { + settings: { + cors: { + origin: 'https://example.com', + credentials: true, + }, + }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'OPTIONS', + headers: { Origin: 'https://example.com' }, + }); + + const response = await source.push(request); + + expect(response.status).toBe(204); + expect(response.headers.get('Access-Control-Allow-Origin')).toBe( + 'https://example.com', + ); + expect(response.headers.get('Access-Control-Allow-Credentials')).toBe( + 'true', + ); + }); + + it('should not set CORS headers when disabled', async () => { + const source = await sourceFetch( + { + settings: { cors: false }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'OPTIONS', + }); + + const response = await source.push(request); + + expect(response.status).toBe(204); + expect(response.headers.get('Access-Control-Allow-Origin')).toBeNull(); + }); + }); + + describe('health check', () => { + it('should respond to health check endpoint', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/health', { + method: 'GET', + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(200); + expect(responseBody).toMatchObject({ + status: 'ok', + source: 'fetch', + timestamp: expect.any(Number), + }); + expect(mockPush).not.toHaveBeenCalled(); + }); + }); + + describe('unsupported methods', () => { + it('should reject PUT requests', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ event: 'page view' }), + }); + + const response = await source.push(request); + const responseBody = await response.json(); + + expect(response.status).toBe(405); + expect(responseBody).toMatchObject({ + success: false, + error: expect.stringContaining('Method not allowed'), + }); + }); + }); + + describe('settings validation', () => { + it('should apply default path', async () => { + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + expect(source.config.settings?.path).toBe('/collect'); + }); + + it('should accept custom path', async () => { + const source = await sourceFetch( + { + settings: { path: '/events' }, + }, + { + push: mockPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + expect(source.config.settings?.path).toBe('/events'); + }); + }); + + describe('example-driven tests', () => { + it('should process examples.inputs.pageView', async () => { + const mockPush = jest + .fn() + .mockResolvedValue({ event: { id: 'test-id' } }); + + const source = await sourceFetch( + {}, + { + push: mockPush as never, + command: jest.fn() as never, + elb: jest.fn() as never, + logger: createMockLogger(), + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(examples.inputs.pageView), + }); + + await source.push(request); + + expect(mockPush).toHaveBeenCalledWith(examples.inputs.pageView); + }); + }); +}); diff --git a/packages/server/sources/fetch/src/__tests__/logging.test.ts b/packages/server/sources/fetch/src/__tests__/logging.test.ts new file mode 100644 index 000000000..eb4a7b570 --- /dev/null +++ b/packages/server/sources/fetch/src/__tests__/logging.test.ts @@ -0,0 +1,57 @@ +import { sourceFetch } from '../index'; +import { createMockLogger } from '@walkeros/core'; + +describe('logger usage', () => { + it('should use logger.throw for validation errors', async () => { + const mockLogger = createMockLogger(); + + const source = await sourceFetch( + {}, + { + push: jest.fn() as never, + command: jest.fn() as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ data: { title: 'Missing name' } }), + }); + + const response = await source.push(request); + + // Should NOT throw (catches internally and returns error response) + expect(response.status).toBe(400); + // But should have logged the error + expect(mockLogger.error).toHaveBeenCalled(); + }); + + it('should NOT log routine operations', async () => { + const mockLogger = createMockLogger(); + + const source = await sourceFetch( + {}, + { + push: jest.fn().mockResolvedValue({ event: { id: 'test' } }) as never, + command: jest.fn() as never, + elb: jest.fn() as never, + logger: mockLogger, + }, + ); + + const request = new Request('https://example.com/collect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: 'page view' }), + }); + + await source.push(request); + + // Should NOT log routine processing + expect(mockLogger.info).not.toHaveBeenCalled(); + expect(mockLogger.debug).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/server/sources/fetch/src/__tests__/validation.test.ts b/packages/server/sources/fetch/src/__tests__/validation.test.ts new file mode 100644 index 000000000..6229cfaa9 --- /dev/null +++ b/packages/server/sources/fetch/src/__tests__/validation.test.ts @@ -0,0 +1,27 @@ +import { EventSchema } from '../schemas/event'; +import { examples } from '../dev'; + +describe('EventSchema validation', () => { + it('should validate example inputs', () => { + const result = EventSchema.safeParse(examples.inputs.pageView); + expect(result.success).toBe(true); + }); + + it('should validate complete event', () => { + const result = EventSchema.safeParse(examples.inputs.completeEvent); + expect(result.success).toBe(true); + }); + + it('should reject event without name', () => { + const result = EventSchema.safeParse({ data: { title: 'Test' } }); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues[0].path).toEqual(['name']); + } + }); + + it('should validate batch array', () => { + const result = EventSchema.array().safeParse(examples.inputs.batch); + expect(result.success).toBe(true); + }); +}); diff --git a/packages/server/sources/fetch/src/dev.ts b/packages/server/sources/fetch/src/dev.ts new file mode 100644 index 000000000..2128576b9 --- /dev/null +++ b/packages/server/sources/fetch/src/dev.ts @@ -0,0 +1,2 @@ +export * from './schemas'; +export * as examples from './examples'; diff --git a/packages/server/sources/fetch/src/examples/env.ts b/packages/server/sources/fetch/src/examples/env.ts new file mode 100644 index 000000000..def8310d0 --- /dev/null +++ b/packages/server/sources/fetch/src/examples/env.ts @@ -0,0 +1,4 @@ +// Mock environment for testing +// Use createMockLogger() from @walkeros/core for actual tests + +export {}; diff --git a/packages/server/sources/fetch/src/examples/index.ts b/packages/server/sources/fetch/src/examples/index.ts new file mode 100644 index 000000000..6da71a202 --- /dev/null +++ b/packages/server/sources/fetch/src/examples/index.ts @@ -0,0 +1,2 @@ +export * as inputs from './inputs'; +export * as requests from './requests'; diff --git a/packages/server/sources/fetch/src/examples/inputs.ts b/packages/server/sources/fetch/src/examples/inputs.ts new file mode 100644 index 000000000..3bc3c63ef --- /dev/null +++ b/packages/server/sources/fetch/src/examples/inputs.ts @@ -0,0 +1,110 @@ +import type { WalkerOS } from '@walkeros/core'; + +/** + * Example walkerOS events that HTTP clients send to this source. + * These are the CONTRACT - tests verify implementation handles these inputs. + */ + +// Simple page view event +export const pageView: WalkerOS.DeepPartialEvent = { + name: 'page view', + data: { + title: 'Home Page', + path: '/', + referrer: 'https://google.com', + }, + user: { + id: 'user-123', + session: 'session-456', + }, + timestamp: 1700000000000, +}; + +// E-commerce event with nested entities +export const productAdd: WalkerOS.DeepPartialEvent = { + name: 'product add', + data: { + id: 'P-123', + name: 'Laptop', + price: 999.99, + quantity: 1, + }, + context: { + stage: ['shopping', 1], + }, + globals: { + language: 'en', + currency: 'USD', + }, + user: { + id: 'user-123', + }, + nested: [ + { + entity: 'category', + data: { + name: 'Electronics', + path: '/electronics', + }, + }, + ], + consent: { + functional: true, + marketing: true, + }, +}; + +// Complete event with all optional fields +export const completeEvent: WalkerOS.DeepPartialEvent = { + name: 'order complete', + data: { + id: 'ORDER-123', + total: 999.99, + currency: 'USD', + }, + context: { + stage: ['checkout', 3], + test: ['variant-A', 0], + }, + globals: { + language: 'en', + country: 'US', + }, + custom: { + campaignId: 'summer-sale', + source: 'email', + }, + user: { + id: 'user-123', + email: 'user@example.com', + session: 'session-456', + }, + nested: [ + { + entity: 'product', + data: { + id: 'P-123', + price: 999.99, + }, + }, + ], + consent: { + functional: true, + marketing: true, + analytics: false, + }, + trigger: 'click', + group: 'ecommerce', +}; + +// Minimal valid event +export const minimal: WalkerOS.DeepPartialEvent = { + name: 'ping', +}; + +// Batch of events +export const batch: WalkerOS.DeepPartialEvent[] = [ + pageView, + productAdd, + { name: 'button click', data: { id: 'cta' } }, +]; diff --git a/packages/server/sources/fetch/src/examples/requests.ts b/packages/server/sources/fetch/src/examples/requests.ts new file mode 100644 index 000000000..1106b8fc6 --- /dev/null +++ b/packages/server/sources/fetch/src/examples/requests.ts @@ -0,0 +1,59 @@ +/** + * HTTP request examples for testing the fetch source. + * Shows what external HTTP clients will send. + */ + +export const validPostRequest = { + method: 'POST', + url: 'https://example.com/collect', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: 'page view', + data: { title: 'Home' }, + }), +}; + +export const batchPostRequest = { + method: 'POST', + url: 'https://example.com/collect', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + batch: [ + { name: 'page view', data: { title: 'Home' } }, + { name: 'button click', data: { id: 'cta' } }, + ], + }), +}; + +export const pixelGetRequest = { + method: 'GET', + url: 'https://example.com/collect?event=page%20view&data[title]=Home&user[id]=user123', +}; + +export const healthCheckRequest = { + method: 'GET', + url: 'https://example.com/health', +}; + +export const optionsRequest = { + method: 'OPTIONS', + url: 'https://example.com/collect', + headers: { Origin: 'https://example.com' }, +}; + +export const invalidJsonRequest = { + method: 'POST', + url: 'https://example.com/collect', + headers: { 'Content-Type': 'application/json' }, + body: 'invalid json{', +}; + +export const oversizedRequest = { + method: 'POST', + url: 'https://example.com/collect', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: 'test', + data: { payload: 'x'.repeat(200000) }, // 200KB + }), +}; diff --git a/packages/server/sources/fetch/src/index.ts b/packages/server/sources/fetch/src/index.ts new file mode 100644 index 000000000..5f74525e0 --- /dev/null +++ b/packages/server/sources/fetch/src/index.ts @@ -0,0 +1,292 @@ +import { requestToData, isObject, isDefined } from '@walkeros/core'; +import type { WalkerOS, Collector } from '@walkeros/core'; +import type { FetchSource, PartialConfig, Types } from './types'; +import { SettingsSchema, EventSchema } from './schemas'; +import { + createCorsHeaders, + createPixelResponse, + createJsonResponse, +} from './utils'; + +export const sourceFetch = async ( + config: PartialConfig, + env: Types['env'], +): Promise => { + const settings = SettingsSchema.parse(config.settings || {}); + const { logger } = env; + + const push = async (request: Request): Promise => { + const startTime = Date.now(); + + try { + const url = new URL(request.url); + const method = request.method.toUpperCase(); + const origin = request.headers.get('Origin'); + const corsHeaders = createCorsHeaders(settings.cors, origin); + + // Health check (no logging - routine operation) + if (settings.healthPath && url.pathname === settings.healthPath) { + return createJsonResponse( + { status: 'ok', timestamp: Date.now(), source: 'fetch' }, + 200, + corsHeaders, + ); + } + + // OPTIONS (no logging - CORS preflight is routine) + if (method === 'OPTIONS') { + return new Response(null, { status: 204, headers: corsHeaders }); + } + + // GET (pixel tracking - no logging, routine) + if (method === 'GET') { + const parsedData = requestToData(url.search); + if (parsedData && isObject(parsedData)) { + await env.push(parsedData); + } + return createPixelResponse(corsHeaders); + } + + // POST + if (method === 'POST') { + // Check request size + const contentLength = request.headers.get('Content-Length'); + if (contentLength) { + const size = parseInt(contentLength, 10); + if (size > settings.maxRequestSize) { + logger.error('Request too large', { + size, + limit: settings.maxRequestSize, + }); + return createJsonResponse( + { + success: false, + error: `Request too large. Maximum size: ${settings.maxRequestSize} bytes`, + }, + 413, + corsHeaders, + ); + } + } + + let eventData: unknown; + let bodyText: string; + + try { + bodyText = await request.text(); + + // Check actual body size + if (bodyText.length > settings.maxRequestSize) { + logger.error('Request body too large', { + size: bodyText.length, + limit: settings.maxRequestSize, + }); + return createJsonResponse( + { + success: false, + error: `Request too large. Maximum size: ${settings.maxRequestSize} bytes`, + }, + 413, + corsHeaders, + ); + } + + eventData = JSON.parse(bodyText); + } catch (error) { + logger.error('Failed to parse JSON', error); + return createJsonResponse( + { success: false, error: 'Invalid JSON body' }, + 400, + corsHeaders, + ); + } + + if (!isDefined(eventData) || !isObject(eventData)) { + logger.error('Invalid event body type'); + return createJsonResponse( + { success: false, error: 'Invalid event: body must be an object' }, + 400, + corsHeaders, + ); + } + + // Check for batch + const isBatch = 'batch' in eventData && Array.isArray(eventData.batch); + + if (isBatch) { + const batch = eventData.batch as unknown[]; + + if (batch.length > settings.maxBatchSize) { + logger.error('Batch too large', { + size: batch.length, + limit: settings.maxBatchSize, + }); + return createJsonResponse( + { + success: false, + error: `Batch too large. Maximum size: ${settings.maxBatchSize} events`, + }, + 400, + corsHeaders, + ); + } + + const results = await processBatch(batch, env.push, logger); + + if (results.failed > 0) { + return createJsonResponse( + { + success: false, + processed: results.successful, + failed: results.failed, + errors: results.errors, + }, + 207, + corsHeaders, + ); + } + + return createJsonResponse( + { + success: true, + processed: results.successful, + ids: results.ids, + }, + 200, + corsHeaders, + ); + } + + // Single event - validate + const validation = EventSchema.safeParse(eventData); + if (!validation.success) { + const errors = validation.error.issues.map((issue) => ({ + path: issue.path.join('.'), + message: issue.message, + })); + + logger.error('Event validation failed', { errors }); + + return createJsonResponse( + { + success: false, + error: 'Event validation failed', + validationErrors: errors, + }, + 400, + corsHeaders, + ); + } + + const result = await processEvent( + validation.data as WalkerOS.DeepPartialEvent, + env.push, + ); + if (result.error) { + logger.error('Event processing failed', { error: result.error }); + return createJsonResponse( + { success: false, error: result.error }, + 400, + corsHeaders, + ); + } + + return createJsonResponse( + { success: true, id: result.id, timestamp: Date.now() }, + 200, + corsHeaders, + ); + } + + return createJsonResponse( + { success: false, error: 'Method not allowed' }, + 405, + corsHeaders, + ); + } catch (error) { + logger.error('Internal server error', error); + const corsHeaders = createCorsHeaders(settings.cors); + return createJsonResponse( + { + success: false, + error: + error instanceof Error ? error.message : 'Internal server error', + }, + 500, + corsHeaders, + ); + } + }; + + return { type: 'fetch', config: { ...config, settings }, push }; +}; + +async function processEvent( + event: WalkerOS.DeepPartialEvent, + push: Collector.PushFn, +): Promise<{ id?: string; error?: string }> { + try { + const result = await push(event); + return { id: result?.event?.id }; + } catch (error) { + return { error: error instanceof Error ? error.message : 'Unknown error' }; + } +} + +async function processBatch( + events: unknown[], + push: Collector.PushFn, + logger: Types['env']['logger'], +): Promise<{ + successful: number; + failed: number; + ids: string[]; + errors: Array<{ index: number; error: string }>; +}> { + const results = { + successful: 0, + failed: 0, + ids: [] as string[], + errors: [] as Array<{ index: number; error: string }>, + }; + + for (let i = 0; i < events.length; i++) { + const event = events[i]; + + const validation = EventSchema.safeParse(event); + if (!validation.success) { + results.failed++; + results.errors.push({ + index: i, + error: `Validation failed: ${validation.error.issues[0].message}`, + }); + logger.error(`Batch event ${i} validation failed`, { + errors: validation.error.issues, + }); + continue; + } + + try { + const result = await push(validation.data as WalkerOS.DeepPartialEvent); + if (result?.event?.id) { + results.ids.push(result.event.id); + } + results.successful++; + } catch (error) { + results.failed++; + results.errors.push({ + index: i, + error: error instanceof Error ? error.message : 'Unknown error', + }); + logger.error(`Batch event ${i} processing failed`, error); + } + } + + return results; +} + +export type * from './types'; +export * as SourceFetch from './types'; +export * from './utils'; +export * as schemas from './schemas'; +export * as examples from './examples'; diff --git a/packages/server/sources/fetch/src/schemas/event.ts b/packages/server/sources/fetch/src/schemas/event.ts new file mode 100644 index 000000000..429edd36b --- /dev/null +++ b/packages/server/sources/fetch/src/schemas/event.ts @@ -0,0 +1,93 @@ +import { z } from '@walkeros/core/dev'; + +// Properties schema - flexible key-value pairs +const PropertiesSchema = z.record( + z.string(), + z.union([z.string(), z.number(), z.boolean(), z.record(z.string(), z.any())]), +); + +// Ordered properties - [value, order] tuples +const OrderedPropertiesSchema = z.record( + z.string(), + z.tuple([ + z.union([ + z.string(), + z.number(), + z.boolean(), + z.record(z.string(), z.any()), + ]), + z.number(), + ]), +); + +// User schema with optional fields +const UserSchema = z + .object({ + id: z.string().optional(), + device: z.string().optional(), + session: z.string().optional(), + email: z.string().optional(), + hash: z.string().optional(), + }) + .passthrough(); + +// Consent schema - boolean flags +const ConsentSchema = z.record(z.string(), z.boolean()); + +// Entity schema (recursive for nested entities) +const EntitySchema: z.ZodTypeAny = z.lazy(() => + z + .object({ + entity: z.string(), + data: PropertiesSchema.optional(), + nested: z.array(EntitySchema).optional(), + context: OrderedPropertiesSchema.optional(), + }) + .passthrough(), +); + +// Version schema +const VersionSchema = z.object({ + source: z.string(), + tagging: z.number(), +}); + +// Source schema +const SourceSchema = z + .object({ + type: z.string(), + id: z.string(), + previous_id: z.string(), + }) + .passthrough(); + +// Main event schema - validates incoming events +export const EventSchema = z + .object({ + // Required + name: z.string().min(1, 'Event name is required'), + + // Core properties + data: PropertiesSchema.optional(), + context: OrderedPropertiesSchema.optional(), + globals: PropertiesSchema.optional(), + custom: PropertiesSchema.optional(), + user: UserSchema.optional(), + nested: z.array(EntitySchema).optional(), + consent: ConsentSchema.optional(), + + // System fields (optional for incoming events) + id: z.string().optional(), + trigger: z.string().optional(), + entity: z.string().optional(), + action: z.string().optional(), + timestamp: z.number().optional(), + timing: z.number().optional(), + group: z.string().optional(), + count: z.number().optional(), + version: VersionSchema.optional(), + source: SourceSchema.optional(), + }) + .passthrough(); // Allow additional fields + +export type ValidatedEvent = z.infer; diff --git a/packages/server/sources/fetch/src/schemas/index.ts b/packages/server/sources/fetch/src/schemas/index.ts new file mode 100644 index 000000000..179204e14 --- /dev/null +++ b/packages/server/sources/fetch/src/schemas/index.ts @@ -0,0 +1,3 @@ +export * from './primitives'; +export * from './settings'; +export * from './event'; diff --git a/packages/server/sources/fetch/src/schemas/primitives.ts b/packages/server/sources/fetch/src/schemas/primitives.ts new file mode 100644 index 000000000..989656533 --- /dev/null +++ b/packages/server/sources/fetch/src/schemas/primitives.ts @@ -0,0 +1,27 @@ +import { z } from '@walkeros/core/dev'; + +export const HttpMethod = z.enum([ + 'GET', + 'POST', + 'PUT', + 'PATCH', + 'DELETE', + 'OPTIONS', + 'HEAD', +]); + +export const CorsOrigin = z.union([ + z.string(), + z.array(z.string()), + z.literal('*'), +]); + +export const CorsOptionsSchema = z.object({ + origin: CorsOrigin.optional(), + methods: z.array(HttpMethod).optional(), + headers: z.array(z.string()).optional(), + credentials: z.boolean().optional(), + maxAge: z.number().int().positive().optional(), +}); + +export type CorsOptions = z.infer; diff --git a/packages/server/sources/fetch/src/schemas/settings.ts b/packages/server/sources/fetch/src/schemas/settings.ts new file mode 100644 index 000000000..bc25170d0 --- /dev/null +++ b/packages/server/sources/fetch/src/schemas/settings.ts @@ -0,0 +1,16 @@ +import { z } from '@walkeros/core/dev'; +import { CorsOptionsSchema } from './primitives'; + +export const SettingsSchema = z.object({ + path: z.string().default('/collect'), + cors: z.union([z.boolean(), CorsOptionsSchema]).default(true), + healthPath: z.string().default('/health'), + maxRequestSize: z + .number() + .int() + .positive() + .default(1024 * 100), // 100KB + maxBatchSize: z.number().int().positive().default(100), // 100 events +}); + +export type Settings = z.infer; diff --git a/packages/server/sources/fetch/src/types.ts b/packages/server/sources/fetch/src/types.ts new file mode 100644 index 000000000..db8f4e090 --- /dev/null +++ b/packages/server/sources/fetch/src/types.ts @@ -0,0 +1,36 @@ +import type { WalkerOS, Source as CoreSource } from '@walkeros/core'; +import type { SettingsSchema, CorsOptionsSchema } from './schemas'; +import { z } from '@walkeros/core/dev'; + +export type Settings = z.infer; +export type CorsOptions = z.infer; +export type InitSettings = Partial; + +export interface Mapping {} + +export type Push = (request: Request) => Response | Promise; + +export interface Env extends CoreSource.Env { + request?: Request; +} + +export type Types = CoreSource.Types< + Settings, + Mapping, + Push, + Env, + InitSettings +>; +export type Config = CoreSource.Config; +export type PartialConfig = CoreSource.PartialConfig; + +export interface FetchSource extends Omit, 'push'> { + push: Push; +} + +export interface EventResponse { + success: boolean; + id?: string; + timestamp?: number; + error?: string; +} diff --git a/packages/server/sources/fetch/src/utils.ts b/packages/server/sources/fetch/src/utils.ts new file mode 100644 index 000000000..1bfefe864 --- /dev/null +++ b/packages/server/sources/fetch/src/utils.ts @@ -0,0 +1,81 @@ +import type { CorsOptions } from './schemas'; + +export function createCorsHeaders( + corsConfig: boolean | CorsOptions = true, + requestOrigin?: string | null, +): Headers { + const headers = new Headers(); + + if (corsConfig === false) return headers; + + if (corsConfig === true) { + headers.set('Access-Control-Allow-Origin', '*'); + headers.set('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); + headers.set('Access-Control-Allow-Headers', 'Content-Type'); + } else { + if (corsConfig.origin) { + let origin: string; + if (Array.isArray(corsConfig.origin)) { + origin = + requestOrigin && corsConfig.origin.includes(requestOrigin) + ? requestOrigin + : corsConfig.origin[0]; + } else { + origin = corsConfig.origin; + } + headers.set('Access-Control-Allow-Origin', origin); + } + + if (corsConfig.methods) { + headers.set( + 'Access-Control-Allow-Methods', + corsConfig.methods.join(', '), + ); + } + + if (corsConfig.headers) { + headers.set( + 'Access-Control-Allow-Headers', + corsConfig.headers.join(', '), + ); + } + + if (corsConfig.credentials) { + headers.set('Access-Control-Allow-Credentials', 'true'); + } + + if (corsConfig.maxAge) { + headers.set('Access-Control-Max-Age', String(corsConfig.maxAge)); + } + } + + return headers; +} + +export const TRANSPARENT_GIF_BASE64 = + 'R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7'; + +export function createPixelResponse(corsHeaders?: Headers): Response { + const binaryString = atob(TRANSPARENT_GIF_BASE64); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + const headers = new Headers(corsHeaders); + headers.set('Content-Type', 'image/gif'); + headers.set('Cache-Control', 'no-cache, no-store, must-revalidate'); + + return new Response(bytes, { status: 200, headers }); +} + +export function createJsonResponse( + body: unknown, + status = 200, + corsHeaders?: Headers, +): Response { + const headers = new Headers(corsHeaders); + headers.set('Content-Type', 'application/json'); + + return new Response(JSON.stringify(body), { status, headers }); +} diff --git a/packages/server/sources/fetch/tsconfig.json b/packages/server/sources/fetch/tsconfig.json new file mode 100644 index 000000000..d548a0926 --- /dev/null +++ b/packages/server/sources/fetch/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@walkeros/config/tsconfig/base.json", + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/server/sources/fetch/tsup.config.ts b/packages/server/sources/fetch/tsup.config.ts new file mode 100644 index 000000000..a83e7945c --- /dev/null +++ b/packages/server/sources/fetch/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig, buildModules } from '@walkeros/config/tsup'; + +export default defineConfig([ + buildModules({ + terserOptions: {}, + }), + buildModules({ + entry: ['src/dev.ts'], + outDir: 'dist', + }), +]); diff --git a/packages/server/sources/gcp/README.md b/packages/server/sources/gcp/README.md index b85641fb9..549fc15b4 100644 --- a/packages/server/sources/gcp/README.md +++ b/packages/server/sources/gcp/README.md @@ -9,6 +9,25 @@ runtime adapters for GCP services. npm install @walkeros/server-source-gcp @google-cloud/functions-framework ``` +## Usage + +```typescript +import { + sourceCloudFunction, + type SourceCloudFunction, +} from '@walkeros/server-source-gcp'; +import { startFlow } from '@walkeros/collector'; +import { http } from '@google-cloud/functions-framework'; + +const { elb } = await startFlow({ + sources: { api: { code: sourceCloudFunction } }, +}); + +http('walkerHandler', elb); +``` + +--- + ## Cloud Functions Source The Cloud Functions source provides an HTTP handler that receives walker events @@ -17,17 +36,39 @@ and forwards them to the walkerOS collector. ### Basic Usage ```typescript -import { sourceCloudFunction } from '@walkeros/server-source-gcp'; +import { + sourceCloudFunction, + type SourceCloudFunction, +} from '@walkeros/server-source-gcp'; +import { startFlow } from '@walkeros/collector'; import { http } from '@google-cloud/functions-framework'; -// Create source with collector -const source = await sourceCloudFunction( - { settings: { cors: true, batch: true } }, - { elb: collector.push }, -); +// Handler singleton - reused across warm invocations +let handler: SourceCloudFunction.Push; + +async function setup() { + if (handler) return handler; + + const { elb } = await startFlow({ + sources: { + api: { + code: sourceCloudFunction, + config: { + settings: { cors: true }, + }, + }, + }, + destinations: { + // Your destinations + }, + }); + + handler = elb; + return handler; +} -// Plug-and-play: source.push IS the Cloud Function handler -http('walkerHandler', source.push); +// Register with Cloud Functions framework +setup().then((h) => http('walkerHandler', h)); ``` ## Bundler Integration diff --git a/packages/web/core/README.md b/packages/web/core/README.md index 76a823c8d..2c4d1e77d 100644 --- a/packages/web/core/README.md +++ b/packages/web/core/README.md @@ -1,6 +1,6 @@

- - + +

@@ -315,7 +315,7 @@ type StorageType = 'localStorage' | 'sessionStorage' | 'cookie'; --- For platform-agnostic utilities, see -[Core Utilities](https://www.elbwalker.com/docs/core). +[Core Utilities](https://www.walkeros.io/docs/core). ## Contribute diff --git a/packages/web/core/src/__tests__/sendWeb.test.ts b/packages/web/core/src/__tests__/sendWeb.test.ts index 571bea3d5..f797207f4 100644 --- a/packages/web/core/src/__tests__/sendWeb.test.ts +++ b/packages/web/core/src/__tests__/sendWeb.test.ts @@ -3,7 +3,7 @@ import { sendWeb } from '..'; describe('sendWeb', () => { const data = { key: 'value' }; const dataStringified = JSON.stringify({ key: 'value' }); - const url = 'https://api.elbwalker.com/'; + const url = 'https://api.walkeros.io/'; const mockFetch = jest.fn( () => diff --git a/packages/web/core/src/__tests__/sessionStorage.test.ts b/packages/web/core/src/__tests__/sessionStorage.test.ts index 629b1a1d9..351c421db 100644 --- a/packages/web/core/src/__tests__/sessionStorage.test.ts +++ b/packages/web/core/src/__tests__/sessionStorage.test.ts @@ -252,7 +252,7 @@ describe('SessionStorage', () => { jest.advanceTimersByTime(1000); const newSession = sessionStorage({ - url: 'https://www.elbwalker.com/?utm_campaign=new', + url: 'https://www.walkeros.io/?utm_campaign=new', }); expect(newSession.id).not.toBe(session.id); // Expect a new session id diff --git a/packages/web/core/src/__tests__/sessionWindow.test.ts b/packages/web/core/src/__tests__/sessionWindow.test.ts index 379971677..3a31f6004 100644 --- a/packages/web/core/src/__tests__/sessionWindow.test.ts +++ b/packages/web/core/src/__tests__/sessionWindow.test.ts @@ -2,7 +2,7 @@ import { sessionWindow } from '..'; describe('SessionStart', () => { const w = window; - const url = 'https://www.elbwalker.com/'; + const url = 'https://www.walkeros.io/'; const referrer = 'https://www.example.com/'; beforeEach(() => { @@ -67,14 +67,14 @@ describe('SessionStart', () => { // Custom domains expect( sessionWindow({ - url: 'https://www.elbwalker.com', - referrer: 'https://another.elbwalker.com', - domains: ['another.elbwalker.com'], + url: 'https://www.walkeros.io', + referrer: 'https://another.walkeros.io', + domains: ['another.walkeros.io'], }), ).toStrictEqual({ isStart: false, storage: false }); expect( sessionWindow({ - url: 'https://www.elbwalker.com', + url: 'https://www.walkeros.io', referrer: '', domains: [''], // Hack to disable direct or hidden referrer }), diff --git a/packages/web/core/src/__tests__/storage.test.ts b/packages/web/core/src/__tests__/storage.test.ts index 539dda8f4..b91b1167f 100644 --- a/packages/web/core/src/__tests__/storage.test.ts +++ b/packages/web/core/src/__tests__/storage.test.ts @@ -33,8 +33,8 @@ describe('Storage', () => { storageDelete(key, Const.Utils.Storage.Cookie); expect(storageRead(key, Const.Utils.Storage.Cookie)).toBe(''); expect(storageRead('foo', Const.Utils.Storage.Cookie)).toBe(''); - storageWrite(key, value, 1, Const.Utils.Storage.Cookie, 'elbwalker.com'); - expect(document.cookie).toContain('domain=elbwalker.com'); + storageWrite(key, value, 1, Const.Utils.Storage.Cookie, 'walkeros.io'); + expect(document.cookie).toContain('domain=walkeros.io'); // Expiration Session expect(storageWrite(key, value, 5)).toBe(value); diff --git a/packages/web/destinations/api/README.md b/packages/web/destinations/api/README.md index 3f0b50611..b85462218 100644 --- a/packages/web/destinations/api/README.md +++ b/packages/web/destinations/api/README.md @@ -1,6 +1,6 @@

- - + +

@@ -35,50 +35,83 @@ npm install @walkeros/web-destination-api | `transform` | `function` | Function to transform event data before sending | No | `(data, config, mapping) => JSON.stringify(data)` | | `transport` | `'fetch' \| 'xhr' \| 'beacon'` | Transport method for sending requests | No | `'fetch'` | -## Usage +## Quick Start + +Configure in your Flow JSON: + +```json +{ + "version": 1, + "flows": { + "default": { + "web": {}, + "destinations": { + "api": { + "package": "@walkeros/web-destination-api", + "config": { + "settings": { + "url": "https://api.example.com/events", + "method": "POST", + "headers": { "Authorization": "Bearer your-token" } + } + } + } + } + } + } +} +``` -### Basic Usage +Or programmatically: ```typescript import { startFlow } from '@walkeros/collector'; import { destinationAPI } from '@walkeros/web-destination-api'; -const { elb } = await startFlow(); - -elb('walker destination', destinationAPI, { - settings: { - url: 'https://api.example.com/events', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer your-token', +const { elb } = await startFlow({ + destinations: [ + { + destination: destinationAPI, + config: { + settings: { + url: 'https://api.example.com/events', + method: 'POST', + headers: { Authorization: 'Bearer your-token' }, + }, + }, }, - }, + ], }); ``` +## Usage + ### Advanced Usage with Transform ```typescript import { startFlow } from '@walkeros/collector'; import { destinationAPI } from '@walkeros/web-destination-api'; -const { elb } = await startFlow(); - -elb('walker destination', destinationAPI, { - settings: { - url: 'https://api.example.com/events', - transport: 'fetch', - transform: (event, config, mapping) => { - // Custom transformation logic - return JSON.stringify({ - timestamp: Date.now(), - event_name: `${event.entity}_${event.action}`, - properties: event.data, - context: event.context, - }); +const { elb } = await startFlow({ + destinations: [ + { + destination: destinationAPI, + config: { + settings: { + url: 'https://api.example.com/events', + transport: 'fetch', + transform: (event, config, mapping) => { + return JSON.stringify({ + timestamp: Date.now(), + event_name: `${event.entity}_${event.action}`, + properties: event.data, + context: event.context, + }); + }, + }, + }, }, - }, + ], }); ``` @@ -86,64 +119,66 @@ elb('walker destination', destinationAPI, { ### Sending to Analytics API -```typescript -import { startFlow } from '@walkeros/collector'; -import { destinationAPI } from '@walkeros/web-destination-api'; - -const { elb } = await startFlow(); - -// Configure for analytics API -elb('walker destination', destinationAPI, { - settings: { - url: 'https://analytics.example.com/track', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-API-Key': 'your-api-key', - }, - transform: (event) => { - return JSON.stringify({ - event_type: `${event.entity}_${event.action}`, - user_id: event.user?.id, - session_id: event.user?.session, - properties: event.data, - timestamp: event.timing, - }); - }, - }, -}); +```json +{ + "destinations": { + "analytics": { + "package": "@walkeros/web-destination-api", + "config": { + "settings": { + "url": "https://analytics.example.com/track", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-API-Key": "your-api-key" + } + } + } + } + } +} ``` ### Using Beacon Transport For critical events that need to be sent even when the page is unloading: -```typescript -elb('walker destination', destinationAPI, { - settings: { - url: 'https://api.example.com/critical-events', - transport: 'beacon', // Reliable for page unload scenarios - }, -}); +```json +{ + "destinations": { + "critical": { + "package": "@walkeros/web-destination-api", + "config": { + "settings": { + "url": "https://api.example.com/critical-events", + "transport": "beacon" + } + } + } + } +} ``` ### Custom Data Mapping Use mapping rules to control which events are sent: -```typescript -elb('walker destination', destinationAPI, { - settings: { - url: 'https://api.example.com/events', - }, - mapping: { - entity: { - action: { - data: 'data', - }, - }, - }, -}); +```json +{ + "destinations": { + "api": { + "package": "@walkeros/web-destination-api", + "config": { + "settings": { "url": "https://api.example.com/events" }, + "mapping": { + "entity": { + "action": { "data": "data" } + } + } + } + } + } +} ``` ## Transport Methods @@ -153,6 +188,15 @@ elb('walker destination', destinationAPI, { - **beacon**: Uses Navigator.sendBeacon() for reliable data transmission during page unload +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces. + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/destinations/web/api/) +- [Destination Interface](../../../core/src/types/destination.ts) + ## Contribute Feel free to contribute by submitting an diff --git a/packages/web/destinations/gtag/README.md b/packages/web/destinations/gtag/README.md index 395f2121d..c8704d531 100644 --- a/packages/web/destinations/gtag/README.md +++ b/packages/web/destinations/gtag/README.md @@ -25,26 +25,52 @@ single destination configuration. npm install @walkeros/web-destination-gtag ``` -## Usage +## Quick Start + +Configure in your Flow JSON: + +```json +{ + "version": 1, + "flows": { + "default": { + "web": {}, + "destinations": { + "gtag": { + "package": "@walkeros/web-destination-gtag", + "config": { + "settings": { + "ga4": { "measurementId": "G-XXXXXXXXXX" }, + "ads": { "conversionId": "AW-XXXXXXXXX" }, + "gtm": { "containerId": "GTM-XXXXXXX" } + } + } + } + } + } + } +} +``` + +Or programmatically: ```typescript import { startFlow } from '@walkeros/collector'; import { destinationGtag } from '@walkeros/web-destination-gtag'; -const { elb } = await startFlow(); - -elb('walker destination', destinationGtag, { - settings: { - ga4: { - measurementId: 'G-XXXXXXXXXX', // Required for GA4 - }, - ads: { - conversionId: 'AW-XXXXXXXXX', // Required for Google Ads - }, - gtm: { - containerId: 'GTM-XXXXXXX', // Required for GTM +const { elb } = await startFlow({ + destinations: [ + { + destination: destinationGtag, + config: { + settings: { + ga4: { measurementId: 'G-XXXXXXXXXX' }, + ads: { conversionId: 'AW-XXXXXXXXX' }, + gtm: { containerId: 'GTM-XXXXXXX' }, + }, + }, }, - }, + ], }); ``` @@ -267,6 +293,15 @@ const rules: DestinationGtag.Rules = { - Check the dataLayer name matches your GTM configuration - Use GTM Preview mode to debug event flow +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces. + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/destinations/web/gtag/) +- [Destination Interface](../../../core/src/types/destination.ts) + ## Contribute Feel free to contribute by submitting an diff --git a/packages/web/destinations/gtag/src/__tests__/index.test.ts b/packages/web/destinations/gtag/src/__tests__/index.test.ts index 7ab0f71fb..3433540a1 100644 --- a/packages/web/destinations/gtag/src/__tests__/index.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/index.test.ts @@ -426,7 +426,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command - destination.on?.('consent', { marketing: true, functional: false }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: true, functional: false }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Call a regular event const event = { name: 'button click', data: { id: 'test-btn' } }; @@ -457,7 +463,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command with denied consent - destination.on?.('consent', { marketing: false, functional: false }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: false, functional: false }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Verify gtag consent default was called with all denied values first expect(mockGtag).toHaveBeenCalledWith('consent', 'default', { @@ -498,7 +510,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // First consent call (should use 'default' then 'update') - destination.on?.('consent', { marketing: false, functional: false }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: false, functional: false }, + env: mockEnvWithGtag, + logger: mockLogger, + }); expect(mockGtag).toHaveBeenCalledWith('consent', 'default', { ad_storage: 'denied', @@ -518,7 +536,13 @@ describe('Unified Gtag Destination', () => { mockGtag.mockClear(); // Second consent call (should only use 'update') - destination.on?.('consent', { marketing: false, functional: true }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: false, functional: true }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Should NOT call default again expect(mockGtag).not.toHaveBeenCalledWith( @@ -547,7 +571,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command with granted consent - destination.on?.('consent', { marketing: true, functional: true }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: true, functional: true }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Verify gtag consent default was called with all denied values first expect(mockGtag).toHaveBeenCalledWith('consent', 'default', { @@ -594,7 +624,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command - destination.on?.('consent', { marketing: true, analytics: false }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: true, analytics: false }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Verify default consent was called with all denied values first expect(mockGtag).toHaveBeenCalledWith('consent', 'default', { @@ -620,7 +656,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command with only marketing consent - destination.on?.('consent', { marketing: true }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: true }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Verify default consent was called with all denied values first expect(mockGtag).toHaveBeenCalledWith('consent', 'default', { @@ -647,7 +689,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command with unknown consent group - destination.on?.('consent', { unknown_group: true }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { unknown_group: true }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Verify default consent was called (because mapping has known parameters) expect(mockGtag).toHaveBeenCalledWith('consent', 'default', { @@ -677,7 +725,13 @@ describe('Unified Gtag Destination', () => { destination.env = mockEnvWithGtag; // Call walker consent command - destination.on?.('consent', { marketing: true }); + destination.on?.('consent', { + collector: mockCollector, + config: destination.config, + data: { marketing: true }, + env: mockEnvWithGtag, + logger: mockLogger, + }); // Verify no gtag consent calls were made (empty mapping = no parameters) expect(mockGtag).not.toHaveBeenCalledWith( diff --git a/packages/web/destinations/gtag/src/index.ts b/packages/web/destinations/gtag/src/index.ts index 89f5a0afa..bb667871e 100644 --- a/packages/web/destinations/gtag/src/index.ts +++ b/packages/web/destinations/gtag/src/index.ts @@ -113,19 +113,17 @@ export const destinationGtag: Destination = { on(type, context) { // Only handle consent events - if (type !== 'consent' || !context) return; + if (type !== 'consent' || !context.data) return; - const consent = context; - - // Access config directly from this destination instance - const settings = this.config?.settings || {}; + const consent = context.data as WalkerOS.Consent; + const settings = (context.config?.settings || {}) as Partial; const { como = true } = settings; // Skip if consent mode is disabled if (!como) return; // Ensure gtag is available - const { window } = getEnv(this.env); + const { window } = getEnv(context.env); const gtag = initializeGtag(window as Window); if (!gtag) return; @@ -160,19 +158,17 @@ export const destinationGtag: Destination = { const gtagConsent: Record = {}; // Map walkerOS consent to gtag consent parameters for update - Object.entries(consent as WalkerOS.Consent).forEach( - ([walkerOSGroup, granted]) => { - const gtagParams = consentMapping[walkerOSGroup]; - if (!gtagParams) return; + Object.entries(consent).forEach(([walkerOSGroup, granted]) => { + const gtagParams = consentMapping[walkerOSGroup]; + if (!gtagParams) return; - const params = Array.isArray(gtagParams) ? gtagParams : [gtagParams]; - const consentValue = granted ? 'granted' : 'denied'; + const params = Array.isArray(gtagParams) ? gtagParams : [gtagParams]; + const consentValue = granted ? 'granted' : 'denied'; - params.forEach((param) => { - gtagConsent[param] = consentValue; - }); - }, - ); + params.forEach((param) => { + gtagConsent[param] = consentValue; + }); + }); // Only proceed if we have consent parameters to update if (Object.keys(gtagConsent).length === 0) return; diff --git a/packages/web/destinations/meta/README.md b/packages/web/destinations/meta/README.md index c613756dc..2c970121d 100644 --- a/packages/web/destinations/meta/README.md +++ b/packages/web/destinations/meta/README.md @@ -1,6 +1,6 @@

- - + +

@@ -25,21 +25,46 @@ events, and audience building data to optimize your Meta advertising campaigns. npm install @walkeros/web-destination-meta ``` -## Usage +## Quick Start + +Configure in your Flow JSON: + +```json +{ + "version": 1, + "flows": { + "default": { + "web": {}, + "destinations": { + "meta": { + "package": "@walkeros/web-destination-meta", + "config": { + "settings": { "pixelId": "1234567890" }, + "loadScript": true + } + } + } + } + } +} +``` -Here's a basic example of how to use the Meta Pixel destination: +Or programmatically: ```typescript import { startFlow } from '@walkeros/collector'; import { destinationMeta } from '@walkeros/web-destination-meta'; -const { elb } = await startFlow(); - -elb('walker destination', destinationMeta, { - settings: { - pixelId: '1234567890', // Your Meta Pixel ID - }, - loadScript: true, // Load Meta Pixel script automatically +const { elb } = await startFlow({ + destinations: [ + { + destination: destinationMeta, + config: { + settings: { pixelId: '1234567890' }, + loadScript: true, + }, + }, + ], }); ``` @@ -50,6 +75,15 @@ elb('walker destination', destinationMeta, { | `pixelId` | `string` | Your Meta Pixel ID from Facebook Business Manager | Yes | `'1234567890'` | | `loadScript` | `boolean` | Whether to automatically load the Meta Pixel script (fbevents.js) | No | `true` | +## Type Definitions + +See [src/types/](./src/types/) for TypeScript interfaces. + +## Related + +- [Website Documentation](https://www.walkeros.io/docs/destinations/web/meta/) +- [Destination Interface](../../../core/src/types/destination.ts) + ## Contribute Feel free to contribute by submitting an diff --git a/packages/web/destinations/piwikpro/README.md b/packages/web/destinations/piwikpro/README.md index 26a15897a..5170479ae 100644 --- a/packages/web/destinations/piwikpro/README.md +++ b/packages/web/destinations/piwikpro/README.md @@ -1,6 +1,6 @@

- - + +

@@ -33,12 +33,17 @@ Here's a basic example of how to use the Piwik PRO destination: import { startFlow } from '@walkeros/collector'; import { destinationPiwikPro } from '@walkeros/web-destination-piwikpro'; -const { elb } = await startFlow(); - -elb('walker destination', destinationPiwikPro, { - settings: { - appId: 'XXX-XXX-XXX-XXX-XXX', // Required - url: 'https://your_account_name.piwik.pro/', // Required +await startFlow({ + destinations: { + piwikpro: { + code: destinationPiwikPro, + config: { + settings: { + appId: 'XXX-XXX-XXX-XXX-XXX', // Required + url: 'https://your_account_name.piwik.pro/', // Required + }, + }, + }, }, }); ``` diff --git a/packages/web/destinations/plausible/README.md b/packages/web/destinations/plausible/README.md index 3fca42478..f35bbaae8 100644 --- a/packages/web/destinations/plausible/README.md +++ b/packages/web/destinations/plausible/README.md @@ -1,6 +1,6 @@

- - + +

@@ -33,20 +33,25 @@ Here's a basic example of how to use the Plausible destination: import { startFlow } from '@walkeros/collector'; import { destinationPlausible } from '@walkeros/web-destination-plausible'; -const { elb } = await startFlow(); - -elb('walker destination', destinationPlausible, { - settings: { - domain: 'elbwalker.com', // Optional, domain of your site as registered +await startFlow({ + destinations: { + plausible: { + code: destinationPlausible, + config: { + settings: { + domain: 'walkeros.io', // Optional, domain of your site as registered + }, + }, + }, }, }); ``` ## Configuration -| Name | Type | Description | Required | Example | -| -------- | -------- | -------------------------------------------------- | -------- | ----------------- | -| `domain` | `string` | The domain of your site as registered in Plausible | No | `'elbwalker.com'` | +| Name | Type | Description | Required | Example | +| -------- | -------- | -------------------------------------------------- | -------- | --------------- | +| `domain` | `string` | The domain of your site as registered in Plausible | No | `'walkeros.io'` | ## Contribute diff --git a/packages/web/destinations/plausible/src/index.test.ts b/packages/web/destinations/plausible/src/index.test.ts index d57a77edd..bd08dd411 100644 --- a/packages/web/destinations/plausible/src/index.test.ts +++ b/packages/web/destinations/plausible/src/index.test.ts @@ -101,7 +101,7 @@ describe('destination plausible', () => { }); test('init with domain', async () => { - const domain = 'elbwalker.com'; + const domain = 'walkeros.io'; const mockScript = { src: '', dataset: {} as Record, diff --git a/packages/web/sources/browser/README.md b/packages/web/sources/browser/README.md index 7a3834c0a..e7e310423 100644 --- a/packages/web/sources/browser/README.md +++ b/packages/web/sources/browser/README.md @@ -1,6 +1,6 @@

- - + +

@@ -12,6 +12,17 @@ The Browser Source is walkerOS's primary web tracking solution that you can use to capture user interactions directly from the browsers DOM. +## Quick Start + +```bash +npm install @walkeros/web-source-browser +``` + +```typescript +import { sourceBrowser } from '@walkeros/web-source-browser'; +await startFlow({ sources: { browser: { code: sourceBrowser } } }); +``` + ## What It Does The Browser Source transforms your website into a comprehensive tracking @@ -115,8 +126,8 @@ Load the source via dynamic import: > Use **separate source creation** for direct access to the enhanced elb > function, or access it via `collector.sources.browser.elb` in the unified API. > -> See [Commands](https://www.elbwalker.com/docs/sources/web/browser/commands) -> for full browser source API documentation. +> See [Commands](https://www.walkeros.io/docs/sources/web/browser/commands) for +> full browser source API documentation. ## Contribute diff --git a/packages/web/sources/dataLayer/README.md b/packages/web/sources/dataLayer/README.md index 321283d93..6fe551feb 100644 --- a/packages/web/sources/dataLayer/README.md +++ b/packages/web/sources/dataLayer/README.md @@ -1,6 +1,6 @@

- - + +

diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 000000000..363018c64 --- /dev/null +++ b/skills/README.md @@ -0,0 +1,54 @@ +# walkerOS Skills + +Skills are the source of truth for AI assistants working with this repository. +Tool-agnostic and accessible to Claude, Cursor, Copilot, and other AI tools. + +## Available Skills + +### Concept Skills (Understanding) + +| Skill | Description | +| ------------------------------------------------------------------- | --------------------------------------------------------- | +| [understanding-development](./understanding-development/SKILL.md) | Build, test, lint, XP principles, folder structure | +| [understanding-flow](./understanding-flow/SKILL.md) | Architecture, composability, Source→Collector→Destination | +| [understanding-events](./understanding-events/SKILL.md) | Event model, entity-action naming, properties | +| [understanding-mapping](./understanding-mapping/SKILL.md) | Event transformation, data/map/loop/condition | +| [understanding-destinations](./understanding-destinations/SKILL.md) | Destination interface, env pattern, configuration | +| [understanding-sources](./understanding-sources/SKILL.md) | Source interface, capture patterns | +| [using-logger](./using-logger/SKILL.md) | Logger access, DRY principles, API logging patterns | + +### Task Skills + +| Skill | Description | +| --------------------------------------------------------- | ---------------------------------------------- | +| [testing-strategy](./testing-strategy/SKILL.md) | Testing patterns, env mocking, dev examples | +| [create-destination](./create-destination/SKILL.md) | Step-by-step destination creation | +| [create-source](./create-source/SKILL.md) | Step-by-step source creation | +| [mapping-configuration](./mapping-configuration/SKILL.md) | Mapping recipes for GA4, Meta, custom APIs | +| [debugging](./debugging/SKILL.md) | Troubleshooting event flow and mapping issues | +| [writing-documentation](./writing-documentation/SKILL.md) | Documentation standards, validation, templates | + +## Architecture + +``` +skills/ <- Primary content (tool-agnostic) +.claude/skills/ <- Claude Code references (auto-discovery) +``` + +## Usage + +### Claude Code + +Skills in `.claude/skills/` are auto-discovered and reference primary files +here. + +### Other AI Tools + +Reference skills in this directory directly. + +## Adding New Skills + +1. Create `skills/[skill-name]/SKILL.md` with full content +2. Create `.claude/skills/[skill-name]/SKILL.md` reference +3. Update this README +4. Update AGENT.md navigation if needed diff --git a/skills/create-destination/SKILL.md b/skills/create-destination/SKILL.md new file mode 100644 index 000000000..51e41a746 --- /dev/null +++ b/skills/create-destination/SKILL.md @@ -0,0 +1,482 @@ +--- +name: create-destination +description: + Use when creating a new walkerOS destination. Example-driven workflow starting + with research and examples before implementation. +--- + +# Create a New Destination + +## Prerequisites + +Before starting, read these skills: + +- [understanding-flow](../understanding-flow/SKILL.md) - How destinations fit in + architecture +- [understanding-destinations](../understanding-destinations/SKILL.md) - + Destination interface +- [understanding-mapping](../understanding-mapping/SKILL.md) - Event + transformation +- [testing-strategy](../testing-strategy/SKILL.md) - How to test with env + pattern +- [writing-documentation](../writing-documentation/SKILL.md) - Documentation + standards (for Phase 7) + +## Process Overview + +``` +1. Research → Find SDK, understand vendor API +2. Examples → Create dev entry with real usage patterns +3. Mapping → Define walkerOS → vendor transformation +4. Scaffold → Copy template and configure +5. Implement → Build using examples as test fixtures +6. Test → Verify against example variations +7. Document → Write README +``` + +--- + +## Phase 1: Research + +**Goal:** Understand the vendor API before writing any code. + +### 1.1 Find Official Resources + +- [ ] **Vendor API Documentation** - Endpoints, authentication, rate limits +- [ ] **Official TypeScript SDK** - Check npm for `@vendor/sdk` or + `vendor-types` +- [ ] **Event Schema** - What fields are required/optional for each event type + +```bash +# Search npm for official packages +npm search [vendor-name] +npm search @[vendor] + +# Check for TypeScript types +npm info @types/[vendor] +``` + +### 1.2 Identify Event Types + +List the vendor's event types and their required fields: + +| Vendor Event | Required Fields | walkerOS Equivalent | +| ------------ | --------------------- | -------------------- | +| `pageview` | `url`, `title` | `page view` | +| `track` | `event`, `properties` | `product view`, etc. | +| `identify` | `userId`, `traits` | User identification | + +### 1.3 Check Existing Patterns + +Review similar destinations in the codebase: + +```bash +# List existing destinations +ls packages/web/destinations/ + +# Reference implementations +# - plausible: Simple, script-based +# - gtag: Complex, multiple services +# - meta: Pixel with custom events +``` + +--- + +## Phase 2: Create Examples (BEFORE Implementation) + +**Goal:** Define expected API calls in `dev` entry FIRST. + +### 2.1 Scaffold Directory Structure + +```bash +mkdir -p packages/web/destinations/[name]/src/{examples,schemas,types} +``` + +### 2.2 Create Output Examples + +**What the vendor API expects when we call it:** + +`src/examples/outputs.ts`: + +```typescript +/** + * Examples of vendor API calls we will make. + * These define the CONTRACT - implementation must produce these outputs. + */ + +// Page view call +export const pageViewCall = { + method: 'track', + args: ['pageview', { url: '/home', title: 'Home Page' }], +}; + +// E-commerce event call +export const purchaseCall = { + method: 'track', + args: [ + 'purchase', + { + transaction_id: 'T-123', + value: 99.99, + currency: 'USD', + items: [{ item_id: 'P-1', item_name: 'Widget', price: 99.99 }], + }, + ], +}; + +// Custom event call +export const customEventCall = { + method: 'track', + args: ['button_click', { button_id: 'cta', button_text: 'Sign Up' }], +}; +``` + +### 2.3 Create Input Examples + +**walkerOS events that will trigger these outputs:** + +`src/examples/events.ts`: + +```typescript +import type { WalkerOS } from '@walkeros/core'; + +/** + * walkerOS events that trigger destination calls. + * Maps to outputs.ts examples. + */ +export const events: Record = { + // Maps to pageViewCall + pageView: { + event: 'page view', + data: { title: 'Home Page', path: '/home' }, + context: {}, + globals: {}, + user: { device: 'device-123' }, + nested: [], + consent: { analytics: true }, + id: '1-abc-1', + trigger: 'load', + entity: 'page', + action: 'view', + timestamp: 1700000000000, + timing: 0, + group: 'group-1', + count: 1, + version: { tagging: 1, config: 1 }, + source: { type: 'web', id: '', previous_id: '' }, + }, + + // Maps to purchaseCall + purchase: { + event: 'order complete', + data: { id: 'T-123', total: 99.99 }, + // ... full event structure + }, + + // Maps to customEventCall + buttonClick: { + event: 'button click', + data: { id: 'cta', text: 'Sign Up' }, + // ... full event structure + }, +}; +``` + +### 2.4 Create Environment Mock + +`src/examples/env.ts`: + +```typescript +import type { DestinationWeb } from '@walkeros/web-core'; + +/** + * Mock environment capturing vendor SDK calls. + */ +export const env: { push: DestinationWeb.Env } = { + push: { + window: { + vendorSdk: jest.fn(), // Captures all calls for verification + } as unknown as Window, + document: {} as Document, + }, +}; +``` + +### 2.5 Export via dev.ts + +`src/dev.ts`: + +```typescript +export * as schemas from './schemas'; +export * as examples from './examples'; +``` + +--- + +## Phase 3: Define Mapping + +**Goal:** Document transformation from walkerOS events to vendor format. + +### 3.1 Create Mapping Examples + +`src/examples/mapping.ts`: + +```typescript +import type { Mapping } from '@walkeros/core'; + +/** + * Default mapping: walkerOS events → vendor format. + */ +export const defaultMapping: Mapping.Rules = { + page: { + view: { + name: 'pageview', // Vendor event name + data: { + map: { + url: 'data.path', + title: 'data.title', + }, + }, + }, + }, + order: { + complete: { + name: 'purchase', + data: { + map: { + transaction_id: 'data.id', + value: 'data.total', + currency: { value: 'USD' }, + }, + }, + }, + }, + button: { + click: { + name: 'button_click', + data: { + map: { + button_id: 'data.id', + button_text: 'data.text', + }, + }, + }, + }, +}; +``` + +### 3.2 Verify Mapping Logic + +Create a mental (or actual) trace: + +``` +Input: events.pageView + ↓ Apply mapping + ↓ page.view rule matches + ↓ name: 'pageview' + ↓ data.path → url, data.title → title +Output: Should match outputs.pageViewCall +``` + +--- + +## Phase 4: Scaffold + +**Template destination:** `packages/web/destinations/plausible/` + +```bash +cp -r packages/web/destinations/plausible packages/web/destinations/[name] +cd packages/web/destinations/[name] + +# Update package.json: name, description, repository.directory +``` + +**Directory structure:** + +``` +packages/web/destinations/[name]/ +├── src/ +│ ├── index.ts # Main destination (init + push) +│ ├── index.test.ts # Tests against examples +│ ├── dev.ts # Exports schemas and examples +│ ├── examples/ +│ │ ├── index.ts # Re-exports +│ │ ├── env.ts # Mock environment +│ │ ├── events.ts # Input events +│ │ ├── outputs.ts # Expected outputs +│ │ └── mapping.ts # Default mapping +│ ├── schemas/ +│ │ └── index.ts # Zod schemas +│ └── types/ +│ └── index.ts # Settings, Config types +├── package.json +├── tsconfig.json +├── tsup.config.ts +├── jest.config.mjs +└── README.md +``` + +--- + +## Phase 5: Implement + +**Now write code to produce the outputs defined in Phase 2.** + +### 5.1 Define Types + +`src/types/index.ts`: + +```typescript +import type { DestinationWeb } from '@walkeros/web-core'; + +export interface Settings { + apiKey?: string; + // Add vendor-specific settings +} + +export interface Config extends DestinationWeb.Config {} +export interface Destination extends DestinationWeb.Destination {} +``` + +### 5.2 Implement Destination + +`src/index.ts`: + +```typescript +import type { Config, Destination } from './types'; +import type { DestinationWeb } from '@walkeros/web-core'; +import { isObject } from '@walkeros/core'; +import { getEnv } from '@walkeros/web-core'; + +export * as DestinationVendor from './types'; + +export const destinationVendor: Destination = { + type: 'vendor', + config: {}, + + init({ config, env }) { + const { window } = getEnv(env); + const settings = config.settings || {}; + + if (config.loadScript) addScript(settings, env); + + // Initialize vendor SDK queue + (window as Window).vendorSdk = + (window as Window).vendorSdk || + function () { + ((window as Window).vendorSdk.q = + (window as Window).vendorSdk.q || []).push(arguments); + }; + + return config; + }, + + push(event, { config, data, env }) { + const params = isObject(data) ? data : {}; + const { window } = getEnv(env); + + // Call vendor API - must match outputs.ts examples + (window as Window).vendorSdk('track', event.name, params); + }, +}; + +function addScript(settings: Settings, env?: DestinationWeb.Env) { + const { document } = getEnv(env); + const script = document.createElement('script'); + script.src = `https://vendor.com/sdk.js?key=${settings.apiKey}`; + document.head.appendChild(script); +} + +export default destinationVendor; +``` + +--- + +## Phase 6: Test Against Examples + +**Verify implementation produces expected outputs.** + +`src/index.test.ts`: + +```typescript +import { destinationVendor } from '.'; +import { examples } from './dev'; + +describe('destinationVendor', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('page view produces correct output', () => { + const mockSdk = jest.fn(); + const env = { + ...examples.env.push, + window: { vendorSdk: mockSdk } as unknown as Window, + }; + + destinationVendor.push(examples.events.pageView, { + config: {}, + data: { url: '/home', title: 'Home Page' }, + env, + }); + + // Verify against expected output + expect(mockSdk).toHaveBeenCalledWith( + examples.outputs.pageViewCall.method, + ...examples.outputs.pageViewCall.args, + ); + }); + + test('purchase produces correct output', () => { + // Similar test against purchaseCall + }); + + test('custom event produces correct output', () => { + // Similar test against customEventCall + }); +}); +``` + +--- + +## Phase 7: Document + +Follow the [writing-documentation](../writing-documentation/SKILL.md) skill for: + +- README structure and templates +- Example validation against `apps/quickstart/` +- Quality checklist before publishing + +Key requirements for destination documentation: + +- [ ] Event mapping table (walkerOS → vendor format) +- [ ] Configuration options table (use PropertyTable if schema exists) +- [ ] Working code example with imports +- [ ] Installation instructions + +--- + +## Validation Checklist + +- [ ] `npm run build` passes +- [ ] `npm run test` passes +- [ ] `npm run lint` passes +- [ ] Examples cover main use cases +- [ ] Tests verify against example outputs +- [ ] README documents configuration + +--- + +## Reference Files + +| What | Where | +| --------------- | -------------------------------------------- | +| Simple template | `packages/web/destinations/plausible/` | +| Complex example | `packages/web/destinations/gtag/` | +| Types | `packages/web/core/src/types/destination.ts` | + +## Related + +- [understanding-destinations skill](../understanding-destinations/SKILL.md) +- [testing-strategy skill](../testing-strategy/SKILL.md) +- [← Back to Hub](../../AGENT.md) diff --git a/skills/create-source/SKILL.md b/skills/create-source/SKILL.md new file mode 100644 index 000000000..44854844c --- /dev/null +++ b/skills/create-source/SKILL.md @@ -0,0 +1,624 @@ +--- +name: create-source +description: + Use when creating a new walkerOS source. Example-driven workflow starting with + research and input examples before implementation. +--- + +# Create a New Source + +## Prerequisites + +Before starting, read these skills: + +- [understanding-flow](../understanding-flow/SKILL.md) - How sources fit in + architecture +- [understanding-sources](../understanding-sources/SKILL.md) - Source interface +- [understanding-events](../understanding-events/SKILL.md) - Event structure + sources emit +- [understanding-mapping](../understanding-mapping/SKILL.md) - Transform raw + input to events +- [testing-strategy](../testing-strategy/SKILL.md) - How to test +- [writing-documentation](../writing-documentation/SKILL.md) - Documentation + standards (for Phase 7) + +## Source Types + +| Type | Platform | Input | Example | +| ------ | -------- | ----------------------- | ----------------------------------- | +| Web | Browser | DOM events, dataLayer | `browser`, `dataLayer` | +| Server | Node.js | HTTP requests, webhooks | `gcp`, `express`, `lambda`, `fetch` | + +## Source Categories + +Sources fall into two categories based on their primary function: + +| Category | Purpose | Examples | Key Concern | +| ------------------ | ----------------------------------------- | ----------------------- | -------------------- | +| **Transformation** | Convert external format → walkerOS events | `dataLayer`, `fetch` | Mapping accuracy | +| **Transport** | Receive events from specific platform | `gcp`, `aws`, `express` | Platform integration | + +**Transformation sources** focus on data conversion - they take input in one +format and produce walkerOS events. The `fetch` source is the purest example. + +**Transport sources** focus on platform integration - they handle +platform-specific concerns (authentication, request parsing, response format) +while delegating transformation. The `gcp` and `aws` sources wrap HTTP handlers +for their respective cloud platforms. + +Many sources are both - they handle platform transport AND transform data. + +## Process Overview + +``` +1. Research → Understand input format, find SDK/types +2. Examples → Create input examples in dev entry FIRST +3. Mapping → Define input → walkerOS event transformation +4. Scaffold → Copy template and configure +5. Implement → Build using examples as test fixtures +6. Test → Verify against example variations +7. Document → Write README +``` + +--- + +## Phase 1: Research + +**Goal:** Understand the input format before writing any code. + +### 1.1 Identify Input Source + +- [ ] **What triggers events?** - HTTP POST, webhook, DOM mutation, dataLayer + push +- [ ] **What data is received?** - Request body, headers, query params +- [ ] **Authentication?** - API keys, signatures, tokens + +### 1.2 Find Official Resources + +```bash +# Search npm for official types +npm search @[platform] +npm info @types/[platform] + +# Check for official SDK +npm search [platform]-sdk +``` + +### 1.3 Document Input Schema + +Capture real examples of incoming data: + +| Field | Type | Required | Description | +| ------------ | ------ | -------- | ---------------------- | +| `event` | string | Yes | Event type from source | +| `properties` | object | No | Event data | +| `userId` | string | No | User identifier | +| `timestamp` | number | No | Event time | + +### 1.4 Map to walkerOS Events + +Plan how input fields become walkerOS events: + +| Source Field | walkerOS Field | Notes | +| ----------------- | -------------- | ----------------------------------- | +| `event` | `name` | May need "entity action" conversion | +| `properties.page` | `data` | Direct mapping | +| `userId` | `user.id` | User identification | + +### 1.5 Check Existing Patterns + +```bash +# List existing sources +ls packages/web/sources/ +ls packages/server/sources/ + +# Reference implementations +# - dataLayer: DOM-based, array interception +# - express: HTTP middleware +# - fetch: Generic HTTP handler (simplest server pattern) +# - gcp: Cloud Functions specific +``` + +--- + +## Phase 2: Create Input Examples (BEFORE Implementation) + +**Goal:** Define realistic input data in `dev` entry FIRST. + +### 2.1 Scaffold Directory Structure + +```bash +mkdir -p packages/server/sources/[name]/src/{examples,schemas,types} +``` + +### 2.2 Create Input Examples + +**Real examples of what the source will receive:** + +`src/examples/inputs.ts`: + +```typescript +/** + * Examples of incoming data this source will receive. + * These define the CONTRACT - implementation must handle these inputs. + */ + +// Page view from external system +export const pageViewInput = { + event: 'page_view', + properties: { + page_title: 'Home Page', + page_path: '/home', + referrer: 'https://google.com', + }, + userId: 'user-123', + timestamp: 1700000000000, +}; + +// E-commerce event +export const purchaseInput = { + event: 'purchase', + properties: { + transaction_id: 'T-123', + value: 99.99, + currency: 'USD', + items: [{ item_id: 'P-1', item_name: 'Widget', price: 99.99 }], + }, + userId: 'user-123', + timestamp: 1700000001000, +}; + +// Custom event +export const customEventInput = { + event: 'button_click', + properties: { + button_id: 'cta', + button_text: 'Sign Up', + }, + timestamp: 1700000002000, +}; + +// Edge cases +export const minimalInput = { + event: 'ping', +}; + +export const invalidInput = { + // Missing event field + properties: { foo: 'bar' }, +}; +``` + +### 2.3 Create Expected Output Examples + +**walkerOS events that should result from inputs:** + +`src/examples/outputs.ts`: + +```typescript +import type { WalkerOS } from '@walkeros/core'; + +/** + * Expected walkerOS events from inputs. + * Tests verify implementation produces these outputs. + */ + +// From pageViewInput → walkerOS event +export const pageViewEvent: Partial = { + event: 'page view', + data: { + title: 'Home Page', + path: '/home', + referrer: 'https://google.com', + }, + user: { id: 'user-123' }, +}; + +// From purchaseInput → walkerOS event +export const purchaseEvent: Partial = { + event: 'order complete', + data: { + id: 'T-123', + total: 99.99, + currency: 'USD', + }, +}; + +// From customEventInput → walkerOS event +export const buttonClickEvent: Partial = { + event: 'button click', + data: { + id: 'cta', + text: 'Sign Up', + }, +}; +``` + +### 2.4 Create HTTP Request Examples (Server Sources) + +`src/examples/requests.ts`: + +```typescript +/** + * HTTP request examples for testing handlers. + */ + +export const validPostRequest = { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': 'test-key', + }, + body: JSON.stringify(inputs.pageViewInput), +}; + +export const batchRequest = { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + batch: [inputs.pageViewInput, inputs.purchaseInput], + }), +}; + +export const invalidRequest = { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: 'invalid json{', +}; +``` + +### 2.5 Export via dev.ts + +`src/dev.ts`: + +```typescript +export * as schemas from './schemas'; +export * as examples from './examples'; +``` + +--- + +## Phase 3: Define Mapping + +**Goal:** Document transformation from input format to walkerOS events. + +### 3.1 Create Mapping Configuration + +`src/examples/mapping.ts`: + +```typescript +import type { Mapping } from '@walkeros/core'; + +/** + * Default mapping: input format → walkerOS events. + */ + +// Event name transformation +export const eventNameMap: Record = { + page_view: 'page view', + purchase: 'order complete', + button_click: 'button click', + add_to_cart: 'product add', +}; + +// Data field mapping +export const defaultMapping: Mapping.Rules = { + page: { + view: { + data: { + map: { + title: 'properties.page_title', + path: 'properties.page_path', + referrer: 'properties.referrer', + }, + }, + }, + }, + order: { + complete: { + data: { + map: { + id: 'properties.transaction_id', + total: 'properties.value', + currency: 'properties.currency', + }, + }, + }, + }, +}; +``` + +### 3.2 Verify Mapping Logic + +Create a trace: + +``` +Input: inputs.pageViewInput + ↓ eventNameMap: 'page_view' → 'page view' + ↓ Entity: 'page', Action: 'view' + ↓ Apply mapping: page.view rule + ↓ properties.page_title → title + ↓ properties.page_path → path +Output: Should match outputs.pageViewEvent +``` + +--- + +## Phase 4: Scaffold + +**Template sources:** + +- Web: `packages/web/sources/dataLayer/` +- Server: `packages/server/sources/fetch/` (simplest pattern) + +```bash +cp -r packages/server/sources/fetch packages/server/sources/[name] +cd packages/server/sources/[name] + +# Update package.json: name, description, repository.directory +``` + +**Directory structure:** + +``` +packages/server/sources/[name]/ +├── src/ +│ ├── index.ts # Main export +│ ├── index.test.ts # Tests against examples +│ ├── dev.ts # Exports schemas and examples +│ ├── examples/ +│ │ ├── index.ts # Re-exports +│ │ ├── inputs.ts # Incoming data examples +│ │ ├── outputs.ts # Expected walkerOS events +│ │ ├── requests.ts # HTTP request examples +│ │ └── mapping.ts # Transformation config +│ ├── schemas/ +│ │ └── index.ts # Zod schemas for input validation +│ └── types/ +│ └── index.ts # Config, Input interfaces +├── package.json +├── tsconfig.json +├── tsup.config.ts +├── jest.config.mjs +└── README.md +``` + +--- + +## Phase 5: Implement + +**Now write code to transform inputs to expected outputs.** + +### 5.1 Define Types + +`src/types/index.ts`: + +```typescript +import type { WalkerOS } from '@walkeros/core'; + +export interface Config { + mapping?: WalkerOS.Mapping; + eventNameMap?: Record; +} + +export interface Input { + event: string; + properties?: Record; + userId?: string; + timestamp?: number; +} + +export interface BatchInput { + batch: Input[]; +} +``` + +### 5.2 Implement Source + +`src/index.ts`: + +```typescript +import type { WalkerOS } from '@walkeros/core'; +import { createEvent, getMappingValue } from '@walkeros/core'; +import type { Config, Input } from './types'; + +export * as SourceName from './types'; + +/** + * Transform incoming input to walkerOS event(s). + */ +export function transformInput( + input: Input, + config: Config = {}, +): WalkerOS.Event | undefined { + if (!input.event) return undefined; + + // Map event name to "entity action" format + const eventName = config.eventNameMap?.[input.event] ?? input.event; + const [entity, action] = eventName.split(' '); + + if (!entity || !action) return undefined; + + // Build event + return createEvent({ + event: eventName, + data: input.properties ?? {}, + user: input.userId ? { id: input.userId } : undefined, + timestamp: input.timestamp, + }); +} + +/** + * Process batch of inputs. + */ +export function transformBatch( + inputs: Input[], + config: Config = {}, +): WalkerOS.Event[] { + return inputs + .map((input) => transformInput(input, config)) + .filter((e): e is WalkerOS.Event => e !== undefined); +} + +/** + * HTTP handler - use directly with any HTTP framework. + * Example: app.post('/events', source.push) + */ +export async function push( + req: { body: Input | { batch: Input[] } }, + config: Config = {}, +): Promise<{ events: WalkerOS.Event[]; error?: string }> { + try { + const body = req.body; + + if ('batch' in body) { + return { events: transformBatch(body.batch, config) }; + } + + const event = transformInput(body, config); + return { events: event ? [event] : [] }; + } catch (error) { + return { events: [], error: 'Invalid input' }; + } +} + +export default { transformInput, transformBatch, push }; +``` + +--- + +## Phase 6: Test Against Examples + +**Verify implementation produces expected outputs.** + +`src/index.test.ts`: + +```typescript +import { transformInput, transformBatch, push } from '.'; +import { examples } from './dev'; + +describe('source transformation', () => { + const config = { + eventNameMap: examples.mapping.eventNameMap, + }; + + test('page view input produces correct event', () => { + const result = transformInput(examples.inputs.pageViewInput, config); + + expect(result).toMatchObject(examples.outputs.pageViewEvent); + }); + + test('purchase input produces correct event', () => { + const result = transformInput(examples.inputs.purchaseInput, config); + + expect(result).toMatchObject(examples.outputs.purchaseEvent); + }); + + test('custom event produces correct event', () => { + const result = transformInput(examples.inputs.customEventInput, config); + + expect(result).toMatchObject(examples.outputs.buttonClickEvent); + }); + + test('handles minimal input', () => { + const result = transformInput(examples.inputs.minimalInput, config); + + // Should handle gracefully (may return undefined or minimal event) + expect(result).toBeDefined(); + }); + + test('handles invalid input gracefully', () => { + const result = transformInput(examples.inputs.invalidInput as any, config); + + expect(result).toBeUndefined(); + }); +}); + +describe('batch processing', () => { + test('transforms multiple inputs', () => { + const inputs = [ + examples.inputs.pageViewInput, + examples.inputs.purchaseInput, + ]; + + const result = transformBatch(inputs, { + eventNameMap: examples.mapping.eventNameMap, + }); + + expect(result).toHaveLength(2); + }); +}); + +describe('HTTP handler', () => { + test('handles single event', async () => { + const req = { body: examples.inputs.pageViewInput }; + const result = await push(req, { + eventNameMap: examples.mapping.eventNameMap, + }); + + expect(result.events).toHaveLength(1); + expect(result.error).toBeUndefined(); + }); + + test('handles batch', async () => { + const req = { + body: { + batch: [examples.inputs.pageViewInput, examples.inputs.purchaseInput], + }, + }; + const result = await push(req, { + eventNameMap: examples.mapping.eventNameMap, + }); + + expect(result.events).toHaveLength(2); + }); +}); +``` + +--- + +## Phase 7: Document + +Follow the [writing-documentation](../writing-documentation/SKILL.md) skill for: + +- README structure and templates +- Example validation against `apps/quickstart/` +- Quality checklist before publishing + +Key requirements for source documentation: + +- [ ] Input format table documenting expected fields +- [ ] Event name mapping table (source format → walkerOS format) +- [ ] Configuration options table +- [ ] Working code example with imports +- [ ] Installation instructions + +--- + +## Validation Checklist + +- [ ] `npm run build` passes +- [ ] `npm run test` passes +- [ ] `npm run lint` passes +- [ ] Examples cover main input patterns +- [ ] Tests verify against example outputs +- [ ] Invalid input handled gracefully +- [ ] README documents input format + +--- + +## Reference Files + +| What | Where | +| --------------- | ----------------------------------- | +| Web template | `packages/web/sources/dataLayer/` | +| Server template | `packages/server/sources/fetch/` | +| Source types | `packages/core/src/types/source.ts` | +| Event creation | `packages/core/src/lib/event.ts` | + +## Related + +- [understanding-sources skill](../understanding-sources/SKILL.md) +- [understanding-events skill](../understanding-events/SKILL.md) +- [testing-strategy skill](../testing-strategy/SKILL.md) +- [← Back to Hub](../../AGENT.md) diff --git a/skills/debugging/SKILL.md b/skills/debugging/SKILL.md new file mode 100644 index 000000000..fe6742e4d --- /dev/null +++ b/skills/debugging/SKILL.md @@ -0,0 +1,276 @@ +--- +name: debugging +description: + Use when events aren't reaching destinations, debugging event flow, or + troubleshooting mapping issues. Covers common problems and debugging + strategies. +--- + +# Debugging walkerOS Events + +## Quick Diagnosis + +| Symptom | Likely Cause | Check | +| ---------------------------------- | ---------------------------- | ---------------------------------------- | +| No events at all | Source not initialized | Console for errors, verify `startFlow()` | +| Events fire but destination silent | Mapping mismatch | Event name matches mapping? | +| Partial data missing | Path doesn't exist | Log event structure, check nested paths | +| Consent blocking | Required consent not granted | Check `consent` config, grant consent | +| Destination error | Vendor API issue | Check network tab, vendor console | + +## Debugging Strategies + +### 1. Console Logging + +**Log all events at collector level:** + +```typescript +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + destinations: { + debug: { + push: async (event, context) => { + console.log('[walkerOS Event]', { + name: event.name, + data: event.data, + context: event.context, + consent: event.consent, + timestamp: event.timestamp, + }); + }, + config: {}, + }, + // ... other destinations + }, +}); +``` + +### 2. Network Tab Inspection + +For destinations that make HTTP calls: + +1. Open DevTools → Network tab +2. Filter by destination domain (e.g., `google-analytics.com`, `facebook.com`) +3. Trigger event +4. Inspect request payload + +**What to look for:** + +- Request being made at all? +- Correct endpoint URL? +- Payload structure matches vendor spec? + +### 3. Vendor Debug Tools + +| Vendor | Debug Tool | +| --------- | --------------------------------------------------------------------------------------------- | +| GA4 | [GA4 DebugView](https://support.google.com/analytics/answer/7201382) | +| Meta | [Facebook Pixel Helper](https://developers.facebook.com/docs/meta-pixel/support/pixel-helper) | +| Plausible | [Plausible Dashboard real-time](https://plausible.io/docs) | + +### 4. Dry Run Mode + +Test mapping without sending to vendor: + +```typescript +const destination = { + ...actualDestination, + config: { + ...actualDestination.config, + dryRun: true, // Events processed but not sent + }, +}; +``` + +## Common Issues + +### Event Name Mismatch + +**Problem:** Event fires but destination doesn't receive it. + +```typescript +// Event pushed +elb('product view', { id: 'P123' }); + +// Mapping expects different name +mapping: { + Product: { + // Wrong: capital P + View: { + // Wrong: capital V + name: 'view_item'; + } + } +} +``` + +**Fix:** Event names are case-sensitive. Use exact match: + +```typescript +mapping: { + product: { + view: { + name: 'view_item'; + } + } +} +``` + +### Missing Nested Data + +**Problem:** `items` array is empty in destination. + +```typescript +// Event structure +{ + name: 'order complete', + data: { total: 100 }, + nested: [ + { type: 'product', data: { id: 'P1' } } + ] +} + +// Mapping tries wrong path +data: { + map: { + items: { + loop: [ + 'data.items', // Wrong: nested is at root, not in data + { map: { id: 'data.id' } } + ] + } + } +} +``` + +**Fix:** Use correct path to nested array: + +```typescript +items: { + loop: [ + 'nested', // Correct: root-level nested + { map: { item_id: 'data.id' } }, + ]; +} +``` + +### Consent Blocking Events + +**Problem:** Events not reaching destination. + +**Check 1:** Does destination require consent? + +```typescript +// Destination config +config: { + consent: { + marketing: true; + } // Requires marketing consent +} +``` + +**Check 2:** Is consent granted? + +```typescript +// Check current consent state +console.log(event.consent); + +// Grant consent +elb('walker consent', { marketing: true }); +``` + +### Vendor SDK Not Loaded + +**Problem:** `TypeError: env.window.gtag is not a function` + +**Cause:** Vendor script not loaded before push. + +**Fix:** Ensure init() loads script: + +```typescript +init: async (config, env) => { + // Wait for script to load + await loadScript('https://vendor.com/sdk.js'); + // Verify SDK available + if (!env.window.vendorSdk) { + throw new Error('Vendor SDK failed to load'); + } +}, +``` + +### Function Mapping Errors + +**Problem:** `Cannot read property 'price' of undefined` + +```typescript +// Mapping with unsafe access +data: { + map: { + value: { + fn: (e) => e.data.price * 100; + } // Fails if data.price undefined + } +} +``` + +**Fix:** Add null checks: + +```typescript +value: { + fn: (e) => (e.data?.price ?? 0) * 100; +} +``` + +## Debugging Checklist + +When events aren't working: + +1. [ ] **Console errors?** Check browser console for exceptions +2. [ ] **Event pushed?** Add debug destination to log all events +3. [ ] **Mapping matched?** Verify entity/action names exactly match +4. [ ] **Data paths correct?** Log full event structure, verify paths exist +5. [ ] **Consent granted?** Check consent requirements and state +6. [ ] **SDK loaded?** Verify vendor script loaded before push +7. [ ] **Network request?** Check DevTools network tab +8. [ ] **Vendor receiving?** Use vendor debug tools + +## Testing in Isolation + +Test destination push directly: + +```typescript +import { push } from '@walkeros/web-destination-gtag'; +import { mockEnv } from '@walkeros/core'; + +// Create test event +const event = { + name: 'product view', + data: { id: 'P123', price: 99 }, + // ... full event +}; + +// Mock env to capture calls +const calls = []; +const testEnv = mockEnv(baseEnv, (path, args) => { + calls.push({ path, args }); +}); + +// Test push directly +await push(event, { config: testConfig, env: testEnv }); + +// Inspect what was called +console.log(calls); +``` + +## Related + +- [understanding-flow skill](../understanding-flow/SKILL.md) - Event flow + architecture +- [understanding-destinations skill](../understanding-destinations/SKILL.md) - + Destination interface +- [mapping-configuration skill](../mapping-configuration/SKILL.md) - Mapping + recipes +- [testing-strategy skill](../testing-strategy/SKILL.md) - Testing with env + pattern +- [← Back to Hub](../../AGENT.md) diff --git a/skills/mapping-configuration/SKILL.md b/skills/mapping-configuration/SKILL.md new file mode 100644 index 000000000..6a4322657 --- /dev/null +++ b/skills/mapping-configuration/SKILL.md @@ -0,0 +1,275 @@ +--- +name: mapping-configuration +description: + Use when configuring event mappings for specific use cases. Provides recipes + for GA4, Meta, custom APIs, and common transformation patterns. +--- + +# Mapping Configuration Recipes + +## Prerequisites + +Read [understanding-mapping](../understanding-mapping/SKILL.md) first for core +concepts. + +## Quick Reference + +| I want to... | Use this pattern | +| -------------------- | ----------------------------------------------------- | +| Rename event | `{ name: 'new_name' }` | +| Extract nested value | `'data.nested.value'` | +| Set static value | `{ value: 'USD' }` | +| Transform value | `{ fn: (e) => transform(e) }` | +| Build object | `{ map: { key: 'source' } }` | +| Process array | `{ loop: ['source', { map: {...} }] }` | +| Gate by consent | `{ key: 'data.email', consent: { marketing: true } }` | + +## Common Recipes + +### GA4 / gtag + +**Product view → view_item:** + +```typescript +product: { + view: { + name: 'view_item', + data: { + map: { + currency: { value: 'USD' }, + value: 'data.price', + items: { + loop: [ + 'nested', + { + map: { + item_id: 'data.id', + item_name: 'data.name', + item_category: 'data.category', + price: 'data.price', + quantity: { value: 1 }, + }, + }, + ], + }, + }, + }, + }, +} +``` + +**Order complete → purchase:** + +```typescript +order: { + complete: { + name: 'purchase', + data: { + map: { + transaction_id: 'data.orderId', + value: 'data.total', + currency: 'data.currency', + items: { + loop: [ + 'nested', + { + map: { + item_id: 'data.id', + item_name: 'data.name', + price: 'data.price', + quantity: 'data.quantity', + }, + }, + ], + }, + }, + }, + }, +} +``` + +### Meta Pixel + +**Product view → ViewContent:** + +```typescript +product: { + view: { + name: 'ViewContent', + data: { + map: { + content_ids: { fn: (e) => [e.data.id] }, + content_type: { value: 'product' }, + content_name: 'data.name', + value: 'data.price', + currency: { value: 'USD' }, + }, + }, + }, +} +``` + +**Order complete → Purchase:** + +```typescript +order: { + complete: { + name: 'Purchase', + data: { + map: { + content_ids: { fn: (e) => e.nested?.map((n) => n.data.id) ?? [] }, + content_type: { value: 'product' }, + value: 'data.total', + currency: 'data.currency', + num_items: { fn: (e) => e.nested?.length ?? 0 }, + }, + }, + }, +} +``` + +### Custom API Destination + +**Transform to REST API format:** + +```typescript +'*': { + '*': { + name: { fn: (e) => `${e.entity}_${e.action}` }, // page_view + data: { + map: { + eventName: 'name', + eventData: 'data', + userId: 'user.id', + sessionId: 'user.session', + timestamp: 'timestamp', + metadata: { + map: { + consent: 'consent', + globals: 'globals', + }, + }, + }, + }, + }, +} +``` + +### Conditional Mapping + +**Different mapping based on event data:** + +```typescript +order: { + complete: [ + // High-value orders get extra tracking + { + condition: (e) => (e.data?.total ?? 0) > 500, + name: 'high_value_purchase', + data: { + map: { + value: 'data.total', + priority: { value: 'high' }, + notify: { value: true }, + }, + }, + }, + // Standard orders + { + name: 'purchase', + data: { map: { value: 'data.total' } }, + }, + ], +} +``` + +### Consent-Gated Fields + +**Only include PII if consent granted:** + +```typescript +user: { + login: { + name: 'login', + data: { + map: { + method: 'data.method', + // Only include email if marketing consent + email: { + key: 'user.email', + consent: { marketing: true }, + }, + // Only include user ID if functional consent + userId: { + key: 'user.id', + consent: { functional: true }, + }, + }, + }, + }, +} +``` + +### Wildcard Patterns + +**Catch-all for unmatched events:** + +```typescript +// Any product action +product: { + '*': { + name: { fn: (e) => `product_${e.action}` }, + data: 'data', + }, +} + +// Any click on any entity +'*': { + click: { + name: 'element_click', + data: { + map: { + element_type: 'entity', + element_id: 'data.id', + }, + }, + }, +} +``` + +## Source-Side Mapping + +**Transform HTTP input to walkerOS event:** + +```typescript +// In source config +{ + mapping: { + // Map incoming field names to walkerOS structure + name: { fn: (input) => `${input.entity} ${input.action}` }, + data: 'payload', + user: { + map: { + id: 'userId', + session: 'sessionId', + }, + }, + }, +} +``` + +## Debugging Tips + +1. **Event not mapping?** Check entity/action match exactly (case-sensitive) +2. **Data missing?** Verify source path exists: `'data.nested.field'` +3. **Function errors?** Add null checks: `e.data?.price ?? 0` +4. **Array empty?** Confirm `nested` array exists and has items + +## Reference + +- [understanding-mapping skill](../understanding-mapping/SKILL.md) - Core + concepts +- [packages/core/src/mapping.ts](../../packages/core/src/mapping.ts) - + Implementation +- [apps/quickstart/src/](../../apps/quickstart/src/) - Validated examples +- [← Back to Hub](../../AGENT.md) diff --git a/skills/testing-strategy/SKILL.md b/skills/testing-strategy/SKILL.md new file mode 100644 index 000000000..6455d2d33 --- /dev/null +++ b/skills/testing-strategy/SKILL.md @@ -0,0 +1,248 @@ +--- +name: testing-strategy +description: + Use when writing tests, reviewing test code, or discussing testing approach + for walkerOS packages. Covers env pattern, dev examples, and package-specific + strategies. +--- + +# walkerOS Testing Strategy + +## Overview + +walkerOS uses a layered testing approach with built-in patterns for mocking and +documentation sync. This skill ensures tests are reliable, efficient, and +maintainable. + +**Core principle:** Test real behavior using the `env` pattern, link to `dev` +examples, verify before claiming complete. + +## The Rules + +### Rule 1: Use `env` for Mocking, Not Jest + +walkerOS has a built-in dependency injection pattern via `env` in context. This +is lighter than Jest mocks, enables documentation generation, and keeps tests in +sync with examples. + +**Wrong:** + +```typescript +jest.mock('../ga4', () => ({ initGA4: jest.fn() })); +expect(initGA4).toHaveBeenCalledWith(...); +``` + +**Right:** + +```typescript +import { examples } from '../dev'; +import { mockEnv } from '@walkeros/core'; + +const calls: Array<{ path: string[]; args: unknown[] }> = []; +const testEnv = mockEnv(examples.env.push, (path, args) => { + calls.push({ path, args }); +}); + +await destination.push(event, { ...context, env: testEnv }); +expect(calls).toContainEqual({ + path: ['window', 'gtag'], + args: ['event', 'page_view', { page_title: 'Home' }], +}); +``` + +### Rule 2: Link Tests to `dev` Examples + +The `dev.ts` export provides `examples.env`, `examples.events`, and +`examples.mapping`. Using these in tests ensures documentation stays in sync. + +```typescript +import { examples } from '../dev'; + +// Use examples.env for mock environment +const testEnv = mockEnv(examples.env.push, interceptor); + +// Assert against examples.events (documented expected output) +expect(calls[0].args).toEqual(examples.events.ga4PageView()); + +// Test with examples.mapping configurations +const config = { mapping: examples.mapping.ecommerce }; +``` + +### Rule 3: Test Real Behavior, Not Mock Behavior + +If you're asserting that a mock was called, you're testing the mock works, not +the code. + +**Red flags:** + +- `expect(mockFn).toHaveBeenCalled()` without verifying the mock produces real + effects +- Assertions on `*-mock` test IDs +- Tests that pass when mock is present, fail when removed + +**Fix:** Test what the code actually does. If external APIs must be mocked, +verify the real API would receive correct data. + +### Rule 4: Test First, Watch It Fail + +If you didn't see the test fail, you don't know it tests the right thing. + +**Process:** + +1. Write failing test +2. Verify it fails for expected reason (missing feature, not typo) +3. Write minimal code to pass +4. Verify it passes +5. Refactor if needed + +**Red flags:** + +- Test passes immediately when written +- Can't explain why test failed +- "I'll add tests later" + +### Rule 5: No Test-Only Methods in Production Code + +Production classes shouldn't have methods only tests use. + +**Wrong:** + +```typescript +class Session { + destroy() { + /* only used in tests */ + } +} +``` + +**Right:** + +```typescript +// In test-utils/ +export function cleanupSession(session: Session) { ... } +``` + +### Rule 6: Verify Before Claiming Complete + +"Should pass now" is not verification. + +**Process:** + +1. Run the actual test command +2. Read the output +3. Confirm pass/fail count +4. Only then claim status + +## When to Use Each Test Type + +| Type | When to Add | Example | +| --------------- | ------------------------------------------------------------------- | -------------------------------------------------------------- | +| **Integration** | New usage pattern, new external API interaction, new data flow path | Collector → Destination → gtag() | +| **Unit** | Combinatorics, edge cases, pure function logic | Mapping variations, core utilities | +| **Contract** | Boundary validation | Destination output matches vendor API, source input validation | + +**Guideline:** Integration tests prove things work when stuck together. Unit +tests efficiently cover variations. Contract tests catch API drift. + +## Package-Specific Approaches + +| Package | Approach | +| ----------------------- | ------------------------------------------------------------------------------- | +| **core** | Unit tests only - pure functions, no env needed | +| **collector** | Integration tests critical - input/output consistency is paramount | +| **browser source** | Maintain walker algorithm coverage | +| **web destinations** | Integration tests per unique pattern + unit tests for mappings, use env pattern | +| **server destinations** | Same as web destinations | +| **cli/docker** | Integration tests for spawn behavior, explore dev pattern to reduce duplication | +| **sources** | Contract tests for input validation, integration tests for event capture | + +## The env Pattern Deep Dive + +### How env Works + +Each destination/source defines an `env` type that specifies external +dependencies: + +```typescript +// Destination-specific env type +export interface Env extends DestinationWeb.Env { + window: { + gtag: Gtag.Gtag; + dataLayer: unknown[]; + }; + document: { + createElement: (tagName: string) => HTMLElement; + head: { appendChild: (node: unknown) => void }; + }; +} +``` + +### mockEnv() Function + +The `mockEnv()` function from `@walkeros/core` creates a Proxy that intercepts +all function calls: + +```typescript +import { mockEnv } from '@walkeros/core'; + +const calls: Array<{ path: string[]; args: unknown[] }> = []; +const testEnv = mockEnv(examples.env.push, (path, args) => { + calls.push({ path, args }); + // Optionally return a value +}); + +// Now use testEnv in your destination context +await destination.push(event, { ...context, env: testEnv }); + +// Assert on captured calls +expect(calls).toContainEqual({ + path: ['window', 'gtag'], + args: ['event', 'purchase', expect.objectContaining({ value: 99.99 })], +}); +``` + +### dev.ts Structure + +Each package with external dependencies should have: + +```typescript +// src/dev.ts +export * as schemas from './schemas'; +export * as examples from './examples'; + +// src/examples/index.ts +export * as env from './env'; +export * as events from './events'; +export * as mapping from './mapping'; +``` + +## Red Flags - Stop and Fix + +- Using `jest.mock()` for internal modules when `env` pattern is available +- Tests that don't import from `../dev` +- Assertions only checking mock call counts +- Tests with extensive mock setup (>50% of test is setup) +- Test-only methods added to production classes +- Claiming tests pass without running them + +## Commands + +```bash +# Run all tests +npm run test + +# Run tests for specific package +cd packages/[name] && npm run test + +# Run single test file +npm run test -- path/to/file.test.ts + +# Watch mode +npm run test -- --watch +``` + +## Related + +- [AGENT.md](../../AGENT.md) - Development guide +- [.claude/skills/testing-strategy/](../../.claude/skills/testing-strategy/) - + Claude Code auto-discovery reference diff --git a/skills/understanding-destinations/SKILL.md b/skills/understanding-destinations/SKILL.md new file mode 100644 index 000000000..70f911fca --- /dev/null +++ b/skills/understanding-destinations/SKILL.md @@ -0,0 +1,125 @@ +--- +name: understanding-destinations +description: + Use when working with destinations, understanding the destination interface, + or learning about env pattern and configuration. Covers interface, lifecycle, + env mocking, and paths. +--- + +# Understanding walkerOS Destinations + +## Overview + +Destinations receive processed events from the collector and deliver them to +third-party tools (analytics, marketing, data warehouses). + +**Core principle:** Destinations transform and deliver. They don't capture or +process—that's sources and collector. + +## Destination Interface + +See +[packages/core/src/types/destination.ts](../../packages/core/src/types/destination.ts) +for canonical interface. + +| Method | Purpose | Required | +| --------------------------- | -------------------------- | ------------ | +| `init(context)` | Load scripts, authenticate | Optional | +| `push(event, context)` | Transform and send event | **Required** | +| `pushBatch(batch, context)` | Batch processing | Optional | +| `config` | Settings, mapping, consent | **Required** | + +## The env Pattern + +Destinations use dependency injection via `env` for external APIs. This enables +testing without mocking. + +```typescript +// Destination defines its env type +export interface Env extends DestinationWeb.Env { + window: { + gtag: Gtag.Gtag; + dataLayer: unknown[]; + }; +} + +// Destination uses env, not globals +async function push(event, context) { + const { env } = context; + env.window.gtag('event', mappedName, mappedData); +} +``` + +### Testing with env + +**REQUIRED SKILL:** See `testing-strategy` for full testing patterns. + +```typescript +import { mockEnv } from '@walkeros/core'; +import { examples } from '../dev'; + +const calls: Array<{ path: string[]; args: unknown[] }> = []; +const testEnv = mockEnv(examples.env.push, (path, args) => { + calls.push({ path, args }); +}); + +await destination.push(event, { ...context, env: testEnv }); + +expect(calls).toContainEqual({ + path: ['window', 'gtag'], + args: ['event', 'purchase', expect.any(Object)], +}); +``` + +## Destination Config + +```typescript +config: { + settings: { /* destination-specific */ }, + mapping: { /* event transformation rules */ }, + data: { /* global data mapping */ }, + consent: { /* required consent states */ }, + policy: { /* processing rules */ }, + queue: boolean, // queue events + dryRun: boolean, // test mode +} +``` + +## Destination Paths + +| Type | Path | Examples | +| ------ | ------------------------------- | ------------------------------------ | +| Web | `packages/web/destinations/` | gtag, meta, api, piwikpro, plausible | +| Server | `packages/server/destinations/` | aws, gcp, meta | + +## Template Destination + +Use as starting point: `packages/web/destinations/plausible/` + +## Related + +**Skills:** + +- [understanding-mapping skill](../understanding-mapping/SKILL.md) - Configure + transformations +- [testing-strategy skill](../testing-strategy/SKILL.md) - Test with env pattern +- [create-destination skill](../create-destination/SKILL.md) - Create new + destination + +**Source Files:** + +- [packages/core/src/types/destination.ts](../../packages/core/src/types/destination.ts) - + Interface + +**Package READMEs:** + +- [packages/web/destinations/gtag/README.md](../../packages/web/destinations/gtag/README.md) - + gtag example +- [packages/web/destinations/plausible/README.md](../../packages/web/destinations/plausible/README.md) - + Plausible (template) + +**Documentation:** + +- [Website: Destinations](../../website/docs/destinations/index.mdx) - Overview +- [Website: Create Your Own](../../website/docs/destinations/create-your-own.mdx) - + Guide diff --git a/skills/understanding-development/SKILL.md b/skills/understanding-development/SKILL.md new file mode 100644 index 000000000..f2b2a780e --- /dev/null +++ b/skills/understanding-development/SKILL.md @@ -0,0 +1,111 @@ +--- +name: understanding-development +description: + Use when contributing to walkerOS, before writing code, or when unsure about + project conventions. Covers build/test/lint workflow, XP principles, folder + structure, and package usage. +--- + +# Understanding walkerOS Development + +## Overview + +walkerOS follows extreme programming principles with strict conventions. This +skill is your foundation before writing any code. + +**Core principle:** DRY, KISS, YAGNI. Test first. Verify before claiming +complete. + +## Commands + +| Command | Purpose | +| ---------------- | --------------------------- | +| `npm install` | Install all dependencies | +| `npm run dev` | Watch mode for all packages | +| `npm run build` | Build all packages | +| `npm run test` | Run all tests | +| `npm run lint` | ESLint + TypeScript check | +| `npm run format` | Prettier formatting | + +**Validation before commit:** `npm run build && npm run test && npm run lint` + +## XP Principles (Non-Negotiable) + +| Principle | In Practice | +| ------------ | -------------------------------------------------------- | +| **DRY** | Use `@walkeros/core` utilities, don't reimplement | +| **KISS** | Minimal code to solve the problem | +| **YAGNI** | Only implement what's requested | +| **TDD** | Test first, watch it fail, then implement | +| **No `any`** | Never use `any` in production code (tests are exception) | + +## Folder Structure + +``` +packages/ +├── core/ # Platform-agnostic types, utilities, schemas +├── collector/ # Central event processing engine +├── config/ # Shared config (eslint, jest, tsconfig, tsup) +├── web/ +│ ├── core/ # Web-specific utilities +│ ├── sources/ # browser, dataLayer +│ └── destinations/ # gtag, meta, api, piwikpro, plausible +└── server/ + ├── core/ # Server-specific utilities + ├── sources/ # gcp + └── destinations/ # aws, gcp, meta + +apps/ +├── walkerjs/ # Ready-to-use browser bundle +├── quickstart/ # Code examples (source of truth for patterns) +└── demos/ # Demo applications +``` + +## Core Package Usage + +**Always import from `@walkeros/core`:** + +```typescript +// Types +import type { WalkerOS } from '@walkeros/core'; + +// Utilities +import { + getEvent, + createEvent, // Event creation + getMappingEvent, + getMappingValue, // Transformations + isString, + isObject, + isDefined, // Type checking + assign, + clone, // Object operations + tryCatch, + tryCatchAsync, // Error handling +} from '@walkeros/core'; +``` + +**Config package for shared tooling:** + +- ESLint config: `@walkeros/config/eslint` +- Jest config: `@walkeros/config/jest` +- TSConfig: `@walkeros/config/tsconfig` +- Tsup config: `@walkeros/config/tsup` + +## Testing + +**REQUIRED SKILL:** Use `testing-strategy` for detailed testing patterns. + +Quick reference: + +- Use `env` pattern for mocking (not Jest mocks) +- Import from `dev.ts` for examples +- Test first, watch it fail +- Verify before claiming complete + +## Related + +- [testing-strategy skill](../testing-strategy/SKILL.md) +- [packages/core/](../../packages/core/) - Core utilities +- [packages/config/](../../packages/config/) - Shared configuration +- [apps/quickstart/](../../apps/quickstart/) - Validated examples diff --git a/skills/understanding-events/SKILL.md b/skills/understanding-events/SKILL.md new file mode 100644 index 000000000..d7ff95869 --- /dev/null +++ b/skills/understanding-events/SKILL.md @@ -0,0 +1,177 @@ +--- +name: understanding-events +description: + Use when creating events, understanding event structure, or working with event + properties. Covers entity-action naming, event properties, statelessness, and + vendor-agnostic design. +--- + +# Understanding walkerOS Events + +## Overview + +walkerOS events are self-describing, stateless, vendor-agnostic data structures. +They capture user interactions in a standardized format that can be transformed +for any destination. + +**Core principle:** Events describe WHAT happened, not WHERE it goes. Stateless. +Self-describing. Industry-agnostic. + +## Entity-Action Naming (Critical) + +**STRICT REQUIREMENT:** All events use "entity action" format with space +separation. + +```typescript +// Correct +'page view'; +'product add'; +'order complete'; +'button click'; + +// Wrong +'page_view'; // underscore +'pageview'; // no separator +'purchase'; // no entity +'add_to_cart'; // wrong format +``` + +**Parsing:** `const [entity, action] = event.split(' ')` + +- **Entity:** Noun (page, product, user, order, button) +- **Action:** Verb (view, add, complete, click, login) + +## Event Properties + +See +[packages/core/src/types/walkeros.ts](../../packages/core/src/types/walkeros.ts) +for canonical types (Event interface). + +| Property | Type | Purpose | Example | +| ----------- | ------ | -------------------------- | ------------------------------------- | +| `name` | string | "entity action" format | `"product view"` | +| `data` | object | Entity-specific properties | `{ id: "P123", price: 99 }` | +| `context` | object | State/environment info | `{ stage: ["checkout", 1] }` | +| `globals` | object | Global properties | `{ language: "en" }` | +| `user` | object | User identification | `{ id: "user123" }` | +| `nested` | array | Related entities | `[{ type: "category", data: {...} }]` | +| `consent` | object | Consent states | `{ marketing: true }` | +| `id` | string | Auto-generated unique ID | `"1647261462000-01b5e2-2"` | +| `timestamp` | number | Auto-generated Unix ms | `1647261462000` | +| `entity` | string | Parsed from name | `"product"` | +| `action` | string | Parsed from name | `"view"` | + +### data Property + +Entity-specific properties. Schema-free but consistent within entity type. + +```typescript +// product entity +data: { id: "P123", name: "Laptop", price: 999, currency: "USD" } + +// page entity +data: { title: "Home", path: "/", referrer: "https://..." } +``` + +### context Property + +Hierarchical state information. Format: `{ name: [value, order] }` + +```typescript +context: { + stage: ["checkout", 1], // checkout stage, first step + test: ["variant-A", 0], // A/B test variant + group: ["premium", 2] // user segment +} +``` + +### globals Property + +Properties that apply to ALL events in the session. + +```typescript +globals: { + language: "en", + currency: "USD", + environment: "production" +} +``` + +### nested Property + +Related entities captured together. + +```typescript +// Order with line items +nested: [ + { type: 'product', data: { id: 'P1', quantity: 2 } }, + { type: 'product', data: { id: 'P2', quantity: 1 } }, +]; +``` + +### user Property + +User identification across sessions. + +```typescript +user: { + id: "user123", // Your user ID + device: "device456", // Device fingerprint + session: "sess789" // Session ID +} +``` + +## Design Principles + +### Statelessness + +Events are immutable snapshots. They don't reference previous events or maintain +state. + +### Self-Describing + +Events contain all context needed to understand them. No external lookups +required. + +### Vendor-Agnostic + +Events use generic concepts (product, order) not vendor-specific (GA4 item, FB +content). + +Transformation to vendor formats happens in **mapping**, not in event creation. + +## Creating Events + +```typescript +import { elb } from '@walkeros/collector'; + +// Basic event +await elb('page view', { title: 'Home', path: '/' }); + +// With all properties +await elb( + 'product add', + { id: 'P123', price: 99 }, // data + { stage: ['cart', 1] }, // context (optional) + { currency: 'USD' }, // globals (optional) +); +``` + +## Related + +**Skills:** + +- [understanding-mapping skill](../understanding-mapping/SKILL.md) - Transform + events for destinations + +**Source Files:** + +- [packages/core/src/types/walkeros.ts](../../packages/core/src/types/walkeros.ts) - + Event types +- [packages/core/src/schemas/](../../packages/core/src/schemas/) - Event schemas + +**Documentation:** + +- [Website: Event Model](../../website/docs/getting-started/event-model.mdx) - + User-facing docs +- [walkeros.io/docs](https://www.walkeros.io/docs/) - Public documentation diff --git a/skills/understanding-flow/SKILL.md b/skills/understanding-flow/SKILL.md new file mode 100644 index 000000000..de71ece60 --- /dev/null +++ b/skills/understanding-flow/SKILL.md @@ -0,0 +1,119 @@ +--- +name: understanding-flow +description: + Use when learning walkerOS architecture, understanding data flow, or designing + composable event pipelines. Covers Source→Collector→Destination pattern and + separation of concerns. +--- + +# Understanding walkerOS Flow + +## Overview + +walkerOS follows a **Source → Collector → Destination(s)** architecture for +composable, modular event processing. + +**Core principle:** Separation of concerns. Each component has one job. +Components are composable and replaceable. + +## The Flow Pattern + +``` +Sources → Collector → Destinations +(Data Capture) (Processing) (Delivery) + +- Browser DOM - Validation - Google Analytics +- DataLayer - Enrichment - Meta Pixel +- Server HTTP - Consent check - Custom API +- Cloud Functions - Routing - Data Warehouse +``` + +## Key Concepts + +### Composability + +A Flow combines components. You can: + +- Use multiple sources feeding one collector +- Route events to multiple destinations +- Swap components without changing others + +### The Flow Type + +See [packages/core/src/types/flow.ts](../../packages/core/src/types/flow.ts) for +the canonical interface. + +```typescript +// Conceptual structure (see source for full type) +interface Flow { + sources?: Record; + collector: Collector; + destinations?: Record; +} +``` + +### Universal Push Interface + +**All components communicate via `push` functions:** + +| Component | Push Signature | Purpose | +| ----------- | ----------------------------- | --------------------- | +| Source | `push(input) → events` | Capture external data | +| Collector | `push(event) → void` | Process and route | +| Destination | `push(event, context) → void` | Transform and deliver | + +The `elb()` function is an alias for `collector.push` - used for component +wiring. + +### startFlow Helper + +See [packages/collector/src/flow.ts](../../packages/collector/src/flow.ts) for +the `startFlow` function. + +```typescript +import { startFlow } from '@walkeros/collector'; + +const { collector, elb } = await startFlow({ + sources: { + /* ... */ + }, + destinations: { + /* ... */ + }, +}); +``` + +## Separation of Concerns + +| Concern | Handled By | NOT Handled By | +| ---------------- | -------------- | ----------------------- | +| Event capture | Sources | Collector, Destinations | +| Event structure | Event model | Components | +| Consent checking | Collector | Sources, Destinations | +| Transformation | Mapping system | Raw push calls | +| Delivery | Destinations | Sources, Collector | + +## Related + +**Skills:** + +- [understanding-events skill](../understanding-events/SKILL.md) - Event model +- [understanding-destinations skill](../understanding-destinations/SKILL.md) - + Destination interface +- [understanding-sources skill](../understanding-sources/SKILL.md) - Source + interface + +**Package READMEs:** + +- [packages/collector/README.md](../../packages/collector/README.md) - Collector + details + +**Source Files:** + +- [packages/collector/src/flow.ts](../../packages/collector/src/flow.ts) - + startFlow implementation + +**Documentation:** + +- [Website: Flow](../../website/docs/getting-started/flow.mdx) - Flow concept +- [Website: Collector](../../website/docs/collector/index.mdx) - Collector docs diff --git a/skills/understanding-mapping/SKILL.md b/skills/understanding-mapping/SKILL.md new file mode 100644 index 000000000..3ff2c20c5 --- /dev/null +++ b/skills/understanding-mapping/SKILL.md @@ -0,0 +1,278 @@ +--- +name: understanding-mapping +description: + Use when transforming events at any point in the flow (source→collector or + collector→destination), configuring data/map/loop/condition, or understanding + value extraction. Covers all mapping strategies. +--- + +# Understanding walkerOS Mapping + +## Overview + +Mapping transforms data at multiple points in the walkerOS flow: + +1. **Source → Collector**: Transform raw input (HTTP requests, dataLayer pushes) + into walkerOS events +2. **Collector → Destination**: Transform walkerOS events into vendor-specific + formats + +**Core principle:** Mapping is the universal transformation layer. Same +strategies work everywhere in the flow. + +## Core Functions + +See [packages/core/src/mapping.ts](../../packages/core/src/mapping.ts) for +implementation. + +| Function | Purpose | +| -------------------------------- | -------------------------------------- | +| `getMappingEvent(event, rules)` | Find mapping config for an event | +| `getMappingValue(value, config)` | Transform a value using mapping config | + +## Event Mapping + +Match events to transformation rules by entity and action. + +```typescript +const mapping = { + // Exact match + product: { + view: { name: 'view_item' }, + add: { name: 'add_to_cart' }, + }, + + // Wildcard: any action + foo: { + '*': { name: 'foo_interaction' }, + }, + + // Wildcard: any entity + '*': { + bar: { name: 'generic_bar' }, + }, +}; +``` + +### Conditional Mapping + +Array of conditions, first match wins: + +```typescript +order: { + complete: [ + { + condition: (event) => event.data?.value > 100, + name: 'high_value_purchase', + }, + { name: 'purchase' }, // Fallback + ], +} +``` + +## Value Mapping Strategies + +### Key Extraction (string) + +Extract nested property from event: + +```typescript +'user.id'; // → event.user.id +'data.price'; // → event.data.price +'context.stage.0'; // → first element of stage array +``` + +### Static Value + +Fixed value regardless of event: + +```typescript +{ + value: 'USD'; +} +{ + value: 99.99; +} +{ + value: true; +} +``` + +### Function Transform + +Custom transformation logic: + +```typescript +{ + fn: (event) => event.data.price * 100; +} // cents +{ + fn: (event) => event.user.email?.split('@')[1]; +} // domain +``` + +### Object Map + +Transform to new structure: + +```typescript +{ + map: { + item_id: 'data.id', + item_name: 'data.name', + price: 'data.price', + currency: { value: 'USD' }, + category: { fn: (e) => e.nested?.[0]?.data?.name } + } +} +``` + +### Array Loop + +Process arrays (e.g., nested entities): + +```typescript +{ + loop: [ + 'nested', // Source array path + { + map: { + item_id: 'data.id', + quantity: 'data.quantity', + }, + }, + ]; +} +``` + +### Consent-Gated + +Only return value if consent granted: + +```typescript +{ + key: 'user.email', + consent: { marketing: true } +} +``` + +## Complete Example + +```typescript +const destinationConfig = { + mapping: { + product: { + view: { + name: 'view_item', + data: { + map: { + currency: { value: 'USD' }, + value: 'data.price', + items: { + loop: [ + 'nested', + { + map: { + item_id: 'data.id', + item_name: 'data.name', + }, + }, + ], + }, + }, + }, + }, + }, + }, +}; +``` + +## Example-Driven Development + +When creating sources or destinations, define mapping examples BEFORE +implementation: + +### 1. Create Input/Output Examples First + +```typescript +// src/examples/inputs.ts - What we receive +export const pageViewInput = { + event: 'page_view', + properties: { page_title: 'Home', page_path: '/home' }, +}; + +// src/examples/outputs.ts - What we must produce +export const pageViewOutput = { + method: 'track', + args: ['pageview', { url: '/home', title: 'Home' }], +}; +``` + +### 2. Define Mapping to Connect Them + +```typescript +// src/examples/mapping.ts +export const defaultMapping = { + page: { + view: { + name: 'pageview', + data: { + map: { + url: 'data.path', + title: 'data.title', + }, + }, + }, + }, +}; +``` + +### 3. Test Against Examples + +```typescript +test('produces expected output', () => { + const result = transform(examples.inputs.pageViewInput); + expect(result).toMatchObject(examples.outputs.pageViewOutput); +}); +``` + +**See:** + +- [create-destination skill](../create-destination/SKILL.md) - Full workflow +- [create-source skill](../create-source/SKILL.md) - Full workflow + +## Where Mapping Lives + +| Location | Purpose | +| ------------------------------ | ----------------------------------------- | +| Source config | Transform raw input → walkerOS events | +| Destination config | Transform walkerOS events → vendor format | +| `src/examples/mapping.ts` | Default mapping examples (example-driven) | +| `packages/core/src/mapping.ts` | Core mapping functions | +| `apps/quickstart/src/` | Validated examples | + +## Related + +**Skills:** + +- [understanding-events skill](../understanding-events/SKILL.md) - Event + structure to map from/to +- [understanding-sources skill](../understanding-sources/SKILL.md) - Source-side + mapping +- [understanding-destinations skill](../understanding-destinations/SKILL.md) - + Destination-side mapping + +**Source Files:** + +- [packages/core/src/mapping.ts](../../packages/core/src/mapping.ts) - + Implementation + +**Examples:** + +- [apps/quickstart/src/](../../apps/quickstart/src/) - Validated examples + +**Documentation:** + +- [Website: Mapping](../../website/docs/mapping.mdx) - User-facing docs +- [walkeros.io/docs/destinations/event-mapping](https://www.walkeros.io/docs/destinations/event-mapping) - + Public documentation diff --git a/skills/understanding-sources/SKILL.md b/skills/understanding-sources/SKILL.md new file mode 100644 index 000000000..e394f58d0 --- /dev/null +++ b/skills/understanding-sources/SKILL.md @@ -0,0 +1,115 @@ +--- +name: understanding-sources +description: + Use when working with sources, understanding event capture, or learning about + the push interface. Covers browser, dataLayer, and server source patterns. +--- + +# Understanding walkerOS Sources + +## Overview + +Sources capture events from the external world (browser DOM, dataLayer, HTTP +requests, cloud functions) and feed them to the collector. + +**Core principle:** Sources capture. They don't process or deliver—that's +collector and destinations. + +## Source Interface + +See [packages/core/src/types/source.ts](../../packages/core/src/types/source.ts) +for canonical interface. + +| Method | Purpose | +| ------------- | ----------------------------------- | +| `push(input)` | Receive external input, emit events | + +## Push Signatures by Type + +| Source Type | Signature | Example | +| -------------- | ----------------------------------- | ------------ | +| Cloud Function | `push(req, res) → Promise` | HTTP handler | +| Browser | `push(event, data) → Promise` | DOM events | +| DataLayer | `push(event, data) → Promise` | GTM-style | + +**Key insight:** Source `push` IS the handler. No wrappers needed. + +```typescript +// Direct deployment +http('handler', source.push); +``` + +## Source Paths + +| Type | Path | Examples | +| ------ | -------------------------- | ------------------ | +| Web | `packages/web/sources/` | browser, dataLayer | +| Server | `packages/server/sources/` | gcp | + +## Browser Source + +The browser source captures events from DOM using data attributes. + +```html + +``` + +See [packages/web/sources/browser/](../../packages/web/sources/browser/) for +implementation. + +## DataLayer Source + +Captures events from a GTM-style dataLayer array. + +```typescript +window.dataLayer.push({ + event: 'product view', + product: { id: 'P123', name: 'Laptop' }, +}); +``` + +See [packages/web/sources/dataLayer/](../../packages/web/sources/dataLayer/) for +implementation. + +## Server Sources + +Handle HTTP requests in cloud functions. + +```typescript +// GCP Cloud Function +export const handler = source.push; +``` + +See [packages/server/sources/gcp/](../../packages/server/sources/gcp/) for +implementation. + +## Related + +**Skills:** + +- [understanding-flow skill](../understanding-flow/SKILL.md) - How sources fit + in architecture +- [understanding-events skill](../understanding-events/SKILL.md) - Events that + sources emit + +**Source Files:** + +- [packages/core/src/types/source.ts](../../packages/core/src/types/source.ts) - + Interface + +**Package READMEs:** + +- [packages/web/sources/browser/README.md](../../packages/web/sources/browser/README.md) - + Browser source +- [packages/web/sources/dataLayer/README.md](../../packages/web/sources/dataLayer/README.md) - + DataLayer source + +**Documentation:** + +- [Website: Sources](../../website/docs/sources/index.mdx) - Overview +- [Website: Browser Source](../../website/docs/sources/web/browser/index.mdx) - + Browser docs +- [Website: Create Your Own](../../website/docs/sources/create-your-own.mdx) - + Guide diff --git a/skills/using-logger/SKILL.md b/skills/using-logger/SKILL.md new file mode 100644 index 000000000..f026b97d8 --- /dev/null +++ b/skills/using-logger/SKILL.md @@ -0,0 +1,334 @@ +--- +name: using-logger +description: + Use when working with sources/destinations to understand standard logging + patterns, replace console.log, or add logging to external API calls. Covers + DRY principles, when to log, and migration patterns. +--- + +# Using the walkerOS Logger + +## Overview + +The logger is walkerOS's standard logging system, available in all sources and +destinations via `env.logger` or `logger` parameter. It provides scoped, +level-aware logging that replaces console.log. + +**Core principle:** Don't log what the collector already logs. Only log +meaningful operations like external API calls, transformations, and validation +errors. + +## Logger Access + +### In Sources + +```typescript +export const sourceFetch = async ( + config: PartialConfig, + env: Types['env'], // env.logger is available here +): Promise => { + // Logger is scoped automatically by collector: [type:sourceId] + env.logger.info('Server listening on port 3000'); +}; +``` + +### In Destinations + +```typescript +export const destinationDataManager: DestinationInterface = { + async init({ config, env, logger }) { + // logger parameter is scoped automatically: [datamanager] + logger.debug('Auth client created'); + }, + + async push(event, { config, data, env, logger }) { + // logger parameter is scoped: [datamanager] + logger.debug('API response', { status: 200 }); + }, +}; +``` + +**Note:** You don't need to create or configure the logger—it's provided +automatically with proper scoping. + +## Logger Methods + +```typescript +interface Logger.Instance { + error(message: string | Error, context?: unknown | Error): void; + info(message: string | Error, context?: unknown | Error): void; + debug(message: string | Error, context?: unknown | Error): void; + throw(message: string | Error, context?: unknown): never; + scope(name: string): Logger.Instance; +} +``` + +### Log Levels + +- **ERROR (0)**: Always visible—use for errors only +- **INFO (1)**: High-level operations (server startup, event processed) +- **DEBUG (2)**: Low-level details (API calls, transformations) + +**Default**: ERROR only (must configure to see INFO/DEBUG) + +### Context Parameter + +All methods accept optional structured context: + +```typescript +logger.debug('Sending to API', { + endpoint: '/events', + method: 'POST', + eventCount: 5, +}); +// Output: DEBUG [datamanager] Sending to API { endpoint: '/events', method: 'POST', eventCount: 5 } +``` + +## When to Log (and When NOT to) + +### ❌ DON'T Log These (Collector Handles) + +- **Init status**: "Initializing...", "Init started", "Init complete" +- **Push status**: "Processing event...", "Event received" +- **Generic status**: "Settings validated", "Config loaded" +- **Duplicate scoping**: Don't add source/dest name (already in scope) + +**Why:** Collector can log these automatically since it calls init/push and has +scoped logger. + +### ✅ DO Log These (Meaningful Operations) + +- **External API calls**: Before/after with request/response details +- **Auth operations**: Token refresh, client creation/failures +- **Transformations**: Complex mappings or data processing +- **Validation errors**: Always use `logger.throw` for fatal errors + +## Usage Patterns + +### Pattern 1: Validation Errors (Always Use logger.throw) + +```typescript +// ✅ GOOD - Fatal configuration error +async init({ config, logger }) { + const { apiKey, projectId } = config.settings || {}; + + if (!apiKey) { + logger.throw('Config settings apiKey missing'); + } + + if (!projectId) { + logger.throw('Config settings projectId missing'); + } +} +``` + +**Why logger.throw:** + +- Logs the error at ERROR level (always visible) +- Throws Error automatically (no separate throw needed) +- Collector catches and handles gracefully +- Never returns (TypeScript type: `never`) + +### Pattern 2: External API Calls + +```typescript +// ✅ GOOD - Log external calls with context +async push(event, { config, logger }) { + const endpoint = 'https://api.vendor.com/events'; + + // Log before call + logger.debug('Calling API', { + endpoint, + method: 'POST', + eventId: event.id, + }); + + const response = await fetch(endpoint, { + method: 'POST', + body: JSON.stringify(event), + }); + + // Log after call + logger.debug('API response', { + status: response.status, + ok: response.ok, + }); + + if (!response.ok) { + const errorText = await response.text(); + logger.throw(`API error (${response.status}): ${errorText}`); + } +} +``` + +### Pattern 3: Auth Operations + +```typescript +// ✅ GOOD - Log auth client creation +async init({ config, logger }) { + try { + const authClient = await createAuthClient(config.settings); + logger.debug('Auth client created'); + + return { + env: { authClient }, + }; + } catch (error) { + logger.throw( + `Authentication failed: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } +} +``` + +### Pattern 4: Server Startup (Sources) + +```typescript +// ✅ GOOD - Log server listening (high-level info) +if (settings.port !== undefined) { + server = app.listen(settings.port, () => { + env.logger.info( + `Express source listening on port ${settings.port}\n` + + ` POST ${settings.path} - Event collection (JSON body)\n` + + ` GET ${settings.path} - Pixel tracking (query params)\n` + + ` OPTIONS ${settings.path} - CORS preflight`, + ); + }); +} +``` + +## Anti-Patterns (What NOT to Do) + +### ❌ BAD: Verbose Init Logging + +```typescript +async init({ logger }) { + logger.debug('Data Manager init started'); // Redundant + logger.info('Data Manager initializing...'); // Redundant + logger.debug('Settings validated'); // Redundant + + const authClient = await createAuthClient(); + logger.debug('Auth client created'); // OK + + logger.info('Data Manager ready'); // Redundant +} +``` + +**Problem:** Collector knows when init is called. Only log meaningful operations +(auth client creation). + +### ❌ BAD: Redundant Push Logging + +```typescript +async push(event, { logger }) { + logger.debug('Processing event', { + // Redundant + name: event.name, + id: event.id, + }); + + // Do work... + + logger.info('Event processed'); // Redundant +} +``` + +**Problem:** Collector knows when push is called and can log automatically. + +### ❌ BAD: Using console.log + +```typescript +// ❌ NEVER use console.log in sources/destinations +console.log('Processing event:', event.name); + +// ✅ Use logger instead +logger.debug('API call', { endpoint }); +``` + +## Migration Checklist + +When updating a source/destination to use the logger: + +- [ ] Remove all `console.log`, `console.warn`, `console.error` statements +- [ ] Remove verbose init/push status logging (let collector handle) +- [ ] Add `logger.throw` for all validation errors (apiKey missing, etc.) +- [ ] Add `logger.debug` before external API calls (with endpoint, method) +- [ ] Add `logger.debug` after external API calls (with response status) +- [ ] Add `logger.debug` for auth operations (client creation, token refresh) +- [ ] Use structured context (objects) instead of string concatenation +- [ ] Verify tests still pass with logger mocked + +## Testing with Logger + +Use `createMockLogger` from `@walkeros/core` in tests: + +```typescript +import { createMockLogger } from '@walkeros/core'; + +test('throws on missing apiKey', () => { + const logger = createMockLogger(); + + expect(() => { + destination.init({ config: {}, logger }); + }).toThrow('Config settings apiKey missing'); + + expect(logger.throw).toHaveBeenCalledWith('Config settings apiKey missing'); +}); +``` + +## Log Level Configuration + +Default log level is ERROR. To see INFO/DEBUG logs: + +```typescript +import { startFlow } from '@walkeros/collector'; + +const { elb } = await startFlow({ + logger: { + level: 'DEBUG', // Show all logs + }, + destinations: { + /* ... */ + }, +}); +``` + +**Levels:** + +- `'ERROR'`: Only errors (default) +- `'INFO'`: Errors + info +- `'DEBUG'`: Everything (errors + info + debug) + +## Related + +**Key Files:** + +- [packages/core/src/logger.ts](../../packages/core/src/logger.ts) - Logger + implementation +- [packages/core/src/types/logger.ts](../../packages/core/src/types/logger.ts) - + Logger types +- [packages/core/src/mockLogger.ts](../../packages/core/src/mockLogger.ts) - + Testing utilities + +**Best Practice Examples:** + +- [packages/server/destinations/datamanager/src/push.ts](../../packages/server/destinations/datamanager/src/push.ts) - + API logging patterns +- [packages/server/sources/express/src/index.ts](../../packages/server/sources/express/src/index.ts) - + Server startup logging + +**Needs Improvement:** + +- [packages/server/sources/fetch/src/index.ts](../../packages/server/sources/fetch/src/index.ts) - + Missing all logging +- [packages/server/destinations/meta/src/push.ts](../../packages/server/destinations/meta/src/push.ts) - + Missing push logging +- [packages/server/destinations/aws/src/firehose/push.ts](../../packages/server/destinations/aws/src/firehose/push.ts) - + Missing push logging + +**Skills:** + +- [understanding-destinations](../understanding-destinations/SKILL.md) - + Destination interface +- [understanding-sources](../understanding-sources/SKILL.md) - Source interface +- [testing-strategy](../testing-strategy/SKILL.md) - Testing with mockLogger diff --git a/skills/writing-documentation/SKILL.md b/skills/writing-documentation/SKILL.md new file mode 100644 index 000000000..cbba95192 --- /dev/null +++ b/skills/writing-documentation/SKILL.md @@ -0,0 +1,354 @@ +--- +name: writing-documentation +description: + Use when writing or updating any documentation - README, website docs, or + skills. Covers quality standards, example validation, and DRY patterns. +--- + +# Writing Documentation + +## When to Use This Skill + +- Creating a new package README +- Writing website documentation (MDX) +- Creating or updating skills +- Reviewing documentation for quality +- Documenting Phase 7 of create-destination or create-source + +## Prerequisites + +- [understanding-flow](../understanding-flow/SKILL.md) - Architecture context +- [understanding-events](../understanding-events/SKILL.md) - Event naming rules + +--- + +## Documentation Types + +### Where Content Belongs + +| Type | Purpose | Audience | +| ------------------ | --------------------------------------------- | --------------------------- | +| **Package README** | Installation, basic usage, API reference | Package users | +| **Website docs** | Guides, integration examples, detailed config | Integrators | +| **Skills** | Process knowledge, workflows | AI assistants, contributors | + +### Divio Documentation Types + +Keep these separate - don't mix tutorials with reference: + +| Type | Purpose | User State | +| ---------------- | ------------------ | ----------------------------- | +| **Tutorial** | Learning | Studying, beginner | +| **How-To Guide** | Problem-solving | Working, knows what they need | +| **Reference** | Information lookup | Working, needs facts | +| **Explanation** | Understanding | Studying, needs context | + +--- + +## Example Validation (CRITICAL) + +### The Problem + +AI-generated examples can be: + +- Syntactically correct but use non-existent APIs +- Plausible-looking but don't match actual exports +- Outdated, referencing deprecated patterns + +### Source of Truth Hierarchy + +```text +TIER 1: apps/quickstart/ + ✓ Tested ✓ Compiled ✓ CI-validated + → USE FOR: All code examples + +TIER 2: packages/core/src/eventGenerator.ts + ✓ Canonical events ✓ Real data structures + → USE FOR: Event examples + +TIER 3: packages/*/src/index.ts exports + ✓ Actual public API + → USE FOR: Verifying API names exist + +TIER 4: Package READMEs & Website docs + ⚠ May contain errors + → VERIFY against Tier 1-3 before trusting +``` + +### Validation Checklist + +Before publishing ANY code example: + +- [ ] **API exists?** Check `packages/*/src/index.ts` exports +- [ ] **Pattern validated?** Compare against `apps/quickstart/` +- [ ] **Events canonical?** Use patterns from `eventGenerator.ts` +- [ ] **Example compiles?** TypeScript check passes +- [ ] **Imports correct?** Package names match actual packages + +### Red Flags + +| Red Flag | What It Indicates | +| -------------------------------------- | ----------------------------------- | +| API name not in package exports | Hallucinated or outdated API | +| Import path doesn't match package.json | Wrong package reference | +| Event name with underscore | Wrong format (should be space) | +| No imports shown | Context missing, harder to validate | + +--- + +## DRY Patterns + +### PropertyTable for Configuration + +**When to use:** Any page documenting package configuration with Zod schemas. + +```mdx +import { schemas } from '@walkeros/web-destination-gtag/dev'; +; + +; +``` + +**When NOT to use:** + +- Pages without package configuration +- Reference tables (Logger API, CLI commands) +- Conceptual explanations + +### Schema Exports (dev.ts) + +Every destination/source should export schemas: + +```typescript +// src/dev.ts +export * as schemas from './schemas'; +export * as examples from './examples'; +``` + +### Don't Duplicate + +- Link to source files instead of copying type definitions +- Reference `apps/quickstart/` examples instead of writing from scratch +- Use PropertyTable instead of hardcoded markdown tables + +--- + +## Quality Checklist + +### Structure + +- [ ] Follows appropriate Divio type (Tutorial/How-To/Reference/Explanation) +- [ ] Code example within first 100 words +- [ ] First example under 20 lines +- [ ] Uses `
` for advanced content +- [ ] Has "Next Steps" or "Related" section + +### Content + +- [ ] All event names use `"entity action"` format with space +- [ ] Flow config shown as primary usage pattern +- [ ] Examples are complete and copy-pasteable +- [ ] Includes imports in code examples + +### AI Readability + +- [ ] Clear semantic headers (H2, H3, H4 hierarchy - no skipped levels) +- [ ] Tables for structured data +- [ ] Links to source of truth TypeScript files +- [ ] Static fallback content alongside dynamic components + +### Consistency + +- [ ] Uses standard table formats +- [ ] Follows package/skill/website templates +- [ ] Terminology matches: walkerOS, collector, destination, source + +--- + +## Templates + +### Package README Template + +````markdown +# @walkeros/[package-name] + +[1-sentence description] + +[Source Code](link) | [NPM](link) | [Documentation](link) + +## Quick Start + +```json +{ + "version": 1, + "flows": { + "default": { + "web": {}, + "[sources|destinations]": { + "[name]": { + "package": "@walkeros/[package-name]", + "config": { ... } + } + } + } + } +} +``` +```` + +## Features + +- **Feature 1**: Brief description +- **Feature 2**: Brief description + +## Installation + +```bash +npm install @walkeros/[package-name] +``` + +## Configuration Reference + +| Name | Type | Description | Required | Default | +| ---- | ---- | ----------- | -------- | ------- | + +## Examples + +### Basic + +[Simple example] + +
+Advanced: Custom Mapping + +[Complex example] + +
+ +## Type Definitions + +See [src/types.ts](./src/types.ts) for TypeScript interfaces. + +## Related + +- [Documentation](website/docs/...) + +```` + +### Website Doc Template (MDX) + +```mdx +--- +title: [Title] +description: [SEO description] +sidebar_position: [N] +--- + +# [Title] + + + +[1-sentence description] + +## Quick Start + +```json +// Flow config example (<15 lines) +```` + +## Features + +- **Feature 1**: Description + +## Installation + + + + ```bash + npm install @walkeros/[package] + ``` + + + +## Configuration + + + +## Next Steps + +- [Related guide 1](/docs/...) + +````mdx +--- + +## Priority Matrix + +### Issue Classification + +| Priority | Criteria | Action | +|----------|----------|--------| +| **P0 Critical** | Incorrect examples, wrong APIs, security issues | Fix immediately | +| **P1 High** | Missing PropertyTable, outdated domains, missing sections | Fix soon | +| **P2 Medium** | Inconsistent terminology, skipped headings | Plan to fix | +| **P3 Low** | Style issues, minor wording | Backlog | + +--- + +## Non-Negotiables + +### Event Naming + +```text + +CORRECT: "page view", "product add", "order complete" WRONG: "page_view", +"pageView", "PAGE VIEW" +``` +```` + +### Package References + +```text + +CORRECT: `@walkeros/collector` (with backticks) WRONG: @walkeros/collector (no +backticks) + +``` + +### Domain References + +```text + +CORRECT: `www.walkeros.io` or relative paths DO NOT USE: legacy domain +references + +``` + +--- + +## Process + +### For New Package Documentation + +1. **Verify examples exist** in `apps/quickstart/` or create them first +2. **Write README** using template above +3. **Write website doc** using MDX template +4. **Run quality checklist** +5. **Verify all code examples** against Tier 1-3 sources + +### For Documentation Updates + +1. **Identify issue priority** using matrix above +2. **Check current state** against source of truth +3. **Make minimal changes** - don't over-engineer +4. **Verify examples still compile** +5. **Run quality checklist** + +--- + +## Related + +- [← Back to Hub](../../AGENT.md) + +``` + +``` diff --git a/turbo.json b/turbo.json index b1e3ed02d..9daf51f0a 100644 --- a/turbo.json +++ b/turbo.json @@ -1,7 +1,7 @@ { "$schema": "https://turbo.build/schema.json", "globalDependencies": ["**/.env.*local"], - "concurrency": "3", + "concurrency": "1", "tasks": { "build": { "dependsOn": ["^build"], diff --git a/website/README.md b/website/README.md index 5a79e2673..ee9891727 100644 --- a/website/README.md +++ b/website/README.md @@ -1,5 +1,6 @@ # Website -Visit [www.elbwalker.com](https://www.elbwalker.com/) +Visit [www.walkeros.io](https://www.walkeros.io/) -This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator. +This website is built using [Docusaurus](https://docusaurus.io/), a modern +static website generator. diff --git a/website/docs/apps/cli.mdx b/website/docs/apps/cli.mdx index 6d56e2c77..672ecaa98 100644 --- a/website/docs/apps/cli.mdx +++ b/website/docs/apps/cli.mdx @@ -37,7 +37,7 @@ npx walkeros --version`} language="bash" /> -## Commands Overview +## Commands overview | Command | Purpose | Use Case | |---------|---------|----------| @@ -48,7 +48,7 @@ npx walkeros --version`} | `run serve` | Serve web bundles as static files | Host browser tracking scripts | | `cache` | Manage CLI package and build caches | Clear stale caches, view cache statistics | -## Configuration Types +## Configuration types The CLI uses types from `@walkeros/core`: @@ -58,7 +58,7 @@ The CLI uses types from `@walkeros/core`: The CLI transforms `Flow.Setup` → `Flow.Config` (per flow) → bundled code that uses `Collector.InitConfig` at runtime. -## Local Packages +## Local packages By default, the CLI downloads packages from npm. For development or testing unpublished packages, you can use local packages instead by specifying a `path` property. @@ -142,7 +142,7 @@ In this example, even though `@walkeros/collector` depends on `@walkeros/core`, When ready for production, simply remove the `path` property to use the published npm version. -## Getting Started +## Getting started Before using the CLI, you need a [flow configuration file](/docs/getting-started/flow). Here's a minimal example: @@ -182,7 +182,7 @@ Before using the CLI, you need a [flow configuration file](/docs/getting-started Save this as `flow.json`. -## Bundle Command +## Bundle command The `bundle` command builds production-ready JavaScript bundles from flow configurations. @@ -318,7 +318,7 @@ walkeros bundle config.json --all`} language="bash" /> -## Simulate Command +## Simulate command The `simulate` command tests your flow configuration with sample events without deploying to production. @@ -465,7 +465,7 @@ walkeros simulate flow.json --event-file events.json`} language="bash" /> -## Push Command +## Push command The `push` command executes your flow with a real event, making actual API calls to your configured destinations. Unlike `simulate` which mocks API calls for safe testing, `push` performs real HTTP requests—ideal for integration testing and production validation. @@ -659,7 +659,7 @@ walkeros push flow.json --event event.json --flow production --local`} language="bash" /> -## Run Collect Command +## Run collect command The `run collect` command starts an HTTP server that accepts events and processes them through your flow. @@ -822,7 +822,7 @@ walkeros run collect dist/bundle.mjs --port 8080`} language="bash" /> -## Run Serve Command +## Run serve command The `run serve` command serves web bundles as static files for browser-side tracking. @@ -948,7 +948,7 @@ Open `test.html` in your browser. Events will be sent to `http://localhost:8080/ | `--local` | Execute locally without Docker | | `-v, --verbose` | Detailed logging | -## Complete Example: Web → Server Flow +## Complete example: Web → Server flow This example demonstrates a complete analytics pipeline: - Browser events captured by web flow @@ -1087,7 +1087,7 @@ Open in browser. Terminal 1 shows: [Server Logger] custom event ``` -## Cache Command +## Cache command The `cache` command manages the CLI's package and build caches. @@ -1176,7 +1176,7 @@ This downloads fresh packages and rebuilds without using or updating the cache. | `--packages` | Clear only the package cache | | `--builds` | Clear only the build cache | -## Global Options +## Global options These options work with all commands: @@ -1194,7 +1194,7 @@ These options work with all commands: | `--help` | Show help for command | | `--version` | Show CLI version | -## Execution Modes +## Execution modes ### Docker Mode (Default) @@ -1230,7 +1230,7 @@ Use `--local` to execute in your current Node.js environment: **Requirements**: - Node.js 18+ installed locally -## CI/CD Integration +## CI/CD integration ### GitHub Actions @@ -1317,7 +1317,7 @@ lsof -ti:8080 | xargs kill`} language="bash" /> -## Next Steps +## Next steps - **[Flow Configuration](/docs/getting-started/flow)** - Learn about flow config structure - **[Docker Deployment](/docs/apps/docker)** - Deploy flows to production diff --git a/website/docs/apps/docker.mdx b/website/docs/apps/docker.mdx index c9fe51873..f0cc067f8 100644 --- a/website/docs/apps/docker.mdx +++ b/website/docs/apps/docker.mdx @@ -32,7 +32,7 @@ Think of it as the **deployment target**, not the build tool. - Docker installed ([Get Docker](https://docs.docker.com/get-docker/)) - (Optional) Pre-built flow bundle from the [CLI](/docs/apps/cli) -## Quick Start with Demos +## Quick start with demos The Docker image includes demo bundles for instant testing. @@ -310,7 +310,7 @@ services: language="yaml" /> -## Cloud Deployment +## Cloud deployment ### Google Cloud Run @@ -657,7 +657,7 @@ kubectl get services walkeros-collector`} language="bash" /> -## Health Checks +## Health checks The Docker container provides a `/health` endpoint: @@ -684,7 +684,7 @@ Use this for: - Load balancer health checks - Monitoring systems -## Monitoring and Logs +## Monitoring and logs ### View Docker Logs @@ -856,7 +856,7 @@ docker run -e FLOW=/app/flow.mjs walkeros/docker`} language="bash" /> -## Next Steps +## Next steps - **[CLI](/docs/apps/cli)** - Learn how to build flows - **[Flow Configuration](/docs/getting-started/flow)** - Understand flow structure diff --git a/website/docs/apps/walkerjs.mdx b/website/docs/apps/walkerjs.mdx index 0992c544e..ae4d8ffd0 100644 --- a/website/docs/apps/walkerjs.mdx +++ b/website/docs/apps/walkerjs.mdx @@ -36,9 +36,9 @@ complex setup or configuration. language="html" /> -## Basic Setup +## Basic setup -### 1. Add Event Queueing (Recommended) +### 1. Add event queueing (recommended) Add this script before walker.js loads to queue events during initialization: @@ -83,7 +83,7 @@ Add this script before walker.js loads to queue events during initialization: language="html" /> -## Configuration Options +## Configuration options Walker.js supports multiple configuration approaches with different priorities: @@ -234,7 +234,7 @@ value: 25.42, language="javascript" /> -## Advanced Features +## Advanced features ### Async Loading & Event Queueing @@ -286,7 +286,7 @@ pageview: true, language="javascript" /> -## Destination Configuration +## Destination configuration Configure multiple destinations for your events: @@ -329,7 +329,7 @@ For comprehensive destination options, see the **Configuration not applied:** Verify `data-elbconfig` points to the correct object name. -## API Reference +## API reference ### Factory Function @@ -359,7 +359,7 @@ const globals = getGlobals();`} language="javascript" /> -## Related Documentation +## Related documentation * **[Browser Source](/docs/sources/web/browser/)** - Detailed DOM tracking capabilities diff --git a/website/docs/collector/commands.mdx b/website/docs/collector/commands.mdx index a9ca4968b..c4d99900e 100644 --- a/website/docs/collector/commands.mdx +++ b/website/docs/collector/commands.mdx @@ -22,41 +22,31 @@ Commands. ## destination -Add destinations to the collector for event processing. Destinations handle the -actual delivery of events to third-party services. +Add destinations to the collector. The recommended approach is to configure +destinations during initialization with `startFlow()`: - -:::tip - -Destinations are typically configured during collector initialization: - -See destination-specific documentation for configuration options. +:::tip + +For dynamic scenarios requiring runtime destination addition, use +`elb('walker destination')` command or `collector.addDestination()`. See destination-specific documentation for +configuration options. ::: diff --git a/website/docs/collector/index.mdx b/website/docs/collector/index.mdx index c12de22e7..3ed1c1d45 100644 --- a/website/docs/collector/index.mdx +++ b/website/docs/collector/index.mdx @@ -75,9 +75,11 @@ browser: { }, destinations: { console: { - code: { - type: 'console', - push: (event) => console.log('Event:', event), + code: true, // Built-in code destination + config: { + settings: { + push: "console.log('Event:', event)", + }, }, }, }, diff --git a/website/docs/core/index.mdx b/website/docs/core/index.mdx index 3e0eced36..4f7091085 100644 --- a/website/docs/core/index.mdx +++ b/website/docs/core/index.mdx @@ -18,7 +18,7 @@ Import the core utilities directly from the `@walkeros/core` package: -## Data Manipulation +## Data manipulation ### assign @@ -115,7 +115,7 @@ getId(10); // Returns 10-character string`} language="javascript" /> -## Event Processing +## Event processing ### getMappingValue @@ -170,7 +170,7 @@ getMarketingParameters(url, { utm_custom: 'custom', partner: 'partnerId' });`} language="javascript" /> -## Type Validation +## Type validation ### Type Checkers @@ -187,7 +187,7 @@ A comprehensive set of type checking functions: * `filterValues(object)` - Filters object to valid properties only * `isPropertyType(value)` - Type guard for property validation -## Request Handling +## Request handling ### requestToData @@ -211,7 +211,7 @@ URL-encoded query strings. language="javascript" /> -## User Agent Parsing +## User agent parsing ### parseUserAgent @@ -232,7 +232,7 @@ Individual functions are also available: * `getOSVersion(userAgent)` - Returns OS version * `getDeviceType(userAgent)` - Returns 'Desktop', 'Tablet', or 'Mobile' -## Error Handling +## Error handling ### tryCatch @@ -259,7 +259,7 @@ async operations. language="javascript" /> -## Performance Optimization +## Performance optimization ### debounce diff --git a/website/docs/core/server.mdx b/website/docs/core/server.mdx index c9cc21503..9c77f78ab 100644 --- a/website/docs/core/server.mdx +++ b/website/docs/core/server.mdx @@ -18,7 +18,7 @@ Import server utilities from the `@walkeros/server-core` package: -## Server Communication +## Server communication ### sendServer @@ -72,7 +72,7 @@ error?: string; // Error message (if request failed) language="typescript" /> -## Cryptographic Operations +## Cryptographic operations ### getHashServer @@ -103,7 +103,7 @@ This function is commonly used for: * **Data Deduplication**: Creating consistent identifiers * **Privacy Compliance**: Hashing PII for GDPR/CCPA compliance -## Usage Examples +## Usage examples ### Event Processing Pipeline @@ -158,7 +158,7 @@ return await getHashServer(fingerprint, 20); language="javascript" /> -## Error Handling +## Error handling Server utilities include comprehensive error handling: @@ -180,7 +180,7 @@ console.error('Network error:', error.message); language="javascript" /> -## Performance Considerations +## Performance considerations ### Timeout Configuration @@ -222,7 +222,7 @@ timeout: 10000, The underlying Node.js HTTP agent automatically reuses connections for better performance with multiple requests to the same host. -## Security Notes +## Security notes * **HTTPS Only**: Use HTTPS URLs in production for encrypted transmission * **API Keys**: Store sensitive credentials in environment variables diff --git a/website/docs/core/web.mdx b/website/docs/core/web.mdx index 44db716c4..5ab834318 100644 --- a/website/docs/core/web.mdx +++ b/website/docs/core/web.mdx @@ -18,7 +18,7 @@ Import web utilities from the `@walkeros/web-core` package: -## DOM Utilities +## DOM utilities ### getAttribute @@ -64,7 +64,7 @@ configuration strings from HTML attributes. language="javascript" /> -## Browser Information +## Browser information ### getLanguage @@ -95,7 +95,7 @@ dimensions. language="javascript" /> -## Element Visibility +## Element visibility ### isVisible @@ -117,7 +117,7 @@ This function considers: * Parent element visibility * Intersection with the visible area -## Storage Management +## Storage management ### Storage Operations @@ -162,7 +162,7 @@ storageDelete('session_temp', 'sessionStorage');`} language="javascript" /> -## Session Management +## Session management ### sessionStart @@ -197,7 +197,7 @@ Session data includes: * `sessionStorage` - Session-specific storage operations * `sessionWindow` - Window/tab session management -## Web Communication +## Web communication ### sendWeb @@ -260,7 +260,7 @@ const response = sendWebAsXhr(url, data, { method: 'POST' });`} language="javascript" /> -## Web Hashing +## Web hashing ### getHashWeb @@ -277,7 +277,7 @@ navigator.userAgent + navigator.language + screen.width, language="javascript" /> -## Configuration Types +## Configuration types ### SendWebOptions @@ -311,7 +311,7 @@ sampling?: number; // Sampling rate (0-1) -## Usage Notes +## Usage notes * **Consent Required**: Browser information functions may require user consent depending on privacy regulations diff --git a/website/docs/destinations/code.mdx b/website/docs/destinations/code.mdx new file mode 100644 index 000000000..e2b665aad --- /dev/null +++ b/website/docs/destinations/code.mdx @@ -0,0 +1,281 @@ +--- +title: Code +description: Built-in destination for executing custom code strings +sidebar_position: 0 +--- + +# Code Destination + +The code destination is a built-in, platform-agnostic destination that executes +custom JavaScript code strings. It provides a lightweight alternative to tag +managers like GTM, allowing you to run arbitrary code in response to events +without external dependencies. + +## Setup + +Use `code: true` to enable the built-in code destination: + + + +## Configuration reference + +### Settings + +| Property | Type | Description | +| ----------- | -------- | ------------------------------------------------ | +| `init` | `string` | Code to run once when the destination initializes | +| `on` | `string` | Code to run on lifecycle events (consent, etc.) | +| `push` | `string` | Default code to run for each event | +| `pushBatch` | `string` | Default code to run for batched events | + +### Mapping + +Event-specific code can override settings via mapping: + +| Property | Type | Description | +| ----------- | -------- | ------------------------------------- | +| `push` | `string` | Code to run for this specific event | +| `pushBatch` | `string` | Code to run for batched events | + +## Context variables + +Each code string has access to specific variables: + +### init + +- `context.collector` - The collector instance +- `context.config` - Destination configuration +- `context.env` - Environment variables +- `context.logger` - Scoped logger instance + +### push + +- `event` - The WalkerOS event object +- `context.collector` - The collector instance +- `context.config` - Destination configuration +- `context.data` - Transformed event data (from mapping) +- `context.env` - Environment variables +- `context.logger` - Scoped logger instance +- `context.mapping` - The event mapping rule + +### pushBatch + +- `batch.key` - The batch key (event name) +- `batch.events` - Array of events in the batch +- `batch.data` - Array of transformed data +- `context.collector` - The collector instance +- `context.config` - Destination configuration +- `context.env` - Environment variables +- `context.logger` - Scoped logger instance +- `context.mapping` - The event mapping rule + +### on + +- `type` - The event type (`'consent'`, `'ready'`, etc.) +- `context.collector` - The collector instance +- `context.config` - Destination configuration +- `context.data` - Event-specific data +- `context.env` - Environment variables +- `context.logger` - Scoped logger instance + +## Examples + +### Basic logging + + + +### API calls + + + +### Consent handling + + + +### Event-specific overrides + +Use mapping to override the default push code for specific events: + + + +### Batched events + + ({ + name: e.name, + data: e.data + })) + }) + }) + \`, + }, + mapping: { + '*': { + '*': { + batch: 1000, // Batch events with 1 second debounce + }, + }, + }, + }, + }, + }, +});`} + language="typescript" +/> + +## Error handling + +All code execution is wrapped in try-catch blocks. Errors are logged using the +destination's scoped logger and don't affect other destinations or event +processing. + + + +## Security considerations + +The code destination uses `new Function()` to execute code strings. This is +similar to `eval()` and should only be used with trusted code. Never execute +user-provided code strings directly. + +For production environments, consider: + +- Only using code strings defined in your source code +- Validating and sanitizing any dynamic configuration +- Using Content Security Policy headers where appropriate diff --git a/website/docs/destinations/create-your-own.mdx b/website/docs/destinations/create-your-own.mdx index 03ed505ba..341e7c8b5 100644 --- a/website/docs/destinations/create-your-own.mdx +++ b/website/docs/destinations/create-your-own.mdx @@ -8,12 +8,12 @@ sidebar_position: 99 This guide provides the essentials for building a custom walkerOS destination. -## What is a Destination? +## What is a destination? A destination is a function that receives events from walkerOS and sends them to an external service, such as an analytics platform, an API, or a database. -## The Destination Interface +## The destination interface A destination is an object that implements the `Destination` interface. The most important property is the `push` function, which is called for every event. @@ -50,7 +50,7 @@ settings?: Settings; language="typescript" /> -## Example: A Simple Webhook Destination +## Example: A simple webhook destination Here is an example of a simple destination that sends events to a webhook URL. @@ -84,7 +84,7 @@ body: JSON.stringify(event), language="typescript" /> -## Schema Validation (Optional) +## Schema validation (optional) Destinations can export Zod schemas to provide runtime validation and TypeScript IDE support for configuration options. Export a `schemas` namespace containing @@ -156,7 +156,7 @@ config: { language="typescript" /> -## Advanced Example: Session Management +## Advanced example: Session management Here's a more advanced example that demonstrates session handling and cleanup: @@ -209,7 +209,7 @@ body: JSON.stringify(event), language="typescript" /> -## TypeScript Integration +## TypeScript integration To get full TypeScript support for your destination's configuration, you can extend the `WalkerOS.Destinations` interface. @@ -229,7 +229,7 @@ webhook: Destination.Config; language="typescript" /> -## Environment Dependencies (Testing) +## Environment dependencies (testing) The `env` parameter enables dependency injection for external APIs and SDKs. This allows you to test your destination logic without making actual API calls diff --git a/website/docs/destinations/index.mdx b/website/docs/destinations/index.mdx index 97a187dc4..7d42f832c 100644 --- a/website/docs/destinations/index.mdx +++ b/website/docs/destinations/index.mdx @@ -36,9 +36,16 @@ Each destination operates independently, so one failed destination won't affect others. This ensures reliable data delivery even when individual services have issues. -## Types of Destinations +## Types of destinations -### Web Destinations +### Built-in destinations + +Platform-agnostic destinations included in the collector: + +* **Code** - Execute custom JavaScript + for maximum flexibility without external dependencies + +### Web destinations Client-side integrations that send events directly from the browser: @@ -53,7 +60,7 @@ Client-side integrations that send events directly from the browser: * **Piwik PRO** - Privacy-focused analytics platform -### Server Destinations +### Server destinations Server-side integrations for enhanced privacy and data control: diff --git a/website/docs/destinations/server/aws.mdx b/website/docs/destinations/server/aws.mdx index 784d914d7..d577d9f56 100644 --- a/website/docs/destinations/server/aws.mdx +++ b/website/docs/destinations/server/aws.mdx @@ -25,22 +25,29 @@ data lakes, and downstream processing. ## Setup diff --git a/website/docs/destinations/server/datamanager.mdx b/website/docs/destinations/server/datamanager.mdx index 9078bc318..4161c999e 100644 --- a/website/docs/destinations/server/datamanager.mdx +++ b/website/docs/destinations/server/datamanager.mdx @@ -25,21 +25,32 @@ through a single unified API endpoint. ## Setup @@ -42,7 +49,7 @@ tableId: 'YOUR_TABLE_ID', -## Default Table Schema +## Default table schema By default, the destination sends the full walkerOS event to BigQuery. All object and array fields are JSON stringified before insertion. @@ -69,7 +76,7 @@ and array fields are JSON stringified before insertion. | `version` | STRING | JSON stringified version | | `source` | STRING | JSON stringified source | -### Create Table Query +### Create table query Use this SQL query to create the default table schema in BigQuery: @@ -98,7 +105,7 @@ Use this SQL query to create the default table schema in BigQuery: language="sql" /> -## Custom Schema Mapping +## Custom schema mapping You can send a custom schema by using the `data` configuration to map specific fields. This is useful when you only need a subset of the event data. @@ -108,21 +115,28 @@ fields. This is useful when you only need a subset of the event data. This example sends only `name`, `id`, `data`, and `timestamp`: diff --git a/website/docs/destinations/web/api.mdx b/website/docs/destinations/web/api.mdx index d02a71740..a1a4e6ada 100644 --- a/website/docs/destinations/web/api.mdx +++ b/website/docs/destinations/web/api.mdx @@ -201,7 +201,7 @@ const { collector, elb } = await startFlow({ language="typescript" /> -## Transport Methods +## Transport methods - **fetch** (default): Modern, promise-based HTTP requests - **xhr**: Traditional XMLHttpRequest for older browser compatibility diff --git a/website/docs/destinations/web/gtag/ads.mdx b/website/docs/destinations/web/gtag/ads.mdx index 186094b69..2c798def0 100644 --- a/website/docs/destinations/web/gtag/ads.mdx +++ b/website/docs/destinations/web/gtag/ads.mdx @@ -39,21 +39,26 @@ function calls. code={`import { startFlow } from '@walkeros/collector'; import { destinationGtag } from '@walkeros/web-destination-gtag'; -const { elb } = await startFlow(); - -elb('walker destination', destinationGtag, { - settings: { - ads: { - conversionId: 'AW-XXXXXXXXX', // Required - currency: 'EUR', // Default currency for conversions - }, - }, - mapping: { - order: { - complete: { +const { elb } = await startFlow({ + destinations: { + gtag: { + code: destinationGtag, + config: { settings: { ads: { - label: 'purchase_conversion', // Conversion label for order complete + conversionId: 'AW-XXXXXXXXX', // Required + currency: 'EUR', // Default currency for conversions + }, + }, + mapping: { + order: { + complete: { + settings: { + ads: { + label: 'purchase_conversion', // Conversion label for order complete + }, + }, + }, }, }, }, diff --git a/website/docs/destinations/web/gtag/ga4.mdx b/website/docs/destinations/web/gtag/ga4.mdx index 53fd45be9..5be342490 100644 --- a/website/docs/destinations/web/gtag/ga4.mdx +++ b/website/docs/destinations/web/gtag/ga4.mdx @@ -39,12 +39,17 @@ function calls. code={`import { startFlow } from '@walkeros/collector'; import { destinationGtag } from '@walkeros/web-destination-gtag'; -const { elb } = await startFlow(); - -elb('walker destination', destinationGtag, { - settings: { - ga4: { - measurementId: 'G-XXXXXXXXXX', +await startFlow({ + destinations: { + gtag: { + code: destinationGtag, + config: { + settings: { + ga4: { + measurementId: 'G-XXXXXXXXXX', + }, + }, + }, }, }, });`} diff --git a/website/docs/destinations/web/gtag/gtm.mdx b/website/docs/destinations/web/gtag/gtm.mdx index dba486215..e0c1ea286 100644 --- a/website/docs/destinations/web/gtag/gtm.mdx +++ b/website/docs/destinations/web/gtag/gtm.mdx @@ -35,13 +35,18 @@ and add third-party integrations to your website. code={`import { startFlow } from '@walkeros/collector'; import { destinationGtag } from '@walkeros/web-destination-gtag'; -const { elb } = await startFlow(); - -elb('walker destination', destinationGtag, { - settings: { - gtm: { - containerId: 'GTM-XXXXXXX', // Required - dataLayer: 'dataLayer', // Optional custom dataLayer name +await startFlow({ + destinations: { + gtag: { + code: destinationGtag, + config: { + settings: { + gtm: { + containerId: 'GTM-XXXXXXX', // Required + dataLayer: 'dataLayer', // Optional custom dataLayer name + }, + }, + }, }, }, });`} diff --git a/website/docs/destinations/web/piwikpro.mdx b/website/docs/destinations/web/piwikpro.mdx index d84a4c073..e47d46b64 100644 --- a/website/docs/destinations/web/piwikpro.mdx +++ b/website/docs/destinations/web/piwikpro.mdx @@ -40,12 +40,17 @@ function calls. code={`import { startFlow } from '@walkeros/collector'; import { destinationPiwikPro } from '@walkeros/web-destination-piwikpro'; -const { elb } = await startFlow(); - -elb('walker destination', destinationPiwikPro, { - settings: { - appId: 'XXX-XXX-XXX-XXX-XXX', // Required - url: 'https://your_account_name.piwik.pro/', // Required +await startFlow({ + destinations: { + piwikpro: { + code: destinationPiwikPro, + config: { + settings: { + appId: 'XXX-XXX-XXX-XXX-XXX', // Required + url: 'https://your_account_name.piwik.pro/', // Required + }, + }, + }, }, });`} language="typescript" diff --git a/website/docs/destinations/web/plausible.mdx b/website/docs/destinations/web/plausible.mdx index 311a76476..5ceb622d3 100644 --- a/website/docs/destinations/web/plausible.mdx +++ b/website/docs/destinations/web/plausible.mdx @@ -39,11 +39,16 @@ care of loading the script and the function calls. code={`import { startFlow } from '@walkeros/collector'; import { destinationPlausible } from '@walkeros/web-destination-plausible'; -const { elb } = await startFlow(); - -elb('walker destination', destinationPlausible, { - settings: { - domain: 'elbwalker.com', // Optional, domain of your site as registered +await startFlow({ + destinations: { + plausible: { + code: destinationPlausible, + config: { + settings: { + domain: 'walkeros.io', // Optional, domain of your site as registered + }, + }, + }, }, });`} language="typescript" @@ -56,7 +61,7 @@ elb('walker destination', destinationPlausible, { @@ -77,7 +82,7 @@ events manually using the JavaScript function