feat: Implement Unified TAP Framework with 8+ Multi-Vendor Adapters

.gitignore

@@ -52,3 +52,11 @@ credentials/
 *.swp
 
 .pancake_db_port
+
+farmers_registry.json
+# --- Local Testing Data ---
+# Keeps the test output out of the repo
+pancake_data_lake/
+jhon-tap.py
+test_sync.py
+

implementation/tap_adapter_base.py

@@ -16,8 +16,12 @@
 from typing import Dict, List, Any, Optional
 from datetime import datetime
 from enum import Enum
+from dotenv import load_dotenv
 import importlib
 import yaml
+import os
+import json
+load_dotenv()
 
 
 class SIRUPType(Enum):
@@ -32,16 +36,23 @@ class SIRUPType(Enum):
     PEST_DISEASE = "pest_disease"
     MARKET_PRICE = "market_price"
     CUSTOM = "custom"
+    WEATHER_DATA = "weather_data"
+    OEM_DATA = "oem_data"
+    FINANCIAL_BENCHMARK = "financial_benchmark"
+    SOIL_DATA = "soil_data"
+
 
 
 class AuthMethod(Enum):
     """Supported authentication methods"""
-    NONE = "none"  # No authentication required
+    NONE = "none"
     API_KEY = "api_key"
     OAUTH2 = "oauth2"
     BASIC = "basic"
     BEARER_TOKEN = "bearer_token"
     CUSTOM = "custom"
+    PUBLIC = "public"
+    CUSTOM_HMAC = "custom_hmac"
 
 
 class TAPAdapter(ABC):
@@ -94,11 +105,44 @@ def __init__(self, config: Dict[str, Any]):
 
         # Initialize vendor-specific state
         self._initialize()
+
+
+
+    def get_bite(self, geoid: str, params: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """The standard Pancake orchestration flow"""
+        # 1. Fetch raw data from the vendor
+        raw_data = self.get_vendor_data(geoid, params)
+        if not raw_data:
+            return None
+
+        # 2. Transform raw data into a SIRUP payload
+        # Grabs the first sirup type defined in your YAML/Enum
+        sirup_type = self.sirup_types[0] if hasattr(self, 'sirup_types') and self.sirup_types else None
+        sirup = self.transform_to_sirup(raw_data, sirup_type)
+        if not sirup:
+            return None
+
+        # 3. Wrap the SIRUP into a BITE envelope
+        return self.sirup_to_bite(sirup, geoid, params)
 
     def _initialize(self):
         """Optional: Vendor-specific initialization (auth, validation, etc.)"""
         pass
 
+    def load_registry(self) -> Dict[str, Any]:
+        """Generic helper to load farmer tokens/data"""
+        path = 'farmers_registry.json'
+        if not os.path.exists(path): return {}
+        with open(path, 'r') as f:
+            try: return json.load(f)
+            except: return {}
+
+    def save_registry(self, registry: Dict[str, Any]):
+        """Generic helper to save farmer tokens/data"""
+        path = 'farmers_registry.json'
+        with open(path, 'w') as f:
+            json.dump(registry, f, indent=4)
+
     @abstractmethod
     def get_vendor_data(self, geoid: str, params: Dict[str, Any]) -> Optional[Dict[str, Any]]:
         """
@@ -196,6 +240,24 @@ def get_capabilities(self) -> Dict[str, Any]:
             "metadata": self.metadata
         }
 
+class OAuth2TAPAdapter(TAPAdapter):
+    """Base class for all OAuth2-based OEM adapters (JD, CNH, etc.)"""
+
+    def load_registry(self) -> Dict[str, Any]:
+        registry_path = 'farmers_registry.json'
+        if os.path.exists(registry_path):
+            with open(registry_path, 'r') as f:
+                return json.load(f)
+        return {}
+
+    def save_registry(self, registry: Dict[str, Any]):
+        with open('farmers_registry.json', 'w') as f:
+            json.dump(registry, f, indent=4)
+
+    @abstractmethod
+    def refresh_token(self, farmer_id: str) -> bool:
+        """Each vendor has a different refresh URL/logic"""
+        pass
 
 class TAPAdapterFactory:
     """
@@ -222,6 +284,24 @@ def __init__(self, config_path: str = None):
 
         if config_path:
             self.load_from_config(config_path)
+
+    def _inject_env_vars(self, config_data):
+        """Recursively replaces ${VAR} with environment variables."""
+        if isinstance(config_data, dict):
+            for k, v in config_data.items():
+                if isinstance(v, str) and v.startswith("${") and v.endswith("}"):
+                    env_var = v[2:-1]
+                    # Update with env value, fallback to original if not found
+                    config_data[k] = os.getenv(env_var, v)
+                else:
+                    self._inject_env_vars(v)
+        elif isinstance(config_data, list):
+            for i in range(len(config_data)):
+                if isinstance(config_data[i], (dict, list)):
+                    self._inject_env_vars(config_data[i])
+                elif isinstance(config_data[i], str) and config_data[i].startswith("${"):
+                    env_var = config_data[i][2:-1]
+                    config_data[i] = os.getenv(env_var, config_data[i])
 
     def load_from_config(self, config_path: str):
         """
@@ -244,6 +324,7 @@ def load_from_config(self, config_path: str):
         """
         with open(config_path, 'r') as f:
             config = yaml.safe_load(f)
+        self._inject_env_vars(config)
 
         for vendor_config in config.get('vendors', []):
             self.register_adapter(vendor_config)
@@ -262,17 +343,21 @@ def register_adapter(self, config: Dict[str, Any]):
             raise ValueError("vendor_name and adapter_class are required")
 
         # Dynamically import the adapter class
-        module_path, class_name = adapter_class_path.rsplit('.', 1)
-        module = importlib.import_module(module_path)
-        adapter_class = getattr(module, class_name)
-
-        # Instantiate the adapter
-        adapter = adapter_class(config)
-        self.adapters[vendor_name] = adapter
-
-        print(f"✓ Registered TAP adapter: {vendor_name}")
-        print(f"  SIRUP types: {[t.value for t in adapter.sirup_types]}")
-
+        try:
+            module_path, class_name = adapter_class_path.rsplit('.', 1)
+            module = importlib.import_module(module_path)
+            adapter_class = getattr(module, class_name)
+
+            # Instantiate the adapter
+            adapter = adapter_class(config)
+            self.adapters[vendor_name] = adapter
+
+            print(f"✓ Registered TAP adapter: {vendor_name}")
+            print(f"  SIRUP types: {[t.value for t in adapter.sirup_types]}")
+
+        except (ImportError, AttributeError, ModuleNotFoundError) as e:
+            print(f"⚠️  Skipping vendor '{vendor_name}': {e}")    
+
     def get_adapter(self, vendor_name: str) -> Optional[TAPAdapter]:
         """Get adapter by vendor name"""
         return self.adapters.get(vendor_name)

implementation/tap_adapters/John_Deer_README.md

@@ -0,0 +1,68 @@
+# John Deere TAP Adapter
+
+This directory contains the production-ready implementation of the John Deere adapter for the **Third-party Agentic-Pipeline (TAP)**. This adapter enables the automated discovery of organizations and machinery assets from the John Deere Operations Center, standardizing them into the SIRUP/BITE format used by the PANCAKE ecosystem.
+
+---
+
+## 🚀 Overview
+
+The John Deere adapter is designed to bridge the gap between proprietary OEM data and standardized agricultural intelligence. It handles the complexities of OAuth2 authentication, token rotation, and multi-version API endpoints.
+
+### Key Capabilities
+- **Automated Token Management**: Implements proactive and reactive OAuth2 token refresh logic.
+- **Multi-Endpoint Discovery**: Support for both modern `/equipment` and legacy `/machines` endpoints ensures compatibility across different organization types.
+- **Asset Standardization**: Transforms raw JSON into SIRUP `oem_data`, mapping specific fields like `modelName` and `vin` to a unified asset structure.
+
+---
+
+## 🛠 Detailed Implementation Steps
+
+We followed a modular approach to ensure the adapter is robust and maintainable. Below are the specific steps taken during development:
+
+### 1. Base Class Integration
+We inherited from the `TAPAdapter` base class in `tap_adapter_base.py`. This enforced a standard interface for fetching, transforming, and packaging data.
+
+### 2. OAuth2 with Token Rotation
+Because John Deere access tokens are short-lived, we implemented a sophisticated authentication handler:
+* **Registry Integration**: The adapter loads credentials and tokens from a local `farmers_registry.json` file.
+* **401 Unauthorized Handling**: If an API call fails with a `401`, the adapter automatically triggers `refresh_token()`, updates the local registry with new tokens, and retries the original request seamlessly.
+
+### 3. Smart Data Discovery (`get_vendor_data`)
+The adapter performs a two-stage discovery process:
+* **Organization Fetching**: It first retrieves all organizations the authenticated user has access to.
+* **Asset Probing**: For each organization, it attempts to fetch data from the latest `/equipment` endpoint. If that fails or returns no data, it falls back to the `/machines` endpoint to ensure no machinery is missed.
+
+### 4. SIRUP & BITE Transformation
+To make the data "AI-ready," we implemented two transformation layers:
+* **`transform_to_sirup`**: Normalizes various machine attributes (ID, Brand, Model, Serial Number) into a flat, predictable JSON structure.
+* **`sirup_to_bite`**: Wraps the normalized data in a BITE (Basic Intelligence Terminal Entity) packet, complete with unique ULID headers, metadata, and cryptographic hashes for data integrity.
+
+---
+
+## 📂 Project Structure
+
+| File | Purpose |
+| :--- | :--- |
+| `johndeere_adapter.py` | Main logic for JD API interaction and data transformation. |
+| `tap_adapter_base.py` | The universal interface and factory for all TAP adapters. |
+| `tap_vendors.yaml` | Configuration file where the JD adapter is registered with its API base URL and credentials. |
+| `TESTING.md` | Comprehensive guide for running unit and integration tests. |
+
+---
+
+## 🔧 Configuration
+
+To enable the adapter, ensure your `tap_vendors.yaml` includes the following entry:
+
+```yaml
+- vendor_name: johndeere
+  adapter_class: tap_adapters.johndeere_adapter.JohnDeereAdapter
+  base_url: [https://sandboxapi.deere.com/platform](https://sandboxapi.deere.com/platform)
+  auth_method: oauth2
+  credentials:
+    client_id: ${DEERE_CLIENT_ID}
+    client_secret: ${DEERE_CLIENT_SECRET}
+
+
+🧪 Testing
+For detailed instructions on verifying the implementation—including how to use the john-tap.py utility for initial authorization—please refer to the TESTING.md file.

implementation/tap_adapters/John_deer_TESTING.md

@@ -0,0 +1,105 @@
+# John Deere Adapter: Testing & Integration Guide
+
+This document provides developers and maintainers with the steps required to verify the John Deere TAP adapter, ranging from mocked unit tests to real-world integration.
+
+---
+
+## 1. Functional Overview
+The John Deere adapter integrates with the Operations Center to discover machinery and organization data.
+* **Discovery**: Probes both `/equipment` and `/machines` endpoints for cross-version compatibility.
+* **Authentication**: Implements OAuth2 with automated token rotation.
+* **Standardization**: Maps raw JSON to the SIRUP `oem_data` and BITE standard formats.
+
+---
+
+## 2. Unit Testing (No API Key Required)
+Reviewers can verify the transformation logic and the 401-refresh trigger without an API account. These tests use a mock API response to ensure stability.
+
+**Run command:**
+```bash
+python3 -m unittest implementation.tests.test_johndeere_adapter
+```
+
+---
+
+## 3. Integration Testing (Real API Access)
+To test the full lifecycle (OAuth2 refresh -> API Fetch -> BITE Storage), you can use the consolidated script below. This single block handles credentials, creates the necessary auth script, and runs the pipeline.
+
+**Copy and run this entire block in your terminal:**
+
+```bash
+# --- STEP A: EXPORT CREDENTIALS ---
+# Replace these with your actual John Deere developer keys
+export DEERE_CLIENT_ID='your_client_id'
+export DEERE_CLIENT_SECRET='your_client_secret'
+
+# --- STEP B: CREATE THE AUTH UTILITY (jhon-tap.py) ---
+cat << 'EOF' > john-tap.py
+import os
+import json
+from requests_oauthlib import OAuth2Session
+
+# 1. Configuration
+client_id = os.getenv('DEERE_CLIENT_ID')
+client_secret = os.getenv('DEERE_CLIENT_SECRET')
+auth_url = "[https://signin.johndeere.com/oauth2/aus78av9p4u0uW7sj357/v1/authorize](https://signin.johndeere.com/oauth2/aus78av9p4u0uW7sj357/v1/authorize)"
+token_url = "[https://signin.johndeere.com/oauth2/aus78av9p4u0uW7sj357/v1/token](https://signin.johndeere.com/oauth2/aus78av9p4u0uW7sj357/v1/token)"
+scope = ['ag1', 'eq1', 'offline_access']
+
+# 2. Authorization Request
+deere = OAuth2Session(client_id, scope=scope, redirect_uri='http://localhost:8080/callback')
+authorization_url, state = deere.authorization_url(auth_url)
+
+print(f'\n1. Please authorize here: {authorization_url}')
+redirect_response = input('2. Paste the full redirect URL here: ')
+
+# 3. Token Exchange
+token = deere.fetch_token(token_url, client_secret=client_secret, authorization_response=redirect_response)
+
+# 4. Save to Registry
+registry = {
+    "Test_FARMS_001": {
+        "access_token": token['access_token'],
+        "refresh_token": token['refresh_token']
+    }
+}
+
+with open('farmers_registry.json', 'w') as f:
+    json.dump(registry, f, indent=4)
+
+print("✓ Created farmers_registry.json")
+EOF
+
+
+# --- STEP C: CREATE TEST RUNNER (test_sync.py) ---
+cat << 'EOF' > test_sync.py
+import os, json
+from datetime import datetime
+from implementation.tap_adapter_base import TAPAdapterFactory, SIRUPType
+factory = TAPAdapterFactory('implementation/tap_vendors.yaml')
+adapter = factory.get_adapter('johndeere')
+with open('farmers_registry.json', 'r') as f:
+    farmers = json.load(f)
+os.makedirs('pancake_data_lake', exist_ok=True)
+for f_id in farmers:
+    bite = adapter.fetch_and_transform("TEST_FIELD_001", SIRUPType.CUSTOM, {"farmer_id": f_id})
+    if bite:
+        path = f"pancake_data_lake/{f_id}_test.json"
+        with open(path, 'w') as f: json.dump(bite, f, indent=4)
+        print(f"🚀 SUCCESS: Data stored at {path}")
+EOF
+
+# --- STEP D: EXECUTE ---
+python3 jhon-tap.py
+python3 test_sync.py
+```
+
+### Expected Output
+```text
+✓ Registered TAP adapter: johndeere
+🔎 Found 1 farmers in registry.
+
+📡 Starting sync for PRUDHVI_FARMS_001...
+🚀 SUCCESS: PRUDHVI_FARMS_001 data is now AI-ready in Pancake.
+📂 Stored at: ./pancake_data_lake/TEST_FARMS_001_20260203.json
+```

implementation/tap_adapters/__init__.py

@@ -0,0 +1,3 @@
+from .johndeere_adapter import JohnDeereAdapter
+from .cnh_industrial_adapter import CNHIndustrialAdapter
+from .usda_nass_adapter import USDANASSAdapter