New database layer and data dashboard

.env.example

@@ -0,0 +1,17 @@
+# Database Configuration
+# Copy this file to .env and fill in your values
+
+# Database Backend Selection
+# Options: 'postgres' or 'duckdb'
+DB_BACKEND=duckdb
+
+# PostgreSQL Configuration (used when DB_BACKEND=postgres)
+POSTGRES_DB=tracking_analytics
+POSTGRES_USER=dima
+POSTGRES_PASSWORD=
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+
+# DuckDB Configuration (used when DB_BACKEND=duckdb)
+DUCKDB_PATH=tracking.duckdb
+DUCKDB_READ_ONLY=false

.gcloudignore

@@ -0,0 +1,87 @@
+# .gcloudignore - Exclude files from Cloud Build uploads
+# This works like .dockerignore but for gcloud builds submit
+
+# Data directories (IMPORTANT - exclude large data files)
+# Use / prefix to match only at root level
+/data/
+/simulated_data/
+/trained_models/
+/meshes/
+*.duckdb
+*.duckdb.wal
+
+# Git
+.git/
+.gitignore
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+
+# Virtual environments
+.venv/
+.venv-*/
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Documentation
+docs/
+*.md
+*.rst
+
+# Tests
+tests/
+.pytest_cache/
+
+# Notebooks
+*.ipynb
+.ipynb_checkpoints/
+
+# Cache
+.cache/
+*.cache
+
+# Models and weights (if you have any)
+*.pth
+*.pt
+*.pkl
+*.h5
+*.onnx
+
+# Docker (keep only what we need)
+Dockerfile
+.dockerignore
+
+# Misc
+.DS_Store
+*.log
+*.tmp
+
+# Keep only essential files for dashboard deployment
+!requirements-db.txt
+!pyproject.toml
+!README.rst
+!collab_env/
+!schema/
+!scripts/deploy/
+
+.mypy_cache/
+submodules/
+sim-output/
+config-local/
+results/
+analysis/
+.ruff_cache/

TESTING_SAFETY_README.md

@@ -0,0 +1,59 @@
+# 🚨 TESTING SAFETY - READ THIS FIRST! 🚨
+
+## CRITICAL WARNING
+
+**Database tests will DROP ALL TABLES and DELETE ALL DATA!**
+
+## Before Running Tests
+
+### ✅ SAFE: Using Test Database
+```bash
+export POSTGRES_DB=tracking_analytics_test  # Ends with _test
+pytest tests/db/
+```
+
+### ❌ DANGEROUS: Using Production Database
+```bash
+export POSTGRES_DB=tracking_analytics  # Production database!
+pytest tests/db/  # ← This will DELETE ALL YOUR DATA!
+```
+
+## Safety Checks
+
+Tests include automatic safety checks that will REFUSE to run if:
+
+1. Database name is `tracking_analytics`, `production`, `prod`, or `main`
+2. Database name doesn't end with `_test`
+
+**If tests are skipped, this is PROTECTING YOUR DATA!**
+
+## Setup Test Database
+
+```bash
+# Create test database
+createdb tracking_analytics_test
+
+# Configure environment
+export POSTGRES_DB=tracking_analytics_test
+export POSTGRES_USER=your_user
+export POSTGRES_PASSWORD=your_password
+
+# Now tests are safe to run
+pytest tests/db/ -v
+```
+
+## Full Documentation
+
+See [docs/data/db/testing_safety.md](docs/data/db/testing_safety.md) for complete testing guidelines.
+
+## Quick Reference
+
+| Database Name | Tests Will Run? | Safe? |
+|---------------|-----------------|-------|
+| `tracking_analytics_test` | ✅ Yes | ✅ Safe |
+| `mydb_test` | ✅ Yes | ✅ Safe |
+| `tracking_analytics` | ❌ BLOCKED | 🚨 Would destroy data |
+| `production` | ❌ BLOCKED | 🚨 Would destroy data |
+| `mydb` | ❌ BLOCKED | 🚨 No _test suffix |
+
+**When in doubt, tests being skipped is GOOD - it means your production data is protected!**

collab_env/dashboard/analysis_widgets.yaml

@@ -0,0 +1,58 @@
+# Analysis Widget Configuration
+#
+# This file defines which analysis widgets are enabled and their default parameters.
+# Widgets are loaded in order and appear as tabs in the spatial analysis GUI.
+
+# Default shared parameters (used across all widgets unless overridden)
+defaults:
+  spatial_bin_size: 10.0        # Spatial discretization for heatmaps
+  temporal_window_size: 10     # Time window for windowed analyses
+  min_samples: 100              # Minimum samples for statistical validity (matches QueryBackend default)
+
+# Widget registry
+# Each widget must specify:
+#   - class: Full Python module path to widget class
+#   - enabled: true/false to enable/disable widget
+#   - order: Display order (lower numbers appear first)
+#   - category: Grouping category (spatial, temporal, behavioral)
+
+widgets:
+  - class: collab_env.dashboard.widgets.basic_data_viewer_widget.BasicDataViewerWidget
+    enabled: true
+    order: 0
+    category: visualization
+    description: "Episode viewer with animated agent tracks and spatial density heatmap"
+
+  - class: collab_env.dashboard.widgets.extended_properties_viewer_widget.ExtendedPropertiesViewerWidget
+    enabled: true
+    order: 1
+    category: visualization
+    description: "Time series and histogram visualization of extended properties"
+
+  - class: collab_env.dashboard.widgets.velocity_widget.VelocityStatsWidget
+    enabled: true
+    order: 2
+    category: temporal
+    description: "Comprehensive velocity statistics: individual speed, mean velocity magnitude, relative velocity magnitude"
+
+  - class: collab_env.dashboard.widgets.distance_widget.DistanceStatsWidget
+    enabled: true
+    order: 3
+    category: spatial
+    description: "Relative locations: pairwise distances between agents"
+
+  - class: collab_env.dashboard.widgets.correlation_widget.CorrelationWidget
+    enabled: false
+    order: 4
+    category: behavioral
+    description: "Agent velocity correlations"
+
+# Example: To disable a widget, set enabled to false
+#   - class: collab_env.dashboard.widgets.correlation_widget.CorrelationWidget
+#     enabled: false
+
+# Example: To add a custom widget
+#   - class: my_package.widgets.custom_widget.CustomWidget
+#     enabled: true
+#     order: 5
+#     category: custom

collab_env/dashboard/spatial_analysis_app.py

@@ -0,0 +1,24 @@
+"""
+Entry point for Spatial Analysis Dashboard.
+
+Run with:
+    panel serve collab_env/dashboard/spatial_analysis_app.py --show --port 5008 --static-dirs dashboard-static=collab_env/dashboard/static
+
+Development mode with autoreload:
+    panel serve collab_env/dashboard/spatial_analysis_app.py --dev --show --port 5008 --static-dirs dashboard-static=collab_env/dashboard/static
+
+Note: The --static-dirs argument maps the dashboard's static files (JS/CSS)
+      to the /dashboard-static/ URL route. This is required for the animation viewer.
+"""
+
+import panel as pn
+import holoviews as hv
+from collab_env.dashboard.spatial_analysis_gui import create_app
+
+# Enable Panel extensions (removed "modal" - not needed for HTML-based overlay)
+pn.extension("tabulator", "plotly")
+hv.extension("plotly", "bokeh")  # plotly first for Scatter3D support
+
+# Create and serve the app
+app = create_app()
+app.servable()