diff --git a/.gitignore b/.gitignore index def949e6..30cb72a1 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ internal/contentviewer/dist/* # Claude Code session files .claude/ + +# Resolved dev ports written by dev/start.sh (auto-relocation) +dev/.dev-ports.env diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index ca288775..d13e8b7f 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -20,7 +20,9 @@ services: # connects via localhost, and an all-interfaces 0.0.0.0 bind both exposes # the DB on the LAN and collides with any other specific-address :5432 # listener (e.g. a kubefwd-forwarded cluster Postgres on a 127.x alias). - - "127.0.0.1:5432:5432" + # Host port is auto-relocated by dev/start.sh (DEV_PG_PORT) when 5432 is + # held by another stack; container port stays 5432. Defaults to 5432. + - "127.0.0.1:${DEV_PG_PORT:-5432}:5432" volumes: - acme_postgres_data:/var/lib/postgresql/data # Init script creates the mcp_test and apitest databases used by the @@ -123,7 +125,7 @@ services: command: 'server -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -dir=/data -volume.max=5' ports: # Loopback only, matching the other dev services (see postgres above). - - "127.0.0.1:9000:8333" + - "127.0.0.1:${DEV_S3_PORT:-9000}:8333" volumes: - acme_seaweed_data:/data - ./seaweedfs-s3.json:/etc/seaweedfs/s3.json:ro @@ -170,7 +172,7 @@ services: image: ollama/ollama:0.30.6@sha256:fbaaecc6c632a73c2b4ca25ef52dbee0b68d07783fcb850f916404c004040f90 container_name: acme-dev-ollama ports: - - "127.0.0.1:11434:11434" + - "127.0.0.1:${DEV_OLLAMA_PORT:-11434}:11434" volumes: - acme_ollama_data:/root/.ollama healthcheck: diff --git a/dev/info.sh b/dev/info.sh index 76bfaa12..dfac3801 100755 --- a/dev/info.sh +++ b/dev/info.sh @@ -7,6 +7,17 @@ # of truth is dev/platform.yaml (api_keys) and the dev Keycloak realm. set -euo pipefail +# Load the ports resolved by the last `make dev`. start.sh auto-relocates the +# Go API and Ollama (among others) when their defaults are held by another +# stack; without these the banner below would print stale URLs. Falls back to +# the defaults when the file is absent (dev never started this session). +DEV_API_PORT=8080 +DEV_OLLAMA_PORT=11434 +if [ -f dev/.dev-ports.env ]; then + # shellcheck disable=SC1091 + source dev/.dev-ports.env +fi + # Colors only when stdout is a terminal, so `make dev-info | pbcopy` and # log redirects stay clean. if [ -t 1 ]; then @@ -18,12 +29,12 @@ fi echo -e "${BOLD}${GREEN}Local dev login${NC}" echo "" echo -e " Portal UI: ${CYAN}http://localhost:5173/portal/${NC}" -echo -e " Go API: ${CYAN}http://localhost:8080${NC}" +echo -e " Go API: ${CYAN}http://localhost:${DEV_API_PORT}${NC}" echo -e " API key: ${CYAN}acme-dev-key-2024${NC} (send as ${BOLD}X-API-Key${NC} header)" echo "" echo -e " ${BOLD}Portal sign-in (Keycloak OIDC)${NC}" echo -e " ${CYAN}admin@example.com${NC} / ${CYAN}admin-password${NC} (dp_admin)" echo -e " ${CYAN}analyst@example.com${NC} / ${CYAN}analyst-password${NC} (dp_analyst)" echo -e " Keycloak admin console: ${CYAN}http://localhost:9090/admin${NC} (admin / admin)" -echo -e " Ollama embedder: ${CYAN}http://localhost:11434${NC} (model: nomic-embed-text — powers semantic indexing)" +echo -e " Ollama embedder: ${CYAN}http://localhost:${DEV_OLLAMA_PORT}${NC} (model: nomic-embed-text — powers semantic indexing)" echo "" diff --git a/dev/platform.yaml b/dev/platform.yaml index 1c21ed38..d6ab3d28 100644 --- a/dev/platform.yaml +++ b/dev/platform.yaml @@ -13,7 +13,9 @@ apiVersion: v1 server: name: acme-data-platform transport: http - address: ":8080" + # DEV_API_PORT is auto-relocated by dev/start.sh when :8080 is held by + # another stack (e.g. a running DataHub). Defaults to 8080 when free. + address: ":${DEV_API_PORT:-8080}" description: | ACME Corp's unified data platform providing governed access to the enterprise data warehouse (Trino/Iceberg), data catalog (DataHub), and object storage (S3). @@ -99,7 +101,7 @@ oauth: enabled: false database: - dsn: "postgres://platform:platform_secret@localhost:5432/mcp_platform?sslmode=disable" + dsn: "postgres://platform:platform_secret@localhost:${DEV_PG_PORT:-5432}/mcp_platform?sslmode=disable" max_open_conns: 10 # Six ACME personas @@ -283,7 +285,7 @@ toolkits: instances: dev-s3: enabled: true - endpoint: "http://localhost:9000" + endpoint: "http://localhost:${DEV_S3_PORT:-9000}" region: "us-east-1" access_key_id: "dev-access-key" secret_access_key: "dev-secret-key" @@ -317,7 +319,7 @@ tuning: portal: enabled: true - public_base_url: "http://localhost:8080" + public_base_url: "http://localhost:${DEV_API_PORT:-8080}" s3_connection: dev-s3 s3_bucket: portal-assets @@ -325,7 +327,7 @@ memory: embedding: provider: ollama ollama: - url: "http://localhost:11434" + url: "http://localhost:${DEV_OLLAMA_PORT:-11434}" model: "nomic-embed-text" knowledge: diff --git a/dev/seed-s3.sh b/dev/seed-s3.sh index aea03722..e2562494 100755 --- a/dev/seed-s3.sh +++ b/dev/seed-s3.sh @@ -6,7 +6,8 @@ # This prevents clearing generated thumbnails on every dev restart. set -euo pipefail -API="http://localhost:8080" +# DEV_API_PORT is exported by dev/start.sh (auto-relocated when 8080 is busy). +API="http://localhost:${DEV_API_PORT:-8080}" API_KEY="acme-dev-key-2024" CONTENT_DIR="dev/seed-content" @@ -33,3 +34,23 @@ upload "asset-003" "$CONTENT_DIR/asset-003.jsx" upload "asset-004" "$CONTENT_DIR/asset-004.md" upload "asset-005" "$CONTENT_DIR/asset-005.svg" upload "asset-006" "$CONTENT_DIR/asset-006.html" + +# Content for the 120 generated demo assets (seed-0001..0120 from seed.sql). +# Reuse the five content files, cycled by the same (n % 5) index used in the +# SQL so each generated asset's bytes match its content_type. Uploaded in +# parallel batches so this stays a few seconds, not a minute of serial curls. +# The GET-404 skip in upload() makes reruns a no-op (preserves thumbnails). +GENERATED_CONTENT=( + "$CONTENT_DIR/asset-001.html" # n % 5 == 0 -> text/html + "$CONTENT_DIR/asset-002.csv" # n % 5 == 1 -> text/csv + "$CONTENT_DIR/asset-003.jsx" # n % 5 == 2 -> text/jsx + "$CONTENT_DIR/asset-004.md" # n % 5 == 3 -> text/markdown + "$CONTENT_DIR/asset-005.svg" # n % 5 == 4 -> image/svg+xml +) +for n in $(seq 1 120); do + id=$(printf 'seed-%04d' "$n") + upload "$id" "${GENERATED_CONTENT[$((n % 5))]}" & + # Cap concurrency so we do not open 120 sockets at once. + if [ "$((n % 16))" -eq 0 ]; then wait; fi +done +wait diff --git a/dev/seed.sql b/dev/seed.sql index d2207bc6..541ad5ea 100644 --- a/dev/seed.sql +++ b/dev/seed.sql @@ -11,7 +11,8 @@ -- 50 fresh audit events within the last hour (new each restart) -- 8 knowledge insights in various states -- 2 knowledge changesets (1 applied, 1 rolled back) --- 6 portal assets with versions and shares +-- 6 curated portal assets (with versions and shares) + 120 generated demo +-- assets so the library exercises pagination (126 total under "Mine") -- 3 portal collections with sections, items, and a public share -- ============================================================================ @@ -561,6 +562,36 @@ ON CONFLICT (id) DO UPDATE SET content_type = EXCLUDED.content_type, tags = EXCLUDED.tags; +-- Bulk demo assets (120) so the portal library has enough volume to exercise +-- pagination / "Load more" out of the box — six assets never surfaced the +-- 50-per-page behaviour. All owned by apikey:admin (the acme-dev-key-2024 API +-- key, so they show under "Mine"), content type cycled through the five kinds, +-- created_at staggered by the hour so created_at DESC ordering is stable. +-- Content is uploaded per asset by seed-s3.sh (reusing the five content files), +-- so the placeholder s3_key here is replaced by a real versioned key on upload. +INSERT INTO portal_assets ( + id, owner_id, owner_email, name, description, content_type, + s3_bucket, s3_key, size_bytes, tags, provenance, session_id, + current_version, created_at, updated_at +) +SELECT + 'seed-' || lpad(n::text, 4, '0'), + 'apikey:admin', 'admin@apikey.local', + (ARRAY['Sales Dashboard','Inventory Export','Store Comparison','Pipeline Notes','Regional Heatmap'])[(n % 5) + 1] + || ' ' || lpad(n::text, 4, '0'), + 'Generated demo asset #' || n || ' — seeded so the portal library exercises pagination.', + (ARRAY['text/html','text/csv','text/jsx','text/markdown','image/svg+xml'])[(n % 5) + 1], + 'portal-assets', + 'portal/apikey:admin/seed-' || lpad(n::text, 4, '0') || '/v1/content' + || (ARRAY['.html','.csv','.jsx','.md','.svg'])[(n % 5) + 1], + (ARRAY[4280, 15720, 6340, 2890, 8150])[(n % 5) + 1], + jsonb_build_array('demo', (ARRAY['sales','inventory','stores','pipeline','regional'])[(n % 5) + 1]), + '{"tool": "save_artifact", "session_id": "sess-demo"}'::jsonb, + 'sess-demo-' || n, 1, + NOW() - (n || ' hours')::interval, NOW() - (n || ' hours')::interval +FROM generate_series(1, 120) AS n +ON CONFLICT (id) DO NOTHING; + -- Asset versions (one per asset, matching the current_version=1) INSERT INTO portal_asset_versions ( id, asset_id, version, s3_key, s3_bucket, content_type, size_bytes, diff --git a/dev/start.sh b/dev/start.sh index cd7268e5..7d1a8cae 100755 --- a/dev/start.sh +++ b/dev/start.sh @@ -107,16 +107,67 @@ if [ ! -d ui/node_modules ]; then fi ok "UI dependencies ready" -# Port checks (8080 = platform, 5173 = vite, 5432 = postgres, -# 9000 = seaweedfs, 9090 = keycloak, 9091 = prometheus, 9180 = dev-mcp-mock -# OAuth, 9181 = dev-mcp-mock MCP, 9281 = mcp-test fixture, 9282 = api-test -# fixture, 9464 = platform /metrics scrape endpoint, 11434 = ollama embedder) -for port in 5432 8080 5173 9000 9090 9091 9180 9181 9281 9282 9464 11434; do +# ─── Auto-relocate the conflicting host ports ─────────────────────── +# +# The four infrastructure ports below are the ones that commonly collide +# with OTHER stacks a developer runs alongside `make dev` — a leftover e2e +# Postgres/SeaweedFS on 5432/9000, a DataHub quickstart on 8080, a native +# Ollama on 11434. Those neighbours bind 0.0.0.0, so a specific-IP bind on +# the same port number cannot dodge them: the only reliable fix is to move +# to different port NUMBERS. When any of the four is busy we shift all four +# together by a uniform offset to the first free window, and thread the +# resolved ports through docker-compose, platform.yaml, the fixtures, Vite, +# and every curl below via the exported DEV_*_PORT vars. +# +# Keycloak (9090), Vite (5173), Prometheus (9091), the metrics endpoint +# (9464), and the mock/fixtures (9180/9181/9281/9282) are deliberately NOT +# relocated: they are rarely contended, and their ports are baked into the +# Keycloak realm's OIDC redirect URIs and the Prometheus scrape target. +# Moving the API port stays invisible to OIDC because the browser always +# reaches the server through Vite's proxy on :5173. +port_free() { ! lsof -nP -iTCP:"$1" -sTCP:LISTEN > /dev/null 2>&1; } + +RELOC_BASE=(5432 8080 9000 11434) # pg, api, s3, ollama +DEV_OFFSET=0 +NEED_SHIFT=0 +for p in "${RELOC_BASE[@]}"; do port_free "$p" || NEED_SHIFT=1; done +if [ "$NEED_SHIFT" = 1 ]; then + for off in 20000 30000 40000 50000; do + all_free=1 + for p in "${RELOC_BASE[@]}"; do port_free $((p + off)) || { all_free=0; break; }; done + if [ "$all_free" = 1 ]; then DEV_OFFSET=$off; break; fi + done + if [ "$DEV_OFFSET" = 0 ]; then + fail "Could not find a free port window for the dev stack (tried offsets 20000-50000). Stop a conflicting stack or free ports 5432/8080/9000/11434." + fi +fi +export DEV_PG_PORT=$((5432 + DEV_OFFSET)) +export DEV_API_PORT=$((8080 + DEV_OFFSET)) +export DEV_S3_PORT=$((9000 + DEV_OFFSET)) +export DEV_OLLAMA_PORT=$((11434 + DEV_OFFSET)) +# Persist the resolved ports so `make dev-info` (a separate process that does +# not inherit these exports) reprints the correct, possibly-relocated URLs. +# Gitignored; overwritten each run. +cat > dev/.dev-ports.env < /dev/null 2>&1; then fail "$(port_conflict_msg "$port")" fi done -ok "Ports 5432, 8080, 5173, 9000, 9090, 9091, 9180, 9181, 9281, 9282, 9464, 11434 are free" +ok "Dev ports free (pg:$DEV_PG_PORT api:$DEV_API_PORT vite:5173 s3:$DEV_S3_PORT ollama:$DEV_OLLAMA_PORT + keycloak/prometheus/fixtures)" echo "" @@ -145,7 +196,7 @@ for i in $(seq 1 60); do fi sleep 1 done -ok "PostgreSQL ready on :5432" +ok "PostgreSQL ready on :$DEV_PG_PORT" # Idempotently ensure the auxiliary databases exist. The fixture init # script creates these on first volume bring-up; this block handles @@ -171,7 +222,7 @@ docker compose -f dev/docker-compose.yml up -d 2>&1 | grep -v "^$" | sed 's/^/ # Wait for SeaweedFS (S3 returns 403 on GET /, so check for any HTTP response) info "Waiting for SeaweedFS..." for i in $(seq 1 30); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9000/ 2>/dev/null || echo "000") + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$DEV_S3_PORT/ 2>/dev/null || echo "000") if [ "$HTTP_CODE" != "000" ]; then break fi @@ -180,7 +231,7 @@ for i in $(seq 1 30); do fi sleep 1 done -ok "SeaweedFS ready on :9000" +ok "SeaweedFS ready on :$DEV_S3_PORT" # Wait for Prometheus (powers the admin Dashboard's API Gateway tab via # the platform's PromQL proxy). Scrapes the platform's /metrics on the @@ -248,14 +299,14 @@ ok "api-test fixture ready on :9282" info "Waiting for Ollama embedder..." OLLAMA_READY="" for i in $(seq 1 60); do - if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then + if curl -sf http://localhost:$DEV_OLLAMA_PORT/api/tags > /dev/null 2>&1; then OLLAMA_READY=1 break fi sleep 1 done if [ -n "$OLLAMA_READY" ]; then - ok "Ollama ready on :11434" + ok "Ollama ready on :$DEV_OLLAMA_PORT" # Pull the embedding model the platform is configured to use. Cached in the # acme_ollama_data volume after the first run, so this is a fast no-op on # subsequent `make dev` runs. @@ -276,9 +327,9 @@ if which aws > /dev/null 2>&1; then info "Creating S3 bucket..." for i in $(seq 1 30); do if AWS_ACCESS_KEY_ID=dev-access-key AWS_SECRET_ACCESS_KEY=dev-secret-key \ - aws --endpoint-url http://localhost:9000 s3 ls s3://portal-assets 2>/dev/null || \ + aws --endpoint-url http://localhost:$DEV_S3_PORT s3 ls s3://portal-assets 2>/dev/null || \ AWS_ACCESS_KEY_ID=dev-access-key AWS_SECRET_ACCESS_KEY=dev-secret-key \ - aws --endpoint-url http://localhost:9000 s3 mb s3://portal-assets 2>/dev/null; then + aws --endpoint-url http://localhost:$DEV_S3_PORT s3 mb s3://portal-assets 2>/dev/null; then break fi if [ "$i" -eq 30 ]; then @@ -328,7 +379,7 @@ PIDS+=($!) # Wait for server health info "Building and starting server (this may take a moment on first run)..." for i in $(seq 1 60); do - if curl -sf http://localhost:8080/healthz > /dev/null 2>&1; then + if curl -sf http://localhost:$DEV_API_PORT/healthz > /dev/null 2>&1; then break fi if [ "$i" -eq 60 ]; then @@ -338,7 +389,7 @@ for i in $(seq 1 60); do fi sleep 1 done -ok "Go server ready on :8080" +ok "Go server ready on :$DEV_API_PORT" echo "" @@ -370,7 +421,7 @@ DEVMOCK_HTTP=$(curl -s -o /dev/null -w "%{http_code}" -X PUT \ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$DEVMOCK_BODY" \ - http://localhost:8080/api/v1/admin/connection-instances/mcp/dev-mock || echo "000") + http://localhost:$DEV_API_PORT/api/v1/admin/connection-instances/mcp/dev-mock || echo "000") if [ "$DEVMOCK_HTTP" = "200" ] || [ "$DEVMOCK_HTTP" = "201" ]; then ok "dev-mock gateway connection registered (tools: dev-mock__echo, dev-mock__add, dev-mock__now)" else @@ -405,7 +456,7 @@ MCPTEST_HTTP=$(curl -s -o /dev/null -w "%{http_code}" -X PUT \ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$MCPTEST_BODY" \ - http://localhost:8080/api/v1/admin/connection-instances/mcp/mcp-test-fixture || echo "000") + http://localhost:$DEV_API_PORT/api/v1/admin/connection-instances/mcp/mcp-test-fixture || echo "000") if [ "$MCPTEST_HTTP" = "200" ] || [ "$MCPTEST_HTTP" = "201" ]; then ok "mcp-test fixture connection registered (tools: mcp-test-fixture__*)" else @@ -461,7 +512,7 @@ print(json.dumps({ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$APITEST_CATALOG_BODY" \ - http://localhost:8080/api/v1/admin/api-catalogs || echo "000") + http://localhost:$DEV_API_PORT/api/v1/admin/api-catalogs || echo "000") case "$CATALOG_HTTP" in 200|201) ok "api-test-fixture catalog created" ;; 409) ok "api-test-fixture catalog already exists (reusing)" ;; @@ -481,7 +532,7 @@ print(json.dumps({ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$APITEST_SPEC_BODY" \ - "http://localhost:8080/api/v1/admin/api-catalogs/${APITEST_CATALOG_ID}/specs/default" || echo "000") + "http://localhost:$DEV_API_PORT/api/v1/admin/api-catalogs/${APITEST_CATALOG_ID}/specs/default" || echo "000") case "$SPEC_HTTP" in 200|201|204) ok "default spec upserted into api-test-fixture catalog" @@ -524,7 +575,7 @@ APITEST_HTTP=$(curl -s -o /dev/null -w "%{http_code}" -X PUT \ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$APITEST_BODY" \ - http://localhost:8080/api/v1/admin/connection-instances/api/api-test-fixture || echo "000") + http://localhost:$DEV_API_PORT/api/v1/admin/connection-instances/api/api-test-fixture || echo "000") if [ "$APITEST_HTTP" = "200" ] || [ "$APITEST_HTTP" = "201" ]; then if [ "$APITEST_CATALOG_READY" = "1" ]; then ok "api-test fixture connection registered with catalog ${APITEST_CATALOG_ID} (api_list_endpoints will resolve)" @@ -549,7 +600,7 @@ OAUTH_MCP_HTTP=$(curl -s -o /dev/null -w "%{http_code}" -X PUT \ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$OAUTH_MCP_BODY" \ - http://localhost:8080/api/v1/admin/connection-instances/mcp/oauth-mcp-dev || echo "000") + http://localhost:$DEV_API_PORT/api/v1/admin/connection-instances/mcp/oauth-mcp-dev || echo "000") if [ "$OAUTH_MCP_HTTP" = "200" ] || [ "$OAUTH_MCP_HTTP" = "201" ]; then ok "oauth-mcp-dev connection registered (kind=mcp, authorization_code, IdP :9180)" else @@ -587,7 +638,7 @@ OAUTH_API_HTTP=$(curl -s -o /dev/null -w "%{http_code}" -X PUT \ -H "X-API-Key: acme-dev-key-2024" \ -H "Content-Type: application/json" \ -d "$OAUTH_API_BODY" \ - http://localhost:8080/api/v1/admin/connection-instances/api/oauth-api-dev || echo "000") + http://localhost:$DEV_API_PORT/api/v1/admin/connection-instances/api/oauth-api-dev || echo "000") if [ "$OAUTH_API_HTTP" = "200" ] || [ "$OAUTH_API_HTTP" = "201" ]; then ok "oauth-api-dev connection registered (kind=api, authorization_code, IdP :9180)" else @@ -637,7 +688,7 @@ print(json.dumps({"config": cfg, "description": "Dev demo: %s (api-test fixture) ') code=$(curl -s -o /dev/null -w "%{http_code}" -X PUT \ -H "X-API-Key: acme-dev-key-2024" -H "Content-Type: application/json" \ - -d "$body" "http://localhost:8080/api/v1/admin/connection-instances/api/$name" || echo "000") + -d "$body" "http://localhost:$DEV_API_PORT/api/v1/admin/connection-instances/api/$name" || echo "000") case "$code" in 200|201) ok "demo connection '$name' registered" ;; *) echo -e " ${YELLOW}⚠${NC} demo connection '$name' register returned HTTP $code" ;; @@ -653,12 +704,12 @@ gw() { # conn method path [bodyjson] curl -s --max-time 5 -o /dev/null -X POST \ -H "X-API-Key: acme-dev-key-2024" -H "Content-Type: application/json" \ -d "{\"method\":\"$2\",\"path\":\"$3\"${4:+,\"body\":$4}}" \ - "http://localhost:8080/api/v1/gateway/$1/invoke" 2>/dev/null || true + "http://localhost:$DEV_API_PORT/api/v1/gateway/$1/invoke" 2>/dev/null || true } gw_bad() { # conn -> malformed body, shim returns 400 (status_class 4xx) curl -s --max-time 5 -o /dev/null -X POST \ -H "X-API-Key: acme-dev-key-2024" -H "Content-Type: application/json" \ - -d 'not-json' "http://localhost:8080/api/v1/gateway/$1/invoke" 2>/dev/null || true + -d 'not-json' "http://localhost:$DEV_API_PORT/api/v1/gateway/$1/invoke" 2>/dev/null || true } gw_burst() { # conn gw "$1" GET /v1/whoami; gw "$1" GET /v1/headers; gw "$1" GET /v1/fixed/demo @@ -680,7 +731,7 @@ gw_burst() { # conn code=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" -X POST \ -H "X-API-Key: acme-dev-key-2024" -H "Content-Type: application/json" \ -d '{"method":"GET","path":"/v1/whoami"}' \ - "http://localhost:8080/api/v1/gateway/$c/invoke" 2>/dev/null || echo "000") + "http://localhost:$DEV_API_PORT/api/v1/gateway/$c/invoke" 2>/dev/null || echo "000") [ "$code" = "200" ] && break sleep 1 done @@ -717,7 +768,10 @@ echo "" # ─── Start Vite dev server ────────────────────────────────────────── echo -e "${BOLD}Starting Vite UI${NC}" -(cd ui && npm run dev -- --clearScreen false 2>&1) & +# VITE_API_TARGET points Vite's /api and /portal/auth proxies at the (possibly +# relocated) Go server. Vite itself stays on :5173 so the Keycloak realm's +# OIDC redirect URIs (localhost:5173) keep matching regardless of the API port. +(cd ui && VITE_API_TARGET="http://localhost:$DEV_API_PORT" npm run dev -- --clearScreen false 2>&1) & PIDS+=($!) # Wait for Vite @@ -741,8 +795,13 @@ echo -e "${BOLD}${GREEN} Development environment ready${NC}" echo -e "${BOLD}${GREEN}══════════════════════════════════════════${NC}" echo "" echo -e " Portal UI: ${CYAN}http://localhost:5173/portal/${NC}" -echo -e " Go API: ${CYAN}http://localhost:8080${NC}" +echo -e " Go API: ${CYAN}http://localhost:$DEV_API_PORT${NC}" +echo -e " Postgres: ${CYAN}localhost:$DEV_PG_PORT${NC} (platform / platform_secret, db mcp_platform)" +echo -e " S3 (SeaweedFS): ${CYAN}http://localhost:$DEV_S3_PORT${NC}" echo -e " API Key: ${CYAN}acme-dev-key-2024${NC}" +if [ "$DEV_OFFSET" != 0 ]; then + echo -e " ${YELLOW}Ports relocated +$DEV_OFFSET${NC} (5432/8080/9000/11434 were busy) — coexisting with your other stacks." +fi echo -e " Metrics: ${CYAN}http://localhost:9464/metrics${NC} (scraped by Prometheus)" echo -e " Prometheus: ${CYAN}http://localhost:9091${NC}" echo "" diff --git a/ui/src/api/admin/hooks/personas-assets.ts b/ui/src/api/admin/hooks/personas-assets.ts index 543b56f0..602ee163 100644 --- a/ui/src/api/admin/hooks/personas-assets.ts +++ b/ui/src/api/admin/hooks/personas-assets.ts @@ -1,5 +1,6 @@ import { useQuery, + useInfiniteQuery, useMutation, useQueryClient, keepPreviousData, @@ -14,7 +15,14 @@ import type { AdminAssetListResponse, } from "../types"; import type { Asset, AssetVersion, PaginatedResponse } from "@/api/portal/types"; -import { REFETCH_INTERVAL, ADMIN_LARGE_ASSET_THRESHOLD } from "./shared"; +import { + ASSET_PAGE_SIZE, + assetKey, + nextOffset, + useInfiniteResult, + type InfiniteAssetsResult, +} from "@/api/portal/hooks/assets"; +import { ADMIN_LARGE_ASSET_THRESHOLD } from "./shared"; // --------------------------------------------------------------------------- // Personas @@ -102,26 +110,34 @@ export function useTestPersonaAccess() { interface AdminAssetsParams { search?: string; - limit?: number; - offset?: number; } -export function useAdminAssets(params: AdminAssetsParams = {}) { - const searchParams = new URLSearchParams(); - if (params.search) searchParams.set("search", params.search); - if (params.limit) searchParams.set("limit", String(params.limit)); - if (params.offset) searchParams.set("offset", String(params.offset)); - - const qs = searchParams.toString(); - return useQuery({ - queryKey: ["admin", "assets", params], - queryFn: () => - apiFetch( - `/assets${qs ? `?${qs}` : ""}`, - ), - refetchInterval: REFETCH_INTERVAL, +// useInfiniteAdminAssets is the admin-scoped, paginated asset list: it +// accumulates pages so an admin viewing a deployment with more than one page of +// assets can load them all, exposing a single merged page plus the +// fetchNextPage/hasNextPage controls a "Load more" affordance needs. +export function useInfiniteAdminAssets( + params: AdminAssetsParams = {}, +): InfiniteAssetsResult { + // No refetchInterval: an infinite query re-polls every accumulated page on + // each tick, so a periodic refetch would multiply request volume as the admin + // loads more. Freshness comes from query invalidation on mutations and + // window-focus refetch instead. + const q = useInfiniteQuery({ + queryKey: ["admin", "assets", "infinite", params], + initialPageParam: 0, + queryFn: ({ pageParam }) => { + const sp = new URLSearchParams(); + if (params.search) sp.set("search", params.search); + sp.set("limit", String(ASSET_PAGE_SIZE)); + sp.set("offset", String(pageParam)); + return apiFetch(`/assets?${sp.toString()}`); + }, + getNextPageParam: (_last, all) => nextOffset(all), placeholderData: keepPreviousData, }); + + return useInfiniteResult(q, assetKey); } export function useAdminAsset(id: string | null) { diff --git a/ui/src/api/portal/hooks/assets.test.ts b/ui/src/api/portal/hooks/assets.test.ts new file mode 100644 index 00000000..e05165c6 --- /dev/null +++ b/ui/src/api/portal/hooks/assets.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect } from "vitest"; +import type { InfiniteData } from "@tanstack/react-query"; +import { flattenPages, nextOffset, assetKey, sharedKey } from "./assets"; +import type { Asset, PaginatedResponse, SharedAsset } from "../types"; + +function asset(id: string, overrides: Partial = {}): Asset { + return { + id, + owner_id: "u1", + owner_email: "u@example.com", + name: `asset-${id}`, + content_type: "text/html", + s3_bucket: "b", + s3_key: "k", + size_bytes: 1, + tags: [], + created_at: "2025-01-01T00:00:00Z", + updated_at: "2025-01-01T00:00:00Z", + ...overrides, + } as Asset; +} + +function page( + data: Asset[], + total: number, + extra: Partial> = {}, +): PaginatedResponse { + return { data, total, limit: 50, offset: 0, ...extra }; +} + +function infinite(pages: PaginatedResponse[]): InfiniteData> { + return { pages, pageParams: pages.map((_, i) => i * 50) }; +} + +describe("flattenPages", () => { + it("returns undefined before the first page resolves", () => { + expect(flattenPages(undefined, assetKey)).toBeUndefined(); + expect(flattenPages(infinite([]), assetKey)).toBeUndefined(); + }); + + it("concatenates rows across pages in fetch order", () => { + const merged = flattenPages( + infinite([page([asset("a"), asset("b")], 3), page([asset("c")], 3)]), + assetKey, + ); + expect(merged?.data.map((a) => a.id)).toEqual(["a", "b", "c"]); + }); + + it("de-duplicates rows that reappear across pages (window shift on insert)", () => { + // Offset paging over a created_at DESC list can re-emit a row when an + // insert shifts the window; the merged list must keep only the first. + const merged = flattenPages( + infinite([page([asset("a"), asset("b")], 3), page([asset("b"), asset("c")], 4)]), + assetKey, + ); + expect(merged?.data.map((a) => a.id)).toEqual(["a", "b", "c"]); + }); + + it("takes total from the latest page and limit from the first", () => { + const merged = flattenPages( + infinite([page([asset("a")], 200, { limit: 50 }), page([asset("b")], 230, { limit: 25 })]), + assetKey, + ); + expect(merged?.total).toBe(230); + expect(merged?.limit).toBe(50); + }); + + it("unions share summaries across pages", () => { + const merged = flattenPages( + infinite([ + page([asset("a")], 2, { share_summaries: { a: { has_user_share: true, has_public_link: false } } }), + page([asset("b")], 2, { share_summaries: { b: { has_user_share: false, has_public_link: true } } }), + ]), + assetKey, + ); + expect(Object.keys(merged?.share_summaries ?? {}).sort()).toEqual(["a", "b"]); + }); +}); + +describe("nextOffset", () => { + it("returns the fetched-row count while more rows remain", () => { + expect(nextOffset([page([asset("a"), asset("b")], 5)])).toBe(2); + }); + + it("returns undefined once every row is fetched", () => { + expect(nextOffset([page([asset("a")], 2), page([asset("b")], 2)])).toBeUndefined(); + }); + + it("uses the latest page's total so rows added after the first fetch stay reachable", () => { + // First page reported total 2; a later page reports 4 (rows added). Fetched + // count is 3, below the fresh total of 4, so paging continues (offset 3). + // Under a first-page-total cap of 2 it would have wrongly stopped. + expect(nextOffset([page([asset("a"), asset("b")], 2), page([asset("c")], 4)])).toBe(3); + }); + + it("stops on an empty trailing page even if total is stale-high", () => { + // total says 3 but the last fetch returned nothing (rows deleted); do not + // spin the Load-more button forever. + expect(nextOffset([page([asset("a")], 3), page([], 3)])).toBeUndefined(); + }); +}); + +describe("key extractors", () => { + it("assetKey keys by asset id; sharedKey keys by the underlying asset id", () => { + expect(assetKey(asset("x"))).toBe("x"); + const shared = { asset: asset("y"), share_id: "s", shared_by: "z", shared_at: "", permission: "viewer" } as SharedAsset; + expect(sharedKey(shared)).toBe("y"); + }); +}); diff --git a/ui/src/api/portal/hooks/assets.ts b/ui/src/api/portal/hooks/assets.ts index 4e7cd1fb..50404d23 100644 --- a/ui/src/api/portal/hooks/assets.ts +++ b/ui/src/api/portal/hooks/assets.ts @@ -1,4 +1,11 @@ -import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; +import { + useQuery, + useInfiniteQuery, + useMutation, + useQueryClient, + type InfiniteData, +} from "@tanstack/react-query"; +import { useMemo } from "react"; import { apiFetch, apiFetchRaw } from "../client"; import type { Asset, @@ -113,19 +120,146 @@ export function useShares(assetId: string) { }); } -export function useSharedWithMe(params?: { limit?: number; offset?: number }) { - const searchParams = new URLSearchParams(); - if (params?.limit) searchParams.set("limit", String(params.limit)); - if (params?.offset) searchParams.set("offset", String(params.offset)); - const qs = searchParams.toString(); +// --- Paginated (infinite) queries --- - return useQuery({ - queryKey: ["shared-with-me", params], - queryFn: () => - apiFetch>( - `/shared-with-me${qs ? `?${qs}` : ""}`, - ), +// ASSET_PAGE_SIZE is the number of assets requested per page. It matches the +// server-side default and stays well under the API's max limit (200), so the +// asset library loads incrementally rather than capping at a single page. +export const ASSET_PAGE_SIZE = 50; + +// assetKey / sharedKey extract the stable identity of a row for de-duplication. +// They are module-level constants so flattenPages memoization stays stable +// across renders. A shared asset is keyed by the underlying asset id. +export const assetKey = (a: Asset): string => a.id; +export const sharedKey = (s: SharedAsset): string => s.asset.id; + +// nextOffset returns the offset for the next page: the count of rows already +// fetched (the server offset, not the de-duplicated row count), or undefined +// once every row has been fetched. The cap comes from the LATEST page's total +// so rows inserted after the first fetch are still reachable, and an empty +// trailing page ends pagination even if total is stale-high (rows deleted +// between the count and the fetch), so "Load more" can't spin forever. +export function nextOffset(pages: PaginatedResponse[]): number | undefined { + const last = pages[pages.length - 1]; + if (last && last.data.length === 0) return undefined; + const fetched = pages.reduce((n, p) => n + p.data.length, 0); + return fetched < (last?.total ?? 0) ? fetched : undefined; +} + +// InfiniteAssetsResult is the flattened view an infinite asset query exposes to +// list pages: a single accumulated page (all loaded rows merged) plus the +// controls needed to render a "Load more" affordance. +export interface InfiniteAssetsResult { + data: PaginatedResponse | undefined; + isLoading: boolean; + hasNextPage: boolean; + isFetchingNextPage: boolean; + fetchNextPage: () => void; +} + +// flattenPages merges every fetched page into one PaginatedResponse: rows are +// concatenated in fetch order and de-duplicated by keyOf (offset paging over a +// created_at DESC list can re-emit a row when inserts shift the window, which +// would otherwise collide React keys and overcount), total comes from the +// latest page (most current count), limit from the first, and per-asset share +// summaries are unioned across pages. Returns undefined before the first page +// resolves so callers can distinguish "loading" from "empty". +export function flattenPages( + pages: InfiniteData> | undefined, + keyOf: (item: T) => string, +): PaginatedResponse | undefined { + const list = pages?.pages; + const first = list?.[0]; + if (!list || !first) return undefined; + const last = list[list.length - 1] ?? first; + + const seen = new Set(); + const data: T[] = []; + for (const page of list) { + for (const item of page.data) { + const k = keyOf(item); + if (seen.has(k)) continue; + seen.add(k); + data.push(item); + } + } + + return { + data, + total: last.total, + limit: first.limit, + offset: 0, + share_summaries: Object.assign( + {}, + ...list.map((p) => p.share_summaries ?? {}), + ), + }; +} + +// useInfiniteResult adapts a TanStack infinite query into the flattened +// InfiniteAssetsResult the list pages consume, so useInfiniteAssets, +// useInfiniteSharedWithMe, and the admin variant share one merge/return path. +export function useInfiniteResult( + q: { + data: InfiniteData> | undefined; + isLoading: boolean; + hasNextPage: boolean; + isFetchingNextPage: boolean; + fetchNextPage: () => unknown; + }, + keyOf: (item: T) => string, +): InfiniteAssetsResult { + const data = useMemo(() => flattenPages(q.data, keyOf), [q.data, keyOf]); + return { + data, + isLoading: q.isLoading, + hasNextPage: q.hasNextPage, + isFetchingNextPage: q.isFetchingNextPage, + fetchNextPage: () => void q.fetchNextPage(), + }; +} + +// useInfiniteAssets is the paginated counterpart of useAssets: it accumulates +// pages so a caller with more than one page of assets can load them all, +// exposing a single merged page plus fetchNextPage/hasNextPage controls. +export function useInfiniteAssets(params?: { + content_type?: string; + tag?: string; +}): InfiniteAssetsResult { + const q = useInfiniteQuery({ + queryKey: ["assets", "infinite", params], + initialPageParam: 0, + queryFn: ({ pageParam }) => { + const sp = new URLSearchParams(); + if (params?.content_type) sp.set("content_type", params.content_type); + if (params?.tag) sp.set("tag", params.tag); + sp.set("limit", String(ASSET_PAGE_SIZE)); + sp.set("offset", String(pageParam)); + return apiFetch>(`/assets?${sp.toString()}`); + }, + getNextPageParam: (_last, all) => nextOffset(all), }); + + return useInfiniteResult(q, assetKey); +} + +// useInfiniteSharedWithMe is the paginated shared-with-me list. +export function useInfiniteSharedWithMe(): InfiniteAssetsResult { + const q = useInfiniteQuery({ + queryKey: ["shared-with-me", "infinite"], + initialPageParam: 0, + queryFn: ({ pageParam }) => { + const sp = new URLSearchParams(); + sp.set("limit", String(ASSET_PAGE_SIZE)); + sp.set("offset", String(pageParam)); + return apiFetch>( + `/shared-with-me?${sp.toString()}`, + ); + }, + getNextPageParam: (_last, all) => nextOffset(all), + }); + + return useInfiniteResult(q, sharedKey); } // --- Mutations --- diff --git a/ui/src/hooks/useInfiniteScroll.test.ts b/ui/src/hooks/useInfiniteScroll.test.ts new file mode 100644 index 00000000..420b7914 --- /dev/null +++ b/ui/src/hooks/useInfiniteScroll.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { renderHook } from "@testing-library/react"; +import { act } from "react"; +import type { RefObject } from "react"; +import { useInfiniteScroll } from "./useInfiniteScroll"; + +// Build a scroll container (overflow-auto) with a sentinel child and mocked +// scroll geometry, then render the hook against the sentinel. jsdom reports 0 +// for scroll* by default, so each dimension is defined explicitly. +function setup(opts: { + hasMore: boolean; + isLoading: boolean; + scrollTop: number; + clientHeight: number; + scrollHeight: number; +}) { + const scroller = document.createElement("div"); + scroller.style.overflowY = "auto"; + const sentinel = document.createElement("div"); + scroller.appendChild(sentinel); + document.body.appendChild(scroller); + + Object.defineProperty(scroller, "scrollHeight", { value: opts.scrollHeight, configurable: true }); + Object.defineProperty(scroller, "clientHeight", { value: opts.clientHeight, configurable: true }); + Object.defineProperty(scroller, "scrollTop", { value: opts.scrollTop, writable: true, configurable: true }); + + const onLoadMore = vi.fn(); + const ref = { current: sentinel } as RefObject; + renderHook(() => + useInfiniteScroll(ref, { hasMore: opts.hasMore, isLoading: opts.isLoading, onLoadMore, threshold: 100 }), + ); + return { scroller, onLoadMore }; +} + +afterEach(() => { + document.body.innerHTML = ""; +}); + +describe("useInfiniteScroll", () => { + it("loads immediately when the list already ends within view (near bottom on mount)", () => { + const { onLoadMore } = setup({ hasMore: true, isLoading: false, scrollTop: 900, clientHeight: 100, scrollHeight: 1000 }); + expect(onLoadMore).toHaveBeenCalledTimes(1); + }); + + it("loads when a scroll brings the list near the bottom", () => { + const { scroller, onLoadMore } = setup({ hasMore: true, isLoading: false, scrollTop: 0, clientHeight: 100, scrollHeight: 1000 }); + expect(onLoadMore).not.toHaveBeenCalled(); // top of a long list + act(() => { + Object.defineProperty(scroller, "scrollTop", { value: 850, configurable: true }); + scroller.dispatchEvent(new Event("scroll")); + }); + expect(onLoadMore).toHaveBeenCalledTimes(1); + }); + + it("does nothing when there is no next page", () => { + const { scroller, onLoadMore } = setup({ hasMore: false, isLoading: false, scrollTop: 950, clientHeight: 100, scrollHeight: 1000 }); + act(() => scroller.dispatchEvent(new Event("scroll"))); + expect(onLoadMore).not.toHaveBeenCalled(); + }); + + it("does nothing while a load is already in flight", () => { + const { scroller, onLoadMore } = setup({ hasMore: true, isLoading: true, scrollTop: 950, clientHeight: 100, scrollHeight: 1000 }); + act(() => scroller.dispatchEvent(new Event("scroll"))); + expect(onLoadMore).not.toHaveBeenCalled(); + }); + + it("does not fire on a degenerate zero-height layout (jsdom / unmeasured)", () => { + const { onLoadMore } = setup({ hasMore: true, isLoading: false, scrollTop: 0, clientHeight: 0, scrollHeight: 0 }); + expect(onLoadMore).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/src/hooks/useInfiniteScroll.ts b/ui/src/hooks/useInfiniteScroll.ts new file mode 100644 index 00000000..50217ee6 --- /dev/null +++ b/ui/src/hooks/useInfiniteScroll.ts @@ -0,0 +1,74 @@ +import { useEffect, useRef, type RefObject } from "react"; + +interface InfiniteScrollOptions { + /** Whether there is another page to fetch. */ + hasMore: boolean; + /** Whether a page fetch is currently in flight. */ + isLoading: boolean; + /** Invoked when the list is scrolled near the bottom and a load is warranted. */ + onLoadMore: () => void; + /** How far (px) from the bottom to pre-fetch (default 600). */ + threshold?: number; +} + +/** + * nearestScrollParent walks up from `el` to the first ancestor that scrolls + * vertically, or null if none. The portal renders its list inside a scrolling + * `
`, so the scroll happens on that container rather than + * the window — the listener must attach there. + */ +function nearestScrollParent(el: HTMLElement | null): HTMLElement | null { + let node = el?.parentElement ?? null; + while (node) { + const oy = getComputedStyle(node).overflowY; + if (oy === "auto" || oy === "scroll" || oy === "overlay") return node; + node = node.parentElement; + } + return null; +} + +/** + * useInfiniteScroll auto-invokes `onLoadMore` when the list scrolls within + * `threshold` px of the bottom of its scroll container. A scroll listener is + * used rather than IntersectionObserver because the portal's list lives inside + * a nested `overflow-auto` container where an observer proved unreliable. It is + * a no-op when there is nothing more to load or a load is already in flight, and + * degenerate layouts (zero scrollHeight, e.g. jsdom in tests) never trigger a + * load — callers keep a manual "Load more" control as the fallback/indicator. + * + * The latest `onLoadMore` is read through a ref so its closure (which may depend + * on the active scope/filters) stays current without re-attaching the listener. + */ +export function useInfiniteScroll( + ref: RefObject, + { hasMore, isLoading, onLoadMore, threshold = 600 }: InfiniteScrollOptions, +): void { + const onLoadMoreRef = useRef(onLoadMore); + onLoadMoreRef.current = onLoadMore; + + useEffect(() => { + if (!hasMore || isLoading) return; + const scroller = nearestScrollParent(ref.current); + + const nearBottom = (): boolean => { + const st = scroller ? scroller.scrollTop : window.scrollY; + const ch = scroller ? scroller.clientHeight : window.innerHeight; + const sh = scroller + ? scroller.scrollHeight + : document.documentElement.scrollHeight; + // sh > 0 guards degenerate layouts (jsdom reports 0) so tests do not + // auto-fire; ch < sh ensures there is actually something to scroll to. + return sh > 0 && ch < sh && st + ch >= sh - threshold; + }; + + const check = (): void => { + if (nearBottom()) onLoadMoreRef.current(); + }; + + const target: HTMLElement | Window = scroller ?? window; + target.addEventListener("scroll", check, { passive: true }); + // Fire once in case the list already ends within the viewport (short page). + check(); + return () => target.removeEventListener("scroll", check); + }, [ref, hasMore, isLoading, threshold]); +} diff --git a/ui/src/pages/assets/AdminAssetsPage.tsx b/ui/src/pages/assets/AdminAssetsPage.tsx index cc37b8fa..cd7b2326 100644 --- a/ui/src/pages/assets/AdminAssetsPage.tsx +++ b/ui/src/pages/assets/AdminAssetsPage.tsx @@ -1,6 +1,7 @@ -import { useState, useEffect } from "react"; +import { useState, useEffect, useRef } from "react"; import { Search, FileText, Image, Code, File, Users, Globe, Table2 } from "lucide-react"; -import { useAdminAssets } from "@/api/admin/hooks"; +import { useInfiniteAdminAssets } from "@/api/admin/hooks"; +import { useInfiniteScroll } from "@/hooks/useInfiniteScroll"; import { formatBytes, formatOwner } from "@/lib/format"; interface Props { @@ -25,12 +26,22 @@ export function AdminAssetsPage({ onNavigate }: Props) { return () => clearTimeout(timer); }, [search]); - const { data, isLoading } = useAdminAssets({ - search: debouncedSearch || undefined, - }); + const { data, isLoading, hasNextPage, isFetchingNextPage, fetchNextPage } = + useInfiniteAdminAssets({ + search: debouncedSearch || undefined, + }); const assets = data?.data ?? []; + // Infinite scroll: auto-load the next page when the sentinel scrolls into + // view. The Load-more button below stays as a manual fallback / indicator. + const sentinelRef = useRef(null); + useInfiniteScroll(sentinelRef, { + hasMore: hasNextPage, + isLoading: isFetchingNextPage, + onLoadMore: fetchNextPage, + }); + return (
{/* Search */} @@ -114,7 +125,24 @@ export function AdminAssetsPage({ onNavigate }: Props) {
)} - {data && data.total > data.limit && ( + {hasNextPage && ( + <> + {/* Sentinel: observed to auto-load the next page on scroll. */} +